From 1f98eae1cf916a18007a26e1740b0e65aa7ff752 Mon Sep 17 00:00:00 2001 From: aus-social <42644106+aus-social@users.noreply.github.com> Date: Thu, 4 Oct 2018 20:36:53 +1000 Subject: Lint pass (#8876) --- db/migrate/20160306172223_create_doorkeeper_tables.rb | 4 ++-- db/migrate/20161006213403_rails_settings_migration.rb | 2 +- db/migrate/20170606113804_change_tag_search_index_to_btree.rb | 1 - db/migrate/20170716191202_add_hide_notifications_to_mute.rb | 2 +- db/migrate/20170920032311_fix_reblogs_in_feeds.rb | 2 +- db/migrate/20171028221157_add_reblogs_to_follows.rb | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) (limited to 'db/migrate') diff --git a/db/migrate/20160306172223_create_doorkeeper_tables.rb b/db/migrate/20160306172223_create_doorkeeper_tables.rb index 9e173a43f..462343e88 100644 --- a/db/migrate/20160306172223_create_doorkeeper_tables.rb +++ b/db/migrate/20160306172223_create_doorkeeper_tables.rb @@ -34,12 +34,12 @@ class CreateDoorkeeperTables < ActiveRecord::Migration[4.2] # https://github.com/doorkeeper-gem/doorkeeper/tree/v3.0.0.rc1#custom-access-token-generator # # t.text :token, null: false - t.string :token, null: false + t.string :token, null: false t.string :refresh_token t.integer :expires_in t.datetime :revoked_at - t.datetime :created_at, null: false + t.datetime :created_at, null: false t.string :scopes end diff --git a/db/migrate/20161006213403_rails_settings_migration.rb b/db/migrate/20161006213403_rails_settings_migration.rb index 3b2e637fc..42875d7cb 100644 --- a/db/migrate/20161006213403_rails_settings_migration.rb +++ b/db/migrate/20161006213403_rails_settings_migration.rb @@ -7,7 +7,7 @@ end class RailsSettingsMigration < MIGRATION_BASE_CLASS def self.up create_table :settings do |t| - t.string :var, :null => false + t.string :var, :null => false t.text :value t.references :target, :null => false, :polymorphic => true t.timestamps :null => true diff --git a/db/migrate/20170606113804_change_tag_search_index_to_btree.rb b/db/migrate/20170606113804_change_tag_search_index_to_btree.rb index 5248e1720..979df2e74 100644 --- a/db/migrate/20170606113804_change_tag_search_index_to_btree.rb +++ b/db/migrate/20170606113804_change_tag_search_index_to_btree.rb @@ -1,5 +1,4 @@ class ChangeTagSearchIndexToBtree < ActiveRecord::Migration[5.1] - def up remove_index :tags, name: :hashtag_search_index execute 'CREATE INDEX hashtag_search_index ON tags (name text_pattern_ops);' diff --git a/db/migrate/20170716191202_add_hide_notifications_to_mute.rb b/db/migrate/20170716191202_add_hide_notifications_to_mute.rb index 0410938c9..a498396b7 100644 --- a/db/migrate/20170716191202_add_hide_notifications_to_mute.rb +++ b/db/migrate/20170716191202_add_hide_notifications_to_mute.rb @@ -8,7 +8,7 @@ class AddHideNotificationsToMute < ActiveRecord::Migration[5.1] def up add_column_with_default :mutes, :hide_notifications, :boolean, default: true, allow_null: false end - + def down remove_column :mutes, :hide_notifications end diff --git a/db/migrate/20170920032311_fix_reblogs_in_feeds.rb b/db/migrate/20170920032311_fix_reblogs_in_feeds.rb index 439c5fca0..5654bf6f8 100644 --- a/db/migrate/20170920032311_fix_reblogs_in_feeds.rb +++ b/db/migrate/20170920032311_fix_reblogs_in_feeds.rb @@ -28,7 +28,7 @@ class FixReblogsInFeeds < ActiveRecord::Migration[5.1] -- So, first, we iterate over the user's feed to find any reblogs. local items = redis.call('zrange', timeline_key, 0, -1, 'withscores') - + for i = 1, #items, 2 do local reblogged_id = items[i] local reblogging_id = items[i + 1] diff --git a/db/migrate/20171028221157_add_reblogs_to_follows.rb b/db/migrate/20171028221157_add_reblogs_to_follows.rb index 4b5d5b7ff..3b2e46ed8 100644 --- a/db/migrate/20171028221157_add_reblogs_to_follows.rb +++ b/db/migrate/20171028221157_add_reblogs_to_follows.rb @@ -11,7 +11,7 @@ class AddReblogsToFollows < ActiveRecord::Migration[5.1] add_column_with_default :follow_requests, :show_reblogs, :boolean, default: true, allow_null: false end end - + def down remove_column :follows, :show_reblogs remove_column :follow_requests, :show_reblogs -- cgit From 0a4739c7324a96cee148373ccc7b57b9c7097b42 Mon Sep 17 00:00:00 2001 From: aus-social <42644106+aus-social@users.noreply.github.com> Date: Fri, 5 Oct 2018 01:38:04 +1000 Subject: lint pass 2 (#8878) * Code quality pass * Typofix * Update applications_controller_spec.rb * Update applications_controller_spec.rb --- config/environments/production.rb | 2 +- config/initializers/http_client_proxy.rb | 2 +- config/initializers/omniauth.rb | 2 +- config/initializers/open_uri_redirection.rb | 2 +- config/initializers/rack_attack.rb | 2 +- config/initializers/sidekiq.rb | 2 +- config/puma.rb | 2 +- .../20160314164231_add_owner_to_application.rb | 2 +- ...105224407_add_shortcode_to_media_attachments.rb | 4 +- spec/controllers/admin/statuses_controller_spec.rb | 2 +- .../concerns/export_controller_concern_spec.rb | 1 + .../settings/applications_controller_spec.rb | 64 +++++++++++----------- spec/models/account_spec.rb | 2 +- spec/models/user_spec.rb | 2 +- spec/views/stream_entries/show.html.haml_spec.rb | 2 +- 15 files changed, 47 insertions(+), 46 deletions(-) (limited to 'db/migrate') diff --git a/config/environments/production.rb b/config/environments/production.rb index 30239671c..ed2d885b0 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -42,7 +42,7 @@ Rails.application.configure do config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX # Allow to specify public IP of reverse proxy if it's needed - config.action_dispatch.trusted_proxies = ENV['TRUSTED_PROXY_IP'].split.map { |item| IPAddr.new(item) } unless ENV['TRUSTED_PROXY_IP'].blank? + config.action_dispatch.trusted_proxies = ENV['TRUSTED_PROXY_IP'].split.map { |item| IPAddr.new(item) } if ENV['TRUSTED_PROXY_IP'].present? # Use the lowest log level to ensure availability of diagnostic information # when problems arise. diff --git a/config/initializers/http_client_proxy.rb b/config/initializers/http_client_proxy.rb index e607aff3c..9d7b16e69 100644 --- a/config/initializers/http_client_proxy.rb +++ b/config/initializers/http_client_proxy.rb @@ -6,7 +6,7 @@ Rails.application.configure do raise "No proxy host" unless proxy.host host = proxy.host - host = host[1...-1] if host[0] == '[' #for IPv6 address + host = host[1...-1] if host[0] == '[' # for IPv6 address config.x.http_client_proxy[:proxy] = { proxy_address: host, proxy_port: proxy.port, proxy_username: proxy.user, proxy_password: proxy.password }.compact end diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb index 21b58eb5b..254e751d4 100644 --- a/config/initializers/omniauth.rb +++ b/config/initializers/omniauth.rb @@ -3,7 +3,7 @@ Rails.application.config.middleware.use OmniAuth::Builder do end Devise.setup do |config| - # Devise omniauth strategies + # Devise omniauth strategies options = {} options[:redirect_at_sign_in] = ENV['OAUTH_REDIRECT_AT_SIGN_IN'] == 'true' diff --git a/config/initializers/open_uri_redirection.rb b/config/initializers/open_uri_redirection.rb index e24fdecab..ea2dcffea 100644 --- a/config/initializers/open_uri_redirection.rb +++ b/config/initializers/open_uri_redirection.rb @@ -1,7 +1,7 @@ require 'open-uri' module OpenURI - def OpenURI.redirectable?(uri1, uri2) # :nodoc: + def self.redirectable?(uri1, uri2) # :nodoc: uri1.scheme.downcase == uri2.scheme.downcase || (/\A(?:http|https|ftp)\z/i =~ uri1.scheme && /\A(?:http|https|ftp)\z/i =~ uri2.scheme) end diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 0ca0a7e7f..8756b8fbf 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -42,7 +42,7 @@ class Rack::Attack # (blocklist & throttles are skipped) Rack::Attack.safelist('allow from localhost') do |req| # Requests are allowed if the return value is truthy - '127.0.0.1' == req.ip || '::1' == req.ip + req.ip == '127.0.0.1' || req.ip == '::1' end throttle('throttle_authenticated_api', limit: 300, period: 5.minutes) do |req| diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 05c804100..7f8a40d7b 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -19,4 +19,4 @@ Sidekiq.configure_client do |config| config.redis = redis_params end -Sidekiq::Logging.logger.level = ::Logger::const_get(ENV.fetch('RAILS_LOG_LEVEL', 'info').upcase.to_s) +Sidekiq::Logging.logger.level = ::Logger.const_get(ENV.fetch('RAILS_LOG_LEVEL', 'info').upcase.to_s) diff --git a/config/puma.rb b/config/puma.rb index 0397b8920..5ebf5ed19 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -1,7 +1,7 @@ threads_count = ENV.fetch('MAX_THREADS') { 5 }.to_i threads threads_count, threads_count -if ENV['SOCKET'] then +if ENV['SOCKET'] bind 'unix://' + ENV['SOCKET'] else port ENV.fetch('PORT') { 3000 } diff --git a/db/migrate/20160314164231_add_owner_to_application.rb b/db/migrate/20160314164231_add_owner_to_application.rb index 1919f09a1..553c18b5e 100644 --- a/db/migrate/20160314164231_add_owner_to_application.rb +++ b/db/migrate/20160314164231_add_owner_to_application.rb @@ -4,4 +4,4 @@ class AddOwnerToApplication < ActiveRecord::Migration[4.2] add_column :oauth_applications, :owner_type, :string, null: true add_index :oauth_applications, [:owner_id, :owner_type] end -end \ No newline at end of file +end diff --git a/db/migrate/20170105224407_add_shortcode_to_media_attachments.rb b/db/migrate/20170105224407_add_shortcode_to_media_attachments.rb index 2685ae150..fba46a4b6 100644 --- a/db/migrate/20170105224407_add_shortcode_to_media_attachments.rb +++ b/db/migrate/20170105224407_add_shortcode_to_media_attachments.rb @@ -8,7 +8,7 @@ class AddShortcodeToMediaAttachments < ActiveRecord::Migration[5.0] end def down - remove_index :media_attachments, :shortcode - remove_column :media_attachments, :shortcode + remove_index :media_attachments, :shortcode + remove_column :media_attachments, :shortcode end end diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index 6afcc1442..1a08c10b7 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -24,7 +24,7 @@ describe Admin::StatusesController do end it 'returns http success with media' do - get :index, params: { account_id: account.id , media: true } + get :index, params: { account_id: account.id, media: true } statuses = assigns(:statuses).to_a expect(statuses.size).to eq 1 diff --git a/spec/controllers/concerns/export_controller_concern_spec.rb b/spec/controllers/concerns/export_controller_concern_spec.rb index 6a13db69d..e5861c801 100644 --- a/spec/controllers/concerns/export_controller_concern_spec.rb +++ b/spec/controllers/concerns/export_controller_concern_spec.rb @@ -8,6 +8,7 @@ describe ApplicationController, type: :controller do def index send_export_file end + def export_data @export.account.username end diff --git a/spec/controllers/settings/applications_controller_spec.rb b/spec/controllers/settings/applications_controller_spec.rb index fd4b10071..29c278148 100644 --- a/spec/controllers/settings/applications_controller_spec.rb +++ b/spec/controllers/settings/applications_controller_spec.rb @@ -47,13 +47,13 @@ describe Settings::ApplicationsController do context 'success (passed scopes as a String)' do def call_create post :create, params: { - doorkeeper_application: { - name: 'My New App', - redirect_uri: 'urn:ietf:wg:oauth:2.0:oob', - website: 'http://google.com', - scopes: 'read write follow' - } - } + doorkeeper_application: { + name: 'My New App', + redirect_uri: 'urn:ietf:wg:oauth:2.0:oob', + website: 'http://google.com', + scopes: 'read write follow' + } + } response end @@ -69,13 +69,13 @@ describe Settings::ApplicationsController do context 'success (passed scopes as an Array)' do def call_create post :create, params: { - doorkeeper_application: { - name: 'My New App', - redirect_uri: 'urn:ietf:wg:oauth:2.0:oob', - website: 'http://google.com', - scopes: [ 'read', 'write', 'follow' ] - } - } + doorkeeper_application: { + name: 'My New App', + redirect_uri: 'urn:ietf:wg:oauth:2.0:oob', + website: 'http://google.com', + scopes: [ 'read', 'write', 'follow' ] + } + } response end @@ -91,13 +91,13 @@ describe Settings::ApplicationsController do context 'failure' do before do post :create, params: { - doorkeeper_application: { - name: '', - redirect_uri: '', - website: '', - scopes: [] - } - } + doorkeeper_application: { + name: '', + redirect_uri: '', + website: '', + scopes: [] + } + } end it 'returns http success' do @@ -120,9 +120,9 @@ describe Settings::ApplicationsController do def call_update patch :update, params: { - id: app.id, - doorkeeper_application: opts - } + id: app.id, + doorkeeper_application: opts + } response end @@ -139,14 +139,14 @@ describe Settings::ApplicationsController do context 'failure' do before do patch :update, params: { - id: app.id, - doorkeeper_application: { - name: '', - redirect_uri: '', - website: '', - scopes: [] - } - } + id: app.id, + doorkeeper_application: { + name: '', + redirect_uri: '', + website: '', + scopes: [] + } + } end it 'returns http success' do diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index ae19251ae..60d13d32e 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -275,7 +275,7 @@ RSpec.describe Account, type: :model do subject { Fabricate(:account) } - context 'when the status is a reblog of another status'do + context 'when the status is a reblog of another status' do let(:original_reblog) do author = Fabricate(:account, username: 'original_reblogger') Fabricate(:status, reblog: original_status, account: author) diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 015e90edc..42198cb4d 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -254,7 +254,7 @@ RSpec.describe User, type: :model do it_behaves_like 'Settings-extended' do def create! - User.create!(account: Fabricate(:account), email: 'foo@mastodon.space', password: 'abcd1234' ) + User.create!(account: Fabricate(:account), email: 'foo@mastodon.space', password: 'abcd1234') end def fabricate diff --git a/spec/views/stream_entries/show.html.haml_spec.rb b/spec/views/stream_entries/show.html.haml_spec.rb index e08419596..93f0adb99 100644 --- a/spec/views/stream_entries/show.html.haml_spec.rb +++ b/spec/views/stream_entries/show.html.haml_spec.rb @@ -49,7 +49,7 @@ describe 'stream_entries/show.html.haml', without_verify_partial_doubles: true d assign(:stream_entry, reply.stream_entry) assign(:account, alice) assign(:type, reply.stream_entry.activity_type.downcase) - assign(:ancestors, reply.stream_entry.activity.ancestors(1, bob) ) + assign(:ancestors, reply.stream_entry.activity.ancestors(1, bob)) assign(:descendant_threads, [{ statuses: reply.stream_entry.activity.descendants(1) }]) render -- cgit From 886af5ce199d4fa178aab280a582cd3a4edd7745 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 7 Oct 2018 04:40:37 +0200 Subject: Add fallback for PostgreSQL without upsert in CopyStatusStats (#8903) Fix #8590 --- db/migrate/20180812173710_copy_status_stats.rb | 50 +++++++++++++++++++++----- 1 file changed, 41 insertions(+), 9 deletions(-) (limited to 'db/migrate') diff --git a/db/migrate/20180812173710_copy_status_stats.rb b/db/migrate/20180812173710_copy_status_stats.rb index 850aa9c13..ff10c18d9 100644 --- a/db/migrate/20180812173710_copy_status_stats.rb +++ b/db/migrate/20180812173710_copy_status_stats.rb @@ -3,15 +3,10 @@ class CopyStatusStats < ActiveRecord::Migration[5.2] def up safety_assured do - Status.unscoped.select('id').find_in_batches(batch_size: 5_000) do |statuses| - execute <<-SQL.squish - INSERT INTO status_stats (status_id, reblogs_count, favourites_count, created_at, updated_at) - SELECT id, reblogs_count, favourites_count, created_at, updated_at - FROM statuses - WHERE id IN (#{statuses.map(&:id).join(', ')}) - ON CONFLICT (status_id) DO UPDATE - SET reblogs_count = EXCLUDED.reblogs_count, favourites_count = EXCLUDED.favourites_count - SQL + if supports_upsert? + up_fast + else + up_slow end end end @@ -19,4 +14,41 @@ class CopyStatusStats < ActiveRecord::Migration[5.2] def down # Nothing end + + private + + def supports_upsert? + version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i + version >= 90500 + end + + def up_fast + say 'Upsert is available, importing counters using the fast method' + + Status.unscoped.select('id').find_in_batches(batch_size: 5_000) do |statuses| + execute <<-SQL.squish + INSERT INTO status_stats (status_id, reblogs_count, favourites_count, created_at, updated_at) + SELECT id, reblogs_count, favourites_count, created_at, updated_at + FROM statuses + WHERE id IN (#{statuses.map(&:id).join(', ')}) + ON CONFLICT (status_id) DO UPDATE + SET reblogs_count = EXCLUDED.reblogs_count, favourites_count = EXCLUDED.favourites_count + SQL + end + end + + def up_slow + say 'Upsert is not available in PostgreSQL below 9.5, falling back to slow import of counters' + + # We cannot use bulk INSERT or overarching transactions here because of possible + # uniqueness violations that we need to skip over + Status.unscoped.select('id, reblogs_count, favourites_count, created_at, updated_at').find_each do |status| + begin + params = [[nil, status.id], [nil, status.reblogs_count], [nil, status.favourites_count], [nil, status.created_at], [nil, status.updated_at]] + exec_insert('INSERT INTO status_stats (status_id, reblogs_count, favourites_count, created_at, updated_at) VALUES ($1, $2, $3, $4, $5)', nil, params) + rescue ActiveRecord::RecordNotUnique + next + end + end + end end -- cgit From 774ac473736cbf348827cf6d861e7fbbb72d7623 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 7 Oct 2018 23:44:58 +0200 Subject: Add conversations API (#8832) * Add conversations API * Add web UI for conversations * Add test for conversations API * Add tests for ConversationAccount * Improve web UI * Rename ConversationAccount to AccountConversation * Remove conversations on block and mute * Change last_status_id to be a denormalization of status_ids * Add optimistic locking --- app/controllers/api/v1/conversations_controller.rb | 55 ++++++++++ app/javascript/mastodon/actions/conversations.js | 59 +++++++++++ app/javascript/mastodon/actions/streaming.js | 4 + app/javascript/mastodon/actions/timelines.js | 1 - app/javascript/mastodon/components/display_name.js | 11 +- .../direct_timeline/components/conversation.js | 85 ++++++++++++++++ .../components/conversations_list.js | 68 +++++++++++++ .../containers/conversation_container.js | 15 +++ .../containers/conversations_list_container.js | 15 +++ .../mastodon/features/direct_timeline/index.js | 27 ++--- app/javascript/mastodon/reducers/conversations.js | 79 +++++++++++++++ app/javascript/mastodon/reducers/index.js | 2 + app/javascript/mastodon/reducers/notifications.js | 2 +- app/javascript/styles/mastodon/components.scss | 42 ++++++++ app/lib/inline_renderer.rb | 2 + app/models/account_conversation.rb | 111 +++++++++++++++++++++ app/models/status.rb | 13 +++ app/serializers/rest/conversation_serializer.rb | 7 ++ app/services/after_block_service.rb | 37 ++++++- app/services/fan_out_on_write_service.rb | 6 ++ app/services/mute_service.rb | 4 +- app/services/notify_service.rb | 22 ++-- app/workers/block_worker.rb | 5 +- app/workers/mute_worker.rb | 12 +++ app/workers/push_conversation_worker.rb | 15 +++ config/environments/development.rb | 2 +- config/routes.rb | 1 + .../20180929222014_create_account_conversations.rb | 14 +++ db/schema.rb | 17 +++- .../api/v1/conversations_controller_spec.rb | 37 +++++++ .../fabricators/conversation_account_fabricator.rb | 6 ++ spec/models/account_conversation_spec.rb | 72 +++++++++++++ streaming/index.js | 12 ++- 33 files changed, 816 insertions(+), 44 deletions(-) create mode 100644 app/controllers/api/v1/conversations_controller.rb create mode 100644 app/javascript/mastodon/actions/conversations.js create mode 100644 app/javascript/mastodon/features/direct_timeline/components/conversation.js create mode 100644 app/javascript/mastodon/features/direct_timeline/components/conversations_list.js create mode 100644 app/javascript/mastodon/features/direct_timeline/containers/conversation_container.js create mode 100644 app/javascript/mastodon/features/direct_timeline/containers/conversations_list_container.js create mode 100644 app/javascript/mastodon/reducers/conversations.js create mode 100644 app/models/account_conversation.rb create mode 100644 app/serializers/rest/conversation_serializer.rb create mode 100644 app/workers/mute_worker.rb create mode 100644 app/workers/push_conversation_worker.rb create mode 100644 db/migrate/20180929222014_create_account_conversations.rb create mode 100644 spec/controllers/api/v1/conversations_controller_spec.rb create mode 100644 spec/fabricators/conversation_account_fabricator.rb create mode 100644 spec/models/account_conversation_spec.rb (limited to 'db/migrate') diff --git a/app/controllers/api/v1/conversations_controller.rb b/app/controllers/api/v1/conversations_controller.rb new file mode 100644 index 000000000..736cb21ca --- /dev/null +++ b/app/controllers/api/v1/conversations_controller.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +class Api::V1::ConversationsController < Api::BaseController + LIMIT = 20 + + before_action -> { doorkeeper_authorize! :read, :'read:statuses' } + before_action :require_user! + after_action :insert_pagination_headers + + respond_to :json + + def index + @conversations = paginated_conversations + render json: @conversations, each_serializer: REST::ConversationSerializer + end + + private + + def paginated_conversations + AccountConversation.where(account: current_account) + .paginate_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id)) + end + + def insert_pagination_headers + set_pagination_headers(next_path, prev_path) + end + + def next_path + if records_continue? + api_v1_conversations_url pagination_params(max_id: pagination_max_id) + end + end + + def prev_path + unless @conversations.empty? + api_v1_conversations_url pagination_params(min_id: pagination_since_id) + end + end + + def pagination_max_id + @conversations.last.last_status_id + end + + def pagination_since_id + @conversations.first.last_status_id + end + + def records_continue? + @conversations.size == limit_param(LIMIT) + end + + def pagination_params(core_params) + params.slice(:limit).permit(:limit).merge(core_params) + end +end diff --git a/app/javascript/mastodon/actions/conversations.js b/app/javascript/mastodon/actions/conversations.js new file mode 100644 index 000000000..3840d23ca --- /dev/null +++ b/app/javascript/mastodon/actions/conversations.js @@ -0,0 +1,59 @@ +import api, { getLinks } from '../api'; +import { + importFetchedAccounts, + importFetchedStatuses, + importFetchedStatus, +} from './importer'; + +export const CONVERSATIONS_FETCH_REQUEST = 'CONVERSATIONS_FETCH_REQUEST'; +export const CONVERSATIONS_FETCH_SUCCESS = 'CONVERSATIONS_FETCH_SUCCESS'; +export const CONVERSATIONS_FETCH_FAIL = 'CONVERSATIONS_FETCH_FAIL'; +export const CONVERSATIONS_UPDATE = 'CONVERSATIONS_UPDATE'; + +export const expandConversations = ({ maxId } = {}) => (dispatch, getState) => { + dispatch(expandConversationsRequest()); + + const params = { max_id: maxId }; + + if (!maxId) { + params.since_id = getState().getIn(['conversations', 0, 'last_status']); + } + + api(getState).get('/api/v1/conversations', { params }) + .then(response => { + const next = getLinks(response).refs.find(link => link.rel === 'next'); + + dispatch(importFetchedAccounts(response.data.reduce((aggr, item) => aggr.concat(item.accounts), []))); + dispatch(importFetchedStatuses(response.data.map(item => item.last_status).filter(x => !!x))); + dispatch(expandConversationsSuccess(response.data, next ? next.uri : null)); + }) + .catch(err => dispatch(expandConversationsFail(err))); +}; + +export const expandConversationsRequest = () => ({ + type: CONVERSATIONS_FETCH_REQUEST, +}); + +export const expandConversationsSuccess = (conversations, next) => ({ + type: CONVERSATIONS_FETCH_SUCCESS, + conversations, + next, +}); + +export const expandConversationsFail = error => ({ + type: CONVERSATIONS_FETCH_FAIL, + error, +}); + +export const updateConversations = conversation => dispatch => { + dispatch(importFetchedAccounts(conversation.accounts)); + + if (conversation.last_status) { + dispatch(importFetchedStatus(conversation.last_status)); + } + + dispatch({ + type: CONVERSATIONS_UPDATE, + conversation, + }); +}; diff --git a/app/javascript/mastodon/actions/streaming.js b/app/javascript/mastodon/actions/streaming.js index 32fc67e67..8cf055540 100644 --- a/app/javascript/mastodon/actions/streaming.js +++ b/app/javascript/mastodon/actions/streaming.js @@ -6,6 +6,7 @@ import { disconnectTimeline, } from './timelines'; import { updateNotifications, expandNotifications } from './notifications'; +import { updateConversations } from './conversations'; import { fetchFilters } from './filters'; import { getLocale } from '../locales'; @@ -31,6 +32,9 @@ export function connectTimelineStream (timelineId, path, pollingRefresh = null) case 'notification': dispatch(updateNotifications(JSON.parse(data.payload), messages, locale)); break; + case 'conversation': + dispatch(updateConversations(JSON.parse(data.payload))); + break; case 'filters_changed': dispatch(fetchFilters()); break; diff --git a/app/javascript/mastodon/actions/timelines.js b/app/javascript/mastodon/actions/timelines.js index e8fd441e1..c4fc6448c 100644 --- a/app/javascript/mastodon/actions/timelines.js +++ b/app/javascript/mastodon/actions/timelines.js @@ -76,7 +76,6 @@ export function expandTimeline(timelineId, path, params = {}, done = noOp) { export const expandHomeTimeline = ({ maxId } = {}, done = noOp) => expandTimeline('home', '/api/v1/timelines/home', { max_id: maxId }, done); export const expandPublicTimeline = ({ maxId, onlyMedia } = {}, done = noOp) => expandTimeline(`public${onlyMedia ? ':media' : ''}`, '/api/v1/timelines/public', { max_id: maxId, only_media: !!onlyMedia }, done); export const expandCommunityTimeline = ({ maxId, onlyMedia } = {}, done = noOp) => expandTimeline(`community${onlyMedia ? ':media' : ''}`, '/api/v1/timelines/public', { local: true, max_id: maxId, only_media: !!onlyMedia }, done); -export const expandDirectTimeline = ({ maxId } = {}, done = noOp) => expandTimeline('direct', '/api/v1/timelines/direct', { max_id: maxId }, done); export const expandAccountTimeline = (accountId, { maxId, withReplies } = {}) => expandTimeline(`account:${accountId}${withReplies ? ':with_replies' : ''}`, `/api/v1/accounts/${accountId}/statuses`, { exclude_replies: !withReplies, max_id: maxId }); export const expandAccountFeaturedTimeline = accountId => expandTimeline(`account:${accountId}:pinned`, `/api/v1/accounts/${accountId}/statuses`, { pinned: true }); export const expandAccountMediaTimeline = (accountId, { maxId } = {}) => expandTimeline(`account:${accountId}:media`, `/api/v1/accounts/${accountId}/statuses`, { max_id: maxId, only_media: true }); diff --git a/app/javascript/mastodon/components/display_name.js b/app/javascript/mastodon/components/display_name.js index a1c56ae35..c3a9ab921 100644 --- a/app/javascript/mastodon/components/display_name.js +++ b/app/javascript/mastodon/components/display_name.js @@ -1,18 +1,25 @@ import React from 'react'; import ImmutablePropTypes from 'react-immutable-proptypes'; +import PropTypes from 'prop-types'; export default class DisplayName extends React.PureComponent { static propTypes = { account: ImmutablePropTypes.map.isRequired, + withAcct: PropTypes.bool, + }; + + static defaultProps = { + withAcct: true, }; render () { - const displayNameHtml = { __html: this.props.account.get('display_name_html') }; + const { account, withAcct } = this.props; + const displayNameHtml = { __html: account.get('display_name_html') }; return ( - @{this.props.account.get('acct')} + {withAcct && @{account.get('acct')}} ); } diff --git a/app/javascript/mastodon/features/direct_timeline/components/conversation.js b/app/javascript/mastodon/features/direct_timeline/components/conversation.js new file mode 100644 index 000000000..f9a8d4f72 --- /dev/null +++ b/app/javascript/mastodon/features/direct_timeline/components/conversation.js @@ -0,0 +1,85 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import ImmutablePureComponent from 'react-immutable-pure-component'; +import StatusContent from '../../../components/status_content'; +import RelativeTimestamp from '../../../components/relative_timestamp'; +import DisplayName from '../../../components/display_name'; +import Avatar from '../../../components/avatar'; +import AttachmentList from '../../../components/attachment_list'; +import { HotKeys } from 'react-hotkeys'; + +export default class Conversation extends ImmutablePureComponent { + + static contextTypes = { + router: PropTypes.object, + }; + + static propTypes = { + conversationId: PropTypes.string.isRequired, + accounts: ImmutablePropTypes.list.isRequired, + lastStatus: ImmutablePropTypes.map.isRequired, + onMoveUp: PropTypes.func, + onMoveDown: PropTypes.func, + }; + + handleClick = () => { + if (!this.context.router) { + return; + } + + const { lastStatus } = this.props; + this.context.router.history.push(`/statuses/${lastStatus.get('id')}`); + } + + handleHotkeyMoveUp = () => { + this.props.onMoveUp(this.props.conversationId); + } + + handleHotkeyMoveDown = () => { + this.props.onMoveDown(this.props.conversationId); + } + + render () { + const { accounts, lastStatus, lastAccount } = this.props; + + if (lastStatus === null) { + return null; + } + + const handlers = { + moveDown: this.handleHotkeyMoveDown, + moveUp: this.handleHotkeyMoveUp, + open: this.handleClick, + }; + + let media; + + if (lastStatus.get('media_attachments').size > 0) { + media = ; + } + + return ( + +
+
+
+
{accounts.map(account => )}
+
+ +
+ +
+ +
+
+ + + + {media} +
+ + ); + } + +} diff --git a/app/javascript/mastodon/features/direct_timeline/components/conversations_list.js b/app/javascript/mastodon/features/direct_timeline/components/conversations_list.js new file mode 100644 index 000000000..4684548e0 --- /dev/null +++ b/app/javascript/mastodon/features/direct_timeline/components/conversations_list.js @@ -0,0 +1,68 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import ImmutablePureComponent from 'react-immutable-pure-component'; +import ConversationContainer from '../containers/conversation_container'; +import ScrollableList from '../../../components/scrollable_list'; +import { debounce } from 'lodash'; + +export default class ConversationsList extends ImmutablePureComponent { + + static propTypes = { + conversationIds: ImmutablePropTypes.list.isRequired, + hasMore: PropTypes.bool, + isLoading: PropTypes.bool, + onLoadMore: PropTypes.func, + shouldUpdateScroll: PropTypes.func, + }; + + getCurrentIndex = id => this.props.conversationIds.indexOf(id) + + handleMoveUp = id => { + const elementIndex = this.getCurrentIndex(id) - 1; + this._selectChild(elementIndex); + } + + handleMoveDown = id => { + const elementIndex = this.getCurrentIndex(id) + 1; + this._selectChild(elementIndex); + } + + _selectChild (index) { + const element = this.node.node.querySelector(`article:nth-of-type(${index + 1}) .focusable`); + + if (element) { + element.focus(); + } + } + + setRef = c => { + this.node = c; + } + + handleLoadOlder = debounce(() => { + const last = this.props.conversationIds.last(); + + if (last) { + this.props.onLoadMore(last); + } + }, 300, { leading: true }) + + render () { + const { conversationIds, onLoadMore, ...other } = this.props; + + return ( + + {conversationIds.map(item => ( + + ))} + + ); + } + +} diff --git a/app/javascript/mastodon/features/direct_timeline/containers/conversation_container.js b/app/javascript/mastodon/features/direct_timeline/containers/conversation_container.js new file mode 100644 index 000000000..4166ee2ac --- /dev/null +++ b/app/javascript/mastodon/features/direct_timeline/containers/conversation_container.js @@ -0,0 +1,15 @@ +import { connect } from 'react-redux'; +import Conversation from '../components/conversation'; + +const mapStateToProps = (state, { conversationId }) => { + const conversation = state.getIn(['conversations', 'items']).find(x => x.get('id') === conversationId); + const lastStatus = state.getIn(['statuses', conversation.get('last_status')], null); + + return { + accounts: conversation.get('accounts').map(accountId => state.getIn(['accounts', accountId], null)), + lastStatus, + lastAccount: lastStatus === null ? null : state.getIn(['accounts', lastStatus.get('account')], null), + }; +}; + +export default connect(mapStateToProps)(Conversation); diff --git a/app/javascript/mastodon/features/direct_timeline/containers/conversations_list_container.js b/app/javascript/mastodon/features/direct_timeline/containers/conversations_list_container.js new file mode 100644 index 000000000..81ea812ad --- /dev/null +++ b/app/javascript/mastodon/features/direct_timeline/containers/conversations_list_container.js @@ -0,0 +1,15 @@ +import { connect } from 'react-redux'; +import ConversationsList from '../components/conversations_list'; +import { expandConversations } from '../../../actions/conversations'; + +const mapStateToProps = state => ({ + conversationIds: state.getIn(['conversations', 'items']).map(x => x.get('id')), + isLoading: state.getIn(['conversations', 'isLoading'], true), + hasMore: state.getIn(['conversations', 'hasMore'], false), +}); + +const mapDispatchToProps = dispatch => ({ + onLoadMore: maxId => dispatch(expandConversations({ maxId })), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(ConversationsList); diff --git a/app/javascript/mastodon/features/direct_timeline/index.js b/app/javascript/mastodon/features/direct_timeline/index.js index 3c7e2d007..4c8485690 100644 --- a/app/javascript/mastodon/features/direct_timeline/index.js +++ b/app/javascript/mastodon/features/direct_timeline/index.js @@ -1,23 +1,19 @@ import React from 'react'; import { connect } from 'react-redux'; import PropTypes from 'prop-types'; -import StatusListContainer from '../ui/containers/status_list_container'; import Column from '../../components/column'; import ColumnHeader from '../../components/column_header'; -import { expandDirectTimeline } from '../../actions/timelines'; +import { expandConversations } from '../../actions/conversations'; import { addColumn, removeColumn, moveColumn } from '../../actions/columns'; -import { defineMessages, injectIntl, FormattedMessage } from 'react-intl'; +import { defineMessages, injectIntl } from 'react-intl'; import { connectDirectStream } from '../../actions/streaming'; +import ConversationsListContainer from './containers/conversations_list_container'; const messages = defineMessages({ title: { id: 'column.direct', defaultMessage: 'Direct messages' }, }); -const mapStateToProps = state => ({ - hasUnread: state.getIn(['timelines', 'direct', 'unread']) > 0, -}); - -export default @connect(mapStateToProps) +export default @connect() @injectIntl class DirectTimeline extends React.PureComponent { @@ -52,7 +48,7 @@ class DirectTimeline extends React.PureComponent { componentDidMount () { const { dispatch } = this.props; - dispatch(expandDirectTimeline()); + dispatch(expandConversations()); this.disconnect = dispatch(connectDirectStream()); } @@ -68,11 +64,11 @@ class DirectTimeline extends React.PureComponent { } handleLoadMore = maxId => { - this.props.dispatch(expandDirectTimeline({ maxId })); + this.props.dispatch(expandConversations({ maxId })); } render () { - const { intl, shouldUpdateScroll, hasUnread, columnId, multiColumn } = this.props; + const { intl, hasUnread, columnId, multiColumn, shouldUpdateScroll } = this.props; const pinned = !!columnId; return ( @@ -88,14 +84,7 @@ class DirectTimeline extends React.PureComponent { multiColumn={multiColumn} /> - } - shouldUpdateScroll={shouldUpdateScroll} - /> + ); } diff --git a/app/javascript/mastodon/reducers/conversations.js b/app/javascript/mastodon/reducers/conversations.js new file mode 100644 index 000000000..f339abf56 --- /dev/null +++ b/app/javascript/mastodon/reducers/conversations.js @@ -0,0 +1,79 @@ +import { Map as ImmutableMap, List as ImmutableList } from 'immutable'; +import { + CONVERSATIONS_FETCH_REQUEST, + CONVERSATIONS_FETCH_SUCCESS, + CONVERSATIONS_FETCH_FAIL, + CONVERSATIONS_UPDATE, +} from '../actions/conversations'; +import compareId from '../compare_id'; + +const initialState = ImmutableMap({ + items: ImmutableList(), + isLoading: false, + hasMore: true, +}); + +const conversationToMap = item => ImmutableMap({ + id: item.id, + accounts: ImmutableList(item.accounts.map(a => a.id)), + last_status: item.last_status.id, +}); + +const updateConversation = (state, item) => state.update('items', list => { + const index = list.findIndex(x => x.get('id') === item.id); + const newItem = conversationToMap(item); + + if (index === -1) { + return list.unshift(newItem); + } else { + return list.set(index, newItem); + } +}); + +const expandNormalizedConversations = (state, conversations, next) => { + let items = ImmutableList(conversations.map(conversationToMap)); + + return state.withMutations(mutable => { + if (!items.isEmpty()) { + mutable.update('items', list => { + list = list.map(oldItem => { + const newItemIndex = items.findIndex(x => x.get('id') === oldItem.get('id')); + + if (newItemIndex === -1) { + return oldItem; + } + + const newItem = items.get(newItemIndex); + items = items.delete(newItemIndex); + + return newItem; + }); + + list = list.concat(items); + + return list.sortBy(x => x.get('last_status'), (a, b) => compareId(a, b) * -1); + }); + } + + if (!next) { + mutable.set('hasMore', false); + } + + mutable.set('isLoading', false); + }); +}; + +export default function conversations(state = initialState, action) { + switch (action.type) { + case CONVERSATIONS_FETCH_REQUEST: + return state.set('isLoading', true); + case CONVERSATIONS_FETCH_FAIL: + return state.set('isLoading', false); + case CONVERSATIONS_FETCH_SUCCESS: + return expandNormalizedConversations(state, action.conversations, action.next); + case CONVERSATIONS_UPDATE: + return updateConversation(state, action.conversation); + default: + return state; + } +}; diff --git a/app/javascript/mastodon/reducers/index.js b/app/javascript/mastodon/reducers/index.js index 4a981fada..d3b98d4f6 100644 --- a/app/javascript/mastodon/reducers/index.js +++ b/app/javascript/mastodon/reducers/index.js @@ -27,6 +27,7 @@ import custom_emojis from './custom_emojis'; import lists from './lists'; import listEditor from './list_editor'; import filters from './filters'; +import conversations from './conversations'; const reducers = { dropdown_menu, @@ -57,6 +58,7 @@ const reducers = { lists, listEditor, filters, + conversations, }; export default combineReducers(reducers); diff --git a/app/javascript/mastodon/reducers/notifications.js b/app/javascript/mastodon/reducers/notifications.js index 0b29f19fa..d71ae00ae 100644 --- a/app/javascript/mastodon/reducers/notifications.js +++ b/app/javascript/mastodon/reducers/notifications.js @@ -69,7 +69,7 @@ const expandNormalizedNotifications = (state, notifications, next) => { } if (!next) { - mutable.set('hasMore', true); + mutable.set('hasMore', false); } mutable.set('isLoading', false); diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 90e2ed5a5..129bde856 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -825,6 +825,7 @@ &.status-direct { background: lighten($ui-base-color, 8%); + border-bottom-color: lighten($ui-base-color, 12%); } &.light { @@ -5496,3 +5497,44 @@ noscript { } } } + +.conversation { + padding: 14px 10px; + border-bottom: 1px solid lighten($ui-base-color, 8%); + cursor: pointer; + + &__header { + display: flex; + margin-bottom: 15px; + } + + &__avatars { + overflow: hidden; + flex: 1 1 auto; + + & > div { + display: flex; + flex-wrap: none; + width: 900px; + } + + .account__avatar { + margin-right: 10px; + } + } + + &__time { + flex: 0 0 auto; + font-size: 14px; + color: $darker-text-color; + text-align: right; + + .display-name { + color: $secondary-text-color; + } + } + + .attachment-list.compact { + margin-top: 15px; + } +} diff --git a/app/lib/inline_renderer.rb b/app/lib/inline_renderer.rb index 7cd9758ec..761a8822d 100644 --- a/app/lib/inline_renderer.rb +++ b/app/lib/inline_renderer.rb @@ -13,6 +13,8 @@ class InlineRenderer serializer = REST::StatusSerializer when :notification serializer = REST::NotificationSerializer + when :conversation + serializer = REST::ConversationSerializer else return end diff --git a/app/models/account_conversation.rb b/app/models/account_conversation.rb new file mode 100644 index 000000000..a7205ec1a --- /dev/null +++ b/app/models/account_conversation.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: account_conversations +# +# id :bigint(8) not null, primary key +# account_id :bigint(8) +# conversation_id :bigint(8) +# participant_account_ids :bigint(8) default([]), not null, is an Array +# status_ids :bigint(8) default([]), not null, is an Array +# last_status_id :bigint(8) +# lock_version :integer default(0), not null +# + +class AccountConversation < ApplicationRecord + after_commit :push_to_streaming_api + + belongs_to :account + belongs_to :conversation + belongs_to :last_status, class_name: 'Status' + + before_validation :set_last_status + + def participant_account_ids=(arr) + self[:participant_account_ids] = arr.sort + end + + def participant_accounts + if participant_account_ids.empty? + [account] + else + Account.where(id: participant_account_ids) + end + end + + class << self + def paginate_by_id(limit, options = {}) + if options[:min_id] + paginate_by_min_id(limit, options[:min_id]).reverse + else + paginate_by_max_id(limit, options[:max_id], options[:since_id]) + end + end + + def paginate_by_min_id(limit, min_id = nil) + query = order(arel_table[:last_status_id].asc).limit(limit) + query = query.where(arel_table[:last_status_id].gt(min_id)) if min_id.present? + query + end + + def paginate_by_max_id(limit, max_id = nil, since_id = nil) + query = order(arel_table[:last_status_id].desc).limit(limit) + query = query.where(arel_table[:last_status_id].lt(max_id)) if max_id.present? + query = query.where(arel_table[:last_status_id].gt(since_id)) if since_id.present? + query + end + + def add_status(recipient, status) + conversation = find_or_initialize_by(account: recipient, conversation_id: status.conversation_id, participant_account_ids: participants_from_status(recipient, status)) + conversation.status_ids << status.id + conversation.save + conversation + rescue ActiveRecord::StaleObjectError + retry + end + + def remove_status(recipient, status) + conversation = find_by(account: recipient, conversation_id: status.conversation_id, participant_account_ids: participants_from_status(recipient, status)) + + return if conversation.nil? + + conversation.status_ids.delete(status.id) + + if conversation.status_ids.empty? + conversation.destroy + else + conversation.save + end + + conversation + rescue ActiveRecord::StaleObjectError + retry + end + + private + + def participants_from_status(recipient, status) + ((status.mentions.pluck(:account_id) + [status.account_id]).uniq - [recipient.id]).sort + end + end + + private + + def set_last_status + self.status_ids = status_ids.sort + self.last_status_id = status_ids.last + end + + def push_to_streaming_api + return if destroyed? || !subscribed_to_timeline? + PushConversationWorker.perform_async(id) + end + + def subscribed_to_timeline? + Redis.current.exists("subscribed:#{streaming_channel}") + end + + def streaming_channel + "timeline:direct:#{account_id}" + end +end diff --git a/app/models/status.rb b/app/models/status.rb index f2b5cc6ce..f61bd0fee 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -24,6 +24,8 @@ # class Status < ApplicationRecord + before_destroy :unlink_from_conversations + include Paginable include Streamable include Cacheable @@ -473,4 +475,15 @@ class Status < ApplicationRecord reblog&.decrement_count!(:reblogs_count) if reblog? thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?) end + + def unlink_from_conversations + return unless direct_visibility? + + mentioned_accounts = mentions.includes(:account).map(&:account) + inbox_owners = mentioned_accounts.select(&:local?) + (account.local? ? [account] : []) + + inbox_owners.each do |inbox_owner| + AccountConversation.remove_status(inbox_owner, self) + end + end end diff --git a/app/serializers/rest/conversation_serializer.rb b/app/serializers/rest/conversation_serializer.rb new file mode 100644 index 000000000..08cea47d2 --- /dev/null +++ b/app/serializers/rest/conversation_serializer.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class REST::ConversationSerializer < ActiveModel::Serializer + attribute :id + has_many :participant_accounts, key: :accounts, serializer: REST::AccountSerializer + has_one :last_status, serializer: REST::StatusSerializer +end diff --git a/app/services/after_block_service.rb b/app/services/after_block_service.rb index a7dce08c7..706db0d63 100644 --- a/app/services/after_block_service.rb +++ b/app/services/after_block_service.rb @@ -2,16 +2,43 @@ class AfterBlockService < BaseService def call(account, target_account) - FeedManager.instance.clear_from_timeline(account, target_account) + clear_home_feed(account, target_account) clear_notifications(account, target_account) + clear_conversations(account, target_account) end private + def clear_home_feed(account, target_account) + FeedManager.instance.clear_from_timeline(account, target_account) + end + + def clear_conversations(account, target_account) + AccountConversation.where(account: account) + .where('? = ANY(participant_account_ids)', target_account.id) + .in_batches + .destroy_all + end + def clear_notifications(account, target_account) - Notification.where(account: account).joins(:follow).where(activity_type: 'Follow', follows: { account_id: target_account.id }).delete_all - Notification.where(account: account).joins(mention: :status).where(activity_type: 'Mention', statuses: { account_id: target_account.id }).delete_all - Notification.where(account: account).joins(:favourite).where(activity_type: 'Favourite', favourites: { account_id: target_account.id }).delete_all - Notification.where(account: account).joins(:status).where(activity_type: 'Status', statuses: { account_id: target_account.id }).delete_all + Notification.where(account: account) + .joins(:follow) + .where(activity_type: 'Follow', follows: { account_id: target_account.id }) + .delete_all + + Notification.where(account: account) + .joins(mention: :status) + .where(activity_type: 'Mention', statuses: { account_id: target_account.id }) + .delete_all + + Notification.where(account: account) + .joins(:favourite) + .where(activity_type: 'Favourite', favourites: { account_id: target_account.id }) + .delete_all + + Notification.where(account: account) + .joins(:status) + .where(activity_type: 'Status', statuses: { account_id: target_account.id }) + .delete_all end end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 5efd3edb2..ab520276b 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -13,6 +13,7 @@ class FanOutOnWriteService < BaseService if status.direct_visibility? deliver_to_mentioned_followers(status) deliver_to_direct_timelines(status) + deliver_to_own_conversation(status) else deliver_to_followers(status) deliver_to_lists(status) @@ -99,6 +100,11 @@ class FanOutOnWriteService < BaseService status.mentions.includes(:account).each do |mention| Redis.current.publish("timeline:direct:#{mention.account.id}", @payload) if mention.account.local? end + Redis.current.publish("timeline:direct:#{status.account.id}", @payload) if status.account.local? end + + def deliver_to_own_conversation(status) + AccountConversation.add_status(status.account, status) + end end diff --git a/app/services/mute_service.rb b/app/services/mute_service.rb index c6122a152..676804cb9 100644 --- a/app/services/mute_service.rb +++ b/app/services/mute_service.rb @@ -5,11 +5,13 @@ class MuteService < BaseService return if account.id == target_account.id mute = account.mute!(target_account, notifications: notifications) + if mute.hide_notifications? BlockWorker.perform_async(account.id, target_account.id) else - FeedManager.instance.clear_from_timeline(account, target_account) + MuteWorker.perform_async(account.id, target_account.id) end + mute end end diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index 7d0dcc7ad..63bf8f17a 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -8,9 +8,10 @@ class NotifyService < BaseService return if recipient.user.nil? || blocked? - create_notification - push_notification if @notification.browserable? - send_email if email_enabled? + create_notification! + push_notification! if @notification.browserable? + push_to_conversation! if direct_message? + send_email! if email_enabled? rescue ActiveRecord::RecordInvalid return end @@ -100,18 +101,23 @@ class NotifyService < BaseService end end - def create_notification + def create_notification! @notification.save! end - def push_notification + def push_notification! return if @notification.activity.nil? Redis.current.publish("timeline:#{@recipient.id}", Oj.dump(event: :notification, payload: InlineRenderer.render(@notification, @recipient, :notification))) - send_push_notifications + send_push_notifications! end - def send_push_notifications + def push_to_conversation! + return if @notification.activity.nil? + AccountConversation.add_status(@recipient, @notification.target_status) + end + + def send_push_notifications! subscriptions_ids = ::Web::PushSubscription.where(user_id: @recipient.user.id) .select { |subscription| subscription.pushable?(@notification) } .map(&:id) @@ -121,7 +127,7 @@ class NotifyService < BaseService end end - def send_email + def send_email! return if @notification.activity.nil? NotificationMailer.public_send(@notification.type, @recipient, @notification).deliver_later(wait: 2.minutes) end diff --git a/app/workers/block_worker.rb b/app/workers/block_worker.rb index 0820490d3..25f5dd808 100644 --- a/app/workers/block_worker.rb +++ b/app/workers/block_worker.rb @@ -4,6 +4,9 @@ class BlockWorker include Sidekiq::Worker def perform(account_id, target_account_id) - AfterBlockService.new.call(Account.find(account_id), Account.find(target_account_id)) + AfterBlockService.new.call( + Account.find(account_id), + Account.find(target_account_id) + ) end end diff --git a/app/workers/mute_worker.rb b/app/workers/mute_worker.rb new file mode 100644 index 000000000..7bf0923a5 --- /dev/null +++ b/app/workers/mute_worker.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class MuteWorker + include Sidekiq::Worker + + def perform(account_id, target_account_id) + FeedManager.instance.clear_from_timeline( + Account.find(account_id), + Account.find(target_account_id) + ) + end +end diff --git a/app/workers/push_conversation_worker.rb b/app/workers/push_conversation_worker.rb new file mode 100644 index 000000000..16f538215 --- /dev/null +++ b/app/workers/push_conversation_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class PushConversationWorker + include Sidekiq::Worker + + def perform(conversation_account_id) + conversation = AccountConversation.find(conversation_account_id) + message = InlineRenderer.render(conversation, conversation.account, :conversation) + timeline_id = "timeline:direct:#{conversation.account_id}" + + Redis.current.publish(timeline_id, Oj.dump(event: :conversation, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/config/environments/development.rb b/config/environments/development.rb index b6478f16e..0791b82ab 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -87,7 +87,7 @@ Rails.application.configure do config.x.otp_secret = ENV.fetch('OTP_SECRET', '1fc2b87989afa6351912abeebe31ffc5c476ead9bf8b3d74cbc4a302c7b69a45b40b1bbef3506ddad73e942e15ed5ca4b402bf9a66423626051104f4b5f05109') end -ActiveRecordQueryTrace.enabled = ENV.fetch('QUERY_TRACE_ENABLED') { false } +ActiveRecordQueryTrace.enabled = ENV['QUERY_TRACE_ENABLED'] == 'true' module PrivateAddressCheck def self.private_address?(*) diff --git a/config/routes.rb b/config/routes.rb index 50c1d57fc..a2468c9bd 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -261,6 +261,7 @@ Rails.application.routes.draw do resources :streaming, only: [:index] resources :custom_emojis, only: [:index] resources :suggestions, only: [:index, :destroy] + resources :conversations, only: [:index] get '/search', to: 'search#index', as: :search diff --git a/db/migrate/20180929222014_create_account_conversations.rb b/db/migrate/20180929222014_create_account_conversations.rb new file mode 100644 index 000000000..53fa137e1 --- /dev/null +++ b/db/migrate/20180929222014_create_account_conversations.rb @@ -0,0 +1,14 @@ +class CreateAccountConversations < ActiveRecord::Migration[5.2] + def change + create_table :account_conversations do |t| + t.belongs_to :account, foreign_key: { on_delete: :cascade } + t.belongs_to :conversation, foreign_key: { on_delete: :cascade } + t.bigint :participant_account_ids, array: true, null: false, default: [] + t.bigint :status_ids, array: true, null: false, default: [] + t.bigint :last_status_id, null: true, default: nil + t.integer :lock_version, null: false, default: 0 + end + + add_index :account_conversations, [:account_id, :conversation_id, :participant_account_ids], unique: true, name: 'index_unique_conversations' + end +end diff --git a/db/schema.rb b/db/schema.rb index 577296a6a..1458bd70f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,10 +10,23 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2018_08_20_232245) do +ActiveRecord::Schema.define(version: 2018_09_29_222014) do + # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "account_conversations", force: :cascade do |t| + t.bigint "account_id" + t.bigint "conversation_id" + t.bigint "participant_account_ids", default: [], null: false, array: true + t.bigint "status_ids", default: [], null: false, array: true + t.bigint "last_status_id" + t.integer "lock_version", default: 0, null: false + t.index ["account_id", "conversation_id", "participant_account_ids"], name: "index_unique_conversations", unique: true + t.index ["account_id"], name: "index_account_conversations_on_account_id" + t.index ["conversation_id"], name: "index_account_conversations_on_conversation_id" + end + create_table "account_domain_blocks", force: :cascade do |t| t.string "domain" t.datetime "created_at", null: false @@ -597,6 +610,8 @@ ActiveRecord::Schema.define(version: 2018_08_20_232245) do t.index ["user_id"], name: "index_web_settings_on_user_id", unique: true end + add_foreign_key "account_conversations", "accounts", on_delete: :cascade + add_foreign_key "account_conversations", "conversations", on_delete: :cascade add_foreign_key "account_domain_blocks", "accounts", name: "fk_206c6029bd", on_delete: :cascade add_foreign_key "account_moderation_notes", "accounts" add_foreign_key "account_moderation_notes", "accounts", column: "target_account_id" diff --git a/spec/controllers/api/v1/conversations_controller_spec.rb b/spec/controllers/api/v1/conversations_controller_spec.rb new file mode 100644 index 000000000..2e9525855 --- /dev/null +++ b/spec/controllers/api/v1/conversations_controller_spec.rb @@ -0,0 +1,37 @@ +require 'rails_helper' + +RSpec.describe Api::V1::ConversationsController, type: :controller do + render_views + + let!(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) } + let(:other) { Fabricate(:user, account: Fabricate(:account, username: 'bob')) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'GET #index' do + let(:scopes) { 'read:statuses' } + + before do + PostStatusService.new.call(other.account, 'Hey @alice', nil, visibility: 'direct') + end + + it 'returns http success' do + get :index + expect(response).to have_http_status(200) + end + + it 'returns pagination headers' do + get :index, params: { limit: 1 } + expect(response.headers['Link'].links.size).to eq(2) + end + + it 'returns conversations' do + get :index + json = body_as_json + expect(json.size).to eq 1 + end + end +end diff --git a/spec/fabricators/conversation_account_fabricator.rb b/spec/fabricators/conversation_account_fabricator.rb new file mode 100644 index 000000000..f57ffd535 --- /dev/null +++ b/spec/fabricators/conversation_account_fabricator.rb @@ -0,0 +1,6 @@ +Fabricator(:conversation_account) do + account nil + conversation nil + participant_account_ids "" + last_status nil +end diff --git a/spec/models/account_conversation_spec.rb b/spec/models/account_conversation_spec.rb new file mode 100644 index 000000000..70a76281e --- /dev/null +++ b/spec/models/account_conversation_spec.rb @@ -0,0 +1,72 @@ +require 'rails_helper' + +RSpec.describe AccountConversation, type: :model do + let!(:alice) { Fabricate(:account, username: 'alice') } + let!(:bob) { Fabricate(:account, username: 'bob') } + let!(:mark) { Fabricate(:account, username: 'mark') } + + describe '.add_status' do + it 'creates new record when no others exist' do + status = Fabricate(:status, account: alice, visibility: :direct) + status.mentions.create(account: bob) + + conversation = AccountConversation.add_status(alice, status) + + expect(conversation.participant_accounts).to include(bob) + expect(conversation.last_status).to eq status + expect(conversation.status_ids).to eq [status.id] + end + + it 'appends to old record when there is a match' do + last_status = Fabricate(:status, account: alice, visibility: :direct) + conversation = AccountConversation.create!(account: alice, conversation: last_status.conversation, participant_account_ids: [bob.id], status_ids: [last_status.id]) + + status = Fabricate(:status, account: bob, visibility: :direct, thread: last_status) + status.mentions.create(account: alice) + + new_conversation = AccountConversation.add_status(alice, status) + + expect(new_conversation.id).to eq conversation.id + expect(new_conversation.participant_accounts).to include(bob) + expect(new_conversation.last_status).to eq status + expect(new_conversation.status_ids).to eq [last_status.id, status.id] + end + + it 'creates new record when new participants are added' do + last_status = Fabricate(:status, account: alice, visibility: :direct) + conversation = AccountConversation.create!(account: alice, conversation: last_status.conversation, participant_account_ids: [bob.id], status_ids: [last_status.id]) + + status = Fabricate(:status, account: bob, visibility: :direct, thread: last_status) + status.mentions.create(account: alice) + status.mentions.create(account: mark) + + new_conversation = AccountConversation.add_status(alice, status) + + expect(new_conversation.id).to_not eq conversation.id + expect(new_conversation.participant_accounts).to include(bob, mark) + expect(new_conversation.last_status).to eq status + expect(new_conversation.status_ids).to eq [status.id] + end + end + + describe '.remove_status' do + it 'updates last status to a previous value' do + last_status = Fabricate(:status, account: alice, visibility: :direct) + status = Fabricate(:status, account: alice, visibility: :direct) + conversation = AccountConversation.create!(account: alice, conversation: last_status.conversation, participant_account_ids: [bob.id], status_ids: [status.id, last_status.id]) + last_status.mentions.create(account: bob) + last_status.destroy! + conversation.reload + expect(conversation.last_status).to eq status + expect(conversation.status_ids).to eq [status.id] + end + + it 'removes the record if no other statuses are referenced' do + last_status = Fabricate(:status, account: alice, visibility: :direct) + conversation = AccountConversation.create!(account: alice, conversation: last_status.conversation, participant_account_ids: [bob.id], status_ids: [last_status.id]) + last_status.mentions.create(account: bob) + last_status.destroy! + expect(AccountConversation.where(id: conversation.id).count).to eq 0 + end + end +end diff --git a/streaming/index.js b/streaming/index.js index 1c6004b77..debf7c8bf 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -485,7 +485,8 @@ const startWorker = (workerId) => { }); app.get('/api/v1/streaming/direct', (req, res) => { - streamFrom(`timeline:direct:${req.accountId}`, req, streamToHttp(req, res), streamHttpEnd(req), true); + const channel = `timeline:direct:${req.accountId}`; + streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel)), true); }); app.get('/api/v1/streaming/hashtag', (req, res) => { @@ -525,9 +526,11 @@ const startWorker = (workerId) => { ws.isAlive = true; }); + let channel; + switch(location.query.stream) { case 'user': - const channel = `timeline:${req.accountId}`; + channel = `timeline:${req.accountId}`; streamFrom(channel, req, streamToWs(req, ws), streamWsEnd(req, ws, subscriptionHeartbeat(channel))); break; case 'user:notification': @@ -546,7 +549,8 @@ const startWorker = (workerId) => { streamFrom('timeline:public:local:media', req, streamToWs(req, ws), streamWsEnd(req, ws), true); break; case 'direct': - streamFrom(`timeline:direct:${req.accountId}`, req, streamToWs(req, ws), streamWsEnd(req, ws), true); + channel = `timeline:direct:${req.accountId}`; + streamFrom(channel, req, streamToWs(req, ws), streamWsEnd(req, ws, subscriptionHeartbeat(channel)), true); break; case 'hashtag': streamFrom(`timeline:hashtag:${location.query.tag.toLowerCase()}`, req, streamToWs(req, ws), streamWsEnd(req, ws), true); @@ -563,7 +567,7 @@ const startWorker = (workerId) => { return; } - const channel = `timeline:list:${listId}`; + channel = `timeline:list:${listId}`; streamFrom(channel, req, streamToWs(req, ws), streamWsEnd(req, ws, subscriptionHeartbeat(channel))); }); break; -- cgit From 7f9dd92a27e937e3cbc42c1e3c8b844c3583c92b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 9 Oct 2018 19:35:14 +0200 Subject: Track historical space stats in PgHero to determine PostgreSQL growth (#8906) --- app/workers/scheduler/pghero_scheduler.rb | 11 +++++++++++ config/sidekiq.yml | 3 +++ db/migrate/20181007025445_create_pghero_space_stats.rb | 13 +++++++++++++ db/schema.rb | 11 ++++++++++- 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 app/workers/scheduler/pghero_scheduler.rb create mode 100644 db/migrate/20181007025445_create_pghero_space_stats.rb (limited to 'db/migrate') diff --git a/app/workers/scheduler/pghero_scheduler.rb b/app/workers/scheduler/pghero_scheduler.rb new file mode 100644 index 000000000..4453bf2cd --- /dev/null +++ b/app/workers/scheduler/pghero_scheduler.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class Scheduler::PgheroScheduler + include Sidekiq::Worker + + sidekiq_options unique: :until_executed, retry: 0 + + def perform + PgHero.capture_space_stats + end +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 6a09bb6e9..c44af5b6c 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -33,3 +33,6 @@ backup_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::BackupCleanupScheduler + pghero_scheduler: + cron: '0 0 * * *' + class: Scheduler::PgheroScheduler diff --git a/db/migrate/20181007025445_create_pghero_space_stats.rb b/db/migrate/20181007025445_create_pghero_space_stats.rb new file mode 100644 index 000000000..6198a6f84 --- /dev/null +++ b/db/migrate/20181007025445_create_pghero_space_stats.rb @@ -0,0 +1,13 @@ +class CreatePgheroSpaceStats < ActiveRecord::Migration[5.2] + def change + create_table :pghero_space_stats do |t| + t.text :database + t.text :schema + t.text :relation + t.integer :size, limit: 8 + t.timestamp :captured_at + end + + add_index :pghero_space_stats, [:database, :captured_at] + end +end diff --git a/db/schema.rb b/db/schema.rb index 1458bd70f..bf6ab4e68 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2018_09_29_222014) do +ActiveRecord::Schema.define(version: 2018_10_07_025445) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -372,6 +372,15 @@ ActiveRecord::Schema.define(version: 2018_09_29_222014) do t.index ["uid"], name: "index_oauth_applications_on_uid", unique: true end + create_table "pghero_space_stats", force: :cascade do |t| + t.text "database" + t.text "schema" + t.text "relation" + t.bigint "size" + t.datetime "captured_at" + t.index ["database", "captured_at"], name: "index_pghero_space_stats_on_database_and_captured_at" + end + create_table "preview_cards", force: :cascade do |t| t.string "url", default: "", null: false t.string "title", default: "", null: false -- cgit