From 1268277a8c203bcae515e0ccc8b3432119bafed2 Mon Sep 17 00:00:00 2001 From: multiple creatures Date: Fri, 10 Jan 2020 20:07:15 -0600 Subject: add custom filter master toggle, add media gallery mode, & fix various filter logic + caching bugs --- app/controllers/api/v1/filters_controller.rb | 7 +++ .../api/v1/timelines/home_controller.rb | 9 ++-- .../api/v1/timelines/list_controller.rb | 10 ++-- app/controllers/filters_controller.rb | 7 +++ app/controllers/settings/preferences_controller.rb | 18 +++++++ app/helpers/filter_helper.rb | 8 +-- app/lib/status_filter.rb | 17 +++--- app/models/account.rb | 1 - app/models/status.rb | 10 ++-- app/models/user.rb | 3 ++ app/views/settings/preferences/show.html.haml | 60 +++++++++++++--------- app/views/settings/profiles/show.html.haml | 5 -- config/locales/simple_form.en.yml | 13 +++-- db/migrate/20200110202317_add_media_only_mode.rb | 5 ++ ...200110213720_add_filter_undescribed_to_users.rb | 7 +++ ...031_migrate_filter_undescribed_from_accounts.rb | 17 ++++++ .../20200110221801_add_filters_enabled_to_users.rb | 7 +++ ...200110221920_enable_filters_if_filters_exist.rb | 7 +++ db/structure.sql | 15 ++++-- streaming/index.js | 42 ++++++++++----- 20 files changed, 192 insertions(+), 76 deletions(-) create mode 100644 db/migrate/20200110202317_add_media_only_mode.rb create mode 100644 db/migrate/20200110213720_add_filter_undescribed_to_users.rb create mode 100644 db/migrate/20200110214031_migrate_filter_undescribed_from_accounts.rb create mode 100644 db/migrate/20200110221801_add_filters_enabled_to_users.rb create mode 100644 db/migrate/20200110221920_enable_filters_if_filters_exist.rb diff --git a/app/controllers/api/v1/filters_controller.rb b/app/controllers/api/v1/filters_controller.rb index 514f7c8fa..ffef0b920 100644 --- a/app/controllers/api/v1/filters_controller.rb +++ b/app/controllers/api/v1/filters_controller.rb @@ -15,6 +15,7 @@ class Api::V1::FiltersController < Api::BaseController def create @filter = current_account.custom_filters.create!(resource_params) + toggle_filters render json: @filter, serializer: REST::FilterSerializer end @@ -24,16 +25,22 @@ class Api::V1::FiltersController < Api::BaseController def update @filter.update!(resource_params) + toggle_filters render json: @filter, serializer: REST::FilterSerializer end def destroy @filter.destroy! + toggle_filters render_empty end private + def toggle_filters + current_account.user.update!(filters_enabled: !current_account.custom_filters.enabled.blank?) + end + def set_filters @filters = params['all'].to_i == 1 ? current_account.custom_filters : [] end diff --git a/app/controllers/api/v1/timelines/home_controller.rb b/app/controllers/api/v1/timelines/home_controller.rb index bd3bac0fe..589bc3486 100644 --- a/app/controllers/api/v1/timelines/home_controller.rb +++ b/app/controllers/api/v1/timelines/home_controller.rb @@ -23,11 +23,10 @@ class Api::V1::Timelines::HomeController < Api::BaseController end def cached_home_statuses - if current_account&.user&.hide_boosts - cache_collection home_statuses.without_reblogs, Status - else - cache_collection home_statuses, Status - end + statuses = home_statuses + statuses = statuses.without_reblogs if current_account&.user&.hide_boosts + statuses = statuses.with_media if current_account&.user&.media_only + cache_collection statuses, Status end def home_statuses diff --git a/app/controllers/api/v1/timelines/list_controller.rb b/app/controllers/api/v1/timelines/list_controller.rb index 7eb656745..b52f53bf9 100644 --- a/app/controllers/api/v1/timelines/list_controller.rb +++ b/app/controllers/api/v1/timelines/list_controller.rb @@ -25,11 +25,11 @@ class Api::V1::Timelines::ListController < Api::BaseController end def cached_list_statuses - if current_account&.user&.hide_boosts - cache_collection list_statuses.without_reblogs, Status - else - cache_collection list_statuses, Status - end + statuses = list_statuses + statuses = statuses.without_reblogs if current_account&.user&.hide_boosts + statuses = statuses.with_media if current_account&.user&.media_only + + cache_collection statuses, Status end def list_statuses diff --git a/app/controllers/filters_controller.rb b/app/controllers/filters_controller.rb index f5cc00d2f..2b615923f 100644 --- a/app/controllers/filters_controller.rb +++ b/app/controllers/filters_controller.rb @@ -22,6 +22,7 @@ class FiltersController < ApplicationController @filter = current_account.custom_filters.build(resource_params) if @filter.save + toggle_filters redirect_to filters_path else render action: :new @@ -32,6 +33,7 @@ class FiltersController < ApplicationController def update if @filter.update(resource_params) + toggle_filters redirect_to filters_path else render action: :edit @@ -40,11 +42,16 @@ class FiltersController < ApplicationController def destroy @filter.destroy + toggle_filters redirect_to filters_path end private + def toggle_filters + current_user.update!(filters_enabled: !current_account.custom_filters.enabled.blank?) + end + def set_pack use_pack 'settings' end diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb index 2ff84ac80..ce3c84f77 100644 --- a/app/controllers/settings/preferences_controller.rb +++ b/app/controllers/settings/preferences_controller.rb @@ -14,6 +14,9 @@ class Settings::PreferencesController < Settings::BaseController if current_user.update(user_params) I18n.locale = current_user.locale + toggle_filters + remove_cache + update_feeds redirect_to settings_preferences_path, notice: I18n.t('generic.changes_saved_msg') else render :show @@ -22,6 +25,18 @@ class Settings::PreferencesController < Settings::BaseController private + def toggle_filters + current_user.update!(filters_enabled: !current_account.custom_filters.enabled.blank?) + end + + def update_feeds + FilterFeedsWorker.perform_async(current_user.account_id) + end + + def remove_cache + redis.del("filtered_statuses:#{current_user.account_id}") + end + def user_settings UserSettingsDecorator.new(current_user) end @@ -29,8 +44,11 @@ class Settings::PreferencesController < Settings::BaseController def user_params params.require(:user).permit( :locale, + :filters_enabled, :hide_boosts, :only_known, + :media_only, + :filter_undescribed, :invert_filters, :filter_timelines_only, chosen_languages: [] diff --git a/app/helpers/filter_helper.rb b/app/helpers/filter_helper.rb index b9dbc1f4f..2334fc356 100644 --- a/app/helpers/filter_helper.rb +++ b/app/helpers/filter_helper.rb @@ -1,14 +1,14 @@ module FilterHelper include Redisable - def phrase_filtered?(status, receiver_id) - return true if redis.sismember("filtered_statuses:#{receiver_id}", status.id) - return false unless CustomFilter.where(account_id: receiver_id).exists? + def phrase_filtered?(status, receiver_id, skip_redis: false) + return true if !skip_redis && redis.sismember("filtered_statuses:#{receiver_id}", status.id) + return false unless CustomFilter.where(account_id: receiver_id, is_enabled: true).exists? status = status.reblog if status.reblog? if Status.where(id: status.id).search_filtered_by_account(receiver_id).exists? - redis.sadd("filtered_statuses:#{receiver_id}", status.id) + redis.sadd("filtered_statuses:#{receiver_id}", status.id) unless skip_redis return true end diff --git a/app/lib/status_filter.rb b/app/lib/status_filter.rb index 4cb5804da..dbfa24dfe 100644 --- a/app/lib/status_filter.rb +++ b/app/lib/status_filter.rb @@ -15,9 +15,9 @@ class StatusFilter def filtered? return true if status.nil? || account.nil? return false if !account.nil? && account.id == status.account_id - return !account.user.invert_filters if !account.user.filter_timelines_only && redis.sismember("filtered_statuses:#{account.id}", status.id) + return true if redis.sismember("filtered_statuses:#{account.id}", status.id) if blocked_by_policy? || (account_present? && filtered_status?) || silenced_account? - redis.sadd("filtered_statuses:#{account.id}", status.id) unless account.user.filter_timelines_only + redis.sadd("filtered_statuses:#{account.id}", status.id) return true end false @@ -40,7 +40,13 @@ class StatusFilter return true if account.user_hides_replies_of_blocker? && reply_to_blocker? # filtered by user? - return true if !account.user.filter_timelines_only && !account.user.invert_filters && phrase_filtered?(status, account.id) + if account.user.filters_enabled && !account.user.filter_timelines_only + if account.user.invert_filters + return true unless phrase_filtered?(status, account.id) + else + return true if phrase_filtered?(status, account.id) + end + end # kajiht has no filters if status has no mentions return false if status&.mentions.blank? @@ -74,10 +80,7 @@ class StatusFilter return true if !@preloaded_relations[:muting] && account.user_hides_mentions_of_muted? && account.muting?(mentioned_account_ids) return true if !@preloaded_relations[:blocking] && account.user_hides_mentions_of_blocked? && account.blocking?(mentioned_account_ids) return false unless status.reply? && status.private_visibility? && account.user_hides_mentions_outside_scope? - return true if !@preloaded_relations[:following] && (mentioned_account_ids - account.following_ids).any? - - # filtered by user? - !account.user.filter_timelines_only && account.user.invert_filters && !phrase_filtered?(status, account.id) + !@preloaded_relations[:following] && (mentioned_account_ids - account.following_ids).any? end def reply_to_blocked? diff --git a/app/models/account.rb b/app/models/account.rb index 66fe554d5..e43db63bd 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -55,7 +55,6 @@ # force_private :boolean default(FALSE), not null # unboostable :boolean default(FALSE), not null # block_anon :boolean default(FALSE), not null -# filter_undescribed :boolean default(FALSE), not null # class Account < ApplicationRecord diff --git a/app/models/status.rb b/app/models/status.rb index 5a1d38932..5d257ba6c 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -119,9 +119,10 @@ class Status < ApplicationRecord scope :search, ->(needle) { where("tsv @@ websearch_to_tsquery('fedi', ?)", needle) } scope :search_not, ->(needle) { where.not("tsv @@ websearch_to_tsquery('fedi', ?)", needle) } - scope :search_filtered_by_account, ->(account_id) { where('tsv @@ (SELECT tsquery_union(websearch_to_tsquery(phrase)) FROM custom_filters WHERE account_id = ? AND is_enabled)', account_id) } - scope :search_not_filtered_by_account, ->(account_id) { where.not('tsv @@ (SELECT tsquery_union(websearch_to_tsquery(phrase)) FROM custom_filters WHERE account_id = ? AND is_enabled)', account_id) } + scope :search_filtered_by_account, ->(account_id) { where("tsv @@ (SELECT tsquery_union(websearch_to_tsquery('fedi', phrase)) FROM custom_filters WHERE account_id = ? AND is_enabled)", account_id) } + scope :search_not_filtered_by_account, ->(account_id) { where.not("tsv @@ (SELECT tsquery_union(websearch_to_tsquery('fedi', phrase)) FROM custom_filters WHERE account_id = ? AND is_enabled)", account_id) } + scope :with_media, -> { joins(:media_attachments).select('statuses.*') } scope :not_missing_media_desc, -> { left_outer_joins(:media_attachments).select('statuses.*').where('media_attachments.id IS NULL OR media_attachments.description IS NOT NULL') } scope :only_followers_of, ->(account) { where(account: [account] + account.following) } @@ -578,14 +579,15 @@ class Status < ApplicationRecord query = query.in_chosen_languages(account) if account.chosen_languages.present? query = query.reply_not_excluded_by_account(account) unless tag_timeline query = query.mention_not_excluded_by_account(account) - unless account.custom_filters.enabled.empty? + unless !account.user.filters_enabled || account.custom_filters.enabled.blank? if account.user.invert_filters query = query.search_filtered_by_account(account.id) else query = query.search_not_filtered_by_account(account.id) end end - query = query.not_missing_media_desc if account.filter_undescribed? + query = query.with_media if account.user.media_only? + query = query.not_missing_media_desc if account.user.filter_undescribed? query.merge(account_silencing_filter(account)) end diff --git a/app/models/user.rb b/app/models/user.rb index 2ee304e81..b519e9b15 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -43,6 +43,9 @@ # only_known :boolean # invert_filters :boolean default(FALSE), not null # filter_timelines_only :boolean default(FALSE), not null +# media_only :boolean default(FALSE), not null +# filter_undescribed :boolean default(FALSE), not null +# filters_enabled :boolean default(FALSE), not null # class User < ApplicationRecord diff --git a/app/views/settings/preferences/show.html.haml b/app/views/settings/preferences/show.html.haml index 80ad6c109..fdb5fea3f 100644 --- a/app/views/settings/preferences/show.html.haml +++ b/app/views/settings/preferences/show.html.haml @@ -11,48 +11,62 @@ = simple_form_for current_user, url: settings_preferences_path, html: { method: :put } do |f| = render 'shared/error_messages', object: current_user - .fields-row#settings_languages - .fields-group.fields-row__column.fields-row__column-6 - = f.input :locale, collection: I18n.available_locales, wrapper: :with_label, include_blank: false, label_method: lambda { |locale| human_locale(locale) }, selected: I18n.locale - .fields-group.fields-row__column.fields-row__column-6 - = f.input :setting_default_language, collection: [nil] + filterable_languages.sort, wrapper: :with_label, label_method: lambda { |locale| locale.nil? ? I18n.t('statuses.language_detection') : human_locale(locale) }, required: false, include_blank: false - - .fields-group - = f.input :chosen_languages, collection: filterable_languages.sort, wrapper: :with_block_label, include_blank: false, label_method: lambda { |locale| human_locale(locale) }, required: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' - %hr#settings_publishing/ .fields-group = f.input :setting_default_privacy, collection: Status.selectable_visibilities, wrapper: :with_floating_label, include_blank: false, label_method: lambda { |visibility| safe_join([I18n.t("statuses.visibilities.#{visibility}"), content_tag(:span, I18n.t("statuses.visibilities.#{visibility}_long"), class: 'hint')]) }, required: false, as: :radio_buttons, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' - = f.input :setting_default_content_type, collection: ['text/x-bbcode+markdown', 'text/markdown', 'text/x-bbcode', 'text/html', 'text/plain', 'text/console'], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.defaults.setting_default_content_type_#{item.split('/')[1].gsub(/[+-]/, '_')}"), content_tag(:span, t("simple_form.hints.defaults.setting_default_content_type_#{item.split('/')[1].gsub(/[+-]/, '_')}_html"), class: 'hint')]) }, required: false, as: :radio_buttons, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' - + .fields-group + = f.input :setting_max_public_history, collection: [1, 3, 6, 7, 14, 30, 60, 90, 180, 365, 730, 1095, 2190], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.lifespan.#{item}")]) }, selected: current_user.max_public_history + = f.input :setting_roar_lifespan, collection: [0, 1, 3, 6, 7, 14, 30, 60, 90, 180, 365, 730, 1095, 2190], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.lifespan.#{item}")]) }, selected: current_user.roar_lifespan .fields-group + = f.input :setting_default_sensitive, as: :boolean, wrapper: :with_label = f.input :setting_default_local, as: :boolean, wrapper: :with_label = f.input :setting_always_local, as: :boolean, wrapper: :with_label - = f.input :setting_default_sensitive, as: :boolean, wrapper: :with_label + = f.input :setting_hide_public_profile, as: :boolean, wrapper: :with_label + = f.input :setting_hide_public_outbox, as: :boolean, wrapper: :with_label + + %hr/ + + .fields-group + = f.input :setting_default_content_type, collection: ['text/x-bbcode+markdown', 'text/markdown', 'text/x-bbcode', 'text/html', 'text/plain', 'text/console'], wrapper: :with_floating_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.defaults.setting_default_content_type_#{item.split('/')[1].gsub(/[+-]/, '_')}"), content_tag(:span, t("simple_form.hints.defaults.setting_default_content_type_#{item.split('/')[1].gsub(/[+-]/, '_')}_html"), class: 'hint')]) }, required: false, as: :radio_buttons, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' + + %hr/ .fields-group = f.input :setting_delayed_roars, as: :boolean, wrapper: :with_label = f.input :setting_delayed_for, collection: [5, 10, 15, 30, 60, 120, 180, 300, 360, 600, 1800, 3600], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.delayed_for.#{item}")]) }, selected: current_user.delayed_for + %hr/ + .fields-group = f.input :setting_boost_interval, as: :boolean, wrapper: :with_label = f.input :setting_boost_random, as: :boolean, wrapper: :with_label = f.input :setting_boost_interval_from, collection: [1, 2, 3, 4, 5, 6, 10, 15, 30, 60, 120, 180, 300, 360, 720, 1440], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.boost_interval.#{item}")]) }, selected: current_user.boost_interval_from = f.input :setting_boost_interval_to, collection: [1, 2, 3, 4, 5, 6, 10, 15, 30, 60, 120, 180, 300, 360, 720, 1440], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.boost_interval.#{item}")]) }, selected: current_user.boost_interval_to + + + %hr#settings_other/ .fields-group + = f.input :filters_enabled, as: :boolean, wrapper: :with_label = f.input :invert_filters, as: :boolean, wrapper: :with_label = f.input :filter_timelines_only, as: :boolean, wrapper: :with_label + = f.input :setting_filter_mentions, as: :boolean, wrapper: :with_label + + %hr/ .fields-group - = f.input :setting_rawr_federated, as: :boolean, wrapper: :with_label - = f.input :hide_boosts, as: :boolean, wrapper: :with_label = f.input :only_known, as: :boolean, wrapper: :with_label + = f.input :hide_boosts, as: :boolean, wrapper: :with_label + = f.input :media_only, as: :boolean, wrapper: :with_label + = f.input :filter_undescribed, as: :boolean, wrapper: :with_label + + .fields-group + = f.input :setting_rawr_federated, as: :boolean, wrapper: :with_label %hr/ @@ -63,14 +77,6 @@ = f.input :setting_show_application, as: :boolean, wrapper: :with_label = f.input :setting_noindex, as: :boolean, wrapper: :with_label - %hr/ - - .fields-group - = f.input :setting_max_public_history, collection: [1, 3, 6, 7, 14, 30, 60, 90, 180, 365, 730, 1095, 2190], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.lifespan.#{item}")]) }, selected: current_user.max_public_history - = f.input :setting_roar_lifespan, collection: [0, 1, 3, 6, 7, 14, 30, 60, 90, 180, 365, 730, 1095, 2190], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.lifespan.#{item}")]) }, selected: current_user.roar_lifespan - = f.input :setting_hide_public_profile, as: :boolean, wrapper: :with_label - = f.input :setting_hide_public_outbox, as: :boolean, wrapper: :with_label - - unless Setting.hide_followers_count .fields-group = f.input :setting_hide_followers_count, as: :boolean, wrapper: :with_label @@ -91,7 +97,6 @@ .fields-group = f.input :setting_hide_mascot, as: :boolean, wrapper: :with_label - = f.input :setting_filter_mentions, as: :boolean, wrapper: :with_label = f.input :setting_hide_replies_muted, as: :boolean, wrapper: :with_label = f.input :setting_hide_replies_blocked, as: :boolean, wrapper: :with_label = f.input :setting_hide_replies_blocker, as: :boolean, wrapper: :with_label @@ -116,5 +121,14 @@ = f.input :setting_display_media, collection: ['default', 'show_all', 'hide_all'], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.defaults.setting_display_media_#{item}"), content_tag(:span, t("simple_form.hints.defaults.setting_display_media_#{item}"), class: 'hint')]) }, required: false, as: :radio_buttons, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' = f.input :setting_expand_spoilers, as: :boolean, wrapper: :with_label + .fields-row#settings_languages + .fields-group.fields-row__column.fields-row__column-6 + = f.input :locale, collection: I18n.available_locales, wrapper: :with_label, include_blank: false, label_method: lambda { |locale| human_locale(locale) }, selected: I18n.locale + .fields-group.fields-row__column.fields-row__column-6 + = f.input :setting_default_language, collection: [nil] + filterable_languages.sort, wrapper: :with_label, label_method: lambda { |locale| locale.nil? ? I18n.t('statuses.language_detection') : human_locale(locale) }, required: false, include_blank: false + + .fields-group + = f.input :chosen_languages, collection: filterable_languages.sort, wrapper: :with_block_label, include_blank: false, label_method: lambda { |locale| human_locale(locale) }, required: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' + .actions = f.button :button, t('generic.save_changes'), type: :submit diff --git a/app/views/settings/profiles/show.html.haml b/app/views/settings/profiles/show.html.haml index 35135267c..4fabfb9f4 100644 --- a/app/views/settings/profiles/show.html.haml +++ b/app/views/settings/profiles/show.html.haml @@ -43,11 +43,6 @@ %hr.spacer/ - .fields-group - = f.input :filter_undescribed, as: :boolean, wrapper: :with_label - - %hr.spacer/ - .fields-row .fields-row__column.fields-group.fields-row__column-6 .input.with_block_label diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index aa32a3200..7c82538d4 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -46,6 +46,8 @@ en: setting_theme: Affects how Mastodon looks when you're logged in from any device. username: Your username will be unique on %{domain} phrase_html: "Examples
Containing any terms: this OR that
Containing all terms: this that, this AND that
Containing an exact term: "this thing"
Grouping: this OR ("this thing" AND "that thing")" + media_only: Hides text-only posts + filters_enabled: Enables custom timeline filters which can be configured in the Filters tab. featured_tag: name: 'You might want to use one of these:' imports: @@ -102,7 +104,6 @@ en: irreversible: Drop instead of hide exclude_media: Filter roars WITHOUT attachments status_text: Filter roars with matching body text - media_only: Filter roars with attachments thread: Filter the entire thread this match is contained in spoiler: Filter roars with matching content warnings or topics tags: Filter roars with matching tag(s) @@ -118,7 +119,7 @@ en: setting_aggregate_reblogs: Group repeats in timelines setting_auto_play_gif: Auto-play animated GIFs setting_boost_modal: Show confirmation dialog before repeating - setting_default_content_type: Default format for roars + setting_default_content_type: Default text format setting_default_content_type_html: HTML setting_default_content_type_markdown: Markdown setting_default_content_type_plain: Plain text @@ -145,7 +146,7 @@ en: setting_larger_buttons: Increase size and spacing of action buttons setting_larger_drawer: Increase width of compose drawer column setting_larger_emoji: Increase size of emoji - setting_filter_mentions: Apply regex filters to mentions + setting_filter_mentions: Apply filters to mentions setting_hide_replies_muted: Filter replies to those who you are muting setting_hide_replies_blocked: Filter replies to those who you are blocking setting_hide_replies_blocker: Filter replies to those who are blocking you @@ -182,8 +183,8 @@ en: setting_unfollow_modal: Show confirmation dialog before unfollowing someone hide_boosts: Don't show boosts on any timeline only_known: Only show posts from packmates on all timelines - invert_filters: Use allow list mode for filters - filter_timelines_only: Apply filters to timelines only + invert_filters: Use allow list mode for custom filters + filter_timelines_only: Apply custom filters to timelines only severity: Severity type: Import type username: Username @@ -195,6 +196,8 @@ en: override_cw: Override existing content warning is_enabled: Enabled filter_undescribed: Hide roars without media descriptions + media_only: Media gallery mode + filters_enabled: Enable custom filters boost_interval: 1: 1 minute 2: 2 minutes diff --git a/db/migrate/20200110202317_add_media_only_mode.rb b/db/migrate/20200110202317_add_media_only_mode.rb new file mode 100644 index 000000000..93934623a --- /dev/null +++ b/db/migrate/20200110202317_add_media_only_mode.rb @@ -0,0 +1,5 @@ +class AddMediaOnlyMode < ActiveRecord::Migration[5.2] + def change + safety_assured { add_column :users, :media_only, :boolean, null: false, default: false } + end +end diff --git a/db/migrate/20200110213720_add_filter_undescribed_to_users.rb b/db/migrate/20200110213720_add_filter_undescribed_to_users.rb new file mode 100644 index 000000000..e97f28e96 --- /dev/null +++ b/db/migrate/20200110213720_add_filter_undescribed_to_users.rb @@ -0,0 +1,7 @@ +class AddFilterUndescribedToUsers < ActiveRecord::Migration[5.2] + def change + safety_assured { + add_column :users, :filter_undescribed, :boolean, null: false, default: false + } + end +end diff --git a/db/migrate/20200110214031_migrate_filter_undescribed_from_accounts.rb b/db/migrate/20200110214031_migrate_filter_undescribed_from_accounts.rb new file mode 100644 index 000000000..6e29b9d44 --- /dev/null +++ b/db/migrate/20200110214031_migrate_filter_undescribed_from_accounts.rb @@ -0,0 +1,17 @@ +class MigrateFilterUndescribedFromAccounts < ActiveRecord::Migration[5.2] + def up + Account.local.find_each do |account| + account.user.update!(filter_undescribed: account.filter_undescribed) + end + safety_assured { + remove_column :accounts, :filter_undescribed + } + end + + def down + return true + safety_assured { + add_column :accounts, :filter_undescribed, :boolean, null: true, default: false + } + end +end diff --git a/db/migrate/20200110221801_add_filters_enabled_to_users.rb b/db/migrate/20200110221801_add_filters_enabled_to_users.rb new file mode 100644 index 000000000..db15015e7 --- /dev/null +++ b/db/migrate/20200110221801_add_filters_enabled_to_users.rb @@ -0,0 +1,7 @@ +class AddFiltersEnabledToUsers < ActiveRecord::Migration[5.2] + def change + safety_assured do + add_column :users, :filters_enabled, :boolean, null: false, default: false + end + end +end diff --git a/db/migrate/20200110221920_enable_filters_if_filters_exist.rb b/db/migrate/20200110221920_enable_filters_if_filters_exist.rb new file mode 100644 index 000000000..68758a9ef --- /dev/null +++ b/db/migrate/20200110221920_enable_filters_if_filters_exist.rb @@ -0,0 +1,7 @@ +class EnableFiltersIfFiltersExist < ActiveRecord::Migration[5.2] + def up + Account.local.find_each do |account| + account.user.update!(filters_enabled: !account.custom_filters.enabled.blank?) + end + end +end diff --git a/db/structure.sql b/db/structure.sql index de17e54f8..e3da0e923 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -550,8 +550,7 @@ CREATE TABLE public.accounts ( known boolean DEFAULT false NOT NULL, force_private boolean DEFAULT false NOT NULL, unboostable boolean DEFAULT false NOT NULL, - block_anon boolean DEFAULT false NOT NULL, - filter_undescribed boolean DEFAULT false NOT NULL + block_anon boolean DEFAULT false NOT NULL ); @@ -2396,7 +2395,10 @@ CREATE TABLE public.users ( hide_boosts boolean, only_known boolean, invert_filters boolean DEFAULT false NOT NULL, - filter_timelines_only boolean DEFAULT false NOT NULL + filter_timelines_only boolean DEFAULT false NOT NULL, + media_only boolean DEFAULT false NOT NULL, + filter_undescribed boolean DEFAULT false NOT NULL, + filters_enabled boolean DEFAULT false NOT NULL ); @@ -5390,6 +5392,11 @@ INSERT INTO "schema_migrations" (version) VALUES ('20200108051211'), ('20200109191740'), ('20200110072034'), -('20200110195612'); +('20200110195612'), +('20200110202317'), +('20200110213720'), +('20200110214031'), +('20200110221801'), +('20200110221920'); diff --git a/streaming/index.js b/streaming/index.js index f429bbcc1..015546658 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -202,7 +202,7 @@ const startWorker = (workerId) => { return; } - client.query('SELECT oauth_access_tokens.resource_owner_id, users.account_id, users.chosen_languages, users.hide_boosts, users.only_known, users.invert_filters, oauth_access_tokens.scopes FROM oauth_access_tokens INNER JOIN users ON oauth_access_tokens.resource_owner_id = users.id WHERE oauth_access_tokens.token = $1 AND oauth_access_tokens.revoked_at IS NULL LIMIT 1', [token], (err, result) => { + client.query('SELECT oauth_access_tokens.resource_owner_id, users.account_id, users.chosen_languages, users.filters_enabled, users.hide_boosts, users.only_known, users.invert_filters, users.media_only, users.filter_undescribed, oauth_access_tokens.scopes FROM oauth_access_tokens INNER JOIN users ON oauth_access_tokens.resource_owner_id = users.id WHERE oauth_access_tokens.token = $1 AND oauth_access_tokens.revoked_at IS NULL LIMIT 1', [token], (err, result) => { done(); if (err) { @@ -230,9 +230,12 @@ const startWorker = (workerId) => { req.accountId = result.rows[0].account_id; req.chosenLanguages = result.rows[0].chosen_languages; + req.filtersEnabled = result.rows[0].filters_enabled; req.hideBoosts = result.rows[0].hide_boosts; req.onlyKnown = result.rows[0].only_known; req.invertFilters = result.rows[0].invert_filters; + req.mediaOnly = result.rows[0].media_only; + req.filterUndescribed = result.rows[0].filter_undescribed; req.allowNotifications = scopes.some(scope => ['read', 'read:notifications'].includes(scope)); next(); @@ -387,35 +390,44 @@ const startWorker = (workerId) => { // Only messages that may require filtering are statuses, since notifications // are already personalized and deletes do not matter - if (!needsFiltering || event !== 'update') { + if (event !== 'update') { transmit(); return; } const unpackedPayload = payload; - const targetAccountIds = [unpackedPayload.account.id].concat(unpackedPayload.mentions.map(item => item.id)); - const accountDomain = unpackedPayload.account.acct.split('@')[1]; + + // Don't filter user's own events. + if (req.accountId === unpackedPayload.account.id) { + transmit(); + return; + } if (req.hideBoosts && (unpackedPayload.in_reply_to !== undefined || unpackedPayload.in_reply_to !== null)) { return; } + if (req.mediaOnly && (!unpackedPayload.media_attachments || unpackedPayload.media_attachments.length === 0)) { + return; + } + + if (req.filterUndescribed && unpackedPayload.media_attachments && unpackedPayload.media_attachments.every(m => !m.description || m.description.length === 0)) { + return; + } + if (Array.isArray(req.chosenLanguages) && unpackedPayload.language !== null && req.chosenLanguages.indexOf(unpackedPayload.language) === -1) { log.silly(req.requestId, `Message ${unpackedPayload.id} filtered by language (${unpackedPayload.language})`); return; } // When the account is not logged in, it is not necessary to confirm the block or mute - if (!req.accountId) { + if (!needsFiltering || !req.accountId) { transmit(); return; } - // Don't filter user's own events. - if (req.accountId === unpackedPayload.account.id) { - transmit(); - return - } + const targetAccountIds = [unpackedPayload.account.id].concat(unpackedPayload.mentions.map(item => item.id)); + const accountDomain = unpackedPayload.account.acct.split('@')[1]; pgPool.connect((err, client, done) => { if (err) { @@ -424,13 +436,17 @@ const startWorker = (workerId) => { } const queries = [ - client.query(`SELECT 1 FROM blocks WHERE (account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 3)})) OR (account_id = $2 AND target_account_id = $1) UNION SELECT 1 FROM mutes WHERE account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 3)}) UNION SELECT 1 FROM statuses WHERE id = $3 ${req.invertFilters ? 'AND NOT' : 'AND'} tsv @@ (SELECT tsquery_union(websearch_to_tsquery(phrase)) FROM custom_filters WHERE account_id = $1 AND is_enabled) UNION SELECT 1 FROM media_attachments WHERE (1 = (SELECT 1 FROM accounts WHERE id = $1 AND filter_undescribed)) AND status_id = $3 AND description IS NULL LIMIT 1`, [req.accountId, unpackedPayload.account.id, unpackedPayload.id].concat(targetAccountIds)), + client.query(`SELECT 1 FROM blocks WHERE (account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)})) OR (account_id = $2 AND target_account_id = $1) UNION SELECT 1 FROM mutes WHERE account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)})`, [req.accountId, unpackedPayload.account.id].concat(targetAccountIds)), ]; if (accountDomain) { queries.push(client.query('SELECT 1 FROM account_domain_blocks WHERE account_id = $1 AND domain = $2', [req.accountId, accountDomain])); } + if (req.filtersEnabled) { + queries.push(client.query(`SELECT 1 FROM statuses WHERE id = $2 ${req.invertFilters ? 'AND NOT' : 'AND'} tsv @@ (SELECT tsquery_union(websearch_to_tsquery('fedi', phrase)) FROM custom_filters WHERE account_id = $1 AND is_enabled) LIMIT 1`, [req.accountId, unpackedPayload.id])); + } + if (req.onlyKnown) { queries.push(client.query('SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM follows WHERE account_id = $1 AND target_account_id = $2)', [req.accountId, unpackedPayload.account.id])); } @@ -534,7 +550,7 @@ const startWorker = (workerId) => { app.get('/api/v1/streaming/user', (req, res) => { const channel = `timeline:${req.accountId}`; - streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel))); + streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel)), true); }); app.get('/api/v1/streaming/user/notification', (req, res) => { @@ -592,7 +608,7 @@ const startWorker = (workerId) => { } const channel = `timeline:list:${listId}`; - streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel))); + streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel)), true); }); }); -- cgit