From 318d34d528d8d4f9165bf9c1f873da68aadb90a2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 9 Mar 2022 08:51:12 +0100 Subject: Fix data integrity of featured tags (#17712) --- app/models/featured_tag.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/featured_tag.rb b/app/models/featured_tag.rb index e02ae0705..74d62e777 100644 --- a/app/models/featured_tag.rb +++ b/app/models/featured_tag.rb @@ -4,8 +4,8 @@ # Table name: featured_tags # # id :bigint(8) not null, primary key -# account_id :bigint(8) -# tag_id :bigint(8) +# account_id :bigint(8) not null +# tag_id :bigint(8) not null # statuses_count :bigint(8) default(0), not null # last_status_at :datetime # created_at :datetime not null -- cgit From bd53dd521064b12261b82105624cf5f8b9ca9d69 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 9 Mar 2022 08:52:32 +0100 Subject: Change design of federation pages in admin UI (#17704) * Change design of federation pages in admin UI * Fix query performance in instance media attachments measure * Fix reblogs being included in instance languages dimension --- app/controllers/admin/domain_blocks_controller.rb | 4 - app/controllers/admin/instances_controller.rb | 26 +--- .../mastodon/components/admin/Counter.js | 6 +- app/javascript/styles/mastodon/admin.scss | 50 ++++++ app/javascript/styles/mastodon/tables.scss | 18 +++ app/lib/admin/metrics/dimension.rb | 2 + .../dimension/instance_accounts_dimension.rb | 35 +++++ .../dimension/instance_languages_dimension.rb | 37 +++++ app/lib/admin/metrics/measure.rb | 6 + app/lib/admin/metrics/measure/base_measure.rb | 8 + .../metrics/measure/instance_accounts_measure.rb | 58 +++++++ .../metrics/measure/instance_followers_measure.rb | 59 +++++++ .../metrics/measure/instance_follows_measure.rb | 59 +++++++ .../measure/instance_media_attachments_measure.rb | 69 +++++++++ .../metrics/measure/instance_reports_measure.rb | 59 +++++++ .../metrics/measure/instance_statuses_measure.rb | 60 ++++++++ app/lib/delivery_failure_tracker.rb | 2 +- app/models/domain_block.rb | 8 + app/models/instance.rb | 36 ++--- app/models/instance_filter.rb | 16 +- app/serializers/rest/admin/measure_serializer.rb | 10 +- app/views/admin/domain_blocks/edit.html.haml | 4 +- app/views/admin/domain_blocks/new.html.haml | 4 +- app/views/admin/domain_blocks/show.html.haml | 25 --- .../instances/_exhausted_deliveries_days.haml | 2 - app/views/admin/instances/_instance.html.haml | 28 +--- app/views/admin/instances/index.html.haml | 19 +-- app/views/admin/instances/show.html.haml | 169 +++++++++++---------- app/workers/admin/domain_purge_worker.rb | 2 + config/locales/en.yml | 53 ++++--- config/routes.rb | 2 +- .../admin/domain_blocks_controller_spec.rb | 9 -- 32 files changed, 711 insertions(+), 234 deletions(-) create mode 100644 app/lib/admin/metrics/dimension/instance_accounts_dimension.rb create mode 100644 app/lib/admin/metrics/dimension/instance_languages_dimension.rb create mode 100644 app/lib/admin/metrics/measure/instance_accounts_measure.rb create mode 100644 app/lib/admin/metrics/measure/instance_followers_measure.rb create mode 100644 app/lib/admin/metrics/measure/instance_follows_measure.rb create mode 100644 app/lib/admin/metrics/measure/instance_media_attachments_measure.rb create mode 100644 app/lib/admin/metrics/measure/instance_reports_measure.rb create mode 100644 app/lib/admin/metrics/measure/instance_statuses_measure.rb delete mode 100644 app/views/admin/domain_blocks/show.html.haml delete mode 100644 app/views/admin/instances/_exhausted_deliveries_days.haml (limited to 'app/models') diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb index b140c454c..16defc1ea 100644 --- a/app/controllers/admin/domain_blocks_controller.rb +++ b/app/controllers/admin/domain_blocks_controller.rb @@ -56,10 +56,6 @@ module Admin end end - def show - authorize @domain_block, :show? - end - def destroy authorize @domain_block, :destroy? UnblockDomainService.new.call(@domain_block) diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb index 306ec1f53..5c82331de 100644 --- a/app/controllers/admin/instances_controller.rb +++ b/app/controllers/admin/instances_controller.rb @@ -4,28 +4,26 @@ module Admin class InstancesController < BaseController before_action :set_instances, only: :index before_action :set_instance, except: :index - before_action :set_exhausted_deliveries_days, only: :show def index authorize :instance, :index? + preload_delivery_failures! end def show authorize :instance, :show? + @time_period = (6.days.ago.to_date...Time.now.utc.to_date) end def destroy authorize :instance, :destroy? - Admin::DomainPurgeWorker.perform_async(@instance.domain) - log_action :destroy, @instance redirect_to admin_instances_path, notice: I18n.t('admin.instances.destroyed_msg', domain: @instance.domain) end def clear_delivery_errors authorize :delivery, :clear_delivery_errors? - @instance.delivery_failure_tracker.clear_failures! redirect_to admin_instance_path(@instance.domain) end @@ -33,11 +31,9 @@ module Admin def restart_delivery authorize :delivery, :restart_delivery? - last_unavailable_domain = unavailable_domain - - if last_unavailable_domain.present? + if @instance.unavailable? @instance.delivery_failure_tracker.track_success! - log_action :destroy, last_unavailable_domain + log_action :destroy, @instance.unavailable_domain end redirect_to admin_instance_path(@instance.domain) @@ -45,8 +41,7 @@ module Admin def stop_delivery authorize :delivery, :stop_delivery? - - UnavailableDomain.create(domain: @instance.domain) + unavailable_domain = UnavailableDomain.create!(domain: @instance.domain) log_action :create, unavailable_domain redirect_to admin_instance_path(@instance.domain) end @@ -57,12 +52,11 @@ module Admin @instance = Instance.find(params[:id]) end - def set_exhausted_deliveries_days - @exhausted_deliveries_days = @instance.delivery_failure_tracker.exhausted_deliveries_days - end - def set_instances @instances = filtered_instances.page(params[:page]) + end + + def preload_delivery_failures! warning_domains_map = DeliveryFailureTracker.warning_domains_map @instances.each do |instance| @@ -70,10 +64,6 @@ module Admin end end - def unavailable_domain - UnavailableDomain.find_by(domain: @instance.domain) - end - def filtered_instances InstanceFilter.new(whitelist_mode? ? { allowed: true } : filter_params).results end diff --git a/app/javascript/mastodon/components/admin/Counter.js b/app/javascript/mastodon/components/admin/Counter.js index 047e864b2..6edb7bcfc 100644 --- a/app/javascript/mastodon/components/admin/Counter.js +++ b/app/javascript/mastodon/components/admin/Counter.js @@ -68,12 +68,12 @@ export default class Counter extends React.PureComponent { ); } else { const measure = data[0]; - const percentChange = percIncrease(measure.previous_total * 1, measure.total * 1); + const percentChange = measure.previous_total && percIncrease(measure.previous_total * 1, measure.total * 1); content = ( - - 0, negative: percentChange < 0 })}>{percentChange > 0 && '+'} + {measure.human_value || } + {measure.previous_total && ( 0, negative: percentChange < 0 })}>{percentChange > 0 && '+'})} ); } diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss index 42f055618..dc4d65edd 100644 --- a/app/javascript/styles/mastodon/admin.scss +++ b/app/javascript/styles/mastodon/admin.scss @@ -367,6 +367,21 @@ body, } } + .positive-hint, + .negative-hint, + .neutral-hint { + a { + color: inherit; + text-decoration: underline; + + &:focus, + &:hover, + &:active { + text-decoration: none; + } + } + } + .positive-hint { color: $valid-value-color; font-weight: 500; @@ -1596,3 +1611,38 @@ a.sparkline { } } } + +.availability-indicator { + display: flex; + align-items: center; + margin-bottom: 30px; + font-size: 14px; + line-height: 21px; + + &__hint { + padding: 0 15px; + } + + &__graphic { + display: flex; + margin: 0 -2px; + + &__item { + display: block; + flex: 0 0 auto; + width: 4px; + height: 21px; + background: lighten($ui-base-color, 8%); + margin: 0 2px; + border-radius: 2px; + + &.positive { + background: $valid-value-color; + } + + &.negative { + background: $error-value-color; + } + } + } +} diff --git a/app/javascript/styles/mastodon/tables.scss b/app/javascript/styles/mastodon/tables.scss index 1f7e71776..431b8a73a 100644 --- a/app/javascript/styles/mastodon/tables.scss +++ b/app/javascript/styles/mastodon/tables.scss @@ -65,6 +65,24 @@ } } + &.horizontal-table { + border-collapse: collapse; + border-style: hidden; + + & > tbody > tr > th, + & > tbody > tr > td { + padding: 11px 10px; + background: transparent; + border: 1px solid lighten($ui-base-color, 8%); + color: $secondary-text-color; + } + + & > tbody > tr > th { + color: $darker-text-color; + font-weight: 600; + } + } + &.batch-table { & > thead > tr > th { background: $ui-base-color; diff --git a/app/lib/admin/metrics/dimension.rb b/app/lib/admin/metrics/dimension.rb index d8392ddfc..81b89d9b3 100644 --- a/app/lib/admin/metrics/dimension.rb +++ b/app/lib/admin/metrics/dimension.rb @@ -9,6 +9,8 @@ class Admin::Metrics::Dimension software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension, tag_servers: Admin::Metrics::Dimension::TagServersDimension, tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension, + instance_accounts: Admin::Metrics::Dimension::InstanceAccountsDimension, + instance_languages: Admin::Metrics::Dimension::InstanceLanguagesDimension, }.freeze def self.retrieve(dimension_keys, start_at, end_at, limit, params) diff --git a/app/lib/admin/metrics/dimension/instance_accounts_dimension.rb b/app/lib/admin/metrics/dimension/instance_accounts_dimension.rb new file mode 100644 index 000000000..4eac8e611 --- /dev/null +++ b/app/lib/admin/metrics/dimension/instance_accounts_dimension.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::InstanceAccountsDimension < Admin::Metrics::Dimension::BaseDimension + include LanguagesHelper + + def self.with_params? + true + end + + def key + 'instance_accounts' + end + + protected + + def perform_query + sql = <<-SQL.squish + SELECT accounts.username, count(follows.*) AS value + FROM accounts + LEFT JOIN follows ON follows.target_account_id = accounts.id + WHERE accounts.domain = $1 + GROUP BY accounts.id, follows.target_account_id + ORDER BY value DESC + LIMIT $2 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:domain]], [nil, @limit]]) + + rows.map { |row| { key: row['username'], human_key: row['username'], value: row['value'].to_s } } + end + + def params + @params.permit(:domain) + end +end diff --git a/app/lib/admin/metrics/dimension/instance_languages_dimension.rb b/app/lib/admin/metrics/dimension/instance_languages_dimension.rb new file mode 100644 index 000000000..1ede1a56e --- /dev/null +++ b/app/lib/admin/metrics/dimension/instance_languages_dimension.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::InstanceLanguagesDimension < Admin::Metrics::Dimension::BaseDimension + include LanguagesHelper + + def self.with_params? + true + end + + def key + 'instance_languages' + end + + protected + + def perform_query + sql = <<-SQL.squish + SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value + FROM statuses + INNER JOIN accounts ON accounts.id = statuses.account_id + WHERE accounts.domain = $1 + AND statuses.id BETWEEN $2 AND $3 + AND statuses.reblog_of_id IS NULL + GROUP BY COALESCE(statuses.language, 'und') + ORDER BY count(*) DESC + LIMIT $4 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:domain]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]]) + + rows.map { |row| { key: row['language'], human_key: standard_locale_name(row['language']), value: row['value'].to_s } } + end + + def params + @params.permit(:domain) + end +end diff --git a/app/lib/admin/metrics/measure.rb b/app/lib/admin/metrics/measure.rb index a839498a1..0b510eb25 100644 --- a/app/lib/admin/metrics/measure.rb +++ b/app/lib/admin/metrics/measure.rb @@ -10,6 +10,12 @@ class Admin::Metrics::Measure tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure, tag_uses: Admin::Metrics::Measure::TagUsesMeasure, tag_servers: Admin::Metrics::Measure::TagServersMeasure, + instance_accounts: Admin::Metrics::Measure::InstanceAccountsMeasure, + instance_media_attachments: Admin::Metrics::Measure::InstanceMediaAttachmentsMeasure, + instance_reports: Admin::Metrics::Measure::InstanceReportsMeasure, + instance_statuses: Admin::Metrics::Measure::InstanceStatusesMeasure, + instance_follows: Admin::Metrics::Measure::InstanceFollowsMeasure, + instance_followers: Admin::Metrics::Measure::InstanceFollowersMeasure, }.freeze def self.retrieve(measure_keys, start_at, end_at, params) diff --git a/app/lib/admin/metrics/measure/base_measure.rb b/app/lib/admin/metrics/measure/base_measure.rb index ed1df9c7d..e33a6c494 100644 --- a/app/lib/admin/metrics/measure/base_measure.rb +++ b/app/lib/admin/metrics/measure/base_measure.rb @@ -26,6 +26,14 @@ class Admin::Metrics::Measure::BaseMeasure raise NotImplementedError end + def unit + nil + end + + def total_in_time_range? + true + end + def total load[:total] end diff --git a/app/lib/admin/metrics/measure/instance_accounts_measure.rb b/app/lib/admin/metrics/measure/instance_accounts_measure.rb new file mode 100644 index 000000000..4c61a064a --- /dev/null +++ b/app/lib/admin/metrics/measure/instance_accounts_measure.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::InstanceAccountsMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'instance_accounts' + end + + def total_in_time_range? + false + end + + protected + + def perform_total_query + Account.where(domain: params[:domain]).count + end + + def perform_previous_total_query + nil + end + + def perform_data_query + sql = <<-SQL.squish + SELECT axis.*, ( + WITH new_accounts AS ( + SELECT accounts.id + FROM accounts + WHERE date_trunc('day', accounts.created_at)::date = axis.period + AND accounts.domain = $3::text + ) + SELECT count(*) FROM new_accounts + ) AS value + FROM ( + SELECT generate_series(date_trunc('day', $1::timestamp)::date, date_trunc('day', $2::timestamp)::date, interval '1 day') AS period + ) AS axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, params[:domain]]]) + + rows.map { |row| { date: row['period'], value: row['value'].to_s } } + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:domain) + end +end diff --git a/app/lib/admin/metrics/measure/instance_followers_measure.rb b/app/lib/admin/metrics/measure/instance_followers_measure.rb new file mode 100644 index 000000000..caa60013b --- /dev/null +++ b/app/lib/admin/metrics/measure/instance_followers_measure.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::InstanceFollowersMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'instance_followers' + end + + def total_in_time_range? + false + end + + protected + + def perform_total_query + Follow.joins(:account).merge(Account.where(domain: params[:domain])).count + end + + def perform_previous_total_query + nil + end + + def perform_data_query + sql = <<-SQL.squish + SELECT axis.*, ( + WITH new_followers AS ( + SELECT follows.id + FROM follows + INNER JOIN accounts ON follows.account_id = accounts.id + WHERE date_trunc('day', follows.created_at)::date = axis.period + AND accounts.domain = $3::text + ) + SELECT count(*) FROM new_followers + ) AS value + FROM ( + SELECT generate_series(date_trunc('day', $1::timestamp)::date, date_trunc('day', $2::timestamp)::date, interval '1 day') AS period + ) AS axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, params[:domain]]]) + + rows.map { |row| { date: row['period'], value: row['value'].to_s } } + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:domain) + end +end diff --git a/app/lib/admin/metrics/measure/instance_follows_measure.rb b/app/lib/admin/metrics/measure/instance_follows_measure.rb new file mode 100644 index 000000000..b026c7e6d --- /dev/null +++ b/app/lib/admin/metrics/measure/instance_follows_measure.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::InstanceFollowsMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'instance_follows' + end + + def total_in_time_range? + false + end + + protected + + def perform_total_query + Follow.joins(:target_account).merge(Account.where(domain: params[:domain])).count + end + + def perform_previous_total_query + nil + end + + def perform_data_query + sql = <<-SQL.squish + SELECT axis.*, ( + WITH new_follows AS ( + SELECT follows.id + FROM follows + INNER JOIN accounts ON follows.target_account_id = accounts.id + WHERE date_trunc('day', follows.created_at)::date = axis.period + AND accounts.domain = $3::text + ) + SELECT count(*) FROM new_follows + ) AS value + FROM ( + SELECT generate_series(date_trunc('day', $1::timestamp)::date, date_trunc('day', $2::timestamp)::date, interval '1 day') AS period + ) AS axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, params[:domain]]]) + + rows.map { |row| { date: row['period'], value: row['value'].to_s } } + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:domain) + end +end diff --git a/app/lib/admin/metrics/measure/instance_media_attachments_measure.rb b/app/lib/admin/metrics/measure/instance_media_attachments_measure.rb new file mode 100644 index 000000000..2e2154c92 --- /dev/null +++ b/app/lib/admin/metrics/measure/instance_media_attachments_measure.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::InstanceMediaAttachmentsMeasure < Admin::Metrics::Measure::BaseMeasure + include ActionView::Helpers::NumberHelper + + def self.with_params? + true + end + + def key + 'instance_media_attachments' + end + + def unit + 'bytes' + end + + def value_to_human_value(value) + number_to_human_size(value) + end + + def total_in_time_range? + false + end + + protected + + def perform_total_query + MediaAttachment.joins(:account).merge(Account.where(domain: params[:domain])).sum('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)') + end + + def perform_previous_total_query + nil + end + + def perform_data_query + sql = <<-SQL.squish + SELECT axis.*, ( + WITH new_media_attachments AS ( + SELECT COALESCE(media_attachments.file_file_size, 0) + COALESCE(media_attachments.thumbnail_file_size, 0) AS size + FROM media_attachments + INNER JOIN accounts ON accounts.id = media_attachments.account_id + WHERE date_trunc('day', media_attachments.created_at)::date = axis.period + AND accounts.domain = $3::text + ) + SELECT SUM(size) FROM new_media_attachments + ) AS value + FROM ( + SELECT generate_series(date_trunc('day', $1::timestamp)::date, date_trunc('day', $2::timestamp)::date, interval '1 day') AS period + ) AS axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, params[:domain]]]) + + rows.map { |row| { date: row['period'], value: row['value'].to_s } } + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:domain) + end +end diff --git a/app/lib/admin/metrics/measure/instance_reports_measure.rb b/app/lib/admin/metrics/measure/instance_reports_measure.rb new file mode 100644 index 000000000..6b3f35067 --- /dev/null +++ b/app/lib/admin/metrics/measure/instance_reports_measure.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::InstanceReportsMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'instance_reports' + end + + def total_in_time_range? + false + end + + protected + + def perform_total_query + Report.where(target_account: Account.where(domain: params[:domain])).count + end + + def perform_previous_total_query + nil + end + + def perform_data_query + sql = <<-SQL.squish + SELECT axis.*, ( + WITH new_reports AS ( + SELECT reports.id + FROM reports + INNER JOIN accounts ON accounts.id = reports.target_account_id + WHERE date_trunc('day', reports.created_at)::date = axis.period + AND accounts.domain = $3::text + ) + SELECT count(*) FROM new_reports + ) AS value + FROM ( + SELECT generate_series(date_trunc('day', $1::timestamp)::date, date_trunc('day', $2::timestamp)::date, interval '1 day') AS period + ) AS axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, params[:domain]]]) + + rows.map { |row| { date: row['period'], value: row['value'].to_s } } + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:domain) + end +end diff --git a/app/lib/admin/metrics/measure/instance_statuses_measure.rb b/app/lib/admin/metrics/measure/instance_statuses_measure.rb new file mode 100644 index 000000000..86b10da6c --- /dev/null +++ b/app/lib/admin/metrics/measure/instance_statuses_measure.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::InstanceStatusesMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'instance_statuses' + end + + def total_in_time_range? + false + end + + protected + + def perform_total_query + Status.joins(:account).merge(Account.where(domain: params[:domain])).count + end + + def perform_previous_total_query + nil + end + + def perform_data_query + sql = <<-SQL.squish + SELECT axis.*, ( + WITH new_statuses AS ( + SELECT statuses.id + FROM statuses + INNER JOIN accounts ON accounts.id = statuses.account_id + WHERE statuses.id BETWEEN $3 AND $4 + AND accounts.domain = $5::text + AND date_trunc('day', statuses.created_at)::date = axis.period + ) + SELECT count(*) FROM new_statuses + ) AS value + FROM ( + SELECT generate_series(date_trunc('day', $1::timestamp)::date, date_trunc('day', $2::timestamp)::date, interval '1 day') AS period + ) AS axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, params[:domain]]]) + + rows.map { |row| { date: row['period'], value: row['value'].to_s } } + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:domain) + end +end diff --git a/app/lib/delivery_failure_tracker.rb b/app/lib/delivery_failure_tracker.rb index 8907ade4c..7b800fc0b 100644 --- a/app/lib/delivery_failure_tracker.rb +++ b/app/lib/delivery_failure_tracker.rb @@ -30,7 +30,7 @@ class DeliveryFailureTracker end def exhausted_deliveries_days - Redis.current.smembers(exhausted_deliveries_key).sort.map { |date| Date.new(date.slice(0, 4).to_i, date.slice(4, 2).to_i, date.slice(6, 2).to_i) } + @exhausted_deliveries_days ||= Redis.current.smembers(exhausted_deliveries_key).sort.map { |date| Date.new(date.slice(0, 4).to_i, date.slice(4, 2).to_i, date.slice(6, 2).to_i) } end alias reset! track_success! diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb index bba04c603..b06fa09da 100644 --- a/app/models/domain_block.rb +++ b/app/models/domain_block.rb @@ -30,6 +30,14 @@ class DomainBlock < ApplicationRecord scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) } scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), reject_media, domain')) } + def policies + if suspend? + :suspend + else + [severity.to_sym, reject_media? ? :reject_media : nil, reject_reports? ? :reject_reports : nil].reject { |policy| policy == :noop || policy.nil? } + end + end + class << self def suspend?(domain) !!rule_for(domain)&.suspend? diff --git a/app/models/instance.rb b/app/models/instance.rb index 8949be054..7434d3322 100644 --- a/app/models/instance.rb +++ b/app/models/instance.rb @@ -32,35 +32,27 @@ class Instance < ApplicationRecord @delivery_failure_tracker ||= DeliveryFailureTracker.new(domain) end - def following_count - @following_count ||= Follow.where(account: accounts).count + def unavailable? + unavailable_domain.present? end - def followers_count - @followers_count ||= Follow.where(target_account: accounts).count + def failing? + failure_days.present? || unavailable? end - def reports_count - @reports_count ||= Report.where(target_account: accounts).count - end - - def blocks_count - @blocks_count ||= Block.where(target_account: accounts).count - end - - def public_comment - domain_block&.public_comment + def to_param + domain end - def private_comment - domain_block&.private_comment - end + delegate :exhausted_deliveries_days, to: :delivery_failure_tracker - def media_storage - @media_storage ||= MediaAttachment.where(account: accounts).sum(:file_file_size) - end + def availability_over_days(num_days, end_date = Time.now.utc.to_date) + failures_map = exhausted_deliveries_days.index_with { true } + period_end_at = exhausted_deliveries_days.last || end_date + period_start_at = period_end_at - num_days.days - def to_param - domain + (period_start_at..period_end_at).map do |date| + [date, failures_map[date]] + end end end diff --git a/app/models/instance_filter.rb b/app/models/instance_filter.rb index 9e533c4aa..e7e5166a1 100644 --- a/app/models/instance_filter.rb +++ b/app/models/instance_filter.rb @@ -4,8 +4,7 @@ class InstanceFilter KEYS = %i( limited by_domain - warning - unavailable + availability ).freeze attr_reader :params @@ -34,12 +33,21 @@ class InstanceFilter Instance.joins(:domain_allow).reorder(Arel.sql('domain_allows.id desc')) when 'by_domain' Instance.matches_domain(value) - when 'warning' + when 'availability' + availability_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def availability_scope(value) + case value + when 'failing' Instance.where(domain: DeliveryFailureTracker.warning_domains) when 'unavailable' Instance.joins(:unavailable_domain) else - raise "Unknown filter: #{key}" + raise "Unknown availability: #{value}" end end end diff --git a/app/serializers/rest/admin/measure_serializer.rb b/app/serializers/rest/admin/measure_serializer.rb index 81d655c1a..fc16b85f2 100644 --- a/app/serializers/rest/admin/measure_serializer.rb +++ b/app/serializers/rest/admin/measure_serializer.rb @@ -1,12 +1,20 @@ # frozen_string_literal: true class REST::Admin::MeasureSerializer < ActiveModel::Serializer - attributes :key, :total, :previous_total, :data + attributes :key, :unit, :total + + attribute :human_value, if: -> { object.respond_to?(:value_to_human_value) } + attribute :previous_total, if: -> { object.total_in_time_range? } + attribute :data def total object.total.to_s end + def human_value + object.value_to_human_value(object.total) + end + def previous_total object.previous_total.to_s end diff --git a/app/views/admin/domain_blocks/edit.html.haml b/app/views/admin/domain_blocks/edit.html.haml index 6fe2edc82..39c6d108a 100644 --- a/app/views/admin/domain_blocks/edit.html.haml +++ b/app/views/admin/domain_blocks/edit.html.haml @@ -24,10 +24,10 @@ = f.input :obfuscate, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.obfuscate'), hint: I18n.t('admin.domain_blocks.obfuscate_hint') .field-group - = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6 + = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), as: :string .field-group - = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6 + = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), as: :string .actions = f.button :button, t('generic.save_changes'), type: :submit diff --git a/app/views/admin/domain_blocks/new.html.haml b/app/views/admin/domain_blocks/new.html.haml index 8b78f71f2..bcaa331b5 100644 --- a/app/views/admin/domain_blocks/new.html.haml +++ b/app/views/admin/domain_blocks/new.html.haml @@ -24,10 +24,10 @@ = f.input :obfuscate, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.obfuscate'), hint: I18n.t('admin.domain_blocks.obfuscate_hint') .field-group - = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6 + = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), as: :string .field-group - = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6 + = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), as: :string .actions = f.button :button, t('.create'), type: :submit diff --git a/app/views/admin/domain_blocks/show.html.haml b/app/views/admin/domain_blocks/show.html.haml deleted file mode 100644 index e64aaa629..000000000 --- a/app/views/admin/domain_blocks/show.html.haml +++ /dev/null @@ -1,25 +0,0 @@ -- content_for :page_title do - = t('admin.domain_blocks.show.title', domain: @domain_block.domain) - -- if @domain_block.private_comment.present? - .speech-bubble - .speech-bubble__bubble - = simple_format(h(@domain_block.private_comment)) - .speech-bubble__owner= t 'admin.instances.private_comment' - -- if @domain_block.public_comment.present? - .speech-bubble - .speech-bubble__bubble - = simple_format(h(@domain_block.public_comment)) - .speech-bubble__owner= t 'admin.instances.public_comment' - -= simple_form_for @domain_block, url: admin_domain_block_path(@domain_block), method: :delete do |f| - - - unless (@domain_block.noop?) - %p= t(".retroactive.#{@domain_block.severity}") - %p.hint= t(:affected_accounts, - scope: [:admin, :domain_blocks, :show], - count: @domain_block.affected_accounts_count) - - .actions - = f.button :button, t('.undo'), type: :submit diff --git a/app/views/admin/instances/_exhausted_deliveries_days.haml b/app/views/admin/instances/_exhausted_deliveries_days.haml deleted file mode 100644 index e581f542d..000000000 --- a/app/views/admin/instances/_exhausted_deliveries_days.haml +++ /dev/null @@ -1,2 +0,0 @@ -%li.negative-hint - = l(exhausted_deliveries_days) diff --git a/app/views/admin/instances/_instance.html.haml b/app/views/admin/instances/_instance.html.haml index dc81007ac..8a4396002 100644 --- a/app/views/admin/instances/_instance.html.haml +++ b/app/views/admin/instances/_instance.html.haml @@ -1,33 +1,15 @@ .directory__tag = link_to admin_instance_path(instance) do %h4 + = fa_icon 'warning fw' if instance.failing? = instance.domain + %small - if instance.domain_block - - first_item = true - - if !instance.domain_block.noop? - = t("admin.domain_blocks.severity.#{instance.domain_block.severity}") - - first_item = false - - unless instance.domain_block.suspend? - - if instance.domain_block.reject_media? - - unless first_item - • - = t('admin.domain_blocks.rejecting_media') - - first_item = false - - if instance.domain_block.reject_reports? - - unless first_item - • - = t('admin.domain_blocks.rejecting_reports') - - elsif whitelist_mode? + = instance.domain_block.policies.map { |policy| t(policy, scope: 'admin.instances.content_policies.policies') }.join(' • ') + - elsif instance.domain_allow = t('admin.accounts.whitelisted') - else = t('admin.accounts.no_limits_imposed') - - if instance.failure_days - = ' / ' - %span.negative-hint - = t('admin.instances.delivery.warning_message', count: instance.failure_days) - - if instance.unavailable_domain - = ' / ' - %span.negative-hint - = t('admin.instances.delivery.unavailable_message') + .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= friendly_number_to_human instance.accounts_count diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml index 797948d94..f8273718d 100644 --- a/app/views/admin/instances/index.html.haml +++ b/app/views/admin/instances/index.html.haml @@ -17,22 +17,11 @@ %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1' .filter-subset - %strong= t('admin.instances.delivery.title') + %strong= t('admin.instances.availability.title') %ul - %li= filter_link_to t('admin.instances.delivery.all'), warning: nil, unavailable: nil - %li= filter_link_to t('admin.instances.delivery.warning'), warning: '1', unavailable: nil - %li= filter_link_to t('admin.instances.delivery.unavailable'), warning: nil, unavailable: '1' - - .back-link - = link_to admin_instances_path() do - %i.fa.fa-chevron-left.fa-fw - = t('admin.instances.back_to_all') - = link_to admin_instances_path(limited: 1) do - %i.fa.fa-chevron-left.fa-fw - = t('admin.instances.back_to_limited') - = link_to admin_instances_path(warning: 1) do - %i.fa.fa-chevron-left.fa-fw - = t('admin.instances.back_to_warning') + %li= filter_link_to t('admin.instances.delivery.all'), availability: nil + %li= filter_link_to t('admin.instances.delivery.failing'), availability: 'failing' + %li= filter_link_to t('admin.instances.delivery.unavailable'), availability: 'unavailable' - unless whitelist_mode? = form_tag admin_instances_url, method: 'GET', class: 'simple_form' do diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml index 4db8fd15c..bed94f3fe 100644 --- a/app/views/admin/instances/show.html.haml +++ b/app/views/admin/instances/show.html.haml @@ -1,88 +1,95 @@ - content_for :page_title do = @instance.domain -.filters - .back-link - = link_to admin_instances_path() do - %i.fa.fa-chevron-left.fa-fw - = t('admin.instances.back_to_all') - = link_to admin_instances_path(limited: 1) do - %i.fa.fa-chevron-left.fa-fw - = t('admin.instances.back_to_limited') - = link_to admin_instances_path(warning: 1) do - %i.fa.fa-chevron-left.fa-fw - = t('admin.instances.back_to_warning') - -.dashboard__counters - %div - = link_to admin_accounts_path(origin: 'remote', by_domain: @instance.domain) do - .dashboard__counters__num= number_with_delimiter @instance.accounts_count - .dashboard__counters__label= t 'admin.accounts.title' - %div - = link_to admin_reports_path(by_target_domain: @instance.domain) do - .dashboard__counters__num= number_with_delimiter @instance.reports_count - .dashboard__counters__label= t 'admin.instances.total_reported' - %div - %div - .dashboard__counters__num= number_to_human_size @instance.media_storage - .dashboard__counters__label= t 'admin.instances.total_storage' - %div - %div - .dashboard__counters__num= number_with_delimiter @instance.following_count - .dashboard__counters__label= t 'admin.instances.total_followed_by_them' - %div - %div - .dashboard__counters__num= number_with_delimiter @instance.followers_count - .dashboard__counters__label= t 'admin.instances.total_followed_by_us' - %div - %div - .dashboard__counters__num= number_with_delimiter @instance.blocks_count - .dashboard__counters__label= t 'admin.instances.total_blocked_by_us' - - %div - %div - .dashboard__counters__num - - if @instance.delivery_failure_tracker.available? - = fa_icon 'check' - - else - = fa_icon 'times' - .dashboard__counters__label= t 'admin.instances.delivery_available' - -- if @instance.private_comment.present? - .speech-bubble - .speech-bubble__bubble - = simple_format(h(@instance.private_comment)) - .speech-bubble__owner= t 'admin.instances.private_comment' - -- if @instance.public_comment.present? - .speech-bubble - .speech-bubble__bubble - = simple_format(h(@instance.public_comment)) - .speech-bubble__owner= t 'admin.instances.public_comment' - -- unless @exhausted_deliveries_days.empty? - %h4= t 'admin.instances.delivery_error_days' - %ul - = render partial: 'exhausted_deliveries_days', collection: @exhausted_deliveries_days - %p.hint - = t 'admin.instances.delivery_error_hint', count: DeliveryFailureTracker::FAILURE_DAYS_THRESHOLD +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +- content_for :heading_actions do + = l(@time_period.first) + = ' - ' + = l(@time_period.last) + +%p + = fa_icon 'info fw' + = t('admin.instances.totals_time_period_hint_html') + +.dashboard + .dashboard__item + = react_admin_component :counter, measure: 'instance_accounts', start_at: @time_period.first, end_at: @time_period.last, params: { domain: @instance.domain }, label: t('admin.instances.dashboard.instance_accounts_measure'), href: admin_accounts_path(origin: 'remote', by_domain: @instance.domain) + .dashboard__item + = react_admin_component :counter, measure: 'instance_statuses', start_at: @time_period.first, end_at: @time_period.last, params: { domain: @instance.domain }, label: t('admin.instances.dashboard.instance_statuses_measure') + .dashboard__item + = react_admin_component :counter, measure: 'instance_media_attachments', start_at: @time_period.first, end_at: @time_period.last, params: { domain: @instance.domain }, label: t('admin.instances.dashboard.instance_media_attachments_measure') + .dashboard__item + = react_admin_component :counter, measure: 'instance_follows', start_at: @time_period.first, end_at: @time_period.last, params: { domain: @instance.domain }, label: t('admin.instances.dashboard.instance_follows_measure') + .dashboard__item + = react_admin_component :counter, measure: 'instance_followers', start_at: @time_period.first, end_at: @time_period.last, params: { domain: @instance.domain }, label: t('admin.instances.dashboard.instance_followers_measure') + .dashboard__item + = react_admin_component :counter, measure: 'instance_reports', start_at: @time_period.first, end_at: @time_period.last, params: { domain: @instance.domain }, label: t('admin.instances.dashboard.instance_reports_measure'), href: admin_reports_path(by_target_domain: @instance.domain) + .dashboard__item + = react_admin_component :dimension, dimension: 'instance_accounts', start_at: @time_period.first, end_at: @time_period.last, params: { domain: @instance.domain }, limit: 8, label: t('admin.instances.dashboard.instance_accounts_dimension') + .dashboard__item + = react_admin_component :dimension, dimension: 'instance_languages', start_at: @time_period.first, end_at: @time_period.last, params: { domain: @instance.domain }, limit: 8, label: t('admin.instances.dashboard.instance_languages_dimension') %hr.spacer/ -%div.action-buttons - %div - - if @instance.domain_allow - = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@instance.domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete } - - elsif @instance.domain_block - = link_to t('admin.domain_blocks.edit'), edit_admin_domain_block_path(@instance.domain_block), class: 'button' - = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@instance.domain_block), class: 'button' - - else - = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button' - - if @instance.delivery_failure_tracker.available? - - unless @exhausted_deliveries_days.empty? - = link_to t('admin.instances.delivery.clear'), clear_delivery_errors_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post }, class: 'button' - = link_to t('admin.instances.delivery.stop'), stop_delivery_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post }, class: 'button' +%h3= t('admin.instances.content_policies.title') + +- if whitelist_mode? + %p= t('admin.instances.content_policies.limited_federation_mode_description_html') + + - if @instance.domain_allow + = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@instance.domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete } + - else + = link_to t('admin.domain_allows.add_new'), admin_domain_allows_path(domain_allow: { domain: @instance.domain }), class: 'button', method: :post +- else + %p= t('admin.instances.content_policies.description_html') + + - if @instance.domain_block + .table-wrapper + %table.table.horizontal-table + %tbody + %tr + %th= t('admin.instances.content_policies.comment') + %td= @instance.domain_block.private_comment + %tr + %th= t('admin.instances.content_policies.reason') + %td= @instance.domain_block.public_comment + %tr + %th= t('admin.instances.content_policies.policy') + %td= @instance.domain_block.policies.map { |policy| t(policy, scope: 'admin.instances.content_policies.policies') }.join(' • ') + + = link_to t('admin.domain_blocks.edit'), edit_admin_domain_block_path(@instance.domain_block), class: 'button' + = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@instance.domain_block), class: 'button', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete } + - else + = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button' + +%hr.spacer/ + +%h3= t('admin.instances.availability.title') + +%p + = t('admin.instances.availability.description_html', count: DeliveryFailureTracker::FAILURE_DAYS_THRESHOLD) + +.availability-indicator + %ul.availability-indicator__graphic + - @instance.availability_over_days(14).each do |(date, failing)| + %li.availability-indicator__graphic__item{ class: failing ? 'negative' : 'neutral', title: l(date) } + .availability-indicator__hint + - if @instance.unavailable? + %span.negative-hint + = t('admin.instances.availability.failure_threshold_reached', date: l(@instance.unavailable_domain.created_at.to_date)) + = link_to t('admin.instances.delivery.restart'), restart_delivery_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post } + - elsif @instance.exhausted_deliveries_days.empty? + %span.positive-hint + = t('admin.instances.availability.no_failures_recorded') + = link_to t('admin.instances.delivery.stop'), stop_delivery_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post } - else - = link_to t('admin.instances.delivery.restart'), restart_delivery_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post }, class: 'button' - - if !@instance.delivery_failure_tracker.available? || @instance.accounts_count.zero? || @instance.domain_block&.suspend? - = link_to t('admin.instances.purge'), admin_instance_path(@instance), data: { confirm: t('admin.instances.confirm_purge'), method: :delete }, class: 'button' + %span.negative-hint + = t('admin.instances.availability.failures_recorded', count: @instance.delivery_failure_tracker.days) + = link_to t('admin.instances.delivery.clear'), clear_delivery_errors_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post } unless @instance.exhausted_deliveries_days.empty? + +- if @instance.unavailable? + %p= t('admin.instances.purge_description_html') + + = link_to t('admin.instances.purge'), admin_instance_path(@instance), data: { confirm: t('admin.instances.confirm_purge'), method: :delete }, class: 'button button--destructive' diff --git a/app/workers/admin/domain_purge_worker.rb b/app/workers/admin/domain_purge_worker.rb index 7cba2c89e..095232a6d 100644 --- a/app/workers/admin/domain_purge_worker.rb +++ b/app/workers/admin/domain_purge_worker.rb @@ -3,6 +3,8 @@ class Admin::DomainPurgeWorker include Sidekiq::Worker + sidekiq_options queue: 'pull', lock: :until_executed + def perform(domain) PurgeDomainService.new.call(domain) end diff --git a/config/locales/en.yml b/config/locales/en.yml index 1964285b0..acedf84c2 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -450,21 +450,6 @@ en: reject_media_hint: Removes locally stored media files and refuses to download any in the future. Irrelevant for suspensions reject_reports: Reject reports reject_reports_hint: Ignore all reports coming from this domain. Irrelevant for suspensions - rejecting_media: rejecting media files - rejecting_reports: rejecting reports - severity: - silence: limited - suspend: suspended - show: - affected_accounts: - one: One account in the database affected - other: "%{count} accounts in the database affected" - zero: No account in the database is affected - retroactive: - silence: Undo limit of existing affected accounts from this domain - suspend: Unsuspend existing affected accounts from this domain - title: Undo domain block for %{domain} - undo: Undo undo: Undo domain block view: View domain block email_domain_blocks: @@ -495,23 +480,47 @@ en: title: Follow recommendations unsuppress: Restore follow recommendation instances: + availability: + description_html: + one: If delivering to the domain fails %{count} day without succeeding, no further delivery attempts will be made unless a delivery from the domain is received. + other: If delivering to the domain fails on %{count} different days without succeeding, no further delivery attempts will be made unless a delivery from the domain is received. + failure_threshold_reached: Failure threshold reached on %{date}. + failures_recorded: + one: Failed attempt on %{count} day. + other: Failed attempts on %{count} different days. + no_failures_recorded: No failures on record. + title: Availability back_to_all: All back_to_limited: Limited back_to_warning: Warning by_domain: Domain confirm_purge: Are you sure you want to permanently delete data from this domain? + content_policies: + comment: Internal note + description_html: You can define content policies that will be applied to all accounts from this domain and any of its subdomains. + policies: + reject_media: Reject media + reject_reports: Reject reports + silence: Limit + suspend: Suspend + policy: Policy + reason: Public reason + title: Content policies + dashboard: + instance_accounts_dimension: Most followed accounts + instance_accounts_measure: stored accounts + instance_followers_measure: our followers there + instance_follows_measure: their followers here + instance_languages_dimension: Top languages + instance_media_attachments_measure: stored media attachments + instance_reports_measure: reports about them + instance_statuses_measure: stored posts delivery: all: All clear: Clear delivery errors restart: Restart delivery stop: Stop delivery - title: Delivery unavailable: Unavailable - unavailable_message: Delivery unavailable - warning: Warning - warning_message: - one: Delivery failure %{count} day - other: Delivery failure %{count} days delivery_available: Delivery is available delivery_error_days: Delivery error days delivery_error_hint: If delivery is not possible for %{count} days, it will be automatically marked as undeliverable. @@ -528,12 +537,14 @@ en: private_comment: Private comment public_comment: Public comment purge: Purge + purge_description_html: If you believe this domain is offline for good, you can delete all account records and associated data from this domain from your storage. This may take a while. title: Federation total_blocked_by_us: Blocked by us total_followed_by_them: Followed by them total_followed_by_us: Followed by us total_reported: Reports about them total_storage: Media attachments + totals_time_period_hint_html: The totals displayed below include data for all time. invites: deactivate_all: Deactivate all filter: diff --git a/config/routes.rb b/config/routes.rb index 9e2f7a648..c108e2ec4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -191,7 +191,7 @@ Rails.application.routes.draw do get '/dashboard', to: 'dashboard#index' resources :domain_allows, only: [:new, :create, :show, :destroy] - resources :domain_blocks, only: [:new, :create, :show, :destroy, :update, :edit] + resources :domain_blocks, only: [:new, :create, :destroy, :update, :edit] resources :email_domain_blocks, only: [:index, :new, :create] do collection do diff --git a/spec/controllers/admin/domain_blocks_controller_spec.rb b/spec/controllers/admin/domain_blocks_controller_spec.rb index fb23658c0..ecc79292b 100644 --- a/spec/controllers/admin/domain_blocks_controller_spec.rb +++ b/spec/controllers/admin/domain_blocks_controller_spec.rb @@ -16,15 +16,6 @@ RSpec.describe Admin::DomainBlocksController, type: :controller do end end - describe 'GET #show' do - it 'returns http success' do - domain_block = Fabricate(:domain_block) - get :show, params: { id: domain_block.id } - - expect(response).to have_http_status(200) - end - end - describe 'POST #create' do it 'blocks the domain when succeeded to save' do allow(DomainBlockWorker).to receive(:perform_async).and_return(true) -- cgit From d17fb7013116767fc5c7d5eef63218bd8c45b023 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 9 Mar 2022 09:06:17 +0100 Subject: Change how changes to media attachments are stored for edits (#17696) * Change how changes to media attachments are stored for edits Fix not being able to re-order media attachments * Fix not broadcasting updates when polls/media is changed through ActivityPub * Various fixes and improvements * Update app/models/report.rb Co-authored-by: Claire * Add tracking of media attachment description changes * Change poll in status edit to have a structure closer to the real one Co-authored-by: Claire --- app/chewy/statuses_index.rb | 2 +- app/helpers/statuses_helper.rb | 12 ++--- app/lib/feed_manager.rb | 2 +- app/lib/rss/serializer.rb | 2 +- app/models/report.rb | 16 +++++- app/models/status.rb | 63 +++++++++++++--------- app/models/status_edit.rb | 41 +++++++++++--- app/serializers/activitypub/note_serializer.rb | 6 ++- app/serializers/rest/status_edit_serializer.rb | 10 +++- app/serializers/rest/status_serializer.rb | 2 +- .../activitypub/process_status_update_service.rb | 19 +++---- app/services/fan_out_on_write_service.rb | 2 +- app/services/post_status_service.rb | 6 ++- app/services/remove_status_service.rb | 2 +- app/services/update_status_service.rb | 21 ++------ app/views/admin/reports/_status.html.haml | 12 ++--- app/views/admin/reports/index.html.haml | 4 +- app/views/admin/trends/statuses/_status.html.haml | 2 +- app/views/disputes/strikes/show.html.haml | 2 +- app/views/notification_mailer/_status.html.haml | 4 +- app/views/statuses/_detailed_status.html.haml | 6 +-- app/views/statuses/_og_image.html.haml | 2 +- app/views/statuses/_simple_status.html.haml | 6 +-- ...add_ordered_media_attachment_ids_to_statuses.rb | 5 ++ ...ordered_media_attachment_ids_to_status_edits.rb | 8 +++ ..._media_attachments_changed_from_status_edits.rb | 7 +++ db/schema.rb | 7 ++- spec/models/report_spec.rb | 13 +++-- .../process_status_update_service_spec.rb | 17 +++--- spec/services/update_status_service_spec.rb | 14 ++--- 30 files changed, 190 insertions(+), 125 deletions(-) create mode 100644 db/migrate/20220302232632_add_ordered_media_attachment_ids_to_statuses.rb create mode 100644 db/migrate/20220303000827_add_ordered_media_attachment_ids_to_status_edits.rb create mode 100644 db/post_migrate/20220303203437_remove_media_attachments_changed_from_status_edits.rb (limited to 'app/models') diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index 1903c2ea3..65cbb6fcd 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -57,7 +57,7 @@ class StatusesIndex < Chewy::Index field :id, type: 'long' field :account_id, type: 'long' - field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do + field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.ordered_media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do field :stemmed, type: 'text', analyzer: 'content' end diff --git a/app/helpers/statuses_helper.rb b/app/helpers/statuses_helper.rb index 25f079e9d..d328f89b7 100644 --- a/app/helpers/statuses_helper.rb +++ b/app/helpers/statuses_helper.rb @@ -132,7 +132,7 @@ module StatusesHelper end def render_video_component(status, **options) - video = status.media_attachments.first + video = status.ordered_media_attachments.first meta = video.file.meta || {} @@ -150,12 +150,12 @@ module StatusesHelper }.merge(**options) react_component :video, component_params do - render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + render partial: 'statuses/attachment_list', locals: { attachments: status.ordered_media_attachments } end end def render_audio_component(status, **options) - audio = status.media_attachments.first + audio = status.ordered_media_attachments.first meta = audio.file.meta || {} @@ -170,7 +170,7 @@ module StatusesHelper }.merge(**options) react_component :audio, component_params do - render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + render partial: 'statuses/attachment_list', locals: { attachments: status.ordered_media_attachments } end end @@ -178,11 +178,11 @@ module StatusesHelper component_params = { sensitive: sensitized?(status, current_account), autoplay: prefers_autoplay?, - media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }, + media: status.ordered_media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }, }.merge(**options) react_component :media_gallery, component_params do - render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + render partial: 'statuses/attachment_list', locals: { attachments: status.ordered_media_attachments } end end diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index fc35292f2..46a55c7a4 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -448,7 +448,7 @@ class FeedManager Formatter.instance.plaintext(status), status.spoiler_text, status.preloadable_poll ? status.preloadable_poll.options.join("\n\n") : nil, - status.media_attachments.map(&:description).join("\n\n"), + status.ordered_media_attachments.map(&:description).join("\n\n"), ].compact.join("\n\n") combined_regex.match?(combined_text) diff --git a/app/lib/rss/serializer.rb b/app/lib/rss/serializer.rb index fd56c568c..7e3ed1f17 100644 --- a/app/lib/rss/serializer.rb +++ b/app/lib/rss/serializer.rb @@ -11,7 +11,7 @@ class RSS::Serializer .pub_date(status.created_at) .description(status.spoiler_text.presence || Formatter.instance.format(status, inline_poll_options: true).to_str) - status.media_attachments.each do |media| + status.ordered_media_attachments.each do |media| item.enclosure(full_asset_url(media.file.url(:original, false)), media.file.content_type, media.file.size) end end diff --git a/app/models/report.rb b/app/models/report.rb index 8ba2dd8fd..6d4166540 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -63,8 +63,20 @@ class Report < ApplicationRecord Status.with_discarded.where(id: status_ids) end - def media_attachments - MediaAttachment.where(status_id: status_ids) + def media_attachments_count + statuses_to_query = [] + count = 0 + + statuses.pluck(:id, :ordered_media_attachment_ids).each do |id, ordered_ids| + if ordered_ids.nil? + statuses_to_query << id + else + count += ordered_ids.size + end + end + + count += MediaAttachment.where(status_id: statuses_to_query).count unless statuses_to_query.empty? + count end def rules diff --git a/app/models/status.rb b/app/models/status.rb index af3e645dc..db10eedc2 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -3,28 +3,29 @@ # # Table name: statuses # -# id :bigint(8) not null, primary key -# uri :string -# text :text default(""), not null -# created_at :datetime not null -# updated_at :datetime not null -# in_reply_to_id :bigint(8) -# reblog_of_id :bigint(8) -# url :string -# sensitive :boolean default(FALSE), not null -# visibility :integer default("public"), not null -# spoiler_text :text default(""), not null -# reply :boolean default(FALSE), not null -# language :string -# conversation_id :bigint(8) -# local :boolean -# account_id :bigint(8) not null -# application_id :bigint(8) -# in_reply_to_account_id :bigint(8) -# poll_id :bigint(8) -# deleted_at :datetime -# edited_at :datetime -# trendable :boolean +# id :bigint(8) not null, primary key +# uri :string +# text :text default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# in_reply_to_id :bigint(8) +# reblog_of_id :bigint(8) +# url :string +# sensitive :boolean default(FALSE), not null +# visibility :integer default("public"), not null +# spoiler_text :text default(""), not null +# reply :boolean default(FALSE), not null +# language :string +# conversation_id :bigint(8) +# local :boolean +# account_id :bigint(8) not null +# application_id :bigint(8) +# in_reply_to_account_id :bigint(8) +# poll_id :bigint(8) +# deleted_at :datetime +# edited_at :datetime +# trendable :boolean +# ordered_media_attachment_ids :bigint(8) is an Array # class Status < ApplicationRecord @@ -211,11 +212,14 @@ class Status < ApplicationRecord public_visibility? || unlisted_visibility? end - def snapshot!(media_attachments_changed: false, account_id: nil, at_time: nil) + def snapshot!(account_id: nil, at_time: nil) edits.create!( text: text, spoiler_text: spoiler_text, - media_attachments_changed: media_attachments_changed, + sensitive: sensitive, + ordered_media_attachment_ids: ordered_media_attachment_ids || media_attachments.pluck(:id), + media_descriptions: ordered_media_attachments.map(&:description), + poll_options: preloadable_poll&.options, account_id: account_id || self.account_id, created_at: at_time || edited_at ) @@ -228,7 +232,7 @@ class Status < ApplicationRecord alias sign? distributable? def with_media? - media_attachments.any? + ordered_media_attachments.any? end def with_preview_card? @@ -252,6 +256,15 @@ class Status < ApplicationRecord @emojis = CustomEmoji.from_text(fields.join(' '), account.domain) end + def ordered_media_attachments + if ordered_media_attachment_ids.nil? + media_attachments + else + map = media_attachments.index_by(&:id) + ordered_media_attachment_ids.map { |media_attachment_id| map[media_attachment_id] } + end + end + def replies_count status_stat&.replies_count || 0 end diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb index 6e88864e8..94a387c36 100644 --- a/app/models/status_edit.rb +++ b/app/models/status_edit.rb @@ -3,17 +3,29 @@ # # Table name: status_edits # -# id :bigint(8) not null, primary key -# status_id :bigint(8) not null -# account_id :bigint(8) -# text :text default(""), not null -# spoiler_text :text default(""), not null -# media_attachments_changed :boolean default(FALSE), not null -# created_at :datetime not null -# updated_at :datetime not null +# id :bigint(8) not null, primary key +# status_id :bigint(8) not null +# account_id :bigint(8) +# text :text default(""), not null +# spoiler_text :text default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# ordered_media_attachment_ids :bigint(8) is an Array +# media_descriptions :text is an Array +# poll_options :string is an Array +# sensitive :boolean # class StatusEdit < ApplicationRecord + self.ignored_columns = %w( + media_attachments_changed + ) + + class PreservedMediaAttachment < ActiveModelSerializers::Model + attributes :media_attachment, :description + delegate :id, :type, :url, :preview_url, :remote_url, :preview_remote_url, :text_url, :meta, :blurhash, to: :media_attachment + end + belongs_to :status belongs_to :account, optional: true @@ -25,4 +37,17 @@ class StatusEdit < ApplicationRecord return @emojis if defined?(@emojis) @emojis = CustomEmoji.from_text([spoiler_text, text].join(' '), status.account.domain) end + + def ordered_media_attachments + return @ordered_media_attachments if defined?(@ordered_media_attachments) + + @ordered_media_attachments = begin + if ordered_media_attachment_ids.nil? + [] + else + map = status.media_attachments.index_by(&:id) + ordered_media_attachment_ids.map.with_index { |media_attachment_id, index| PreservedMediaAttachment.new(media_attachment: map[media_attachment_id], description: media_descriptions[index]) } + end + end + end end diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb index 12dabc65a..7be2e2647 100644 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@ -13,7 +13,7 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer attribute :content_map, if: :language? attribute :updated, if: :edited? - has_many :media_attachments, key: :attachment + has_many :virtual_attachments, key: :attachment has_many :virtual_tags, key: :tag has_one :replies, serializer: ActivityPub::CollectionSerializer, if: :local? @@ -106,6 +106,10 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer object.account.sensitized? || object.sensitive end + def virtual_attachments + object.ordered_media_attachments + end + def virtual_tags object.active_mentions.to_a.sort_by(&:id) + object.tags + object.emojis end diff --git a/app/serializers/rest/status_edit_serializer.rb b/app/serializers/rest/status_edit_serializer.rb index a1f9e824e..05ccd5e94 100644 --- a/app/serializers/rest/status_edit_serializer.rb +++ b/app/serializers/rest/status_edit_serializer.rb @@ -3,12 +3,18 @@ class REST::StatusEditSerializer < ActiveModel::Serializer has_one :account, serializer: REST::AccountSerializer - attributes :content, :spoiler_text, - :media_attachments_changed, :created_at + attributes :content, :spoiler_text, :sensitive, :created_at + has_many :ordered_media_attachments, key: :media_attachments, serializer: REST::MediaAttachmentSerializer has_many :emojis, serializer: REST::CustomEmojiSerializer + attribute :poll, if: -> { object.poll_options.present? } + def content Formatter.instance.format(object) end + + def poll + { options: object.poll_options.map { |title| { title: title } } } + end end diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index aef51e0f7..7c3dd673e 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -19,7 +19,7 @@ class REST::StatusSerializer < ActiveModel::Serializer belongs_to :application, if: :show_application? belongs_to :account, serializer: REST::AccountSerializer - has_many :media_attachments, serializer: REST::MediaAttachmentSerializer + has_many :ordered_media_attachments, key: :media_attachments, serializer: REST::MediaAttachmentSerializer has_many :ordered_mentions, key: :mentions has_many :tags has_many :emojis, serializer: REST::CustomEmojiSerializer diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb index 7438a7c53..11afa894f 100644 --- a/app/services/activitypub/process_status_update_service.rb +++ b/app/services/activitypub/process_status_update_service.rb @@ -76,13 +76,14 @@ class ActivityPub::ProcessStatusUpdateService < BaseService end end - removed_media_attachments = previous_media_attachments - next_media_attachments - added_media_attachments = next_media_attachments - previous_media_attachments + added_media_attachments = next_media_attachments - previous_media_attachments - MediaAttachment.where(id: removed_media_attachments.map(&:id)).update_all(status_id: nil) MediaAttachment.where(id: added_media_attachments.map(&:id)).update_all(status_id: @status.id) - @media_attachments_changed = true if removed_media_attachments.any? || added_media_attachments.any? + @status.ordered_media_attachment_ids = next_media_attachments.map(&:id) + @status.media_attachments.reload + + @media_attachments_changed = true if @status.ordered_media_attachment_ids_changed? end def update_poll! @@ -215,19 +216,13 @@ class ActivityPub::ProcessStatusUpdateService < BaseService return if @status.edits.any? - @status.snapshot!( - media_attachments_changed: false, - at_time: @status.created_at - ) + @status.snapshot!(at_time: @status.created_at) end def create_edit! return unless significant_changes? - @status.snapshot!( - media_attachments_changed: @media_attachments_changed || @poll_changed, - account_id: @account.id - ) + @status.snapshot!(account_id: @account.id) end def skip_download? diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 2bab91116..76404c6b8 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -110,7 +110,7 @@ class FanOutOnWriteService < BaseService Redis.current.publish('timeline:public', anonymous_payload) Redis.current.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', anonymous_payload) - if @status.media_attachments.any? + if @status.with_media? Redis.current.publish('timeline:public:media', anonymous_payload) Redis.current.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', anonymous_payload) end diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 7508c3b64..c132930a9 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -100,7 +100,10 @@ class PostStatusService < BaseService end def validate_media! - return if @options[:media_ids].blank? || !@options[:media_ids].is_a?(Enumerable) + if @options[:media_ids].blank? || !@options[:media_ids].is_a?(Enumerable) + @media = [] + return + end raise Mastodon::ValidationError, I18n.t('media_attachments.validations.too_many') if @options[:media_ids].size > 4 || @options[:poll].present? @@ -157,6 +160,7 @@ class PostStatusService < BaseService { text: @text, media_attachments: @media || [], + ordered_media_attachment_ids: (@options[:media_ids] || []).map(&:to_i) & @media.map(&:id), thread: @in_reply_to, poll_attributes: poll_attributes, sensitive: @sensitive, diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 7fb9b6301..159aec1f2 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -40,7 +40,7 @@ class RemoveStatusService < BaseService remove_reblogs remove_from_hashtags remove_from_public - remove_from_media if @status.media_attachments.any? + remove_from_media if @status.with_media? remove_media end diff --git a/app/services/update_status_service.rb b/app/services/update_status_service.rb index 973e6ddee..1c63ab656 100644 --- a/app/services/update_status_service.rb +++ b/app/services/update_status_service.rb @@ -17,8 +17,6 @@ class UpdateStatusService < BaseService @status = status @options = options @account_id = account_id - @media_attachments_changed = false - @poll_changed = false Status.transaction do create_previous_edit! @@ -41,14 +39,12 @@ class UpdateStatusService < BaseService def update_media_attachments! previous_media_attachments = @status.media_attachments.to_a next_media_attachments = validate_media! - removed_media_attachments = previous_media_attachments - next_media_attachments added_media_attachments = next_media_attachments - previous_media_attachments - MediaAttachment.where(id: removed_media_attachments.map(&:id)).update_all(status_id: nil) MediaAttachment.where(id: added_media_attachments.map(&:id)).update_all(status_id: @status.id) + @status.ordered_media_attachment_ids = (@options[:media_ids] || []).map(&:to_i) & next_media_attachments.map(&:id) @status.media_attachments.reload - @media_attachments_changed = true if removed_media_attachments.any? || added_media_attachments.any? end def validate_media! @@ -73,19 +69,18 @@ class UpdateStatusService < BaseService # If for some reasons the options were changed, it invalidates all previous # votes, so we need to remove them - @poll_changed = true if @options[:poll][:options] != poll.options || ActiveModel::Type::Boolean.new.cast(@options[:poll][:multiple]) != poll.multiple + poll_changed = true if @options[:poll][:options] != poll.options || ActiveModel::Type::Boolean.new.cast(@options[:poll][:multiple]) != poll.multiple poll.options = @options[:poll][:options] poll.hide_totals = @options[:poll][:hide_totals] || false poll.multiple = @options[:poll][:multiple] || false poll.expires_in = @options[:poll][:expires_in] - poll.reset_votes! if @poll_changed + poll.reset_votes! if poll_changed poll.save! @status.poll_id = poll.id elsif previous_poll.present? previous_poll.destroy - @poll_changed = true @status.poll_id = nil end end @@ -136,16 +131,10 @@ class UpdateStatusService < BaseService return if @status.edits.any? - @status.snapshot!( - media_attachments_changed: false, - at_time: @status.created_at - ) + @status.snapshot!(at_time: @status.created_at) end def create_edit! - @status.snapshot!( - media_attachments_changed: @media_attachments_changed || @poll_changed, - account_id: @account_id - ) + @status.snapshot!(account_id: @account_id) end end diff --git a/app/views/admin/reports/_status.html.haml b/app/views/admin/reports/_status.html.haml index 4e06d4bbf..1c033c4c3 100644 --- a/app/views/admin/reports/_status.html.haml +++ b/app/views/admin/reports/_status.html.haml @@ -11,15 +11,15 @@ %strong> Content warning: #{Formatter.instance.format_spoiler(status.proper)} = Formatter.instance.format(status.proper, custom_emojify: true) - - unless status.proper.media_attachments.empty? - - if status.proper.media_attachments.first.video? - - video = status.proper.media_attachments.first + - unless status.proper.ordered_media_attachments.empty? + - if status.proper.ordered_media_attachments.first.video? + - video = status.proper.ordered_media_attachments.first = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), frameRate: video.file.meta.dig('original', 'frame_rate'), blurhash: video.blurhash, sensitive: status.proper.sensitive?, visible: false, width: 610, height: 343, inline: true, alt: video.description, media: [ActiveModelSerializers::SerializableResource.new(video, serializer: REST::MediaAttachmentSerializer)].as_json - - elsif status.proper.media_attachments.first.audio? - - audio = status.proper.media_attachments.first + - elsif status.proper.ordered_media_attachments.first.audio? + - audio = status.proper.ordered_media_attachments.first = react_component :audio, src: audio.file.url(:original), height: 110, alt: audio.description, duration: audio.file.meta.dig(:original, :duration) - else - = react_component :media_gallery, height: 343, sensitive: status.proper.sensitive?, visible: false, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } + = react_component :media_gallery, height: 343, sensitive: status.proper.sensitive?, visible: false, media: status.proper.ordered_media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } .detailed-status__meta - if status.application diff --git a/app/views/admin/reports/index.html.haml b/app/views/admin/reports/index.html.haml index 619173373..248718a73 100644 --- a/app/views/admin/reports/index.html.haml +++ b/app/views/admin/reports/index.html.haml @@ -59,11 +59,11 @@ %span.report-card__summary__item__content__icon{ title: t('admin.accounts.statuses') } = fa_icon('comment') - = report.statuses.count + = report.status_ids.size %span.report-card__summary__item__content__icon{ title: t('admin.accounts.media_attachments') } = fa_icon('camera') - = report.media_attachments.count + = report.media_attachments_count - if report.forwarded? · diff --git a/app/views/admin/trends/statuses/_status.html.haml b/app/views/admin/trends/statuses/_status.html.haml index edb27b9ff..50a855349 100644 --- a/app/views/admin/trends/statuses/_status.html.haml +++ b/app/views/admin/trends/statuses/_status.html.haml @@ -9,7 +9,7 @@ = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank', class: 'emojify', rel: 'noopener noreferrer' do = one_line_preview(status) - - status.media_attachments.each do |media_attachment| + - status.ordered_media_attachments.each do |media_attachment| %abbr{ title: media_attachment.description } = fa_icon 'link' = media_attachment.file_file_name diff --git a/app/views/disputes/strikes/show.html.haml b/app/views/disputes/strikes/show.html.haml index 7248b2574..0fc32b918 100644 --- a/app/views/disputes/strikes/show.html.haml +++ b/app/views/disputes/strikes/show.html.haml @@ -53,7 +53,7 @@ = link_to short_account_status_url(@strike.target_account, status_id), class: 'emojify' do = one_line_preview(status) - - status.media_attachments.each do |media_attachment| + - status.ordered_media_attachments.each do |media_attachment| %abbr{ title: media_attachment.description } = fa_icon 'link' = media_attachment.file_file_name diff --git a/app/views/notification_mailer/_status.html.haml b/app/views/notification_mailer/_status.html.haml index 31460a76e..f520208e1 100644 --- a/app/views/notification_mailer/_status.html.haml +++ b/app/views/notification_mailer/_status.html.haml @@ -33,9 +33,9 @@ %div.auto-dir = Formatter.instance.format(status) - - if status.media_attachments.size > 0 + - if status.ordered_media_attachments.size > 0 %p - - status.media_attachments.each do |a| + - status.ordered_media_attachments.each do |a| - if status.local? = link_to full_asset_url(a.file.url(:original)), full_asset_url(a.file.url(:original)) - else diff --git a/app/views/statuses/_detailed_status.html.haml b/app/views/statuses/_detailed_status.html.haml index 1922f53ce..6ccf3725a 100644 --- a/app/views/statuses/_detailed_status.html.haml +++ b/app/views/statuses/_detailed_status.html.haml @@ -25,10 +25,10 @@ - if status.preloadable_poll = render_poll_component(status) - - if !status.media_attachments.empty? - - if status.media_attachments.first.video? + - if !status.ordered_media_attachments.empty? + - if status.ordered_media_attachments.first.video? = render_video_component(status, width: 670, height: 380, detailed: true) - - elsif status.media_attachments.first.audio? + - elsif status.ordered_media_attachments.first.audio? = render_audio_component(status, width: 670, height: 380) - else = render_media_gallery_component(status, height: 380, standalone: true) diff --git a/app/views/statuses/_og_image.html.haml b/app/views/statuses/_og_image.html.haml index c8b6147ef..5a647531a 100644 --- a/app/views/statuses/_og_image.html.haml +++ b/app/views/statuses/_og_image.html.haml @@ -1,6 +1,6 @@ - if activity.is_a?(Status) && (activity.non_sensitive_with_media? || (activity.with_media? && Setting.preview_sensitive_media)) - player_card = false - - activity.media_attachments.each do |media| + - activity.ordered_media_attachments.each do |media| - if media.image? = opengraph 'og:image', full_asset_url(media.file.url(:original)) = opengraph 'og:image:type', media.file_content_type diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml index 0139d2016..5dd265b59 100644 --- a/app/views/statuses/_simple_status.html.haml +++ b/app/views/statuses/_simple_status.html.haml @@ -37,10 +37,10 @@ - if status.preloadable_poll = render_poll_component(status) - - if !status.media_attachments.empty? - - if status.media_attachments.first.video? + - if !status.ordered_media_attachments.empty? + - if status.ordered_media_attachments.first.video? = render_video_component(status, width: 610, height: 343) - - elsif status.media_attachments.first.audio? + - elsif status.ordered_media_attachments.first.audio? = render_audio_component(status, width: 610, height: 343) - else = render_media_gallery_component(status, height: 343) diff --git a/db/migrate/20220302232632_add_ordered_media_attachment_ids_to_statuses.rb b/db/migrate/20220302232632_add_ordered_media_attachment_ids_to_statuses.rb new file mode 100644 index 000000000..5443f32a2 --- /dev/null +++ b/db/migrate/20220302232632_add_ordered_media_attachment_ids_to_statuses.rb @@ -0,0 +1,5 @@ +class AddOrderedMediaAttachmentIdsToStatuses < ActiveRecord::Migration[6.1] + def change + add_column :statuses, :ordered_media_attachment_ids, :bigint, array: true + end +end diff --git a/db/migrate/20220303000827_add_ordered_media_attachment_ids_to_status_edits.rb b/db/migrate/20220303000827_add_ordered_media_attachment_ids_to_status_edits.rb new file mode 100644 index 000000000..b1071f359 --- /dev/null +++ b/db/migrate/20220303000827_add_ordered_media_attachment_ids_to_status_edits.rb @@ -0,0 +1,8 @@ +class AddOrderedMediaAttachmentIdsToStatusEdits < ActiveRecord::Migration[6.1] + def change + add_column :status_edits, :ordered_media_attachment_ids, :bigint, array: true + add_column :status_edits, :media_descriptions, :text, array: true + add_column :status_edits, :poll_options, :string, array: true + add_column :status_edits, :sensitive, :boolean + end +end diff --git a/db/post_migrate/20220303203437_remove_media_attachments_changed_from_status_edits.rb b/db/post_migrate/20220303203437_remove_media_attachments_changed_from_status_edits.rb new file mode 100644 index 000000000..09725c74e --- /dev/null +++ b/db/post_migrate/20220303203437_remove_media_attachments_changed_from_status_edits.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class RemoveMediaAttachmentsChangedFromStatusEdits < ActiveRecord::Migration[5.2] + def change + safety_assured { remove_column :status_edits, :media_attachments_changed, :boolean, default: false, null: false } + end +end diff --git a/db/schema.rb b/db/schema.rb index ef1620113..6251fa28c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -391,7 +391,6 @@ ActiveRecord::Schema.define(version: 2022_03_07_094650) do t.bigint "parent_id" t.inet "ips", array: true t.datetime "last_refresh_at" - t.index ["domain"], name: "index_email_domain_blocks_on_domain", unique: true end @@ -845,9 +844,12 @@ ActiveRecord::Schema.define(version: 2022_03_07_094650) do t.bigint "account_id" t.text "text", default: "", null: false t.text "spoiler_text", default: "", null: false - t.boolean "media_attachments_changed", default: false, null: false t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false + t.bigint "ordered_media_attachment_ids", array: true + t.text "media_descriptions", array: true + t.string "poll_options", array: true + t.boolean "sensitive" t.index ["account_id"], name: "index_status_edits_on_account_id" t.index ["status_id"], name: "index_status_edits_on_status_id" end @@ -892,6 +894,7 @@ ActiveRecord::Schema.define(version: 2022_03_07_094650) do t.datetime "deleted_at" t.datetime "edited_at" t.boolean "trendable" + t.bigint "ordered_media_attachment_ids", array: true t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)" t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)" t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))" diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb index df32a7c9d..874be4132 100644 --- a/spec/models/report_spec.rb +++ b/spec/models/report_spec.rb @@ -11,14 +11,13 @@ describe Report do end end - describe 'media_attachments' do - it 'returns media attachments from statuses' do - status = Fabricate(:status) - media_attachment = Fabricate(:media_attachment, status: status) - _other_media_attachment = Fabricate(:media_attachment) - report = Fabricate(:report, status_ids: [status.id]) + describe 'media_attachments_count' do + it 'returns count of media attachments in statuses' do + status1 = Fabricate(:status, ordered_media_attachment_ids: [1, 2]) + status2 = Fabricate(:status, ordered_media_attachment_ids: [5]) + report = Fabricate(:report, status_ids: [status1.id, status2.id]) - expect(report.media_attachments).to eq [media_attachment] + expect(report.media_attachments_count).to eq 3 end end diff --git a/spec/services/activitypub/process_status_update_service_spec.rb b/spec/services/activitypub/process_status_update_service_spec.rb index 6ee1dcb43..40b405217 100644 --- a/spec/services/activitypub/process_status_update_service_spec.rb +++ b/spec/services/activitypub/process_status_update_service_spec.rb @@ -124,7 +124,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do end it 'updates media attachments' do - media_attachment = status.media_attachments.reload.first + media_attachment = status.reload.ordered_media_attachments.first expect(media_attachment).to_not be_nil expect(media_attachment.remote_url).to eq 'https://example.com/foo.png' @@ -135,7 +135,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do end it 'records media change in edit' do - expect(status.edits.reload.last.media_attachments_changed).to be true + expect(status.edits.reload.last.ordered_media_attachment_ids).to_not be_empty end end @@ -173,11 +173,11 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do end it 'updates media attachments' do - expect(status.media_attachments.reload.map(&:remote_url)).to eq %w(https://example.com/foo.png) + expect(status.ordered_media_attachments.map(&:remote_url)).to eq %w(https://example.com/foo.png) end it 'records media change in edit' do - expect(status.edits.reload.last.media_attachments_changed).to be true + expect(status.edits.reload.last.ordered_media_attachment_ids).to_not be_empty end end @@ -193,7 +193,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do end it 'records media change in edit' do - expect(status.edits.reload.last.media_attachments_changed).to be true + expect(status.edits.reload.last.poll_options).to be_nil end end @@ -226,7 +226,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do end it 'records media change in edit' do - expect(status.edits.reload.last.media_attachments_changed).to be true + expect(status.edits.reload.last.poll_options).to eq %w(Foo Bar Baz) end end @@ -239,10 +239,5 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do subject.call(status, json) expect(status.reload.edited_at.to_s).to eq '2021-09-08 22:39:25 UTC' end - - it 'records that no media has been changed in edit' do - subject.call(status, json) - expect(status.edits.reload.last.media_attachments_changed).to be false - end end end diff --git a/spec/services/update_status_service_spec.rb b/spec/services/update_status_service_spec.rb index 4fd4837c6..78cc89cd4 100644 --- a/spec/services/update_status_service_spec.rb +++ b/spec/services/update_status_service_spec.rb @@ -21,7 +21,7 @@ RSpec.describe UpdateStatusService, type: :service do end it 'saves edit history' do - expect(status.edits.pluck(:text, :media_attachments_changed)).to eq [['Foo', false], ['Bar', false]] + expect(status.edits.pluck(:text)).to eq %w(Foo Bar) end end @@ -39,7 +39,7 @@ RSpec.describe UpdateStatusService, type: :service do end it 'saves edit history' do - expect(status.edits.pluck(:text, :spoiler_text, :media_attachments_changed)).to eq [['Foo', '', false], ['Foo', 'Bar', false]] + expect(status.edits.pluck(:text, :spoiler_text)).to eq [['Foo', ''], ['Foo', 'Bar']] end end @@ -54,11 +54,11 @@ RSpec.describe UpdateStatusService, type: :service do end it 'updates media attachments' do - expect(status.media_attachments.to_a).to eq [attached_media_attachment] + expect(status.ordered_media_attachments).to eq [attached_media_attachment] end - it 'detaches detached media attachments' do - expect(detached_media_attachment.reload.status_id).to be_nil + it 'does not detach detached media attachments' do + expect(detached_media_attachment.reload.status_id).to eq status.id end it 'attaches attached media attachments' do @@ -66,7 +66,7 @@ RSpec.describe UpdateStatusService, type: :service do end it 'saves edit history' do - expect(status.edits.pluck(:text, :media_attachments_changed)).to eq [['Foo', false], ['Foo', true]] + expect(status.edits.pluck(:ordered_media_attachment_ids)).to eq [[detached_media_attachment.id], [attached_media_attachment.id]] end end @@ -95,7 +95,7 @@ RSpec.describe UpdateStatusService, type: :service do end it 'saves edit history' do - expect(status.edits.pluck(:text, :media_attachments_changed)).to eq [['Foo', false], ['Foo', true]] + expect(status.edits.pluck(:poll_options)).to eq [%w(Foo Bar), %w(Bar Baz Foo)] end end -- cgit From a6ed6845c9cab3b314ce6434b851cc507a71ee62 Mon Sep 17 00:00:00 2001 From: chandrn7 Date: Wed, 9 Mar 2022 06:07:35 -0500 Subject: Allow login through OpenID Connect (#16221) * added OpenID Connect as an SSO option * minor fixes * added comments, removed an option that shouldn't be set * fixed Gemfile.lock * added newline to end of Gemfile.lock * removed tab from Gemfile.lock * remove chomp * codeclimate changes and small name change to make function's purpose clearer * codeclimate fix * added SSO buttons to /about page * minor refactor * minor style change * removed spurious change * removed unecessary conditional from ensure_valid_username and added support for auth.info.name in user_params_from_auth * minor changes --- Gemfile | 1 + Gemfile.lock | 41 ++++++++++++++++++++++ .../auth/omniauth_callbacks_controller.rb | 6 ++-- app/models/concerns/omniauthable.rb | 23 +++++++----- config/initializers/omniauth.rb | 41 ++++++++++++++++++++-- 5 files changed, 97 insertions(+), 15 deletions(-) (limited to 'app/models') diff --git a/Gemfile b/Gemfile index b5d15da61..549e774f7 100644 --- a/Gemfile +++ b/Gemfile @@ -40,6 +40,7 @@ end gem 'net-ldap', '~> 0.17' gem 'omniauth-cas', '~> 2.0' gem 'omniauth-saml', '~> 1.10' +gem 'gitlab-omniauth-openid-connect', '~>0.5.0', require: 'omniauth_openid_connect' gem 'omniauth', '~> 1.9' gem 'omniauth-rails_csrf_protection', '~> 0.1' diff --git a/Gemfile.lock b/Gemfile.lock index 87539348c..65343a465 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -68,6 +68,7 @@ GEM zeitwerk (~> 2.3) addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) + aes_key_wrap (1.1.0) airbrussh (1.4.0) sshkit (>= 1.6.1, != 1.7.0) android_key_attestation (0.3.0) @@ -77,6 +78,7 @@ GEM ast (2.4.2) attr_encrypted (3.1.0) encryptor (~> 3.0.0) + attr_required (1.0.1) awrence (1.1.1) aws-eventstream (1.2.0) aws-partitions (1.558.0) @@ -260,6 +262,10 @@ GEM fuubar (2.5.1) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) + gitlab-omniauth-openid-connect (0.5.0) + addressable (~> 2.7) + omniauth (~> 1.9) + openid_connect (~> 1.2) globalid (1.0.0) activesupport (>= 5.0) hamlit (2.13.0) @@ -286,6 +292,7 @@ GEM domain_name (~> 0.5) http-form_data (2.3.0) http_accept_language (2.1.1) + httpclient (2.8.3) httplog (1.5.0) rack (>= 1.0) rainbow (>= 2.0.0) @@ -306,6 +313,10 @@ GEM jmespath (1.6.0) json (2.5.1) json-canonicalization (0.3.0) + json-jwt (1.13.0) + activesupport (>= 4.2) + aes_key_wrap + bindata json-ld (3.2.0) htmlentities (~> 4.3) json-canonicalization (~> 0.3) @@ -406,6 +417,16 @@ GEM omniauth-saml (1.10.3) omniauth (~> 1.3, >= 1.3.2) ruby-saml (~> 1.9) + openid_connect (1.2.0) + activemodel + attr_required (>= 1.0.0) + json-jwt (>= 1.5.0) + rack-oauth2 (>= 1.6.1) + swd (>= 1.0.0) + tzinfo + validate_email + validate_url + webfinger (>= 1.0.1) openssl (2.2.0) openssl-signature_algorithm (0.4.0) orm_adapter (0.5.0) @@ -449,6 +470,12 @@ GEM rack (>= 1.0, < 3) rack-cors (1.1.1) rack (>= 2.0.0) + rack-oauth2 (1.16.0) + activesupport + attr_required + httpclient + json-jwt (>= 1.11.0) + rack (>= 2.1.0) rack-proxy (0.7.0) rack rack-test (1.1.0) @@ -608,6 +635,10 @@ GEM stoplight (2.2.1) strong_migrations (0.7.9) activerecord (>= 5) + swd (1.2.0) + activesupport (>= 3) + attr_required (>= 0.0.5) + httpclient (>= 2.4) temple (0.8.2) terminal-table (3.0.2) unicode-display_width (>= 1.1.1, < 3) @@ -642,6 +673,12 @@ GEM unf_ext (0.0.8) unicode-display_width (2.1.0) uniform_notifier (1.14.2) + validate_email (0.1.6) + activemodel (>= 3.0) + mail (>= 2.2.5) + validate_url (1.0.13) + activemodel (>= 3.0.0) + public_suffix warden (1.2.9) rack (>= 2.0.9) webauthn (3.0.0.alpha1) @@ -654,6 +691,9 @@ GEM safety_net_attestation (~> 0.4.0) securecompare (~> 1.0) tpm-key_attestation (~> 0.9.0) + webfinger (1.1.0) + activesupport + httpclient (>= 2.4) webmock (3.14.0) addressable (>= 2.8.0) crack (>= 0.3.2) @@ -717,6 +757,7 @@ DEPENDENCIES fog-core (<= 2.1.0) fog-openstack (~> 0.3) fuubar (~> 2.5) + gitlab-omniauth-openid-connect (~> 0.5.0) hamlit-rails (~> 0.2) hiredis (~> 0.6) htmlentities (~> 4.3) diff --git a/app/controllers/auth/omniauth_callbacks_controller.rb b/app/controllers/auth/omniauth_callbacks_controller.rb index 991a50b03..f9cf6d655 100644 --- a/app/controllers/auth/omniauth_callbacks_controller.rb +++ b/app/controllers/auth/omniauth_callbacks_controller.rb @@ -4,8 +4,6 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController skip_before_action :verify_authenticity_token def self.provides_callback_for(provider) - provider_id = provider.to_s.chomp '_oauth2' - define_method provider do @user = User.find_for_oauth(request.env['omniauth.auth'], current_user) @@ -20,7 +18,7 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController ) sign_in_and_redirect @user, event: :authentication - set_flash_message(:notice, :success, kind: provider_id.capitalize) if is_navigational_format? + set_flash_message(:notice, :success, kind: Devise.omniauth_configs[provider].strategy.display_name.capitalize) if is_navigational_format? else session["devise.#{provider}_data"] = request.env['omniauth.auth'] redirect_to new_user_registration_url @@ -33,7 +31,7 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController end def after_sign_in_path_for(resource) - if resource.email_verified? + if resource.email_present? root_path else auth_setup_path(missing_email: '1') diff --git a/app/models/concerns/omniauthable.rb b/app/models/concerns/omniauthable.rb index 791a94911..a90d5d888 100644 --- a/app/models/concerns/omniauthable.rb +++ b/app/models/concerns/omniauthable.rb @@ -13,7 +13,7 @@ module Omniauthable Devise.omniauth_configs.keys end - def email_verified? + def email_present? email && email !~ TEMP_EMAIL_REGEX end end @@ -40,16 +40,14 @@ module Omniauthable end def create_for_oauth(auth) - # Check if the user exists with provided email if the provider gives us a - # verified email. If no verified email was provided or the user already - # exists, we assign a temporary email and ask the user to verify it on + # Check if the user exists with provided email. If no email was provided, + # we assign a temporary email and ask the user to verify it on # the next step via Auth::SetupController.show strategy = Devise.omniauth_configs[auth.provider.to_sym].strategy assume_verified = strategy&.security&.assume_email_is_verified - email_is_verified = auth.info.verified || auth.info.verified_email || assume_verified + email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified email = auth.info.verified_email || auth.info.email - email = nil unless email_is_verified user = User.find_by(email: email) if email_is_verified @@ -58,7 +56,7 @@ module Omniauthable user = User.new(user_params_from_auth(email, auth)) user.account.avatar_remote_url = auth.info.image if /\A#{URI::DEFAULT_PARSER.make_regexp(%w(http https))}\z/.match?(auth.info.image) - user.skip_confirmation! + user.skip_confirmation! if email_is_verified user.save! user end @@ -71,8 +69,8 @@ module Omniauthable agreement: true, external: true, account_attributes: { - username: ensure_unique_username(auth.uid), - display_name: auth.info.full_name || [auth.info.first_name, auth.info.last_name].join(' '), + username: ensure_unique_username(ensure_valid_username(auth.uid)), + display_name: auth.info.full_name || auth.info.name || [auth.info.first_name, auth.info.last_name].join(' '), }, } end @@ -88,5 +86,12 @@ module Omniauthable username end + + def ensure_valid_username(starting_username) + starting_username = starting_username.split('@')[0] + temp_username = starting_username.gsub(/[^a-z0-9_]+/i, '') + validated_username = temp_username.truncate(30, omission: '') + validated_username + end end end diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb index 1a041ad48..51241e546 100644 --- a/config/initializers/omniauth.rb +++ b/config/initializers/omniauth.rb @@ -8,7 +8,8 @@ Devise.setup do |config| # CAS strategy if ENV['CAS_ENABLED'] == 'true' - cas_options = options + cas_options = {} + cas_options[:display_name] = ENV['CAS_DISPLAY_NAME'] || 'cas' cas_options[:url] = ENV['CAS_URL'] if ENV['CAS_URL'] cas_options[:host] = ENV['CAS_HOST'] if ENV['CAS_HOST'] cas_options[:port] = ENV['CAS_PORT'] if ENV['CAS_PORT'] @@ -36,7 +37,8 @@ Devise.setup do |config| # SAML strategy if ENV['SAML_ENABLED'] == 'true' - saml_options = options + saml_options = {} + saml_options[:display_name] = ENV['SAML_DISPLAY_NAME'] || 'saml' saml_options[:assertion_consumer_service_url] = ENV['SAML_ACS_URL'] if ENV['SAML_ACS_URL'] saml_options[:issuer] = ENV['SAML_ISSUER'] if ENV['SAML_ISSUER'] saml_options[:idp_sso_target_url] = ENV['SAML_IDP_SSO_TARGET_URL'] if ENV['SAML_IDP_SSO_TARGET_URL'] @@ -64,4 +66,39 @@ Devise.setup do |config| saml_options[:allowed_clock_drift] = ENV['SAML_ALLOWED_CLOCK_DRIFT'] if ENV['SAML_ALLOWED_CLOCK_DRIFT'] config.omniauth :saml, saml_options end + + # OpenID Connect Strategy + if ENV['OIDC_ENABLED'] == 'true' + oidc_options = {} + oidc_options[:display_name] = ENV['OIDC_DISPLAY_NAME'] || 'openid_connect' #OPTIONAL + oidc_options[:issuer] = ENV['OIDC_ISSUER'] if ENV['OIDC_ISSUER'] #NEED + oidc_options[:discovery] = ENV['OIDC_DISCOVERY'] == 'true' if ENV['OIDC_DISCOVERY'] #OPTIONAL (default: false) + oidc_options[:client_auth_method] = ENV['OIDC_CLIENT_AUTH_METHOD'] if ENV['OIDC_CLIENT_AUTH_METHOD'] #OPTIONAL (default: basic) + scope_string = ENV['OIDC_SCOPE'] if ENV['OIDC_SCOPE'] #NEED + scopes = scope_string.split(',') + oidc_options[:scope] = scopes.map { |x| x.to_sym } + oidc_options[:response_type] = ENV['OIDC_RESPONSE_TYPE'] if ENV['OIDC_RESPONSE_TYPE'] #OPTIONAL (default: code) + oidc_options[:response_mode] = ENV['OIDC_RESPONSE_MODE'] if ENV['OIDC_RESPONSE_MODE'] #OPTIONAL (default: query) + oidc_options[:display] = ENV['OIDC_DISPLAY'] if ENV['OIDC_DISPLAY'] #OPTIONAL (default: page) + oidc_options[:prompt] = ENV['OIDC_PROMPT'] if ENV['OIDC_PROMPT'] #OPTIONAL + oidc_options[:send_nonce] = ENV['OIDC_SEND_NONCE'] == 'true' if ENV['OIDC_SEND_NONCE'] #OPTIONAL (default: true) + oidc_options[:send_scope_to_token_endpoint] = ENV['OIDC_SEND_SCOPE_TO_TOKEN_ENDPOINT'] == 'true' if ENV['OIDC_SEND_SCOPE_TO_TOKEN_ENDPOINT'] #OPTIONAL (default: true) + oidc_options[:post_logout_redirect_uri] = ENV['OIDC_IDP_LOGOUT_REDIRECT_URI'] if ENV['OIDC_IDP_LOGOUT_REDIRECT_URI'] #OPTIONAL + oidc_options[:uid_field] = ENV['OIDC_UID_FIELD'] if ENV['OIDC_UID_FIELD'] #NEED + oidc_options[:client_options] = {} + oidc_options[:client_options][:identifier] = ENV['OIDC_CLIENT_ID'] if ENV['OIDC_CLIENT_ID'] #NEED + oidc_options[:client_options][:secret] = ENV['OIDC_CLIENT_SECRET'] if ENV['OIDC_CLIENT_SECRET'] #NEED + oidc_options[:client_options][:redirect_uri] = ENV['OIDC_REDIRECT_URI'] if ENV['OIDC_REDIRECT_URI'] #NEED + oidc_options[:client_options][:scheme] = ENV['OIDC_HTTP_SCHEME'] if ENV['OIDC_HTTP_SCHEME'] #OPTIONAL (default: https) + oidc_options[:client_options][:host] = ENV['OIDC_HOST'] if ENV['OIDC_HOST'] #OPTIONAL + oidc_options[:client_options][:port] = ENV['OIDC_PORT'] if ENV['OIDC_PORT'] #OPTIONAL + oidc_options[:client_options][:authorization_endpoint] = ENV['OIDC_AUTH_ENDPOINT'] if ENV['OIDC_AUTH_ENDPOINT'] #NEED when discovery != true + oidc_options[:client_options][:token_endpoint] = ENV['OIDC_TOKEN_ENDPOINT'] if ENV['OIDC_TOKEN_ENDPOINT'] #NEED when discovery != true + oidc_options[:client_options][:userinfo_endpoint] = ENV['OIDC_USER_INFO_ENDPOINT'] if ENV['OIDC_USER_INFO_ENDPOINT'] #NEED when discovery != true + oidc_options[:client_options][:jwks_uri] = ENV['OIDC_JWKS_URI'] if ENV['OIDC_JWKS_URI'] #NEED when discovery != true + oidc_options[:client_options][:end_session_endpoint] = ENV['OIDC_END_SESSION_ENDPOINT'] if ENV['OIDC_END_SESSION_ENDPOINT'] #OPTIONAL + oidc_options[:security] = {} + oidc_options[:security][:assume_email_is_verified] = ENV['OIDC_SECURITY_ASSUME_EMAIL_IS_VERIFIED'] == 'true' #OPTIONAL + config.omniauth :openid_connect, oidc_options + end end -- cgit From b2cd34474b58b8a1f5e01ba73d236551dd0a878f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 9 Mar 2022 20:06:51 +0100 Subject: Add rate limit for editing (#17728) --- app/controllers/api/v1/statuses_controller.rb | 1 + app/models/status.rb | 5 +++-- app/models/status_edit.rb | 4 ++++ app/services/activitypub/process_status_update_service.rb | 4 ++-- app/services/update_status_service.rb | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index f48aeb945..3fe137bfd 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -10,6 +10,7 @@ class Api::V1::StatusesController < Api::BaseController before_action :set_thread, only: [:create] override_rate_limit_headers :create, family: :statuses + override_rate_limit_headers :update, family: :statuses # This API was originally unlimited, pagination cannot be introduced without # breaking backwards-compatibility. Arbitrarily high number to cover most diff --git a/app/models/status.rb b/app/models/status.rb index db10eedc2..12daee2de 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -212,7 +212,7 @@ class Status < ApplicationRecord public_visibility? || unlisted_visibility? end - def snapshot!(account_id: nil, at_time: nil) + def snapshot!(account_id: nil, at_time: nil, rate_limit: true) edits.create!( text: text, spoiler_text: spoiler_text, @@ -221,7 +221,8 @@ class Status < ApplicationRecord media_descriptions: ordered_media_attachments.map(&:description), poll_options: preloadable_poll&.options, account_id: account_id || self.account_id, - created_at: at_time || edited_at + created_at: at_time || edited_at, + rate_limit: rate_limit ) end diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb index 94a387c36..6da9b4b85 100644 --- a/app/models/status_edit.rb +++ b/app/models/status_edit.rb @@ -17,6 +17,8 @@ # class StatusEdit < ApplicationRecord + include RateLimitable + self.ignored_columns = %w( media_attachments_changed ) @@ -26,6 +28,8 @@ class StatusEdit < ApplicationRecord delegate :id, :type, :url, :preview_url, :remote_url, :preview_remote_url, :text_url, :meta, :blurhash, to: :media_attachment end + rate_limit by: :account, family: :statuses + belongs_to :status belongs_to :account, optional: true diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb index 11afa894f..1260c0482 100644 --- a/app/services/activitypub/process_status_update_service.rb +++ b/app/services/activitypub/process_status_update_service.rb @@ -216,13 +216,13 @@ class ActivityPub::ProcessStatusUpdateService < BaseService return if @status.edits.any? - @status.snapshot!(at_time: @status.created_at) + @status.snapshot!(at_time: @status.created_at, rate_limit: false) end def create_edit! return unless significant_changes? - @status.snapshot!(account_id: @account.id) + @status.snapshot!(account_id: @account.id, rate_limit: false) end def skip_download? diff --git a/app/services/update_status_service.rb b/app/services/update_status_service.rb index 1c63ab656..055e5968d 100644 --- a/app/services/update_status_service.rb +++ b/app/services/update_status_service.rb @@ -131,7 +131,7 @@ class UpdateStatusService < BaseService return if @status.edits.any? - @status.snapshot!(at_time: @status.created_at) + @status.snapshot!(at_time: @status.created_at, rate_limit: false) end def create_edit! -- cgit From 2a56a890dabb7cffd1dc14cf8a7aea9cccc7ab09 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 9 Mar 2022 20:49:14 +0100 Subject: Fix rare race condition when rebloged status is deleted (#17693) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix rare race condition when rebloged status is deleted * Use INSERT INTO … SELECT --- app/models/status.rb | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) (limited to 'app/models') diff --git a/app/models/status.rb b/app/models/status.rb index 12daee2de..f2c55090b 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -379,6 +379,63 @@ class Status < ApplicationRecord super || build_status_stat end + # Hack to use a "INSERT INTO ... SELECT ..." query instead of "INSERT INTO ... VALUES ..." query + def self._insert_record(values) + if values.is_a?(Hash) && values['reblog_of_id'].present? + primary_key = self.primary_key + primary_key_value = nil + + if primary_key + primary_key_value = values[primary_key] + + if !primary_key_value && prefetch_primary_key? + primary_key_value = next_sequence_value + values[primary_key] = primary_key_value + end + end + + # The following line is where we differ from stock ActiveRecord implementation + im = _compile_reblog_insert(values) + + # Since we are using SELECT instead of VALUES, a non-error `nil` return is possible. + # For our purposes, it's equivalent to a foreign key constraint violation + result = connection.insert(im, "#{self} Create", primary_key || false, primary_key_value) + raise ActiveRecord::InvalidForeignKey, "(reblog_of_id)=(#{values['reblog_of_id']}) is not present in table \"statuses\"" if result.nil? + + result + else + super + end + end + + def self._compile_reblog_insert(values) + # This is somewhat equivalent to the following code of ActiveRecord::Persistence: + # `arel_table.compile_insert(_substitute_values(values))` + # The main difference is that we use a `SELECT` instead of a `VALUES` clause, + # which means we have to build the `SELECT` clause ourselves and do a bit more + # manual work. + + # Instead of using Arel::InsertManager#values, we are going to use Arel::InsertManager#select + im = Arel::InsertManager.new + im.into(arel_table) + + binds = [] + reblog_bind = nil + values.each do |name, value| + attr = arel_table[name] + bind = predicate_builder.build_bind_attribute(attr.name, value) + + im.columns << attr + binds << bind + + reblog_bind = bind if name == 'reblog_of_id' + end + + im.select(arel_table.where(arel_table[:id].eq(reblog_bind)).where(arel_table[:deleted_at].eq(nil)).project(*binds)) + + im + end + private def update_status_stat!(attrs) -- cgit From 9f2791eb64d5d19418561270f79071c185876d20 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 9 Mar 2022 21:15:24 +0100 Subject: Add polls and media attachments to edit comparison modal in web UI (#17727) --- .../mastodon/components/media_attachments.js | 116 +++++++++++++++++++++ .../features/report/components/status_check_box.js | 52 +-------- .../ui/components/compare_history_modal.js | 20 ++++ app/javascript/styles/mastodon/components.scss | 8 +- app/models/status_edit.rb | 7 +- 5 files changed, 154 insertions(+), 49 deletions(-) create mode 100644 app/javascript/mastodon/components/media_attachments.js (limited to 'app/models') diff --git a/app/javascript/mastodon/components/media_attachments.js b/app/javascript/mastodon/components/media_attachments.js new file mode 100644 index 000000000..d27720de4 --- /dev/null +++ b/app/javascript/mastodon/components/media_attachments.js @@ -0,0 +1,116 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import ImmutablePureComponent from 'react-immutable-pure-component'; +import { MediaGallery, Video, Audio } from 'mastodon/features/ui/util/async-components'; +import Bundle from 'mastodon/features/ui/components/bundle'; +import noop from 'lodash/noop'; + +export default class MediaAttachments extends ImmutablePureComponent { + + static propTypes = { + status: ImmutablePropTypes.map.isRequired, + height: PropTypes.number, + width: PropTypes.number, + }; + + static defaultProps = { + height: 110, + width: 239, + }; + + updateOnProps = [ + 'status', + ]; + + renderLoadingMediaGallery = () => { + const { height, width } = this.props; + + return ( +
+ ); + } + + renderLoadingVideoPlayer = () => { + const { height, width } = this.props; + + return ( +
+ ); + } + + renderLoadingAudioPlayer = () => { + const { height, width } = this.props; + + return ( +
+ ); + } + + render () { + const { status, width, height } = this.props; + const mediaAttachments = status.get('media_attachments'); + + if (mediaAttachments.size === 0) { + return null; + } + + if (mediaAttachments.getIn([0, 'type']) === 'audio') { + const audio = mediaAttachments.get(0); + + return ( + + {Component => ( + + )} + + ); + } else if (mediaAttachments.getIn([0, 'type']) === 'video') { + const video = mediaAttachments.get(0); + + return ( + + {Component => ( + + )} + + ); + } else { + return ( + + {Component => ( + + )} + + ); + } + } + +} diff --git a/app/javascript/mastodon/features/report/components/status_check_box.js b/app/javascript/mastodon/features/report/components/status_check_box.js index a2eb3d6f5..373c60e21 100644 --- a/app/javascript/mastodon/features/report/components/status_check_box.js +++ b/app/javascript/mastodon/features/report/components/status_check_box.js @@ -1,14 +1,12 @@ import React from 'react'; import PropTypes from 'prop-types'; import ImmutablePropTypes from 'react-immutable-proptypes'; -import noop from 'lodash/noop'; import StatusContent from 'mastodon/components/status_content'; -import { MediaGallery, Video } from 'mastodon/features/ui/util/async-components'; -import Bundle from 'mastodon/features/ui/components/bundle'; import Avatar from 'mastodon/components/avatar'; import DisplayName from 'mastodon/components/display_name'; import RelativeTimestamp from 'mastodon/components/relative_timestamp'; import Option from './option'; +import MediaAttachments from 'mastodon/components/media_attachments'; export default class StatusCheckBox extends React.PureComponent { @@ -27,51 +25,10 @@ export default class StatusCheckBox extends React.PureComponent { render () { const { status, checked } = this.props; - let media = null; - if (status.get('reblog')) { return null; } - if (status.get('media_attachments').size > 0) { - if (status.get('media_attachments').some(item => item.get('type') === 'unknown')) { - - } else if (status.getIn(['media_attachments', 0, 'type']) === 'video') { - const video = status.getIn(['media_attachments', 0]); - - media = ( - - {Component => ( - - )} - - ); - } else { - media = ( - - {Component => ( - - )} - - ); - } - } - const labelComponent = (
@@ -79,12 +36,13 @@ export default class StatusCheckBox extends React.PureComponent {
-
·
+
+ · +
- - {media} +
); diff --git a/app/javascript/mastodon/features/ui/components/compare_history_modal.js b/app/javascript/mastodon/features/ui/components/compare_history_modal.js index 40cfba335..ecccc8f7d 100644 --- a/app/javascript/mastodon/features/ui/components/compare_history_modal.js +++ b/app/javascript/mastodon/features/ui/components/compare_history_modal.js @@ -9,6 +9,7 @@ import escapeTextContentForBrowser from 'escape-html'; import InlineAccount from 'mastodon/components/inline_account'; import IconButton from 'mastodon/components/icon_button'; import RelativeTimestamp from 'mastodon/components/relative_timestamp'; +import MediaAttachments from 'mastodon/components/media_attachments'; const mapStateToProps = (state, { statusId }) => ({ versions: state.getIn(['history', statusId, 'items']), @@ -70,6 +71,25 @@ class CompareHistoryModal extends React.PureComponent { )}
+ + {!!currentVersion.get('poll') && ( +
+
    + {currentVersion.getIn(['poll', 'options']).map(option => ( +
  • + + + +
    • + ))} +
+
+ )} + +
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 5d0d44b73..97587b62b 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -1127,7 +1127,7 @@ .media-gallery, .audio-player, .video-player { - margin-top: 8px; + margin-top: 15px; max-width: 250px; } @@ -5609,6 +5609,12 @@ a.status-card.compact:hover { margin: 20px 0; } } + + .media-gallery, + .audio-player, + .video-player { + margin-top: 15px; + } } .loading-bar { diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb index 6da9b4b85..e9c8fbe98 100644 --- a/app/models/status_edit.rb +++ b/app/models/status_edit.rb @@ -25,7 +25,12 @@ class StatusEdit < ApplicationRecord class PreservedMediaAttachment < ActiveModelSerializers::Model attributes :media_attachment, :description - delegate :id, :type, :url, :preview_url, :remote_url, :preview_remote_url, :text_url, :meta, :blurhash, to: :media_attachment + + delegate :id, :type, :url, :preview_url, :remote_url, + :preview_remote_url, :text_url, :meta, :blurhash, + :not_processed?, :needs_redownload?, :local?, + :file, :thumbnail, :thumbnail_remote_url, + :shortcode, to: :media_attachment end rate_limit by: :account, family: :statuses -- cgit From d7fab238a891d2530c9937a2fa627b622972d409 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Thu, 10 Mar 2022 12:10:20 +0900 Subject: Fix a type error in domain_block policies (#17735) --- app/models/domain_block.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb index b06fa09da..2b797ef91 100644 --- a/app/models/domain_block.rb +++ b/app/models/domain_block.rb @@ -32,7 +32,7 @@ class DomainBlock < ApplicationRecord def policies if suspend? - :suspend + [:suspend] else [severity.to_sym, reject_media? ? :reject_media : nil, reject_reports? ? :reject_reports : nil].reject { |policy| policy == :noop || policy.nil? } end -- cgit