From 9a015e43ef173a3afc93cc1d06fd7f76adb8876a Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Mon, 14 Feb 2022 08:17:09 +0900 Subject: Add `from:` query operator to search syntax (#16526) * Add 'by:userhandle' parameter to search api * Use search syntax for "by" prefix * Codeclimate * Use 'from' instead of 'by' --- app/lib/search_query_transformer.rb | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) (limited to 'app/lib') diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index e07ebfffe..c685d7b6f 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -2,19 +2,21 @@ class SearchQueryTransformer < Parslet::Transform class Query - attr_reader :should_clauses, :must_not_clauses, :must_clauses + attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses def initialize(clauses) grouped = clauses.chunk(&:operator).to_h @should_clauses = grouped.fetch(:should, []) @must_not_clauses = grouped.fetch(:must_not, []) @must_clauses = grouped.fetch(:must, []) + @filter_clauses = grouped.fetch(:filter, []) end def apply(search) should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) } must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) } must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) } + filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause)) } search.query.minimum_should_match(1) end @@ -30,6 +32,15 @@ class SearchQueryTransformer < Parslet::Transform raise "Unexpected clause type: #{clause}" end end + + def clause_to_filter(clause) + case clause + when PrefixClause + { term: { clause.filter => clause.term } } + else + raise "Unexpected clause type: #{clause}" + end + end end class Operator @@ -69,11 +80,33 @@ class SearchQueryTransformer < Parslet::Transform end end + class PrefixClause + attr_reader :filter, :operator, :term + + def initialize(prefix, term) + @operator = :filter + case prefix + when 'from' + @filter = :account_id + username, domain = term.split('@') + account = Account.find_remote(username, domain) + + raise "Account not found: #{term}" unless account + + @term = account.id + else + raise "Unknown prefix: #{prefix}" + end + end + end + rule(clause: subtree(:clause)) do prefix = clause[:prefix][:term].to_s if clause[:prefix] operator = clause[:operator]&.to_s - if clause[:term] + if clause[:prefix] + PrefixClause.new(prefix, clause[:term].to_s) + elsif clause[:term] TermClause.new(prefix, operator, clause[:term].to_s) elsif clause[:shortcode] TermClause.new(prefix, operator, ":#{clause[:term]}:") -- cgit From 73fce8d31115f44d6d3f20f563c656b1595e70f8 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 16 Feb 2022 14:28:45 +0100 Subject: Fix performance of server-side filtering (#17575) Fixes #17567 --- app/lib/feed_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 7840afee8..ccd4d3610 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -441,7 +441,7 @@ class FeedManager return false if active_filters.empty? - combined_regex = active_filters.reduce { |memo, obj| Regexp.union(memo, obj) } + combined_regex = Regexp.union(active_filters) status = status.reblog if status.reblog? combined_text = [ -- cgit From 8f537a116802ee39727d8b6c286883b9440ab51a Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 16 Feb 2022 14:36:44 +0100 Subject: Change relays handling to not record boosts (#17571) * Change relays handling to not record boosts * Update tests --- app/lib/activitypub/activity/announce.rb | 1 + spec/lib/activitypub/activity/announce_spec.rb | 32 ++++++++++---------------- 2 files changed, 13 insertions(+), 20 deletions(-) (limited to 'app/lib') diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 1f9319290..12fad8da4 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -8,6 +8,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity original_status = status_from_object return reject_payload! if original_status.nil? || !announceable?(original_status) + return if requested_through_relay? @status = Status.find_by(account: @account, reblog: original_status) diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index b93fcbe66..41806b258 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -113,26 +113,23 @@ RSpec.describe ActivityPub::Activity::Announce do let!(:relay_account) { Fabricate(:account, inbox_url: 'https://relay.example.com/inbox') } let!(:relay) { Fabricate(:relay, inbox_url: 'https://relay.example.com/inbox') } + let(:object_json) { 'https://example.com/actor/hello-world' } + subject { described_class.new(json, sender, relayed_through_account: relay_account) } + before do + stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json)) + end + context 'and the relay is enabled' do before do relay.update(state: :accepted) subject.perform end - let(:object_json) do - { - id: 'https://example.com/actor#bar', - type: 'Note', - content: 'Lorem ipsum', - to: 'http://example.com/followers', - attributedTo: 'https://example.com/actor', - } - end - - it 'creates a reblog by sender of status' do - expect(sender.statuses.count).to eq 2 + it 'fetches the remote status' do + expect(a_request(:get, 'https://example.com/actor/hello-world')).to have_been_made + expect(Status.find_by(uri: 'https://example.com/actor/hello-world').text).to eq 'Hello world' end end @@ -141,14 +138,9 @@ RSpec.describe ActivityPub::Activity::Announce do subject.perform end - let(:object_json) do - { - id: 'https://example.com/actor#bar', - type: 'Note', - content: 'Lorem ipsum', - to: 'http://example.com/followers', - attributedTo: 'https://example.com/actor', - } + it 'does not fetch the remote status' do + expect(a_request(:get, 'https://example.com/actor/hello-world')).not_to have_been_made + expect(Status.find_by(uri: 'https://example.com/actor/hello-world')).to be_nil end it 'does not create anything' do -- cgit From b377022cf9dfac4d98d0d10b511aeb65e540e0a3 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 22 Feb 2022 15:27:08 +0100 Subject: Add caching layer to metrics (#17617) --- app/lib/admin/metrics/dimension/base_dimension.rb | 32 ++++++++++++- .../admin/metrics/dimension/languages_dimension.rb | 4 +- .../admin/metrics/dimension/servers_dimension.rb | 4 +- .../dimension/software_versions_dimension.rb | 6 +-- .../admin/metrics/dimension/sources_dimension.rb | 4 +- .../metrics/dimension/space_usage_dimension.rb | 6 +-- .../metrics/dimension/tag_languages_dimension.rb | 6 +-- .../metrics/dimension/tag_servers_dimension.rb | 6 +-- .../admin/metrics/measure/active_users_measure.rb | 10 ++--- app/lib/admin/metrics/measure/base_measure.rb | 52 ++++++++++++++++++++-- .../admin/metrics/measure/interactions_measure.rb | 10 ++--- app/lib/admin/metrics/measure/new_users_measure.rb | 8 ++-- .../metrics/measure/opened_reports_measure.rb | 8 ++-- .../metrics/measure/resolved_reports_measure.rb | 8 ++-- .../admin/metrics/measure/tag_accounts_measure.rb | 10 ++--- .../admin/metrics/measure/tag_servers_measure.rb | 10 ++--- app/lib/admin/metrics/measure/tag_uses_measure.rb | 10 ++--- app/lib/admin/metrics/retention.rb | 26 +++++++++++ 18 files changed, 165 insertions(+), 55 deletions(-) (limited to 'app/lib') diff --git a/app/lib/admin/metrics/dimension/base_dimension.rb b/app/lib/admin/metrics/dimension/base_dimension.rb index 5872c22cb..bd2e4ecec 100644 --- a/app/lib/admin/metrics/dimension/base_dimension.rb +++ b/app/lib/admin/metrics/dimension/base_dimension.rb @@ -1,23 +1,34 @@ # frozen_string_literal: true class Admin::Metrics::Dimension::BaseDimension + CACHE_TTL = 5.minutes.freeze + def self.with_params? false end + attr_reader :loaded + + alias loaded? loaded + def initialize(start_at, end_at, limit, params) @start_at = start_at&.to_datetime @end_at = end_at&.to_datetime @limit = limit&.to_i @params = params + @loaded = false end def key raise NotImplementedError end + def cache_key + ["metrics/dimension/#{key}", @start_at, @end_at, @limit, canonicalized_params].join(';') + end + def data - raise NotImplementedError + load end def self.model_name @@ -30,11 +41,28 @@ class Admin::Metrics::Dimension::BaseDimension protected + def load + unless loaded? + @values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_query } + @loaded = true + end + + @values + end + + def perform_query + raise NotImplementedError + end + def time_period (@start_at..@end_at) end def params - raise NotImplementedError + {} + end + + def canonicalized_params + params.to_h.to_a.sort_by { |k, _v| k.to_s }.map { |k, v| "#{k}=#{v}" }.join(';') end end diff --git a/app/lib/admin/metrics/dimension/languages_dimension.rb b/app/lib/admin/metrics/dimension/languages_dimension.rb index 1cc5f4120..f1cf82cf2 100644 --- a/app/lib/admin/metrics/dimension/languages_dimension.rb +++ b/app/lib/admin/metrics/dimension/languages_dimension.rb @@ -7,7 +7,9 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension: 'languages' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT locale, count(*) AS value FROM users diff --git a/app/lib/admin/metrics/dimension/servers_dimension.rb b/app/lib/admin/metrics/dimension/servers_dimension.rb index 3e80b6625..91bcce655 100644 --- a/app/lib/admin/metrics/dimension/servers_dimension.rb +++ b/app/lib/admin/metrics/dimension/servers_dimension.rb @@ -5,7 +5,9 @@ class Admin::Metrics::Dimension::ServersDimension < Admin::Metrics::Dimension::B 'servers' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT accounts.domain, count(*) AS value FROM statuses diff --git a/app/lib/admin/metrics/dimension/software_versions_dimension.rb b/app/lib/admin/metrics/dimension/software_versions_dimension.rb index 34917404d..816615f99 100644 --- a/app/lib/admin/metrics/dimension/software_versions_dimension.rb +++ b/app/lib/admin/metrics/dimension/software_versions_dimension.rb @@ -7,12 +7,12 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim 'software_versions' end - def data + protected + + def perform_query [mastodon_version, ruby_version, postgresql_version, redis_version] end - private - def mastodon_version value = Mastodon::Version.to_s diff --git a/app/lib/admin/metrics/dimension/sources_dimension.rb b/app/lib/admin/metrics/dimension/sources_dimension.rb index a9f061809..122807cdc 100644 --- a/app/lib/admin/metrics/dimension/sources_dimension.rb +++ b/app/lib/admin/metrics/dimension/sources_dimension.rb @@ -5,7 +5,9 @@ class Admin::Metrics::Dimension::SourcesDimension < Admin::Metrics::Dimension::B 'sources' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT oauth_applications.name, count(*) AS value FROM users diff --git a/app/lib/admin/metrics/dimension/space_usage_dimension.rb b/app/lib/admin/metrics/dimension/space_usage_dimension.rb index aa00a2e18..5867c5bab 100644 --- a/app/lib/admin/metrics/dimension/space_usage_dimension.rb +++ b/app/lib/admin/metrics/dimension/space_usage_dimension.rb @@ -8,12 +8,12 @@ class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension 'space_usage' end - def data + protected + + def perform_query [postgresql_size, redis_size, media_size] end - private - def postgresql_size value = ActiveRecord::Base.connection.execute('SELECT pg_database_size(current_database())').first['pg_database_size'] diff --git a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb index afbc8cde8..e1349c229 100644 --- a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb +++ b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb @@ -11,7 +11,9 @@ class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimensi 'tag_languages' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value FROM statuses @@ -28,8 +30,6 @@ class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimensi rows.map { |row| { key: row['language'], human_key: standard_locale_name(row['language']), value: row['value'].to_s } } end - private - def params @params.permit(:id) end diff --git a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb index 12c5980d7..7ddf3378c 100644 --- a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb +++ b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb @@ -9,7 +9,9 @@ class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension 'tag_servers' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT accounts.domain, count(*) AS value FROM statuses @@ -27,8 +29,6 @@ class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } } end - private - def params @params.permit(:id) end diff --git a/app/lib/admin/metrics/measure/active_users_measure.rb b/app/lib/admin/metrics/measure/active_users_measure.rb index 513189780..e6f09d4bc 100644 --- a/app/lib/admin/metrics/measure/active_users_measure.rb +++ b/app/lib/admin/metrics/measure/active_users_measure.rb @@ -5,20 +5,20 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas 'active_users' end - def total + protected + + def perform_total_query activity_tracker.sum(time_period.first, time_period.last) end - def previous_total + def perform_previous_total_query activity_tracker.sum(previous_time_period.first, previous_time_period.last) end - def data + def perform_data_query activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } } end - protected - def activity_tracker @activity_tracker ||= ActivityTracker.new('activity:logins', :unique) end diff --git a/app/lib/admin/metrics/measure/base_measure.rb b/app/lib/admin/metrics/measure/base_measure.rb index 0107ffd9c..ed1df9c7d 100644 --- a/app/lib/admin/metrics/measure/base_measure.rb +++ b/app/lib/admin/metrics/measure/base_measure.rb @@ -1,14 +1,25 @@ # frozen_string_literal: true class Admin::Metrics::Measure::BaseMeasure + CACHE_TTL = 5.minutes.freeze + def self.with_params? false end + attr_reader :loaded + + alias loaded? loaded + def initialize(start_at, end_at, params) @start_at = start_at&.to_datetime @end_at = end_at&.to_datetime @params = params + @loaded = false + end + + def cache_key + ["metrics/measure/#{key}", @start_at, @end_at, canonicalized_params].join(';') end def key @@ -16,15 +27,15 @@ class Admin::Metrics::Measure::BaseMeasure end def total - raise NotImplementedError + load[:total] end def previous_total - raise NotImplementedError + load[:previous_total] end def data - raise NotImplementedError + load[:data] end def self.model_name @@ -37,6 +48,35 @@ class Admin::Metrics::Measure::BaseMeasure protected + def load + unless loaded? + @values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_queries }.with_indifferent_access + @loaded = true + end + + @values + end + + def perform_queries + { + total: perform_total_query, + previous_total: perform_previous_total_query, + data: perform_data_query, + } + end + + def perform_total_query + raise NotImplementedError + end + + def perform_previous_total_query + raise NotImplementedError + end + + def perform_data_query + raise NotImplementedError + end + def time_period (@start_at..@end_at) end @@ -50,6 +90,10 @@ class Admin::Metrics::Measure::BaseMeasure end def params - raise NotImplementedError + {} + end + + def canonicalized_params + params.to_h.to_a.sort_by { |k, _v| k.to_s }.map { |k, v| "#{k}=#{v}" }.join(';') end end diff --git a/app/lib/admin/metrics/measure/interactions_measure.rb b/app/lib/admin/metrics/measure/interactions_measure.rb index b928fdb8f..7a2b7e0fa 100644 --- a/app/lib/admin/metrics/measure/interactions_measure.rb +++ b/app/lib/admin/metrics/measure/interactions_measure.rb @@ -5,20 +5,20 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba 'interactions' end - def total + protected + + def perform_total_query activity_tracker.sum(time_period.first, time_period.last) end - def previous_total + def perform_previous_total_query activity_tracker.sum(previous_time_period.first, previous_time_period.last) end - def data + def perform_data_query activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } } end - protected - def activity_tracker @activity_tracker ||= ActivityTracker.new('activity:interactions', :basic) end diff --git a/app/lib/admin/metrics/measure/new_users_measure.rb b/app/lib/admin/metrics/measure/new_users_measure.rb index b31679ad3..71191f1a2 100644 --- a/app/lib/admin/metrics/measure/new_users_measure.rb +++ b/app/lib/admin/metrics/measure/new_users_measure.rb @@ -5,15 +5,17 @@ class Admin::Metrics::Measure::NewUsersMeasure < Admin::Metrics::Measure::BaseMe 'new_users' end - def total + protected + + def perform_total_query User.where(created_at: time_period).count end - def previous_total + def perform_previous_total_query User.where(created_at: previous_time_period).count end - def data + def perform_data_query sql = <<-SQL.squish SELECT axis.*, ( WITH new_users AS ( diff --git a/app/lib/admin/metrics/measure/opened_reports_measure.rb b/app/lib/admin/metrics/measure/opened_reports_measure.rb index 9acc2c33d..4b80a0c8c 100644 --- a/app/lib/admin/metrics/measure/opened_reports_measure.rb +++ b/app/lib/admin/metrics/measure/opened_reports_measure.rb @@ -5,15 +5,17 @@ class Admin::Metrics::Measure::OpenedReportsMeasure < Admin::Metrics::Measure::B 'opened_reports' end - def total + protected + + def perform_total_query Report.where(created_at: time_period).count end - def previous_total + def perform_previous_total_query Report.where(created_at: previous_time_period).count end - def data + def perform_data_query sql = <<-SQL.squish SELECT axis.*, ( WITH new_reports AS ( diff --git a/app/lib/admin/metrics/measure/resolved_reports_measure.rb b/app/lib/admin/metrics/measure/resolved_reports_measure.rb index 00cb24f7e..4ab746c8f 100644 --- a/app/lib/admin/metrics/measure/resolved_reports_measure.rb +++ b/app/lib/admin/metrics/measure/resolved_reports_measure.rb @@ -5,15 +5,17 @@ class Admin::Metrics::Measure::ResolvedReportsMeasure < Admin::Metrics::Measure: 'resolved_reports' end - def total + protected + + def perform_total_query Report.resolved.where(action_taken_at: time_period).count end - def previous_total + def perform_previous_total_query Report.resolved.where(action_taken_at: previous_time_period).count end - def data + def perform_data_query sql = <<-SQL.squish SELECT axis.*, ( WITH resolved_reports AS ( diff --git a/app/lib/admin/metrics/measure/tag_accounts_measure.rb b/app/lib/admin/metrics/measure/tag_accounts_measure.rb index ef773081b..8f4512efe 100644 --- a/app/lib/admin/metrics/measure/tag_accounts_measure.rb +++ b/app/lib/admin/metrics/measure/tag_accounts_measure.rb @@ -9,20 +9,20 @@ class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::Bas 'tag_accounts' end - def total + protected + + def perform_total_query tag.history.aggregate(time_period).accounts end - def previous_total + def perform_previous_total_query tag.history.aggregate(previous_time_period).accounts end - def data + def perform_data_query time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } } end - protected - def tag @tag ||= Tag.find(params[:id]) end diff --git a/app/lib/admin/metrics/measure/tag_servers_measure.rb b/app/lib/admin/metrics/measure/tag_servers_measure.rb index cc064f63f..11f229602 100644 --- a/app/lib/admin/metrics/measure/tag_servers_measure.rb +++ b/app/lib/admin/metrics/measure/tag_servers_measure.rb @@ -9,15 +9,17 @@ class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::Base 'tag_servers' end - def total + protected + + def perform_total_query tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain') end - def previous_total + def perform_previous_total_query tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain') end - def data + def perform_data_query sql = <<-SQL.squish SELECT axis.*, ( SELECT count(distinct accounts.domain) AS value @@ -38,8 +40,6 @@ class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::Base rows.map { |row| { date: row['day'], value: row['value'].to_s } } end - protected - def tag @tag ||= Tag.find(params[:id]) end diff --git a/app/lib/admin/metrics/measure/tag_uses_measure.rb b/app/lib/admin/metrics/measure/tag_uses_measure.rb index b7667bc6c..bce86b89f 100644 --- a/app/lib/admin/metrics/measure/tag_uses_measure.rb +++ b/app/lib/admin/metrics/measure/tag_uses_measure.rb @@ -9,20 +9,20 @@ class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMea 'tag_uses' end - def total + protected + + def perform_total_query tag.history.aggregate(time_period).uses end - def previous_total + def perform_previous_total_query tag.history.aggregate(previous_time_period).uses end - def data + def perform_data_query time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } } end - protected - def tag @tag ||= Tag.find(params[:id]) end diff --git a/app/lib/admin/metrics/retention.rb b/app/lib/admin/metrics/retention.rb index 0179a6e28..f6135ac1e 100644 --- a/app/lib/admin/metrics/retention.rb +++ b/app/lib/admin/metrics/retention.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Admin::Metrics::Retention + CACHE_TTL = 5.minutes.freeze + class Cohort < ActiveModelSerializers::Model attributes :period, :frequency, :data end @@ -9,13 +11,37 @@ class Admin::Metrics::Retention attributes :date, :rate, :value end + attr_reader :loaded + + alias loaded? loaded + def initialize(start_at, end_at, frequency) @start_at = start_at&.to_date @end_at = end_at&.to_date @frequency = %w(day month).include?(frequency) ? frequency : 'day' + @loaded = false + end + + def cache_key + ['metrics/retention', @start_at, @end_at, @frequency].join(';') end def cohorts + load + end + + protected + + def load + unless loaded? + @values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_query } + @loaded = true + end + + @values + end + + def perform_query sql = <<-SQL.squish SELECT axis.*, ( WITH new_users AS ( -- cgit From 166f6e4b500dd84eeffdbf887b2dc21e6d8c0aa6 Mon Sep 17 00:00:00 2001 From: Claire Date: Tue, 22 Feb 2022 17:11:22 +0100 Subject: Fix some media attachments being converted with too high framerates (#17619) Video files with variable framerates are converted to constant framerate videos and the output framerate picked by ffmpeg is based on the original file's container framerate (which can be different from the average framerate). This means that an input video with variable framerate with about 30 frames per second on average, but a maximum of 120 fps will be converted to a constant 120 fps file, which won't be processed by other Mastodon servers. This commit changes it so that input files with VFR and a maximum framerate above the framerate threshold are converted to VFR files with the maximum frame rate enforced. --- app/lib/video_metadata_extractor.rb | 3 ++- app/models/media_attachment.rb | 13 +++++++------ lib/paperclip/transcoder.rb | 16 +++++++++++++++- 3 files changed, 24 insertions(+), 8 deletions(-) (limited to 'app/lib') diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb index 03e40f923..2896620cb 100644 --- a/app/lib/video_metadata_extractor.rb +++ b/app/lib/video_metadata_extractor.rb @@ -2,7 +2,7 @@ class VideoMetadataExtractor attr_reader :duration, :bitrate, :video_codec, :audio_codec, - :colorspace, :width, :height, :frame_rate + :colorspace, :width, :height, :frame_rate, :r_frame_rate def initialize(path) @path = path @@ -42,6 +42,7 @@ class VideoMetadataExtractor @width = video_stream[:width] @height = video_stream[:height] @frame_rate = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate]) + @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate]) end if (audio_stream = audio_streams.first) diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 0a9d05f1d..a3115637e 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -38,6 +38,12 @@ class MediaAttachment < ApplicationRecord MAX_DESCRIPTION_LENGTH = 1_500 + IMAGE_LIMIT = 10.megabytes + VIDEO_LIMIT = 40.megabytes + + MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px + MAX_VIDEO_FRAME_RATE = 60 + IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif).freeze VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp .wma).freeze @@ -75,6 +81,7 @@ class MediaAttachment < ApplicationRecord VIDEO_FORMAT = { format: 'mp4', content_type: 'video/mp4', + vfr_frame_rate_threshold: MAX_VIDEO_FRAME_RATE, convert_options: { output: { 'loglevel' => 'fatal', @@ -152,12 +159,6 @@ class MediaAttachment < ApplicationRecord all: '-quality 90 -strip +set modify-date +set create-date', }.freeze - IMAGE_LIMIT = 10.megabytes - VIDEO_LIMIT = 40.megabytes - - MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px - MAX_VIDEO_FRAME_RATE = 60 - belongs_to :account, inverse_of: :media_attachments, optional: true belongs_to :status, inverse_of: :media_attachments, optional: true belongs_to :scheduled_status, inverse_of: :media_attachments, optional: true diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index ec1305038..afd9f58ff 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -13,6 +13,7 @@ module Paperclip @time = options[:time] || 3 @passthrough_options = options[:passthrough_options] @convert_options = options[:convert_options].dup + @vfr_threshold = options[:vfr_frame_rate_threshold] end def make @@ -41,6 +42,11 @@ module Paperclip when 'mp4' @output_options['acodec'] = 'aac' @output_options['strict'] = 'experimental' + + if high_vfr?(metadata) && !eligible_to_passthrough?(metadata) + @output_options['vsync'] = 'vfr' + @output_options['r'] = @vfr_threshold + end end command_arguments, interpolations = prepare_command(destination) @@ -88,13 +94,21 @@ module Paperclip end def update_options_from_metadata(metadata) - return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace) + return unless eligible_to_passthrough?(metadata) @format = @passthrough_options[:options][:format] || @format @time = @passthrough_options[:options][:time] || @time @convert_options = @passthrough_options[:options][:convert_options].dup end + def high_vfr?(metadata) + @vfr_threshold && metadata.r_frame_rate && metadata.r_frame_rate > @vfr_threshold + end + + def eligible_to_passthrough?(metadata) + @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace) + end + def update_attachment_type(metadata) @attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec end -- cgit From 73f5e4a1d9c6d067ff62c5cc9a65d929a56f0f00 Mon Sep 17 00:00:00 2001 From: luzpaz Date: Tue, 22 Feb 2022 14:14:17 -0500 Subject: Fix various typos (#17621) Found via `codespell -q 3 -S ./CHANGELOG.md,./AUTHORS.md,./config/locales,./app/javascript/mastodon/locales -L ba,keypair,medias,ro` --- app/javascript/mastodon/components/loading_indicator.js | 2 +- app/javascript/mastodon/features/emoji/emoji_compressed.js | 2 +- app/lib/feed_manager.rb | 2 +- app/services/fetch_link_card_service.rb | 2 +- app/services/notify_service.rb | 2 +- config/initializers/preload_link_headers.rb | 2 +- db/migrate/20170920032311_fix_reblogs_in_feeds.rb | 2 +- .../two_factor_authentication/otp_authentication_controller_spec.rb | 2 +- spec/models/web/push_subscription_spec.rb | 2 +- spec/services/block_domain_service_spec.rb | 2 +- spec/services/clear_domain_media_service_spec.rb | 2 +- spec/services/delete_account_service_spec.rb | 2 +- spec/validators/unreserved_username_validator_spec.rb | 4 ++-- 13 files changed, 14 insertions(+), 14 deletions(-) (limited to 'app/lib') diff --git a/app/javascript/mastodon/components/loading_indicator.js b/app/javascript/mastodon/components/loading_indicator.js index 59f721c50..33c59d94c 100644 --- a/app/javascript/mastodon/components/loading_indicator.js +++ b/app/javascript/mastodon/components/loading_indicator.js @@ -6,7 +6,7 @@ export const CircularProgress = ({ size, strokeWidth }) => { const radius = (size - strokeWidth) / 2; return ( - + { let { short_names, search, unified } = emojiMartData.emojis[key]; if (short_names[0] !== key) { - throw new Error('The compresser expects the first short_code to be the ' + + throw new Error('The compressor expects the first short_code to be the ' + 'key. It may need to be rewritten if the emoji change such that this ' + 'is no longer the case.'); } diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index ccd4d3610..fc35292f2 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -491,7 +491,7 @@ class FeedManager end else # A reblog may reach earlier than the original status because of the - # delay of the worker deliverying the original status, the late addition + # delay of the worker delivering the original status, the late addition # by merging timelines, and other reasons. # If such a reblog already exists, just do not re-insert it into the feed. return false unless redis.zscore(reblog_key, status.id).nil? diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 94dc6389f..239ab9b93 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -2,7 +2,7 @@ class FetchLinkCardService < BaseService URL_PATTERN = %r{ - (#{Twitter::TwitterText::Regex[:valid_url_preceding_chars]}) # $1 preceeding chars + (#{Twitter::TwitterText::Regex[:valid_url_preceding_chars]}) # $1 preceding chars ( # $2 URL (https?:\/\/) # $3 Protocol (required) (#{Twitter::TwitterText::Regex[:valid_domain]}) # $4 Domain(s) diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index 039e007f5..ab80a7e4b 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -71,7 +71,7 @@ class NotifyService < BaseService message? && @notification.target_status.direct_visibility? end - # Returns true if the sender has been mentionned by the recipient up the thread + # Returns true if the sender has been mentioned by the recipient up the thread def response_to_recipient? return false if @notification.target_status.in_reply_to_id.nil? diff --git a/config/initializers/preload_link_headers.rb b/config/initializers/preload_link_headers.rb index 9f21c45ec..364a7cc1b 100644 --- a/config/initializers/preload_link_headers.rb +++ b/config/initializers/preload_link_headers.rb @@ -2,7 +2,7 @@ # in the Links header per default. # In our case, that will bloat headers too much and potentially cause -# issues with reverse proxies. Furhermore, we don't need those links, +# issues with reverse proxies. Furthermore, we don't need those links, # as we already output them as HTML link tags. Rails.application.config.action_view.preload_links_header = false diff --git a/db/migrate/20170920032311_fix_reblogs_in_feeds.rb b/db/migrate/20170920032311_fix_reblogs_in_feeds.rb index 5654bf6f8..bcd4b9137 100644 --- a/db/migrate/20170920032311_fix_reblogs_in_feeds.rb +++ b/db/migrate/20170920032311_fix_reblogs_in_feeds.rb @@ -15,7 +15,7 @@ class FixReblogsInFeeds < ActiveRecord::Migration[5.1] # feed, with an entry in a reblog tracking zset (where the score # is once again set to the reblogging status' ID, and the value # is set to the reblogged status' ID). This is safe for Redis' - # float coersion because in this reblog tracking zset, we only + # float conversion because in this reblog tracking zset, we only # need the rebloggging status' ID to be able to stop tracking # entries after they have gotten too far down the feed, which # does not require an exact value. diff --git a/spec/controllers/settings/two_factor_authentication/otp_authentication_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/otp_authentication_controller_spec.rb index 17e8fa9b8..c8ba6f9a8 100644 --- a/spec/controllers/settings/two_factor_authentication/otp_authentication_controller_spec.rb +++ b/spec/controllers/settings/two_factor_authentication/otp_authentication_controller_spec.rb @@ -18,7 +18,7 @@ describe Settings::TwoFactorAuthentication::OtpAuthenticationController do user.update(otp_required_for_login: true) end - it 'redirects to two factor authentciation methods list page' do + it 'redirects to two factor authentication methods list page' do get :show expect(response).to redirect_to settings_two_factor_authentication_methods_path diff --git a/spec/models/web/push_subscription_spec.rb b/spec/models/web/push_subscription_spec.rb index b44904369..bd5719593 100644 --- a/spec/models/web/push_subscription_spec.rb +++ b/spec/models/web/push_subscription_spec.rb @@ -29,7 +29,7 @@ RSpec.describe Web::PushSubscription, type: :model do context "when notification is a #{type}" do let(:notification_type) { type } - it "returns boolean corresonding to alert setting" do + it "returns boolean corresponding to alert setting" do expect(subject.pushable?(notification)).to eq data[:alerts][type] end end diff --git a/spec/services/block_domain_service_spec.rb b/spec/services/block_domain_service_spec.rb index c689b57e3..242b02fff 100644 --- a/spec/services/block_domain_service_spec.rb +++ b/spec/services/block_domain_service_spec.rb @@ -66,7 +66,7 @@ RSpec.describe BlockDomainService, type: :service do expect(Account.find_remote('badguy', 'evil.org').silenced_at).to_not eq DomainBlock.find_by(domain: 'evil.org').created_at end - it 'leaves the domains status and attachements, but clears media' do + it 'leaves the domains status and attachments, but clears media' do expect { bad_status1.reload }.not_to raise_error expect { bad_status2.reload }.not_to raise_error expect { bad_attachment.reload }.not_to raise_error diff --git a/spec/services/clear_domain_media_service_spec.rb b/spec/services/clear_domain_media_service_spec.rb index 8e58c6039..45b92e2c9 100644 --- a/spec/services/clear_domain_media_service_spec.rb +++ b/spec/services/clear_domain_media_service_spec.rb @@ -13,7 +13,7 @@ RSpec.describe ClearDomainMediaService, type: :service do subject.call(DomainBlock.create!(domain: 'evil.org', severity: :silence, reject_media: true)) end - it 'leaves the domains status and attachements, but clears media' do + it 'leaves the domains status and attachments, but clears media' do expect { bad_status1.reload }.not_to raise_error expect { bad_status2.reload }.not_to raise_error expect { bad_attachment.reload }.not_to raise_error diff --git a/spec/services/delete_account_service_spec.rb b/spec/services/delete_account_service_spec.rb index b1da97036..9c785fc17 100644 --- a/spec/services/delete_account_service_spec.rb +++ b/spec/services/delete_account_service_spec.rb @@ -90,7 +90,7 @@ RSpec.describe DeleteAccountService, type: :service do let!(:account) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', protocol: :activitypub) } let!(:local_follower) { Fabricate(:account) } - it 'sends a reject follow to follwer inboxes' do + it 'sends a reject follow to follower inboxes' do subject.call expect(a_request(:post, account.inbox_url)).to have_been_made.once end diff --git a/spec/validators/unreserved_username_validator_spec.rb b/spec/validators/unreserved_username_validator_spec.rb index cabd6d386..746b3866c 100644 --- a/spec/validators/unreserved_username_validator_spec.rb +++ b/spec/validators/unreserved_username_validator_spec.rb @@ -27,7 +27,7 @@ RSpec.describe UnreservedUsernameValidator, type: :validator do context 'reserved_username?' do let(:reserved_username) { true } - it 'calls erros.add' do + it 'calls errors.add' do expect(errors).to have_received(:add).with(:username, :reserved) end end @@ -35,7 +35,7 @@ RSpec.describe UnreservedUsernameValidator, type: :validator do context '!reserved_username?' do let(:reserved_username) { false } - it 'not calls erros.add' do + it 'not calls errors.add' do expect(errors).not_to have_received(:add).with(:username, any_args) end end -- cgit