diff options
Diffstat (limited to 'app/lib')
50 files changed, 1635 insertions, 675 deletions
diff --git a/app/lib/activity_tracker.rb b/app/lib/activity_tracker.rb index 81303b715..6d3401b37 100644 --- a/app/lib/activity_tracker.rb +++ b/app/lib/activity_tracker.rb @@ -1,29 +1,73 @@ # frozen_string_literal: true class ActivityTracker + include Redisable + EXPIRE_AFTER = 6.months.seconds - class << self - include Redisable + def initialize(prefix, type) + @prefix = prefix + @type = type + end - def increment(prefix) - key = [prefix, current_week].join(':') + def add(value = 1, at_time = Time.now.utc) + key = key_at(at_time) - redis.incrby(key, 1) - redis.expire(key, EXPIRE_AFTER) + case @type + when :basic + redis.incrby(key, value) + when :unique + redis.pfadd(key, value) end - def record(prefix, value) - key = [prefix, current_week].join(':') + redis.expire(key, EXPIRE_AFTER) + end - redis.pfadd(key, value) - redis.expire(key, EXPIRE_AFTER) + def get(start_at, end_at = Time.now.utc) + (start_at.to_date...end_at.to_date).map do |date| + key = key_at(date.to_time(:utc)) + + value = begin + case @type + when :basic + redis.get(key).to_i + when :unique + redis.pfcount(key) + end + end + + [date, value] + end + end + + def sum(start_at, end_at = Time.now.utc) + keys = (start_at.to_date...end_at.to_date).flat_map { |date| [key_at(date.to_time(:utc)), legacy_key_at(date)] }.uniq + + case @type + when :basic + redis.mget(*keys).map(&:to_i).sum + when :unique + redis.pfcount(*keys) end + end - private + class << self + def increment(prefix) + new(prefix, :basic).add + end - def current_week - Time.zone.today.cweek + def record(prefix, value) + new(prefix, :unique).add(value) end end + + private + + def key_at(at_time) + "#{@prefix}:#{at_time.beginning_of_day.to_i}" + end + + def legacy_key_at(at_time) + "#{@prefix}:#{at_time.to_date.cweek}" + end end diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index d2ec122a4..706960f92 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -94,51 +94,6 @@ class ActivityPub::Activity equals_or_includes_any?(@object['type'], CONVERTED_TYPES) end - def distribute(status) - crawl_links(status) - - notify_about_reblog(status) if reblog_of_local_account?(status) && !reblog_by_following_group_account?(status) - notify_about_mentions(status) - - # Only continue if the status is supposed to have arrived in real-time. - # Note that if @options[:override_timestamps] isn't set, the status - # may have a lower snowflake id than other existing statuses, potentially - # "hiding" it from paginated API calls - return unless @options[:override_timestamps] || status.within_realtime_window? - - distribute_to_followers(status) - end - - def reblog_of_local_account?(status) - status.reblog? && status.reblog.account.local? - end - - def reblog_by_following_group_account?(status) - status.reblog? && status.account.group? && status.reblog.account.following?(status.account) - end - - def notify_about_reblog(status) - NotifyService.new.call(status.reblog.account, :reblog, status) - end - - def notify_about_mentions(status) - status.active_mentions.includes(:account).each do |mention| - next unless mention.account.local? && audience_includes?(mention.account) - NotifyService.new.call(mention.account, :mention, mention) - end - end - - def crawl_links(status) - return if status.spoiler_text? - - # Spread out crawling randomly to avoid DDoSing the link - LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id) - end - - def distribute_to_followers(status) - ::DistributionWorker.perform_async(status.id) - end - def delete_arrived_first?(uri) redis.exists?("delete_upon_arrival:#{@account.id}:#{uri}") end diff --git a/app/lib/activitypub/activity/accept.rb b/app/lib/activitypub/activity/accept.rb index 7010ff43e..5126e23c6 100644 --- a/app/lib/activitypub/activity/accept.rb +++ b/app/lib/activitypub/activity/accept.rb @@ -3,7 +3,7 @@ class ActivityPub::Activity::Accept < ActivityPub::Activity def perform return accept_follow_for_relay if relay_follow? - return follow_request_from_object.authorize! unless follow_request_from_object.nil? + return accept_follow!(follow_request_from_object) unless follow_request_from_object.nil? case @object['type'] when 'Follow' @@ -19,7 +19,16 @@ class ActivityPub::Activity::Accept < ActivityPub::Activity return if target_account.nil? || !target_account.local? follow_request = FollowRequest.find_by(account: target_account, target_account: @account) - follow_request&.authorize! + accept_follow!(follow_request) + end + + def accept_follow!(request) + return if request.nil? + + is_first_follow = !request.target_account.followers.local.exists? + request.authorize! + + RemoteAccountRefreshWorker.perform_async(request.target_account_id) if is_first_follow end def accept_follow_for_relay diff --git a/app/lib/activitypub/activity/add.rb b/app/lib/activitypub/activity/add.rb index 688ab00b3..845eeaef7 100644 --- a/app/lib/activitypub/activity/add.rb +++ b/app/lib/activitypub/activity/add.rb @@ -4,8 +4,7 @@ class ActivityPub::Activity::Add < ActivityPub::Activity def perform return unless @json['target'].present? && value_or_id(@json['target']) == @account.featured_collection_url - status = status_from_uri(object_uri) - status ||= fetch_remote_original_status + status = status_from_object return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status) diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 9f778ffb9..1f9319290 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -22,11 +22,10 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity visibility: visibility_from_audience ) - original_status.tags.each do |tag| - tag.use!(@account) - end + Trends.tags.register(@status) + Trends.links.register(@status) - distribute(@status) + distribute end @status @@ -34,6 +33,22 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity private + def distribute + # Notify the author of the original status if that status is local + NotifyService.new.call(@status.reblog.account, :reblog, @status) if reblog_of_local_account?(@status) && !reblog_by_following_group_account?(@status) + + # Distribute into home and list feeds + ::DistributionWorker.perform_async(@status.id) if @options[:override_timestamps] || @status.within_realtime_window? + end + + def reblog_of_local_account?(status) + status.reblog? && status.reblog.account.local? + end + + def reblog_by_following_group_account?(status) + status.reblog? && status.account.group? && status.reblog.account.following?(status.account) + end + def audience_to as_array(@json['to']).map { |x| value_or_id(x) } end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index cc2d391fb..ad273c20b 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -69,9 +69,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def process_status - @tags = [] - @mentions = [] - @params = {} + @tags = [] + @mentions = [] + @silenced_account_ids = [] + @params = {} process_status_params process_tags @@ -84,10 +85,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity resolve_thread(@status) fetch_replies(@status) - distribute(@status) + distribute forward_for_reply end + def distribute + # Spread out crawling randomly to avoid DDoSing the link + LinkCrawlWorker.perform_in(rand(1..59).seconds, @status.id) + + # Distribute into home and list feeds and notify mentioned accounts + ::DistributionWorker.perform_async(@status.id, { 'silenced_account_ids' => @silenced_account_ids }) if @options[:override_timestamps] || @status.within_realtime_window? + end + def find_existing_status status = status_from_uri(object_uri) status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present? @@ -95,19 +104,22 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def process_status_params + @status_parser = ActivityPub::Parser::StatusParser.new(@json, followers_collection: @account.followers_url) + @params = begin { - uri: object_uri, - url: object_url || object_uri, + uri: @status_parser.uri, + url: @status_parser.url || @status_parser.uri, account: @account, - text: text_from_content || '', - language: detected_language, - spoiler_text: converted_object_type? ? '' : (text_from_summary || ''), - created_at: @object['published'], + text: converted_object_type? ? converted_text : (@status_parser.text || ''), + language: @status_parser.language || detected_language, + spoiler_text: converted_object_type? ? '' : (@status_parser.spoiler_text || ''), + created_at: @status_parser.created_at, + edited_at: @status_parser.edited_at, override_timestamps: @options[:override_timestamps], - reply: @object['inReplyTo'].present?, - sensitive: @account.sensitized? || @object['sensitive'] || false, - visibility: visibility_from_audience, + reply: @status_parser.reply, + sensitive: @account.sensitized? || @status_parser.sensitive || false, + visibility: @status_parser.visibility, thread: replied_to_status, conversation: conversation_from_uri(@object['conversation']), media_attachment_ids: process_attachments.take(4).map(&:id), @@ -117,56 +129,63 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def process_audience - (audience_to + audience_cc).uniq.each do |audience| - next if ActivityPub::TagManager.instance.public_collection?(audience) + # Unlike with tags, there is no point in resolving accounts we don't already + # know here, because silent mentions would only be used for local access control anyway + accounts_in_audience = (audience_to + audience_cc).uniq.filter_map do |audience| + account_from_uri(audience) unless ActivityPub::TagManager.instance.public_collection?(audience) + end - # Unlike with tags, there is no point in resolving accounts we don't already - # know here, because silent mentions would only be used for local access - # control anyway - account = account_from_uri(audience) + # If the payload was delivered to a specific inbox, the inbox owner must have + # access to it, unless they already have access to it anyway + if @options[:delivered_to_account_id] + accounts_in_audience << delivered_to_account + accounts_in_audience.uniq! + end - next if account.nil? || @mentions.any? { |mention| mention.account_id == account.id } + accounts_in_audience.each do |account| + # This runs after tags are processed, and those translate into non-silent + # mentions, which take precedence + next if @mentions.any? { |mention| mention.account_id == account.id } @mentions << Mention.new(account: account, silent: true) # If there is at least one silent mention, then the status can be considered # as a limited-audience status, and not strictly a direct message, but only # if we considered a direct message in the first place - next unless @params[:visibility] == :direct && direct_message.nil? - - @params[:visibility] = :limited + @params[:visibility] = :limited if @params[:visibility] == :direct && !@object['directMessage'] end - # If the payload was delivered to a specific inbox, the inbox owner must have - # access to it, unless they already have access to it anyway - return if @options[:delivered_to_account_id].nil? || @mentions.any? { |mention| mention.account_id == @options[:delivered_to_account_id] } - - @mentions << Mention.new(account_id: @options[:delivered_to_account_id], silent: true) - - return unless @params[:visibility] == :direct && direct_message.nil? - - @params[:visibility] = :limited + # Accounts that are tagged but are not in the audience are not + # supposed to be notified explicitly + @silenced_account_ids = @mentions.map(&:account_id) - accounts_in_audience.map(&:id) end def postprocess_audience_and_deliver return if @status.mentions.find_by(account_id: @options[:delivered_to_account_id]) - delivered_to_account = Account.find(@options[:delivered_to_account_id]) - @status.mentions.create(account: delivered_to_account, silent: true) - @status.update(visibility: :limited) if @status.direct_visibility? && direct_message.nil? + @status.update(visibility: :limited) if @status.direct_visibility? && !@object['directMessage'] return unless delivered_to_account.following?(@account) - FeedInsertWorker.perform_async(@status.id, delivered_to_account.id, :home) + FeedInsertWorker.perform_async(@status.id, delivered_to_account.id, 'home') + end + + def delivered_to_account + @delivered_to_account ||= Account.find(@options[:delivered_to_account_id]) end def attach_tags(status) @tags.each do |tag| status.tags << tag - tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility? + tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago) end + # If we're processing an old status, this may register tags as being used now + # as opposed to when the status was really published, but this is probably + # not a big deal + Trends.tags.register(status) + @mentions.each do |mention| mention.status = status mention.save @@ -210,21 +229,22 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def process_emoji(tag) return if skip_download? - return if tag['name'].blank? || tag['icon'].blank? || tag['icon']['url'].blank? - shortcode = tag['name'].delete(':') - image_url = tag['icon']['url'] - uri = tag['id'] - updated = tag['updated'] - emoji = CustomEmoji.find_by(shortcode: shortcode, domain: @account.domain) + custom_emoji_parser = ActivityPub::Parser::CustomEmojiParser.new(tag) - return unless emoji.nil? || image_url != emoji.image_remote_url || (updated && updated >= emoji.updated_at) + return if custom_emoji_parser.shortcode.blank? || custom_emoji_parser.image_remote_url.blank? - emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: shortcode, uri: uri) - emoji.image_remote_url = image_url - emoji.save - rescue Seahorse::Client::NetworkingError - nil + emoji = CustomEmoji.find_by(shortcode: custom_emoji_parser.shortcode, domain: @account.domain) + + return unless emoji.nil? || custom_emoji_parser.image_remote_url != emoji.image_remote_url || (custom_emoji_parser.updated_at && custom_emoji_parser.updated_at >= emoji.updated_at) + + begin + emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: custom_emoji_parser.shortcode, uri: custom_emoji_parser.uri) + emoji.image_remote_url = custom_emoji_parser.image_remote_url + emoji.save + rescue Seahorse::Client::NetworkingError => e + Rails.logger.warn "Error storing emoji: #{e}" + end end def process_attachments @@ -233,22 +253,31 @@ class ActivityPub::Activity::Create < ActivityPub::Activity media_attachments = [] as_array(@object['attachment']).each do |attachment| - next if attachment['url'].blank? || media_attachments.size >= 4 + media_attachment_parser = ActivityPub::Parser::MediaAttachmentParser.new(attachment) + + next if media_attachment_parser.remote_url.blank? || media_attachments.size >= 4 begin - href = Addressable::URI.parse(attachment['url']).normalize.to_s - media_attachment = MediaAttachment.create(account: @account, remote_url: href, thumbnail_remote_url: icon_url_from_attachment(attachment), description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) + media_attachment = MediaAttachment.create( + account: @account, + remote_url: media_attachment_parser.remote_url, + thumbnail_remote_url: media_attachment_parser.thumbnail_remote_url, + description: media_attachment_parser.description, + focus: media_attachment_parser.focus, + blurhash: media_attachment_parser.blurhash + ) + media_attachments << media_attachment - next if unsupported_media_type?(attachment['mediaType']) || skip_download? + next if unsupported_media_type?(media_attachment_parser.file_content_type) || skip_download? media_attachment.download_file! media_attachment.download_thumbnail! media_attachment.save rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id) - rescue Seahorse::Client::NetworkingError - nil + rescue Seahorse::Client::NetworkingError => e + Rails.logger.warn "Error storing media attachment: #{e}" end end @@ -258,42 +287,17 @@ class ActivityPub::Activity::Create < ActivityPub::Activity media_attachments end - def icon_url_from_attachment(attachment) - url = attachment['icon'].is_a?(Hash) ? attachment['icon']['url'] : attachment['icon'] - Addressable::URI.parse(url).normalize.to_s if url.present? - rescue Addressable::URI::InvalidURIError - nil - end - def process_poll - return unless @object['type'] == 'Question' && (@object['anyOf'].is_a?(Array) || @object['oneOf'].is_a?(Array)) - - expires_at = begin - if @object['closed'].is_a?(String) - @object['closed'] - elsif !@object['closed'].nil? && !@object['closed'].is_a?(FalseClass) - Time.now.utc - else - @object['endTime'] - end - end - - if @object['anyOf'].is_a?(Array) - multiple = true - items = @object['anyOf'] - else - multiple = false - items = @object['oneOf'] - end + poll_parser = ActivityPub::Parser::PollParser.new(@object) - voters_count = @object['votersCount'] + return unless poll_parser.valid? @account.polls.new( - multiple: multiple, - expires_at: expires_at, - options: items.map { |item| item['name'].presence || item['content'] }.compact, - cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }, - voters_count: voters_count + multiple: poll_parser.multiple, + expires_at: poll_parser.expires_at, + options: poll_parser.options, + cached_tallies: poll_parser.cached_tallies, + voters_count: poll_parser.voters_count ) end @@ -346,29 +350,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end end - def visibility_from_audience - if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) } - :public - elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) } - :unlisted - elsif audience_to.include?(@account.followers_url) - :private - elsif direct_message == false - :limited - else - :direct - end - end - - def audience_includes?(account) - uri = ActivityPub::TagManager.instance.uri_for(account) - audience_to.include?(uri) || audience_cc.include?(uri) - end - - def direct_message - @object['directMessage'] - end - def replied_to_status return @replied_to_status if defined?(@replied_to_status) @@ -385,77 +366,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity value_or_id(@object['inReplyTo']) end - def text_from_content - return Formatter.instance.linkify([[text_from_name, text_from_summary.presence].compact.join("\n\n"), object_url || object_uri].join(' ')) if converted_object_type? - - if @object['content'].present? - @object['content'] - elsif content_language_map? - @object['contentMap'].values.first - end - end - - def text_from_summary - if @object['summary'].present? - @object['summary'] - elsif summary_language_map? - @object['summaryMap'].values.first - end - end - - def text_from_name - if @object['name'].present? - @object['name'] - elsif name_language_map? - @object['nameMap'].values.first - end + def converted_text + Formatter.instance.linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n")) end def detected_language - if content_language_map? - @object['contentMap'].keys.first - elsif name_language_map? - @object['nameMap'].keys.first - elsif summary_language_map? - @object['summaryMap'].keys.first - elsif supported_object_type? - LanguageDetector.instance.detect(text_from_content, @account) - end - end - - def object_url - return if @object['url'].blank? - - url_candidate = url_to_href(@object['url'], 'text/html') - - if invalid_origin?(url_candidate) - nil - else - url_candidate - end - end - - def summary_language_map? - @object['summaryMap'].is_a?(Hash) && !@object['summaryMap'].empty? - end - - def content_language_map? - @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty? - end - - def name_language_map? - @object['nameMap'].is_a?(Hash) && !@object['nameMap'].empty? + LanguageDetector.instance.detect(@status_parser.text, @account) if supported_object_type? end def unsupported_media_type?(mime_type) mime_type.present? && !MediaAttachment.supported_mime_types.include?(mime_type) end - def supported_blurhash?(blurhash) - components = blurhash.blank? ? nil : Blurhash.components(blurhash) - components.present? && components.none? { |comp| comp > 5 } - end - def skip_download? return @skip_download if defined?(@skip_download) diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index 018e2df54..f04ad321b 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -1,32 +1,31 @@ # frozen_string_literal: true class ActivityPub::Activity::Update < ActivityPub::Activity - SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze - def perform dereference_object! - if equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) + if equals_or_includes_any?(@object['type'], %w(Application Group Organization Person Service)) update_account - elsif equals_or_includes_any?(@object['type'], %w(Question)) - update_poll + elsif equals_or_includes_any?(@object['type'], %w(Note Question)) + update_status end end private def update_account - return if @account.uri != object_uri + return reject_payload! if @account.uri != object_uri ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true) end - def update_poll + def update_status return reject_payload! if invalid_origin?(@object['id']) status = Status.find_by(uri: object_uri, account_id: @account.id) - return if status.nil? || status.preloadable_poll.nil? - ActivityPub::ProcessPollService.new.call(status.preloadable_poll, @object) + return if status.nil? + + ActivityPub::ProcessStatusUpdateService.new.call(status, @object) end end diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index ef00a4e2e..d8b0c63b2 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -19,7 +19,6 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' }, conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' }, focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } }, - identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' }, blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' }, discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' }, voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' }, diff --git a/app/lib/activitypub/parser/custom_emoji_parser.rb b/app/lib/activitypub/parser/custom_emoji_parser.rb new file mode 100644 index 000000000..724c60215 --- /dev/null +++ b/app/lib/activitypub/parser/custom_emoji_parser.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ActivityPub::Parser::CustomEmojiParser + include JsonLdHelper + + def initialize(json) + @json = json + end + + def uri + @json['id'] + end + + def shortcode + @json['name']&.delete(':') + end + + def image_remote_url + @json.dig('icon', 'url') + end + + def updated_at + @json['updated']&.to_datetime + rescue ArgumentError + nil + end +end diff --git a/app/lib/activitypub/parser/media_attachment_parser.rb b/app/lib/activitypub/parser/media_attachment_parser.rb new file mode 100644 index 000000000..1798e58a4 --- /dev/null +++ b/app/lib/activitypub/parser/media_attachment_parser.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +class ActivityPub::Parser::MediaAttachmentParser + include JsonLdHelper + + def initialize(json) + @json = json + end + + # @param [MediaAttachment] previous_record + def significantly_changes?(previous_record) + remote_url != previous_record.remote_url || + thumbnail_remote_url != previous_record.thumbnail_remote_url || + description != previous_record.description + end + + def remote_url + Addressable::URI.parse(@json['url'])&.normalize&.to_s + rescue Addressable::URI::InvalidURIError + nil + end + + def thumbnail_remote_url + Addressable::URI.parse(@json['icon'].is_a?(Hash) ? @json['icon']['url'] : @json['icon'])&.normalize&.to_s + rescue Addressable::URI::InvalidURIError + nil + end + + def description + @json['summary'].presence || @json['name'].presence + end + + def focus + @json['focalPoint'] + end + + def blurhash + supported_blurhash? ? @json['blurhash'] : nil + end + + def file_content_type + @json['mediaType'] + end + + private + + def supported_blurhash? + components = begin + blurhash = @json['blurhash'] + + if blurhash.present? && /^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash) + Blurhash.components(blurhash) + end + end + + components.present? && components.none? { |comp| comp > 5 } + end +end diff --git a/app/lib/activitypub/parser/poll_parser.rb b/app/lib/activitypub/parser/poll_parser.rb new file mode 100644 index 000000000..758c03f07 --- /dev/null +++ b/app/lib/activitypub/parser/poll_parser.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class ActivityPub::Parser::PollParser + include JsonLdHelper + + def initialize(json) + @json = json + end + + def valid? + equals_or_includes?(@json['type'], 'Question') && items.is_a?(Array) + end + + # @param [Poll] previous_record + def significantly_changes?(previous_record) + options != previous_record.options || + multiple != previous_record.multiple + end + + def options + items.filter_map { |item| item['name'].presence || item['content'] } + end + + def multiple + @json['anyOf'].is_a?(Array) + end + + def expires_at + if @json['closed'].is_a?(String) + @json['closed'].to_datetime + elsif !@json['closed'].nil? && !@json['closed'].is_a?(FalseClass) + Time.now.utc + else + @json['endTime']&.to_datetime + end + rescue ArgumentError + nil + end + + def voters_count + @json['votersCount'] + end + + def cached_tallies + items.map { |item| item.dig('replies', 'totalItems') || 0 } + end + + private + + def items + @json['anyOf'] || @json['oneOf'] + end +end diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb new file mode 100644 index 000000000..75b8f3d5c --- /dev/null +++ b/app/lib/activitypub/parser/status_parser.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +class ActivityPub::Parser::StatusParser + include JsonLdHelper + + # @param [Hash] json + # @param [Hash] magic_values + # @option magic_values [String] :followers_collection + def initialize(json, magic_values = {}) + @json = json + @object = json['object'] || json + @magic_values = magic_values + end + + def uri + id = @object['id'] + + if id&.start_with?('bear:') + Addressable::URI.parse(id).query_values['u'] + else + id + end + rescue Addressable::URI::InvalidURIError + id + end + + def url + url_to_href(@object['url'], 'text/html') if @object['url'].present? + end + + def text + if @object['content'].present? + @object['content'] + elsif content_language_map? + @object['contentMap'].values.first + end + end + + def spoiler_text + if @object['summary'].present? + @object['summary'] + elsif summary_language_map? + @object['summaryMap'].values.first + end + end + + def title + if @object['name'].present? + @object['name'] + elsif name_language_map? + @object['nameMap'].values.first + end + end + + def created_at + @object['published']&.to_datetime + rescue ArgumentError + nil + end + + def edited_at + @object['updated']&.to_datetime + rescue ArgumentError + nil + end + + def reply + @object['inReplyTo'].present? + end + + def sensitive + @object['sensitive'] + end + + def visibility + if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) } + :public + elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) } + :unlisted + elsif audience_to.include?(@magic_values[:followers_collection]) + :private + elsif direct_message == false + :limited + else + :direct + end + end + + def language + if content_language_map? + @object['contentMap'].keys.first + elsif name_language_map? + @object['nameMap'].keys.first + elsif summary_language_map? + @object['summaryMap'].keys.first + end + end + + def direct_message + @object['directMessage'] + end + + private + + def audience_to + as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) } + end + + def audience_cc + as_array(@object['cc'] || @json['cc']).map { |x| value_or_id(x) } + end + + def summary_language_map? + @object['summaryMap'].is_a?(Hash) && !@object['summaryMap'].empty? + end + + def content_language_map? + @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty? + end + + def name_language_map? + @object['nameMap'].is_a?(Hash) && !@object['nameMap'].empty? + end +end diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index f6b5e10d3..f6b9741fa 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -64,6 +64,10 @@ class ActivityPub::TagManager account_status_replies_url(target.account, target, page_params) end + def followers_uri_for(target) + target.local? ? account_followers_url(target) : target.followers_url.presence + end + # Primary audience of a status # Public statuses go out to primarily the public collection # Unlisted and private statuses go out primarily to the followers collection @@ -80,17 +84,17 @@ class ActivityPub::TagManager account_ids = status.active_mentions.pluck(:account_id) to = status.account.followers.where(id: account_ids).each_with_object([]) do |account, result| result << uri_for(account) - result << account_followers_url(account) if account.group? + result << followers_uri_for(account) if account.group? end to.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result| result << uri_for(request.account) - result << account_followers_url(request.account) if request.account.group? - end) + result << followers_uri_for(request.account) if request.account.group? + end).compact else status.active_mentions.each_with_object([]) do |mention, result| result << uri_for(mention.account) - result << account_followers_url(mention.account) if mention.account.group? - end + result << followers_uri_for(mention.account) if mention.account.group? + end.compact end end end @@ -118,17 +122,17 @@ class ActivityPub::TagManager account_ids = status.active_mentions.pluck(:account_id) cc.concat(status.account.followers.where(id: account_ids).each_with_object([]) do |account, result| result << uri_for(account) - result << account_followers_url(account) if account.group? - end) + result << followers_uri_for(account) if account.group? + end.compact) cc.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result| result << uri_for(request.account) - result << account_followers_url(request.account) if request.account.group? - end) + result << followers_uri_for(request.account) if request.account.group? + end.compact) else cc.concat(status.active_mentions.each_with_object([]) do |mention, result| result << uri_for(mention.account) - result << account_followers_url(mention.account) if mention.account.group? - end) + result << followers_uri_for(mention.account) if mention.account.group? + end.compact) end end diff --git a/app/lib/admin/metrics/dimension.rb b/app/lib/admin/metrics/dimension.rb new file mode 100644 index 000000000..d8392ddfc --- /dev/null +++ b/app/lib/admin/metrics/dimension.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension + DIMENSIONS = { + languages: Admin::Metrics::Dimension::LanguagesDimension, + sources: Admin::Metrics::Dimension::SourcesDimension, + servers: Admin::Metrics::Dimension::ServersDimension, + space_usage: Admin::Metrics::Dimension::SpaceUsageDimension, + software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension, + tag_servers: Admin::Metrics::Dimension::TagServersDimension, + tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension, + }.freeze + + def self.retrieve(dimension_keys, start_at, end_at, limit, params) + Array(dimension_keys).map do |key| + klass = DIMENSIONS[key.to_sym] + klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil) + end.compact + end +end diff --git a/app/lib/admin/metrics/dimension/base_dimension.rb b/app/lib/admin/metrics/dimension/base_dimension.rb new file mode 100644 index 000000000..5872c22cb --- /dev/null +++ b/app/lib/admin/metrics/dimension/base_dimension.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::BaseDimension + def self.with_params? + false + end + + def initialize(start_at, end_at, limit, params) + @start_at = start_at&.to_datetime + @end_at = end_at&.to_datetime + @limit = limit&.to_i + @params = params + end + + def key + raise NotImplementedError + end + + def data + raise NotImplementedError + end + + def self.model_name + self.class.name + end + + def read_attribute_for_serialization(key) + send(key) if respond_to?(key) + end + + protected + + def time_period + (@start_at..@end_at) + end + + def params + raise NotImplementedError + end +end diff --git a/app/lib/admin/metrics/dimension/languages_dimension.rb b/app/lib/admin/metrics/dimension/languages_dimension.rb new file mode 100644 index 000000000..a6aaf5d21 --- /dev/null +++ b/app/lib/admin/metrics/dimension/languages_dimension.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension + include LanguagesHelper + + def key + 'languages' + end + + def data + sql = <<-SQL.squish + SELECT locale, count(*) AS value + FROM users + WHERE current_sign_in_at BETWEEN $1 AND $2 + AND locale IS NOT NULL + GROUP BY locale + ORDER BY count(*) DESC + LIMIT $3 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]]) + + rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } } + end +end diff --git a/app/lib/admin/metrics/dimension/servers_dimension.rb b/app/lib/admin/metrics/dimension/servers_dimension.rb new file mode 100644 index 000000000..3e80b6625 --- /dev/null +++ b/app/lib/admin/metrics/dimension/servers_dimension.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::ServersDimension < Admin::Metrics::Dimension::BaseDimension + def key + 'servers' + end + + def data + sql = <<-SQL.squish + SELECT accounts.domain, count(*) AS value + FROM statuses + INNER JOIN accounts ON accounts.id = statuses.account_id + WHERE statuses.id BETWEEN $1 AND $2 + GROUP BY accounts.domain + ORDER BY count(*) DESC + LIMIT $3 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at)], [nil, Mastodon::Snowflake.id_at(@end_at)], [nil, @limit]]) + + rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } } + end +end diff --git a/app/lib/admin/metrics/dimension/software_versions_dimension.rb b/app/lib/admin/metrics/dimension/software_versions_dimension.rb new file mode 100644 index 000000000..34917404d --- /dev/null +++ b/app/lib/admin/metrics/dimension/software_versions_dimension.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dimension::BaseDimension + include Redisable + + def key + 'software_versions' + end + + def data + [mastodon_version, ruby_version, postgresql_version, redis_version] + end + + private + + def mastodon_version + value = Mastodon::Version.to_s + + { + key: 'mastodon', + human_key: 'Mastodon', + value: value, + human_value: value, + } + end + + def ruby_version + value = "#{RUBY_VERSION}p#{RUBY_PATCHLEVEL}" + + { + key: 'ruby', + human_key: 'Ruby', + value: value, + human_value: value, + } + end + + def postgresql_version + value = ActiveRecord::Base.connection.execute('SELECT VERSION()').first['version'].match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] + + { + key: 'postgresql', + human_key: 'PostgreSQL', + value: value, + human_value: value, + } + end + + def redis_version + value = redis_info['redis_version'] + + { + key: 'redis', + human_key: 'Redis', + value: value, + human_value: value, + } + end + + def redis_info + @redis_info ||= begin + if redis.is_a?(Redis::Namespace) + redis.redis.info + else + redis.info + end + end + end +end diff --git a/app/lib/admin/metrics/dimension/sources_dimension.rb b/app/lib/admin/metrics/dimension/sources_dimension.rb new file mode 100644 index 000000000..a9f061809 --- /dev/null +++ b/app/lib/admin/metrics/dimension/sources_dimension.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::SourcesDimension < Admin::Metrics::Dimension::BaseDimension + def key + 'sources' + end + + def data + sql = <<-SQL.squish + SELECT oauth_applications.name, count(*) AS value + FROM users + LEFT JOIN oauth_applications ON oauth_applications.id = users.created_by_application_id + WHERE users.created_at BETWEEN $1 AND $2 + GROUP BY oauth_applications.name + ORDER BY count(*) DESC + LIMIT $3 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]]) + + rows.map { |row| { key: row['name'] || 'web', human_key: row['name'] || I18n.t('admin.dashboard.website'), value: row['value'].to_s } } + end +end diff --git a/app/lib/admin/metrics/dimension/space_usage_dimension.rb b/app/lib/admin/metrics/dimension/space_usage_dimension.rb new file mode 100644 index 000000000..aa00a2e18 --- /dev/null +++ b/app/lib/admin/metrics/dimension/space_usage_dimension.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension::BaseDimension + include Redisable + include ActionView::Helpers::NumberHelper + + def key + 'space_usage' + end + + def data + [postgresql_size, redis_size, media_size] + end + + private + + def postgresql_size + value = ActiveRecord::Base.connection.execute('SELECT pg_database_size(current_database())').first['pg_database_size'] + + { + key: 'postgresql', + human_key: 'PostgreSQL', + value: value.to_s, + unit: 'bytes', + human_value: number_to_human_size(value), + } + end + + def redis_size + value = redis_info['used_memory'] + + { + key: 'redis', + human_key: 'Redis', + value: value.to_s, + unit: 'bytes', + human_value: number_to_human_size(value), + } + end + + def media_size + value = [ + MediaAttachment.sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')), + CustomEmoji.sum(:image_file_size), + PreviewCard.sum(:image_file_size), + Account.sum(Arel.sql('COALESCE(avatar_file_size, 0) + COALESCE(header_file_size, 0)')), + Backup.sum(:dump_file_size), + Import.sum(:data_file_size), + SiteUpload.sum(:file_file_size), + ].sum + + { + key: 'media', + human_key: I18n.t('admin.dashboard.media_storage'), + value: value.to_s, + unit: 'bytes', + human_value: number_to_human_size(value), + } + end + + def redis_info + @redis_info ||= begin + if redis.is_a?(Redis::Namespace) + redis.redis.info + else + redis.info + end + end + end +end diff --git a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb new file mode 100644 index 000000000..1cfa07478 --- /dev/null +++ b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension + include LanguagesHelper + + def self.with_params? + true + end + + def key + 'tag_languages' + end + + def data + sql = <<-SQL.squish + SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value + FROM statuses + INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id + WHERE statuses_tags.tag_id = $1 + AND statuses.id BETWEEN $2 AND $3 + GROUP BY COALESCE(statuses.language, 'und') + ORDER BY count(*) DESC + LIMIT $4 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]]) + + rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } } + end + + private + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb new file mode 100644 index 000000000..12c5980d7 --- /dev/null +++ b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension + def self.with_params? + true + end + + def key + 'tag_servers' + end + + def data + sql = <<-SQL.squish + SELECT accounts.domain, count(*) AS value + FROM statuses + INNER JOIN accounts ON accounts.id = statuses.account_id + INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id + WHERE statuses_tags.tag_id = $1 + AND statuses.id BETWEEN $2 AND $3 + GROUP BY accounts.domain + ORDER BY count(*) DESC + LIMIT $4 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]]) + + rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } } + end + + private + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/measure.rb b/app/lib/admin/metrics/measure.rb new file mode 100644 index 000000000..a839498a1 --- /dev/null +++ b/app/lib/admin/metrics/measure.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure + MEASURES = { + active_users: Admin::Metrics::Measure::ActiveUsersMeasure, + new_users: Admin::Metrics::Measure::NewUsersMeasure, + interactions: Admin::Metrics::Measure::InteractionsMeasure, + opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure, + resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure, + tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure, + tag_uses: Admin::Metrics::Measure::TagUsesMeasure, + tag_servers: Admin::Metrics::Measure::TagServersMeasure, + }.freeze + + def self.retrieve(measure_keys, start_at, end_at, params) + Array(measure_keys).map do |key| + klass = MEASURES[key.to_sym] + klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil) + end.compact + end +end diff --git a/app/lib/admin/metrics/measure/active_users_measure.rb b/app/lib/admin/metrics/measure/active_users_measure.rb new file mode 100644 index 000000000..513189780 --- /dev/null +++ b/app/lib/admin/metrics/measure/active_users_measure.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::BaseMeasure + def key + 'active_users' + end + + def total + activity_tracker.sum(time_period.first, time_period.last) + end + + def previous_total + activity_tracker.sum(previous_time_period.first, previous_time_period.last) + end + + def data + activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } } + end + + protected + + def activity_tracker + @activity_tracker ||= ActivityTracker.new('activity:logins', :unique) + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end +end diff --git a/app/lib/admin/metrics/measure/base_measure.rb b/app/lib/admin/metrics/measure/base_measure.rb new file mode 100644 index 000000000..0107ffd9c --- /dev/null +++ b/app/lib/admin/metrics/measure/base_measure.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::BaseMeasure + def self.with_params? + false + end + + def initialize(start_at, end_at, params) + @start_at = start_at&.to_datetime + @end_at = end_at&.to_datetime + @params = params + end + + def key + raise NotImplementedError + end + + def total + raise NotImplementedError + end + + def previous_total + raise NotImplementedError + end + + def data + raise NotImplementedError + end + + def self.model_name + self.class.name + end + + def read_attribute_for_serialization(key) + send(key) if respond_to?(key) + end + + protected + + def time_period + (@start_at..@end_at) + end + + def previous_time_period + ((@start_at - length_of_period)..(@end_at - length_of_period)) + end + + def length_of_period + @length_of_period ||= @end_at - @start_at + end + + def params + raise NotImplementedError + end +end diff --git a/app/lib/admin/metrics/measure/interactions_measure.rb b/app/lib/admin/metrics/measure/interactions_measure.rb new file mode 100644 index 000000000..b928fdb8f --- /dev/null +++ b/app/lib/admin/metrics/measure/interactions_measure.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::BaseMeasure + def key + 'interactions' + end + + def total + activity_tracker.sum(time_period.first, time_period.last) + end + + def previous_total + activity_tracker.sum(previous_time_period.first, previous_time_period.last) + end + + def data + activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } } + end + + protected + + def activity_tracker + @activity_tracker ||= ActivityTracker.new('activity:interactions', :basic) + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end +end diff --git a/app/lib/admin/metrics/measure/new_users_measure.rb b/app/lib/admin/metrics/measure/new_users_measure.rb new file mode 100644 index 000000000..b31679ad3 --- /dev/null +++ b/app/lib/admin/metrics/measure/new_users_measure.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::NewUsersMeasure < Admin::Metrics::Measure::BaseMeasure + def key + 'new_users' + end + + def total + User.where(created_at: time_period).count + end + + def previous_total + User.where(created_at: previous_time_period).count + end + + def data + sql = <<-SQL.squish + SELECT axis.*, ( + WITH new_users AS ( + SELECT users.id + FROM users + WHERE date_trunc('day', users.created_at)::date = axis.period + ) + SELECT count(*) FROM new_users + ) AS value + FROM ( + SELECT generate_series(date_trunc('day', $1::timestamp)::date, date_trunc('day', $2::timestamp)::date, interval '1 day') AS period + ) AS axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at]]) + + rows.map { |row| { date: row['period'], value: row['value'].to_s } } + end +end diff --git a/app/lib/admin/metrics/measure/opened_reports_measure.rb b/app/lib/admin/metrics/measure/opened_reports_measure.rb new file mode 100644 index 000000000..9acc2c33d --- /dev/null +++ b/app/lib/admin/metrics/measure/opened_reports_measure.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::OpenedReportsMeasure < Admin::Metrics::Measure::BaseMeasure + def key + 'opened_reports' + end + + def total + Report.where(created_at: time_period).count + end + + def previous_total + Report.where(created_at: previous_time_period).count + end + + def data + sql = <<-SQL.squish + SELECT axis.*, ( + WITH new_reports AS ( + SELECT reports.id + FROM reports + WHERE date_trunc('day', reports.created_at)::date = axis.period + ) + SELECT count(*) FROM new_reports + ) AS value + FROM ( + SELECT generate_series(date_trunc('day', $1::timestamp)::date, date_trunc('day', $2::timestamp)::date, interval '1 day') AS period + ) AS axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at]]) + + rows.map { |row| { date: row['period'], value: row['value'].to_s } } + end +end diff --git a/app/lib/admin/metrics/measure/resolved_reports_measure.rb b/app/lib/admin/metrics/measure/resolved_reports_measure.rb new file mode 100644 index 000000000..00cb24f7e --- /dev/null +++ b/app/lib/admin/metrics/measure/resolved_reports_measure.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::ResolvedReportsMeasure < Admin::Metrics::Measure::BaseMeasure + def key + 'resolved_reports' + end + + def total + Report.resolved.where(action_taken_at: time_period).count + end + + def previous_total + Report.resolved.where(action_taken_at: previous_time_period).count + end + + def data + sql = <<-SQL.squish + SELECT axis.*, ( + WITH resolved_reports AS ( + SELECT reports.id + FROM reports + WHERE date_trunc('day', reports.action_taken_at)::date = axis.period + ) + SELECT count(*) FROM resolved_reports + ) AS value + FROM ( + SELECT generate_series(date_trunc('day', $1::timestamp)::date, date_trunc('day', $2::timestamp)::date, interval '1 day') AS period + ) AS axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at]]) + + rows.map { |row| { date: row['period'], value: row['value'].to_s } } + end +end diff --git a/app/lib/admin/metrics/measure/tag_accounts_measure.rb b/app/lib/admin/metrics/measure/tag_accounts_measure.rb new file mode 100644 index 000000000..ef773081b --- /dev/null +++ b/app/lib/admin/metrics/measure/tag_accounts_measure.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'tag_accounts' + end + + def total + tag.history.aggregate(time_period).accounts + end + + def previous_total + tag.history.aggregate(previous_time_period).accounts + end + + def data + time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } } + end + + protected + + def tag + @tag ||= Tag.find(params[:id]) + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/measure/tag_servers_measure.rb b/app/lib/admin/metrics/measure/tag_servers_measure.rb new file mode 100644 index 000000000..cc064f63f --- /dev/null +++ b/app/lib/admin/metrics/measure/tag_servers_measure.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'tag_servers' + end + + def total + tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain') + end + + def previous_total + tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain') + end + + def data + sql = <<-SQL.squish + SELECT axis.*, ( + SELECT count(distinct accounts.domain) AS value + FROM statuses + INNER JOIN statuses_tags ON statuses.id = statuses_tags.status_id + INNER JOIN accounts ON statuses.account_id = accounts.id + WHERE statuses_tags.tag_id = $1 + AND statuses.id BETWEEN $2 AND $3 + AND date_trunc('day', statuses.created_at)::date = axis.day + ) + FROM ( + SELECT generate_series(date_trunc('day', $4::timestamp)::date, date_trunc('day', $5::timestamp)::date, ('1 day')::interval) AS day + ) as axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id].to_i], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]]) + + rows.map { |row| { date: row['day'], value: row['value'].to_s } } + end + + protected + + def tag + @tag ||= Tag.find(params[:id]) + end + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/measure/tag_uses_measure.rb b/app/lib/admin/metrics/measure/tag_uses_measure.rb new file mode 100644 index 000000000..b7667bc6c --- /dev/null +++ b/app/lib/admin/metrics/measure/tag_uses_measure.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'tag_uses' + end + + def total + tag.history.aggregate(time_period).uses + end + + def previous_total + tag.history.aggregate(previous_time_period).uses + end + + def data + time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } } + end + + protected + + def tag + @tag ||= Tag.find(params[:id]) + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/retention.rb b/app/lib/admin/metrics/retention.rb new file mode 100644 index 000000000..0179a6e28 --- /dev/null +++ b/app/lib/admin/metrics/retention.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +class Admin::Metrics::Retention + class Cohort < ActiveModelSerializers::Model + attributes :period, :frequency, :data + end + + class CohortData < ActiveModelSerializers::Model + attributes :date, :rate, :value + end + + def initialize(start_at, end_at, frequency) + @start_at = start_at&.to_date + @end_at = end_at&.to_date + @frequency = %w(day month).include?(frequency) ? frequency : 'day' + end + + def cohorts + sql = <<-SQL.squish + SELECT axis.*, ( + WITH new_users AS ( + SELECT users.id + FROM users + WHERE date_trunc($3, users.created_at)::date = axis.cohort_period + ), + retained_users AS ( + SELECT users.id + FROM users + INNER JOIN new_users on new_users.id = users.id + WHERE date_trunc($3, users.current_sign_in_at) >= axis.retention_period + ) + SELECT ARRAY[count(*), (count(*))::float / (SELECT GREATEST(count(*), 1) FROM new_users)] AS retention_value_and_rate + FROM retained_users + ) + FROM ( + WITH cohort_periods AS ( + SELECT generate_series(date_trunc($3, $1::timestamp)::date, date_trunc($3, $2::timestamp)::date, ('1 ' || $3)::interval) AS cohort_period + ), + retention_periods AS ( + SELECT cohort_period AS retention_period FROM cohort_periods + ) + SELECT * + FROM cohort_periods, retention_periods + WHERE retention_period >= cohort_period + ) as axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @frequency]]) + + rows.each_with_object([]) do |row, arr| + current_cohort = arr.last + + if current_cohort.nil? || current_cohort.period != row['cohort_period'] + current_cohort = Cohort.new(period: row['cohort_period'], frequency: @frequency, data: []) + arr << current_cohort + end + + value, rate = row['retention_value_and_rate'].delete('{}').split(',') + + current_cohort.data << CohortData.new( + date: row['retention_period'], + rate: rate.to_f, + value: value.to_s + ) + end + end +end diff --git a/app/lib/fast_geometry_parser.rb b/app/lib/fast_geometry_parser.rb index 5209c2bc5..f3395a833 100644 --- a/app/lib/fast_geometry_parser.rb +++ b/app/lib/fast_geometry_parser.rb @@ -2,7 +2,7 @@ class FastGeometryParser def self.from_file(file) - width, height = FastImage.size(file.path) + width, height = FastImage.size(file) raise Paperclip::Errors::NotIdentifiedByImageMagickError if width.nil? diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index d57508ef9..0713aa471 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -55,46 +55,50 @@ class FeedManager # Add a status to a home feed and send a streaming API update # @param [Account] account # @param [Status] status + # @param [Boolean] update # @return [Boolean] - def push_to_home(account, status) + def push_to_home(account, status, update: false) return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?) trim(:home, account.id) - PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}") + PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}", { 'update' => update }) if push_update_required?("timeline:#{account.id}") true end # Remove a status from a home feed and send a streaming API update # @param [Account] account # @param [Status] status + # @param [Boolean] update # @return [Boolean] - def unpush_from_home(account, status) + def unpush_from_home(account, status, update: false) return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?) - redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update true end # Add a status to a list feed and send a streaming API update # @param [List] list # @param [Status] status + # @param [Boolean] update # @return [Boolean] - def push_to_list(list, status) + def push_to_list(list, status, update: false) return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) trim(:list, list.id) - PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}") + PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}", { 'update' => update }) if push_update_required?("timeline:list:#{list.id}") true end # Remove a status from a list feed and send a streaming API update # @param [List] list # @param [Status] status + # @param [Boolean] update # @return [Boolean] - def unpush_from_list(list, status) + def unpush_from_list(list, status, update: false) return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) - redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update true end @@ -102,11 +106,11 @@ class FeedManager # @param [Account] account # @param [Status] status # @return [Boolean] - def push_to_direct(account, status) + def push_to_direct(account, status, update: false) return false unless add_to_feed(:direct, account.id, status) trim(:direct, account.id) - PushUpdateWorker.perform_async(account.id, status.id, "timeline:direct:#{account.id}") + PushUpdateWorker.perform_async(account.id, status.id, "timeline:direct:#{account.id}") unless update true end @@ -114,10 +118,10 @@ class FeedManager # @param [List] list # @param [Status] status # @return [Boolean] - def unpush_from_direct(account, status) + def unpush_from_direct(account, status, update: false) return false unless remove_from_feed(:direct, account.id, status) - redis.publish("timeline:direct:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + redis.publish("timeline:direct:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update true end diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index b26138642..f2c4beed5 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -279,39 +279,10 @@ class Formatter result.flatten.join end - UNICODE_ESCAPE_BLACKLIST_RE = /\p{Z}|\p{P}/ - def utf8_friendly_extractor(text, options = {}) - old_to_new_index = [0] - - escaped = text.chars.map do |c| - output = begin - if c.ord.to_s(16).length > 2 && !UNICODE_ESCAPE_BLACKLIST_RE.match?(c) - CGI.escape(c) - else - c - end - end - - old_to_new_index << old_to_new_index.last + output.length - - output - end.join - # Note: I couldn't obtain list_slug with @user/list-name format # for mention so this requires additional check - special = Extractor.extract_urls_with_indices(escaped, options).map do |extract| - new_indices = [ - old_to_new_index.find_index(extract[:indices].first), - old_to_new_index.find_index(extract[:indices].last), - ] - - next extract.merge( - indices: new_indices, - url: text[new_indices.first..new_indices.last - 1] - ) - end - + special = Extractor.extract_urls_with_indices(text, options) standard = Extractor.extract_entities_with_indices(text, options) extra = Extractor.extract_extra_uris_with_indices(text, options) diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb new file mode 100644 index 000000000..56ad0717b --- /dev/null +++ b/app/lib/link_details_extractor.rb @@ -0,0 +1,249 @@ +# frozen_string_literal: true + +class LinkDetailsExtractor + include ActionView::Helpers::TagHelper + + class StructuredData + SUPPORTED_TYPES = %w( + NewsArticle + WebPage + ).freeze + + def initialize(data) + @data = data + end + + def headline + json['headline'] + end + + def description + json['description'] + end + + def language + json['inLanguage'] + end + + def type + json['@type'] + end + + def image + obj = first_of_value(json['image']) + + return obj['url'] if obj.is_a?(Hash) + + obj + end + + def date_published + json['datePublished'] + end + + def date_modified + json['dateModified'] + end + + def author_name + author['name'] + end + + def author_url + author['url'] + end + + def publisher_name + publisher['name'] + end + + def publisher_logo + publisher.dig('logo', 'url') + end + + private + + def author + first_of_value(json['author']) || {} + end + + def publisher + first_of_value(json['publisher']) || {} + end + + def first_of_value(arr) + arr.is_a?(Array) ? arr.first : arr + end + + def root_array(root) + root.is_a?(Array) ? root : [root] + end + + def json + @json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {} + end + end + + def initialize(original_url, html, html_charset) + @original_url = Addressable::URI.parse(original_url) + @html = html + @html_charset = html_charset + end + + def to_preview_card_attributes + { + title: title || '', + description: description || '', + image_remote_url: image, + type: type, + link_type: link_type, + width: width || 0, + height: height || 0, + html: html || '', + provider_name: provider_name || '', + provider_url: provider_url || '', + author_name: author_name || '', + author_url: author_url || '', + embed_url: embed_url || '', + language: language, + } + end + + def type + player_url.present? ? :video : :link + end + + def link_type + if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article' + :article + else + :unknown + end + end + + def html + player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil + end + + def width + opengraph_tag('twitter:player:width') + end + + def height + opengraph_tag('twitter:player:height') + end + + def title + structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first + end + + def description + structured_data&.description || opengraph_tag('og:description') || meta_tag('description') + end + + def image + valid_url_or_nil(opengraph_tag('og:image')) + end + + def canonical_url + valid_url_or_nil(opengraph_tag('og:url') || link_tag('canonical'), same_origin_only: true) || @original_url.to_s + end + + def provider_name + structured_data&.publisher_name || opengraph_tag('og:site_name') + end + + def provider_url + valid_url_or_nil(host_to_url(opengraph_tag('og:site'))) + end + + def author_name + structured_data&.author_name || opengraph_tag('og:author') || opengraph_tag('og:author:username') + end + + def author_url + structured_data&.author_url + end + + def embed_url + valid_url_or_nil(opengraph_tag('twitter:player:stream')) + end + + def language + valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first) + end + + def icon + valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon')) + end + + private + + def player_url + valid_url_or_nil(opengraph_tag('twitter:player')) + end + + def host_to_url(str) + return if str.blank? + + str.start_with?(/https?:\/\//) ? str : "http://#{str}" + end + + def valid_url_or_nil(str, same_origin_only: false) + return if str.blank? + + url = @original_url + Addressable::URI.parse(str) + + return if url.host.blank? || !%w(http https).include?(url.scheme) || (same_origin_only && url.host != @original_url.host) + + url.to_s + rescue Addressable::URI::InvalidURIError + nil + end + + def valid_locale_or_nil(str) + return nil if str.blank? + + code, = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA + locale = ISO_639.find(code) + locale&.alpha2 + end + + def link_tag(name) + document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first + end + + def opengraph_tag(name) + document.xpath("//meta[@property=\"#{name}\" or @name=\"#{name}\"]").map { |meta| meta['content'] }.first + end + + def meta_tag(name) + document.xpath("//meta[@name=\"#{name}\"]").map { |meta| meta['content'] }.first + end + + def structured_data + @structured_data ||= begin + json_ld = document.xpath('//script[@type="application/ld+json"]').map(&:content).first + json_ld.present? ? StructuredData.new(json_ld) : nil + rescue Oj::ParseError + nil + end + end + + def document + @document ||= Nokogiri::HTML(@html, nil, encoding) + end + + def encoding + @encoding ||= begin + guess = detector.detect(@html, @html_charset) + guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil + end + end + + def detector + @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector| + detector.strip_tags = true + end + end +end diff --git a/app/lib/permalink_redirector.rb b/app/lib/permalink_redirector.rb new file mode 100644 index 000000000..e48bce060 --- /dev/null +++ b/app/lib/permalink_redirector.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +class PermalinkRedirector + include RoutingHelper + + def initialize(path) + @path = path + end + + def redirect_path + if path_segments[0] == 'web' + if path_segments[1].present? && path_segments[1].start_with?('@') && path_segments[2] =~ /\d/ + find_status_url_by_id(path_segments[2]) + elsif path_segments[1].present? && path_segments[1].start_with?('@') + find_account_url_by_name(path_segments[1]) + elsif path_segments[1] == 'statuses' && path_segments[2] =~ /\d/ + find_status_url_by_id(path_segments[2]) + elsif path_segments[1] == 'accounts' && path_segments[2] =~ /\d/ + find_account_url_by_id(path_segments[2]) + elsif path_segments[1] == 'timelines' && path_segments[2] == 'tag' && path_segments[3].present? + find_tag_url_by_name(path_segments[3]) + elsif path_segments[1] == 'tags' && path_segments[2].present? + find_tag_url_by_name(path_segments[2]) + end + end + end + + private + + def path_segments + @path_segments ||= @path.gsub(/\A\//, '').split('/') + end + + def find_status_url_by_id(id) + status = Status.find_by(id: id) + + return unless status&.distributable? + + ActivityPub::TagManager.instance.url_for(status) + end + + def find_account_url_by_id(id) + account = Account.find_by(id: id) + + return unless account + + ActivityPub::TagManager.instance.url_for(account) + end + + def find_account_url_by_name(name) + username, domain = name.gsub(/\A@/, '').split('@') + domain = nil if TagManager.instance.local_domain?(domain) + account = Account.find_remote(username, domain) + + return unless account + + ActivityPub::TagManager.instance.url_for(account) + end + + def find_tag_url_by_name(name) + tag_path(CGI.unescape(name)) + end +end diff --git a/app/lib/proof_provider.rb b/app/lib/proof_provider.rb deleted file mode 100644 index 102c50f4f..000000000 --- a/app/lib/proof_provider.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module ProofProvider - SUPPORTED_PROVIDERS = %w(keybase).freeze - - def self.find(identifier, proof = nil) - case identifier - when 'keybase' - ProofProvider::Keybase.new(proof) - end - end -end diff --git a/app/lib/proof_provider/keybase.rb b/app/lib/proof_provider/keybase.rb deleted file mode 100644 index 8e51d7146..000000000 --- a/app/lib/proof_provider/keybase.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -class ProofProvider::Keybase - BASE_URL = ENV.fetch('KEYBASE_BASE_URL', 'https://keybase.io') - DOMAIN = ENV.fetch('KEYBASE_DOMAIN', Rails.configuration.x.web_domain) - - class Error < StandardError; end - - class ExpectedProofLiveError < Error; end - - class UnexpectedResponseError < Error; end - - def initialize(proof = nil) - @proof = proof - end - - def serializer_class - ProofProvider::Keybase::Serializer - end - - def worker_class - ProofProvider::Keybase::Worker - end - - def validate! - unless @proof.token&.size == 66 - @proof.errors.add(:base, I18n.t('identity_proofs.errors.keybase.invalid_token')) - return - end - - # Do not perform synchronous validation for remote accounts - return if @proof.provider_username.blank? || !@proof.account.local? - - if verifier.valid? - @proof.verified = true - @proof.live = false - else - @proof.errors.add(:base, I18n.t('identity_proofs.errors.keybase.verification_failed', kb_username: @proof.provider_username)) - end - end - - def refresh! - worker_class.new.perform(@proof) - rescue ProofProvider::Keybase::Error - nil - end - - def on_success_path(user_agent = nil) - verifier.on_success_path(user_agent) - end - - def badge - @badge ||= ProofProvider::Keybase::Badge.new(@proof.account.username, @proof.provider_username, @proof.token, domain) - end - - def verifier - @verifier ||= ProofProvider::Keybase::Verifier.new(@proof.account.username, @proof.provider_username, @proof.token, domain) - end - - private - - def domain - if @proof.account.local? - DOMAIN - else - @proof.account.domain - end - end -end diff --git a/app/lib/proof_provider/keybase/badge.rb b/app/lib/proof_provider/keybase/badge.rb deleted file mode 100644 index f587b1cc7..000000000 --- a/app/lib/proof_provider/keybase/badge.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -class ProofProvider::Keybase::Badge - include RoutingHelper - - def initialize(local_username, provider_username, token, domain) - @local_username = local_username - @provider_username = provider_username - @token = token - @domain = domain - end - - def proof_url - "#{ProofProvider::Keybase::BASE_URL}/#{@provider_username}/sigchain\##{@token}" - end - - def profile_url - "#{ProofProvider::Keybase::BASE_URL}/#{@provider_username}" - end - - def icon_url - "#{ProofProvider::Keybase::BASE_URL}/#{@provider_username}/proof_badge/#{@token}?username=#{@local_username}&domain=#{@domain}" - end - - def avatar_url - Rails.cache.fetch("proof_providers/keybase/#{@provider_username}/avatar_url", expires_in: 5.minutes) { remote_avatar_url } || default_avatar_url - end - - private - - def remote_avatar_url - request = Request.new(:get, "#{ProofProvider::Keybase::BASE_URL}/_/api/1.0/user/pic_url.json", params: { username: @provider_username }) - - request.perform do |res| - json = Oj.load(res.body_with_limit, mode: :strict) - json['pic_url'] if json.is_a?(Hash) - end - rescue Oj::ParseError, HTTP::Error, OpenSSL::SSL::SSLError - nil - end - - def default_avatar_url - asset_pack_path('media/images/proof_providers/keybase.png') - end -end diff --git a/app/lib/proof_provider/keybase/config_serializer.rb b/app/lib/proof_provider/keybase/config_serializer.rb deleted file mode 100644 index c6c364d31..000000000 --- a/app/lib/proof_provider/keybase/config_serializer.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -class ProofProvider::Keybase::ConfigSerializer < ActiveModel::Serializer - include RoutingHelper - include ActionView::Helpers::TextHelper - - attributes :version, :domain, :display_name, :username, - :brand_color, :logo, :description, :prefill_url, - :profile_url, :check_url, :check_path, :avatar_path, - :contact - - def version - 1 - end - - def domain - ProofProvider::Keybase::DOMAIN - end - - def display_name - Setting.site_title - end - - def logo - { - svg_black: full_asset_url(asset_pack_path('media/images/logo_transparent_black.svg')), - svg_white: full_asset_url(asset_pack_path('media/images/logo_transparent_white.svg')), - svg_full: full_asset_url(asset_pack_path('media/images/logo.svg')), - svg_full_darkmode: full_asset_url(asset_pack_path('media/images/logo.svg')), - } - end - - def brand_color - '#282c37' - end - - def description - strip_tags(Setting.site_short_description.presence || I18n.t('about.about_mastodon_html')) - end - - def username - { min: 1, max: 30, re: '[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?' } - end - - def prefill_url - params = { - provider: 'keybase', - token: '%{sig_hash}', - provider_username: '%{kb_username}', - username: '%{username}', - user_agent: '%{kb_ua}', - } - - CGI.unescape(new_settings_identity_proof_url(params)) - end - - def profile_url - CGI.unescape(short_account_url('%{username}')) - end - - def check_url - CGI.unescape(api_proofs_url(username: '%{username}', provider: 'keybase')) - end - - def check_path - ['signatures'] - end - - def avatar_path - ['avatar'] - end - - def contact - [Setting.site_contact_email.presence || 'unknown'].compact - end -end diff --git a/app/lib/proof_provider/keybase/serializer.rb b/app/lib/proof_provider/keybase/serializer.rb deleted file mode 100644 index d29283600..000000000 --- a/app/lib/proof_provider/keybase/serializer.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -class ProofProvider::Keybase::Serializer < ActiveModel::Serializer - include RoutingHelper - - attribute :avatar - - has_many :identity_proofs, key: :signatures - - def avatar - full_asset_url(object.avatar_original_url) - end - - class AccountIdentityProofSerializer < ActiveModel::Serializer - attributes :sig_hash, :kb_username - - def sig_hash - object.token - end - - def kb_username - object.provider_username - end - end -end diff --git a/app/lib/proof_provider/keybase/verifier.rb b/app/lib/proof_provider/keybase/verifier.rb deleted file mode 100644 index af69b1bfc..000000000 --- a/app/lib/proof_provider/keybase/verifier.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -class ProofProvider::Keybase::Verifier - def initialize(local_username, provider_username, token, domain) - @local_username = local_username - @provider_username = provider_username - @token = token - @domain = domain - end - - def valid? - request = Request.new(:get, "#{ProofProvider::Keybase::BASE_URL}/_/api/1.0/sig/proof_valid.json", params: query_params) - - request.perform do |res| - json = Oj.load(res.body_with_limit, mode: :strict) - - if json.is_a?(Hash) - json.fetch('proof_valid', false) - else - false - end - end - rescue Oj::ParseError, HTTP::Error, OpenSSL::SSL::SSLError - false - end - - def on_success_path(user_agent = nil) - url = Addressable::URI.parse("#{ProofProvider::Keybase::BASE_URL}/_/proof_creation_success") - url.query_values = query_params.merge(kb_ua: user_agent || 'unknown') - url.to_s - end - - def status - request = Request.new(:get, "#{ProofProvider::Keybase::BASE_URL}/_/api/1.0/sig/proof_live.json", params: query_params) - - request.perform do |res| - raise ProofProvider::Keybase::UnexpectedResponseError unless res.code == 200 - - json = Oj.load(res.body_with_limit, mode: :strict) - - raise ProofProvider::Keybase::UnexpectedResponseError unless json.is_a?(Hash) && json.key?('proof_valid') && json.key?('proof_live') - - json - end - rescue Oj::ParseError, HTTP::Error, OpenSSL::SSL::SSLError - raise ProofProvider::Keybase::UnexpectedResponseError - end - - private - - def query_params - { - domain: @domain, - kb_username: @provider_username, - username: @local_username, - sig_hash: @token, - } - end -end diff --git a/app/lib/proof_provider/keybase/worker.rb b/app/lib/proof_provider/keybase/worker.rb deleted file mode 100644 index bcdd18cc5..000000000 --- a/app/lib/proof_provider/keybase/worker.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -class ProofProvider::Keybase::Worker - include Sidekiq::Worker - - sidekiq_options queue: 'pull', retry: 20, unique: :until_executed - - sidekiq_retry_in do |count, exception| - # Retry aggressively when the proof is valid but not live in Keybase. - # This is likely because Keybase just hasn't noticed the proof being - # served from here yet. - - if exception.class == ProofProvider::Keybase::ExpectedProofLiveError - case count - when 0..2 then 0.seconds - when 2..6 then 1.second - end - end - end - - def perform(proof_id) - proof = proof_id.is_a?(AccountIdentityProof) ? proof_id : AccountIdentityProof.find(proof_id) - status = proof.provider_instance.verifier.status - - # If Keybase thinks the proof is valid, and it exists here in Mastodon, - # then it should be live. Keybase just has to notice that it's here - # and then update its state. That might take a couple seconds. - raise ProofProvider::Keybase::ExpectedProofLiveError if status['proof_valid'] && !status['proof_live'] - - proof.update!(verified: status['proof_valid'], live: status['proof_live']) - end -end diff --git a/app/lib/request.rb b/app/lib/request.rb index 125dee3ea..4289da933 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -94,7 +94,7 @@ class Request end def http_client - HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 2) + HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3) end end diff --git a/app/lib/sidekiq_error_handler.rb b/app/lib/sidekiq_error_handler.rb deleted file mode 100644 index ab555b1be..000000000 --- a/app/lib/sidekiq_error_handler.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -class SidekiqErrorHandler - BACKTRACE_LIMIT = 3 - - def call(*) - yield - rescue Mastodon::HostValidationError - # Do not retry - rescue => e - limit_backtrace_and_raise(e) - ensure - socket = Thread.current[:statsd_socket] - socket&.close - Thread.current[:statsd_socket] = nil - end - - private - - # rubocop:disable Naming/MethodParameterName - def limit_backtrace_and_raise(e) - e.set_backtrace(e.backtrace.first(BACKTRACE_LIMIT)) - raise e - end - # rubocop:enable Naming/MethodParameterName -end diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb index 735d66a4f..98e502bb6 100644 --- a/app/lib/status_reach_finder.rb +++ b/app/lib/status_reach_finder.rb @@ -1,8 +1,12 @@ # frozen_string_literal: true class StatusReachFinder - def initialize(status) - @status = status + # @param [Status] status + # @param [Hash] options + # @option options [Boolean] :unsafe + def initialize(status, options = {}) + @status = status + @options = options end def inboxes @@ -38,7 +42,7 @@ class StatusReachFinder end def replied_to_account_id - @status.in_reply_to_account_id + @status.in_reply_to_account_id if distributable? end def reblog_of_account_id @@ -49,21 +53,26 @@ class StatusReachFinder @status.mentions.pluck(:account_id) end + # Beware: Reblogs can be created without the author having had access to the status def reblogs_account_ids - @status.reblogs.pluck(:account_id) + @status.reblogs.pluck(:account_id) if distributable? || unsafe? end + # Beware: Favourites can be created without the author having had access to the status def favourites_account_ids - @status.favourites.pluck(:account_id) + @status.favourites.pluck(:account_id) if distributable? || unsafe? end + # Beware: Replies can be created without the author having had access to the status def replies_account_ids - @status.replies.pluck(:account_id) + @status.replies.pluck(:account_id) if distributable? || unsafe? end def followers_inboxes - if @status.in_reply_to_local_account? && @status.distributable? + if @status.in_reply_to_local_account? && distributable? @status.account.followers.or(@status.thread.account.followers).inboxes + elsif @status.direct_visibility? || @status.limited_visibility? + [] else @status.account.followers.inboxes end @@ -76,4 +85,12 @@ class StatusReachFinder [] end end + + def distributable? + @status.public_visibility? || @status.unlisted_visibility? + end + + def unsafe? + @options[:unsafe] + end end diff --git a/app/lib/themes.rb b/app/lib/themes.rb index 2147904e4..81e016d4a 100644 --- a/app/lib/themes.rb +++ b/app/lib/themes.rb @@ -14,16 +14,20 @@ class Themes result = Hash.new Dir.glob(Rails.root.join('app', 'javascript', 'flavours', '*', 'theme.yml')) do |path| data = YAML.load_file(path) + next unless data['pack'] + dir = File.dirname(path) name = File.basename(dir) locales = [] screenshots = [] + if data['locales'] Dir.glob(File.join(dir, data['locales'], '*.{js,json}')) do |locale| localeName = File.basename(locale, File.extname(locale)) locales.push(localeName) unless localeName.match(/defaultMessages|whitelist|index/) end end + if data['screenshot'] if data['screenshot'].is_a? Array screenshots = data['screenshot'] @@ -31,38 +35,37 @@ class Themes screenshots.push(data['screenshot']) end end - if data['pack'] - data['name'] = name - data['locales'] = locales - data['screenshot'] = screenshots - data['skin'] = { 'default' => [] } - result[name] = data - end + + data['name'] = name + data['locales'] = locales + data['screenshot'] = screenshots + data['skin'] = { 'default' => [] } + result[name] = data end Dir.glob(Rails.root.join('app', 'javascript', 'skins', '*', '*')) do |path| ext = File.extname(path) skin = File.basename(path) name = File.basename(File.dirname(path)) - if result[name] - if File.directory?(path) - pack = [] - Dir.glob(File.join(path, '*.{css,scss}')) do |sheet| - pack.push(File.basename(sheet, File.extname(sheet))) - end - elsif ext.match(/^\.s?css$/i) - skin = File.basename(path, ext) - pack = ['common'] - end - if skin != 'default' - result[name]['skin'][skin] = pack + next unless result[name] + + if File.directory?(path) + pack = [] + Dir.glob(File.join(path, '*.{css,scss}')) do |sheet| + pack.push(File.basename(sheet, File.extname(sheet))) end + elsif ext.match(/^\.s?css$/i) + skin = File.basename(path, ext) + pack = ['common'] + end + + if skin != 'default' + result[name]['skin'][skin] = pack end end @core = core @conf = result - end def core diff --git a/app/lib/webfinger.rb b/app/lib/webfinger.rb index e0e022cea..1ffb5b4bf 100644 --- a/app/lib/webfinger.rb +++ b/app/lib/webfinger.rb @@ -46,7 +46,9 @@ class Webfinger def body_from_webfinger(url = standard_url, use_fallback = true) webfinger_request(url).perform do |res| if res.code == 200 - res.body_with_limit + body = res.body_with_limit + raise Webfinger::Error, "Request for #{@uri} returned empty response" if body.empty? + body elsif res.code == 404 && use_fallback body_from_host_meta elsif res.code == 410 |