From dfa1be4ab451a4dd099ad3c140502fb0759ef832 Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Mon, 6 Mar 2023 11:33:58 -0500 Subject: Remove unused TOCGenerator (#23986) --- app/lib/toc_generator.rb | 69 ------------------------------------------------ 1 file changed, 69 deletions(-) delete mode 100644 app/lib/toc_generator.rb (limited to 'app/lib') diff --git a/app/lib/toc_generator.rb b/app/lib/toc_generator.rb deleted file mode 100644 index 0c8f766ca..000000000 --- a/app/lib/toc_generator.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -class TOCGenerator - TARGET_ELEMENTS = %w(h1 h2 h3 h4 h5 h6).freeze - LISTED_ELEMENTS = %w(h2 h3).freeze - - class Section - attr_accessor :depth, :title, :children, :anchor - - def initialize(depth, title, anchor) - @depth = depth - @title = title - @children = [] - @anchor = anchor - end - - delegate :<<, to: :children - end - - def initialize(source_html) - @source_html = source_html - @processed = false - @target_html = '' - @headers = [] - @slugs = Hash.new { |h, k| h[k] = 0 } - end - - def html - parse_and_transform unless @processed - @target_html - end - - def toc - parse_and_transform unless @processed - @headers - end - - private - - def parse_and_transform - return if @source_html.blank? - - parsed_html = Nokogiri::HTML.fragment(@source_html) - - parsed_html.traverse do |node| - next unless TARGET_ELEMENTS.include?(node.name) - - anchor = node['id'] || node.text.parameterize.presence || 'sec' - @slugs[anchor] += 1 - anchor = "#{anchor}-#{@slugs[anchor]}" if @slugs[anchor] > 1 - - node['id'] = anchor - - next unless LISTED_ELEMENTS.include?(node.name) - - depth = node.name[1..-1] - latest_section = @headers.last - - if latest_section.nil? || latest_section.depth >= depth - @headers << Section.new(depth, node.text, anchor) - else - latest_section << Section.new(depth, node.text, anchor) - end - end - - @target_html = parsed_html.to_s - @processed = true - end -end -- cgit From 684a970b3c7ce19ec419c4db52b67a3b0903f493 Mon Sep 17 00:00:00 2001 From: Christian Schmidt Date: Wed, 8 Mar 2023 19:56:41 +0100 Subject: Unescape HTML entities (#24019) --- app/lib/plain_text_formatter.rb | 6 +++- spec/lib/plain_text_formatter_spec.rb | 61 ++++++++++++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 6 deletions(-) (limited to 'app/lib') diff --git a/app/lib/plain_text_formatter.rb b/app/lib/plain_text_formatter.rb index 08aa29696..6fa2bc5d2 100644 --- a/app/lib/plain_text_formatter.rb +++ b/app/lib/plain_text_formatter.rb @@ -18,7 +18,7 @@ class PlainTextFormatter if local? text else - strip_tags(insert_newlines).chomp + html_entities.decode(strip_tags(insert_newlines)).chomp end end @@ -27,4 +27,8 @@ class PlainTextFormatter def insert_newlines text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" } end + + def html_entities + HTMLEntities.new + end end diff --git a/spec/lib/plain_text_formatter_spec.rb b/spec/lib/plain_text_formatter_spec.rb index 4a0519aaf..80b3c331a 100644 --- a/spec/lib/plain_text_formatter_spec.rb +++ b/spec/lib/plain_text_formatter_spec.rb @@ -6,7 +6,7 @@ RSpec.describe PlainTextFormatter do describe '#to_s' do subject { described_class.new(status.text, status.local?).to_s } - context 'given a post with local status' do + context 'when status is local' do let(:status) { Fabricate(:status, text: '

a text by a nerd who uses an HTML tag in text

', uri: nil) } it 'returns the raw text' do @@ -14,12 +14,63 @@ RSpec.describe PlainTextFormatter do end end - context 'given a post with remote status' do + context 'when status is remote' do let(:remote_account) { Fabricate(:account, domain: 'remote.test', username: 'bob', url: 'https://remote.test/') } - let(:status) { Fabricate(:status, account: remote_account, text: '

Hello

') } - it 'returns tag-stripped text' do - expect(subject).to eq 'Hello' + context 'when text contains inline HTML tags' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem ipsum') } + + it 'strips the tags' do + expect(subject).to eq 'Lorem ipsum' + end + end + + context 'when text contains

tags' do + let(:status) { Fabricate(:status, account: remote_account, text: '

Lorem

ipsum

') } + + it 'inserts a newline' do + expect(subject).to eq "Lorem\nipsum" + end + end + + context 'when text contains a single
tag' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem
ipsum') } + + it 'inserts a newline' do + expect(subject).to eq "Lorem\nipsum" + end + end + + context 'when text contains consecutive
tag' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem


ipsum') } + + it 'inserts a single newline' do + expect(subject).to eq "Lorem\nipsum" + end + end + + context 'when text contains HTML entity' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem & ipsum ❤') } + + it 'unescapes the entity' do + expect(subject).to eq 'Lorem & ipsum ❤' + end + end + + context 'when text contains ipsum') } + + it 'strips the tag and its contents' do + expect(subject).to eq 'Lorem ipsum' + end + end + + context 'when text contains an HTML comment tags' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem ipsum') } + + it 'strips the comment' do + expect(subject).to eq 'Lorem ipsum' + end end end end -- cgit From 1d0ad558ff51ce3e42ef4e4c9a9bed3c4b722d0d Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 15 Mar 2023 03:45:15 +0100 Subject: Change sidekiq-bulk's batch size from 10,000 to 1,000 jobs in one Redis call (#24034) --- app/lib/activitypub/forwarder.rb | 2 +- app/services/delete_account_service.rb | 4 ++-- app/services/remove_status_service.rb | 2 +- app/services/suspend_account_service.rb | 12 ++++++------ app/services/unsuspend_account_service.rb | 2 +- app/services/update_account_service.rb | 2 +- app/workers/activitypub/distribute_poll_update_worker.rb | 2 +- app/workers/activitypub/move_distribution_worker.rb | 2 +- app/workers/activitypub/raw_distribution_worker.rb | 2 +- lib/cli.rb | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) (limited to 'app/lib') diff --git a/app/lib/activitypub/forwarder.rb b/app/lib/activitypub/forwarder.rb index b01d63e58..3a94f9669 100644 --- a/app/lib/activitypub/forwarder.rb +++ b/app/lib/activitypub/forwarder.rb @@ -12,7 +12,7 @@ class ActivityPub::Forwarder end def forward! - ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [payload, signature_account_id, inbox_url] end end diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb index a2d535d26..190a72e5c 100644 --- a/app/services/delete_account_service.rb +++ b/app/services/delete_account_service.rb @@ -257,11 +257,11 @@ class DeleteAccountService < BaseService end def delete_actor! - ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes, limit: 1_000) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end - ActivityPub::LowPriorityDeliveryWorker.push_bulk(low_priority_delivery_inboxes) do |inbox_url| + ActivityPub::LowPriorityDeliveryWorker.push_bulk(low_priority_delivery_inboxes, limit: 1_000) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 45cfb75f4..ea799db57 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -88,7 +88,7 @@ class RemoveStatusService < BaseService status_reach_finder = StatusReachFinder.new(@status, unsafe: true) - ActivityPub::DeliveryWorker.push_bulk(status_reach_finder.inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(status_reach_finder.inboxes, limit: 1_000) do |inbox_url| [signed_activity_json, @account.id, inbox_url] end end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 211544fea..cfb3eb583 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -31,13 +31,13 @@ class SuspendAccountService < BaseService # counterpart to this operation, i.e. you can't then force a remote # account to re-follow you, so this part is not reversible. - follows = Follow.where(account: @account).to_a + Follow.where(account: @account).find_in_batches do |follows| + ActivityPub::DeliveryWorker.push_bulk(follows) do |follow| + [Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)), follow.target_account_id, @account.inbox_url] + end - ActivityPub::DeliveryWorker.push_bulk(follows) do |follow| - [Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)), follow.target_account_id, @account.inbox_url] + follows.each(&:destroy) end - - follows.each(&:destroy) end def distribute_update_actor! @@ -45,7 +45,7 @@ class SuspendAccountService < BaseService account_reach_finder = AccountReachFinder.new(@account) - ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes, limit: 1_000) do |inbox_url| [signed_activity_json, @account.id, inbox_url] end end diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb index 70667308e..d851a0f70 100644 --- a/app/services/unsuspend_account_service.rb +++ b/app/services/unsuspend_account_service.rb @@ -41,7 +41,7 @@ class UnsuspendAccountService < BaseService account_reach_finder = AccountReachFinder.new(@account) - ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes, limit: 1_000) do |inbox_url| [signed_activity_json, @account.id, inbox_url] end end diff --git a/app/services/update_account_service.rb b/app/services/update_account_service.rb index 71976ab00..4604d71b2 100644 --- a/app/services/update_account_service.rb +++ b/app/services/update_account_service.rb @@ -22,7 +22,7 @@ class UpdateAccountService < BaseService def authorize_all_follow_requests(account) follow_requests = FollowRequest.where(target_account: account) follow_requests = follow_requests.preload(:account).select { |req| !req.account.silenced? } - AuthorizeFollowWorker.push_bulk(follow_requests) do |req| + AuthorizeFollowWorker.push_bulk(follow_requests, limit: 1_000) do |req| [req.account_id, req.target_account_id] end end diff --git a/app/workers/activitypub/distribute_poll_update_worker.rb b/app/workers/activitypub/distribute_poll_update_worker.rb index 25dee4ee2..8c1eefd93 100644 --- a/app/workers/activitypub/distribute_poll_update_worker.rb +++ b/app/workers/activitypub/distribute_poll_update_worker.rb @@ -12,7 +12,7 @@ class ActivityPub::DistributePollUpdateWorker return unless @status.preloadable_poll - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [payload, @account.id, inbox_url] end diff --git a/app/workers/activitypub/move_distribution_worker.rb b/app/workers/activitypub/move_distribution_worker.rb index 65c5c0d1c..1680fcc76 100644 --- a/app/workers/activitypub/move_distribution_worker.rb +++ b/app/workers/activitypub/move_distribution_worker.rb @@ -10,7 +10,7 @@ class ActivityPub::MoveDistributionWorker @migration = AccountMigration.find(migration_id) @account = @migration.account - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [signed_payload, @account.id, inbox_url] end diff --git a/app/workers/activitypub/raw_distribution_worker.rb b/app/workers/activitypub/raw_distribution_worker.rb index 8ecc17db9..c77821e0f 100644 --- a/app/workers/activitypub/raw_distribution_worker.rb +++ b/app/workers/activitypub/raw_distribution_worker.rb @@ -25,7 +25,7 @@ class ActivityPub::RawDistributionWorker def distribute! return if inboxes.empty? - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [payload, source_account_id, inbox_url, options] end end diff --git a/lib/cli.rb b/lib/cli.rb index 157465c4b..ac235cf03 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -131,7 +131,7 @@ module Mastodon json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account)) unless options[:dry_run] - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [json, account.id, inbox_url] end -- cgit From 9875be09feebc52592caca739291288c3ae05b46 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 15 Mar 2023 04:01:49 +0100 Subject: Fix dashboard crash on ElasticSearch server error (#23751) --- app/lib/admin/system_check/elasticsearch_check.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index 5b4c12399..0b55be350 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -31,7 +31,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck def running_version @running_version ||= begin Chewy.client.info['version']['number'] - rescue Faraday::ConnectionFailed + rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error nil end end -- cgit