From 4eeff26533b75b592395e353c6d4506db9958bcf Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 25 Jul 2019 04:17:35 +0200
Subject: Change account domain block to clear out notifications and follows
 (#11393)

---
 app/javascript/mastodon/actions/domain_blocks.js   |  1 +
 app/javascript/mastodon/reducers/conversations.js  | 11 ++++++
 app/javascript/mastodon/reducers/notifications.js  | 11 +++---
 app/javascript/mastodon/reducers/suggestions.js    |  7 ++++
 .../after_block_domain_from_account_service.rb     | 12 +++++++
 app/services/after_block_service.rb                | 42 +++++++---------------
 app/services/follow_service.rb                     |  2 +-
 7 files changed, 51 insertions(+), 35 deletions(-)

(limited to 'app')

diff --git a/app/javascript/mastodon/actions/domain_blocks.js b/app/javascript/mastodon/actions/domain_blocks.js
index 0445a5e10..34a33a654 100644
--- a/app/javascript/mastodon/actions/domain_blocks.js
+++ b/app/javascript/mastodon/actions/domain_blocks.js
@@ -23,6 +23,7 @@ export function blockDomain(domain) {
     api(getState).post('/api/v1/domain_blocks', { domain }).then(() => {
       const at_domain = '@' + domain;
       const accounts = getState().get('accounts').filter(item => item.get('acct').endsWith(at_domain)).valueSeq().map(item => item.get('id'));
+
       dispatch(blockDomainSuccess(domain, accounts));
     }).catch(err => {
       dispatch(blockDomainFail(domain, err));
diff --git a/app/javascript/mastodon/reducers/conversations.js b/app/javascript/mastodon/reducers/conversations.js
index 9564bffcd..390658239 100644
--- a/app/javascript/mastodon/reducers/conversations.js
+++ b/app/javascript/mastodon/reducers/conversations.js
@@ -8,6 +8,8 @@ import {
   CONVERSATIONS_UPDATE,
   CONVERSATIONS_READ,
 } from '../actions/conversations';
+import { ACCOUNT_BLOCK_SUCCESS, ACCOUNT_MUTE_SUCCESS } from 'mastodon/actions/accounts';
+import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks';
 import compareId from '../compare_id';
 
 const initialState = ImmutableMap({
@@ -74,6 +76,10 @@ const expandNormalizedConversations = (state, conversations, next, isLoadingRece
   });
 };
 
+const filterConversations = (state, accountIds) => {
+  return state.update('items', list => list.filterNot(item => item.get('accounts').some(accountId => accountIds.includes(accountId))));
+};
+
 export default function conversations(state = initialState, action) {
   switch (action.type) {
   case CONVERSATIONS_FETCH_REQUEST:
@@ -96,6 +102,11 @@ export default function conversations(state = initialState, action) {
 
       return item;
     }));
+  case ACCOUNT_BLOCK_SUCCESS:
+  case ACCOUNT_MUTE_SUCCESS:
+    return filterConversations(state, [action.relationship.id]);
+  case DOMAIN_BLOCK_SUCCESS:
+    return filterConversations(state, action.accounts);
   default:
     return state;
   }
diff --git a/app/javascript/mastodon/reducers/notifications.js b/app/javascript/mastodon/reducers/notifications.js
index e94a4946b..049c70cb4 100644
--- a/app/javascript/mastodon/reducers/notifications.js
+++ b/app/javascript/mastodon/reducers/notifications.js
@@ -12,6 +12,7 @@ import {
   ACCOUNT_BLOCK_SUCCESS,
   ACCOUNT_MUTE_SUCCESS,
 } from '../actions/accounts';
+import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks';
 import { TIMELINE_DELETE, TIMELINE_DISCONNECT } from '../actions/timelines';
 import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import compareId from '../compare_id';
@@ -83,8 +84,8 @@ const expandNormalizedNotifications = (state, notifications, next, usePendingIte
   });
 };
 
-const filterNotifications = (state, relationship) => {
-  const helper = list => list.filterNot(item => item !== null && item.get('account') === relationship.id);
+const filterNotifications = (state, accountIds) => {
+  const helper = list => list.filterNot(item => item !== null && accountIds.includes(item.get('account')));
   return state.update('items', helper).update('pendingItems', helper);
 };
 
@@ -118,9 +119,11 @@ export default function notifications(state = initialState, action) {
   case NOTIFICATIONS_EXPAND_SUCCESS:
     return expandNormalizedNotifications(state, action.notifications, action.next, action.usePendingItems);
   case ACCOUNT_BLOCK_SUCCESS:
-    return filterNotifications(state, action.relationship);
+    return filterNotifications(state, [action.relationship.id]);
   case ACCOUNT_MUTE_SUCCESS:
-    return action.relationship.muting_notifications ? filterNotifications(state, action.relationship) : state;
+    return action.relationship.muting_notifications ? filterNotifications(state, [action.relationship.id]) : state;
+  case DOMAIN_BLOCK_SUCCESS:
+    return filterNotifications(state, action.accounts);
   case NOTIFICATIONS_CLEAR:
     return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', false);
   case TIMELINE_DELETE:
diff --git a/app/javascript/mastodon/reducers/suggestions.js b/app/javascript/mastodon/reducers/suggestions.js
index 9f4b89d58..834be728f 100644
--- a/app/javascript/mastodon/reducers/suggestions.js
+++ b/app/javascript/mastodon/reducers/suggestions.js
@@ -4,6 +4,8 @@ import {
   SUGGESTIONS_FETCH_FAIL,
   SUGGESTIONS_DISMISS,
 } from '../actions/suggestions';
+import { ACCOUNT_BLOCK_SUCCESS, ACCOUNT_MUTE_SUCCESS } from 'mastodon/actions/accounts';
+import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks';
 import { Map as ImmutableMap, List as ImmutableList, fromJS } from 'immutable';
 
 const initialState = ImmutableMap({
@@ -24,6 +26,11 @@ export default function suggestionsReducer(state = initialState, action) {
     return state.set('isLoading', false);
   case SUGGESTIONS_DISMISS:
     return state.update('items', list => list.filterNot(id => id === action.id));
+  case ACCOUNT_BLOCK_SUCCESS:
+  case ACCOUNT_MUTE_SUCCESS:
+    return state.update('items', list => list.filterNot(id => id === action.relationship.id));
+  case DOMAIN_BLOCK_SUCCESS:
+    return state.update('items', list => list.filterNot(id => action.accounts.includes(id)));
   default:
     return state;
   }
diff --git a/app/services/after_block_domain_from_account_service.rb b/app/services/after_block_domain_from_account_service.rb
index a87c2e792..f50bde261 100644
--- a/app/services/after_block_domain_from_account_service.rb
+++ b/app/services/after_block_domain_from_account_service.rb
@@ -10,12 +10,24 @@ class AfterBlockDomainFromAccountService < BaseService
     @account = account
     @domain  = domain
 
+    clear_notifications!
+    remove_follows!
     reject_existing_followers!
     reject_pending_follow_requests!
   end
 
   private
 
+  def remove_follows!
+    @account.active_relationships.where(account: Account.where(domain: @domain)).includes(:target_account).reorder(nil).find_each do |follow|
+      UnfollowService.new.call(@account, follow.target_account)
+    end
+  end
+
+  def clear_notifications!
+    Notification.where(account: @account).where(from_account: Account.where(domain: @domain)).in_batches.delete_all
+  end
+
   def reject_existing_followers!
     @account.passive_relationships.where(account: Account.where(domain: @domain)).includes(:account).reorder(nil).find_each do |follow|
       reject_follow!(follow)
diff --git a/app/services/after_block_service.rb b/app/services/after_block_service.rb
index 706db0d63..2a0e10a79 100644
--- a/app/services/after_block_service.rb
+++ b/app/services/after_block_service.rb
@@ -2,43 +2,25 @@
 
 class AfterBlockService < BaseService
   def call(account, target_account)
-    clear_home_feed(account, target_account)
-    clear_notifications(account, target_account)
-    clear_conversations(account, target_account)
+    @account        = account
+    @target_account = target_account
+
+    clear_home_feed!
+    clear_notifications!
+    clear_conversations!
   end
 
   private
 
-  def clear_home_feed(account, target_account)
-    FeedManager.instance.clear_from_timeline(account, target_account)
+  def clear_home_feed!
+    FeedManager.instance.clear_from_timeline(@account, @target_account)
   end
 
-  def clear_conversations(account, target_account)
-    AccountConversation.where(account: account)
-                       .where('? = ANY(participant_account_ids)', target_account.id)
-                       .in_batches
-                       .destroy_all
+  def clear_conversations!
+    AccountConversation.where(account: @account).where('? = ANY(participant_account_ids)', @target_account.id).in_batches.destroy_all
   end
 
-  def clear_notifications(account, target_account)
-    Notification.where(account: account)
-                .joins(:follow)
-                .where(activity_type: 'Follow', follows: { account_id: target_account.id })
-                .delete_all
-
-    Notification.where(account: account)
-                .joins(mention: :status)
-                .where(activity_type: 'Mention', statuses: { account_id: target_account.id })
-                .delete_all
-
-    Notification.where(account: account)
-                .joins(:favourite)
-                .where(activity_type: 'Favourite', favourites: { account_id: target_account.id })
-                .delete_all
-
-    Notification.where(account: account)
-                .joins(:status)
-                .where(activity_type: 'Status', statuses: { account_id: target_account.id })
-                .delete_all
+  def clear_notifications!
+    Notification.where(account: @account).where(from_account: @target_account).in_batches.delete_all
   end
 end
diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index 8e118f5d3..101acdaf9 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -13,7 +13,7 @@ class FollowService < BaseService
     target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
 
     raise ActiveRecord::RecordNotFound if target_account.nil? || target_account.id == source_account.id || target_account.suspended?
-    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?)
+    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?) || source_account.domain_blocking?(target_account.domain)
 
     if source_account.following?(target_account)
       # We're already following this account, but we'll call follow! again to
-- 
cgit 


From 92569ffde81a3fe768fd8773fdab5d1a927f3f88 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 26 Jul 2019 18:55:33 +0200
Subject: Fix invites not being disabled upon account suspension (#11412)

* Disable invite links from disabled/suspended users

* Add has_many invites relationship to users

* Destroy unused invites when suspending an account
---
 app/controllers/invites_controller.rb   |  2 +-
 app/models/invite.rb                    |  4 ++--
 app/models/user.rb                      |  1 +
 app/services/suspend_account_service.rb |  1 +
 spec/models/invite_spec.rb              | 16 +++++++++++-----
 5 files changed, 16 insertions(+), 8 deletions(-)

(limited to 'app')

diff --git a/app/controllers/invites_controller.rb b/app/controllers/invites_controller.rb
index fdb3a0962..de5280305 100644
--- a/app/controllers/invites_controller.rb
+++ b/app/controllers/invites_controller.rb
@@ -39,7 +39,7 @@ class InvitesController < ApplicationController
   private
 
   def invites
-    Invite.where(user: current_user).order(id: :desc)
+    current_user.invites.order(id: :desc)
   end
 
   def resource_params
diff --git a/app/models/invite.rb b/app/models/invite.rb
index fe2322462..02ab8e0b2 100644
--- a/app/models/invite.rb
+++ b/app/models/invite.rb
@@ -17,7 +17,7 @@
 class Invite < ApplicationRecord
   include Expireable
 
-  belongs_to :user
+  belongs_to :user, inverse_of: :invites
   has_many :users, inverse_of: :invite
 
   scope :available, -> { where(expires_at: nil).or(where('expires_at >= ?', Time.now.utc)) }
@@ -25,7 +25,7 @@ class Invite < ApplicationRecord
   before_validation :set_code
 
   def valid_for_use?
-    (max_uses.nil? || uses < max_uses) && !expired?
+    (max_uses.nil? || uses < max_uses) && !expired? && !(user.nil? || user.disabled?)
   end
 
   private
diff --git a/app/models/user.rb b/app/models/user.rb
index 474c77293..6806c0362 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -73,6 +73,7 @@ class User < ApplicationRecord
 
   has_many :applications, class_name: 'Doorkeeper::Application', as: :owner
   has_many :backups, inverse_of: :user
+  has_many :invites, inverse_of: :user
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? }
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 00cffcdfc..902af376c 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -64,6 +64,7 @@ class SuspendAccountService < BaseService
       @account.user.destroy
     else
       @account.user.disable!
+      @account.user.invites.where(uses: 0).destroy_all
     end
   end
 
diff --git a/spec/models/invite_spec.rb b/spec/models/invite_spec.rb
index 0ba1dccb3..30abfb86b 100644
--- a/spec/models/invite_spec.rb
+++ b/spec/models/invite_spec.rb
@@ -3,27 +3,33 @@ require 'rails_helper'
 RSpec.describe Invite, type: :model do
   describe '#valid_for_use?' do
     it 'returns true when there are no limitations' do
-      invite = Invite.new(max_uses: nil, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: nil)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns true when not expired' do
-      invite = Invite.new(max_uses: nil, expires_at: 1.hour.from_now)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.from_now)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns false when expired' do
-      invite = Invite.new(max_uses: nil, expires_at: 1.hour.ago)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.ago)
       expect(invite.valid_for_use?).to be false
     end
 
     it 'returns true when uses still available' do
-      invite = Invite.new(max_uses: 250, uses: 249, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: 250, uses: 249, expires_at: nil)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns false when maximum uses reached' do
-      invite = Invite.new(max_uses: 250, uses: 250, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: 250, uses: 250, expires_at: nil)
+      expect(invite.valid_for_use?).to be false
+    end
+
+    it 'returns false when invite creator has been disabled' do
+      invite = Fabricate(:invite, max_uses: nil, expires_at: nil)
+      SuspendAccountService.new.call(invite.user.account)
       expect(invite.valid_for_use?).to be false
     end
   end
-- 
cgit 


From 501148ab912b3bd36dbf0f9f2e10bfde7787012d Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 27 Jul 2019 04:41:55 +0200
Subject: Remove timestamps from converted images to make them deterministic
 (#11408)

---
 app/models/media_attachment.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 05be302af..2fc5ccf27 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -113,7 +113,7 @@ class MediaAttachment < ApplicationRecord
   has_attached_file :file,
                     styles: ->(f) { file_styles f },
                     processors: ->(f) { file_processors f },
-                    convert_options: { all: '-quality 90 -strip' }
+                    convert_options: { all: '-quality 90 -strip +set modify-date +set create-date' }
 
   validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
   validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format?
-- 
cgit 


From b9fbcbfe4e0a15fcf8a457ce17ea080f0eb939fc Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 27 Jul 2019 04:42:08 +0200
Subject: Add search syntax for operators and phrases (#11411)

---
 Gemfile                              |  1 +
 Gemfile.lock                         |  2 +
 app/lib/search_query_parser.rb       | 14 ++++++
 app/lib/search_query_transformer.rb  | 86 ++++++++++++++++++++++++++++++++++++
 app/services/search_service.rb       |  9 ++--
 spec/services/search_service_spec.rb |  6 +--
 6 files changed, 112 insertions(+), 6 deletions(-)
 create mode 100644 app/lib/search_query_parser.rb
 create mode 100644 app/lib/search_query_transformer.rb

(limited to 'app')

diff --git a/Gemfile b/Gemfile
index e969c4ae0..96eb44af7 100644
--- a/Gemfile
+++ b/Gemfile
@@ -64,6 +64,7 @@ gem 'nsa', '~> 0.2'
 gem 'oj', '~> 3.8'
 gem 'ostatus2', '~> 2.0'
 gem 'ox', '~> 2.11'
+gem 'parslet'
 gem 'posix-spawn', git: 'https://github.com/rtomayko/posix-spawn', ref: '58465d2e213991f8afb13b984854a49fcdcc980c'
 gem 'pundit', '~> 2.0'
 gem 'premailer-rails'
diff --git a/Gemfile.lock b/Gemfile.lock
index f435b3a35..5adae6f9c 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -404,6 +404,7 @@ GEM
       parallel
     parser (2.6.3.0)
       ast (~> 2.4.0)
+    parslet (1.8.2)
     pastel (0.7.2)
       equatable (~> 0.5.0)
       tty-color (~> 0.4.0)
@@ -724,6 +725,7 @@ DEPENDENCIES
   paperclip (~> 6.0)
   paperclip-av-transcoder (~> 0.6)
   parallel_tests (~> 2.29)
+  parslet
   pg (~> 1.1)
   pghero (~> 2.2)
   pkg-config (~> 1.3)
diff --git a/app/lib/search_query_parser.rb b/app/lib/search_query_parser.rb
new file mode 100644
index 000000000..405ad15b8
--- /dev/null
+++ b/app/lib/search_query_parser.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class SearchQueryParser < Parslet::Parser
+  rule(:term)     { match('[^\s":]').repeat(1).as(:term) }
+  rule(:quote)    { str('"') }
+  rule(:colon)    { str(':') }
+  rule(:space)    { match('\s').repeat(1) }
+  rule(:operator) { (str('+') | str('-')).as(:operator) }
+  rule(:prefix)   { (term >> colon).as(:prefix) }
+  rule(:phrase)   { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
+  rule(:clause)   { (prefix.maybe >> operator.maybe >> (phrase | term)).as(:clause) }
+  rule(:query)    { (clause >> space.maybe).repeat.as(:query) }
+  root(:query)
+end
diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb
new file mode 100644
index 000000000..2c4144790
--- /dev/null
+++ b/app/lib/search_query_transformer.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+class SearchQueryTransformer < Parslet::Transform
+  class Query
+    attr_reader :should_clauses, :must_not_clauses, :must_clauses
+
+    def initialize(clauses)
+      grouped = clauses.chunk(&:operator).to_h
+      @should_clauses = grouped.fetch(:should, [])
+      @must_not_clauses = grouped.fetch(:must_not, [])
+      @must_clauses = grouped.fetch(:must, [])
+    end
+
+    def apply(search)
+      should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
+      must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
+      must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
+      search.query.minimum_should_match(1)
+    end
+
+    private
+
+    def clause_to_query(clause)
+      case clause
+      when TermClause
+        { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } }
+      when PhraseClause
+        { match_phrase: { text: { query: clause.phrase } } }
+      else
+        raise "Unexpected clause type: #{clause}"
+      end
+    end
+  end
+
+  class Operator
+    class << self
+      def symbol(str)
+        case str
+        when '+'
+          :must
+        when '-'
+          :must_not
+        when nil
+          :should
+        else
+          raise "Unknown operator: #{str}"
+        end
+      end
+    end
+  end
+
+  class TermClause
+    attr_reader :prefix, :operator, :term
+
+    def initialize(prefix, operator, term)
+      @prefix = prefix
+      @operator = Operator.symbol(operator)
+      @term = term
+    end
+  end
+
+  class PhraseClause
+    attr_reader :prefix, :operator, :phrase
+
+    def initialize(prefix, operator, phrase)
+      @prefix = prefix
+      @operator = Operator.symbol(operator)
+      @phrase = phrase
+    end
+  end
+
+  rule(clause: subtree(:clause)) do
+    prefix   = clause[:prefix][:term].to_s if clause[:prefix]
+    operator = clause[:operator]&.to_s
+
+    if clause[:term]
+      TermClause.new(prefix, operator, clause[:term].to_s)
+    elsif clause[:phrase]
+      PhraseClause.new(prefix, operator, clause[:phrase].map { |p| p[:term].to_s }.join(' '))
+    else
+      raise "Unexpected clause type: #{clause}"
+    end
+  end
+
+  rule(query: sequence(:clauses)) { Query.new(clauses) }
+end
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index e0da61dac..769d1ac7a 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -33,8 +33,7 @@ class SearchService < BaseService
   end
 
   def perform_statuses_search!
-    definition = StatusesIndex.filter(term: { searchable_by: @account.id })
-                              .query(multi_match: { type: 'most_fields', query: @query, operator: 'and', fields: %w(text text.stemmed) })
+    definition = parsed_query.apply(StatusesIndex.filter(term: { searchable_by: @account.id }))
 
     if @options[:account_id].present?
       definition = definition.filter(term: { account_id: @options[:account_id] })
@@ -70,7 +69,7 @@ class SearchService < BaseService
   end
 
   def url_query?
-    @options[:type].blank? && @query =~ /\Ahttps?:\/\//
+    @resolve && @options[:type].blank? && @query =~ /\Ahttps?:\/\//
   end
 
   def url_resource_results
@@ -120,4 +119,8 @@ class SearchService < BaseService
       domain_blocking_by_domain: Account.domain_blocking_map_by_domain(domains, account.id),
     }
   end
+
+  def parsed_query
+    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
+  end
 end
diff --git a/spec/services/search_service_spec.rb b/spec/services/search_service_spec.rb
index d064cd9b8..ade306ed2 100644
--- a/spec/services/search_service_spec.rb
+++ b/spec/services/search_service_spec.rb
@@ -27,7 +27,7 @@ describe SearchService, type: :service do
         it 'returns the empty results' do
           service = double(call: nil)
           allow(ResolveURLService).to receive(:new).and_return(service)
-          results = subject.call(@query, nil, 10)
+          results = subject.call(@query, nil, 10, resolve: true)
 
           expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
           expect(results).to eq empty_results
@@ -40,7 +40,7 @@ describe SearchService, type: :service do
           service = double(call: account)
           allow(ResolveURLService).to receive(:new).and_return(service)
 
-          results = subject.call(@query, nil, 10)
+          results = subject.call(@query, nil, 10, resolve: true)
           expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
           expect(results).to eq empty_results.merge(accounts: [account])
         end
@@ -52,7 +52,7 @@ describe SearchService, type: :service do
           service = double(call: status)
           allow(ResolveURLService).to receive(:new).and_return(service)
 
-          results = subject.call(@query, nil, 10)
+          results = subject.call(@query, nil, 10, resolve: true)
           expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
           expect(results).to eq empty_results.merge(statuses: [status])
         end
-- 
cgit 


From 8a4674f2c3d89c998eb5438b96b7977dc2be3167 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 27 Jul 2019 05:49:50 +0200
Subject: Add search results pagination to web UI (#11409)

* Add search results pagination to web UI

Fix #10737

* Fix code style issue
---
 app/javascript/mastodon/actions/search.js          | 54 ++++++++++++++++++++--
 .../features/compose/components/search_results.js  | 18 +++++++-
 .../compose/containers/search_results_container.js |  4 +-
 app/javascript/mastodon/reducers/search.js         |  3 ++
 app/javascript/styles/mastodon/components.scss     |  5 +-
 5 files changed, 76 insertions(+), 8 deletions(-)

(limited to 'app')

diff --git a/app/javascript/mastodon/actions/search.js b/app/javascript/mastodon/actions/search.js
index 0974fdd15..a178faead 100644
--- a/app/javascript/mastodon/actions/search.js
+++ b/app/javascript/mastodon/actions/search.js
@@ -10,6 +10,10 @@ export const SEARCH_FETCH_REQUEST = 'SEARCH_FETCH_REQUEST';
 export const SEARCH_FETCH_SUCCESS = 'SEARCH_FETCH_SUCCESS';
 export const SEARCH_FETCH_FAIL    = 'SEARCH_FETCH_FAIL';
 
+export const SEARCH_EXPAND_REQUEST = 'SEARCH_EXPAND_REQUEST';
+export const SEARCH_EXPAND_SUCCESS = 'SEARCH_EXPAND_SUCCESS';
+export const SEARCH_EXPAND_FAIL    = 'SEARCH_EXPAND_FAIL';
+
 export function changeSearch(value) {
   return {
     type: SEARCH_CHANGE,
@@ -77,8 +81,50 @@ export function fetchSearchFail(error) {
   };
 };
 
-export function showSearch() {
-  return {
-    type: SEARCH_SHOW,
-  };
+export const expandSearch = type => (dispatch, getState) => {
+  const value  = getState().getIn(['search', 'value']);
+  const offset = getState().getIn(['search', 'results', type]).size;
+
+  dispatch(expandSearchRequest());
+
+  api(getState).get('/api/v2/search', {
+    params: {
+      q: value,
+      type,
+      offset,
+    },
+  }).then(({ data }) => {
+    if (data.accounts) {
+      dispatch(importFetchedAccounts(data.accounts));
+    }
+
+    if (data.statuses) {
+      dispatch(importFetchedStatuses(data.statuses));
+    }
+
+    dispatch(expandSearchSuccess(data, value, type));
+    dispatch(fetchRelationships(data.accounts.map(item => item.id)));
+  }).catch(error => {
+    dispatch(expandSearchFail(error));
+  });
 };
+
+export const expandSearchRequest = () => ({
+  type: SEARCH_EXPAND_REQUEST,
+});
+
+export const expandSearchSuccess = (results, searchTerm, searchType) => ({
+  type: SEARCH_EXPAND_SUCCESS,
+  results,
+  searchTerm,
+  searchType,
+});
+
+export const expandSearchFail = error => ({
+  type: SEARCH_EXPAND_FAIL,
+  error,
+});
+
+export const showSearch = () => ({
+  type: SEARCH_SHOW,
+});
diff --git a/app/javascript/mastodon/features/compose/components/search_results.js b/app/javascript/mastodon/features/compose/components/search_results.js
index 2f338dd24..4b4cdff74 100644
--- a/app/javascript/mastodon/features/compose/components/search_results.js
+++ b/app/javascript/mastodon/features/compose/components/search_results.js
@@ -8,6 +8,7 @@ import ImmutablePureComponent from 'react-immutable-pure-component';
 import Hashtag from '../../../components/hashtag';
 import Icon from 'mastodon/components/icon';
 import { searchEnabled } from '../../../initial_state';
+import LoadMore from 'mastodon/components/load_more';
 
 const messages = defineMessages({
   dismissSuggestion: { id: 'suggestions.dismiss', defaultMessage: 'Dismiss suggestion' },
@@ -20,15 +21,24 @@ class SearchResults extends ImmutablePureComponent {
     results: ImmutablePropTypes.map.isRequired,
     suggestions: ImmutablePropTypes.list.isRequired,
     fetchSuggestions: PropTypes.func.isRequired,
+    expandSearch: PropTypes.func.isRequired,
     dismissSuggestion: PropTypes.func.isRequired,
     searchTerm: PropTypes.string,
     intl: PropTypes.object.isRequired,
   };
 
   componentDidMount () {
-    this.props.fetchSuggestions();
+    if (this.props.searchTerm === '') {
+      this.props.fetchSuggestions();
+    }
   }
 
+  handleLoadMoreAccounts = () => this.props.expandSearch('accounts');
+
+  handleLoadMoreStatuses = () => this.props.expandSearch('statuses');
+
+  handleLoadMoreHashtags = () => this.props.expandSearch('hashtags');
+
   render () {
     const { intl, results, suggestions, dismissSuggestion, searchTerm } = this.props;
 
@@ -65,6 +75,8 @@ class SearchResults extends ImmutablePureComponent {
           <h5><Icon id='users' fixedWidth /><FormattedMessage id='search_results.accounts' defaultMessage='People' /></h5>
 
           {results.get('accounts').map(accountId => <AccountContainer key={accountId} id={accountId} />)}
+
+          {results.get('accounts').size >= 5 && <LoadMore visible onClick={this.handleLoadMoreAccounts} />}
         </div>
       );
     }
@@ -76,6 +88,8 @@ class SearchResults extends ImmutablePureComponent {
           <h5><Icon id='quote-right' fixedWidth /><FormattedMessage id='search_results.statuses' defaultMessage='Toots' /></h5>
 
           {results.get('statuses').map(statusId => <StatusContainer key={statusId} id={statusId} />)}
+
+          {results.get('statuses').size >= 5 && <LoadMore visible onClick={this.handleLoadMoreStatuses} />}
         </div>
       );
     } else if(results.get('statuses') && results.get('statuses').size === 0 && !searchEnabled && !(searchTerm.startsWith('@') || searchTerm.startsWith('#') || searchTerm.includes(' '))) {
@@ -97,6 +111,8 @@ class SearchResults extends ImmutablePureComponent {
           <h5><Icon id='hashtag' fixedWidth /><FormattedMessage id='search_results.hashtags' defaultMessage='Hashtags' /></h5>
 
           {results.get('hashtags').map(hashtag => <Hashtag key={hashtag.get('name')} hashtag={hashtag} />)}
+
+          {results.get('hashtags').size >= 5 && <LoadMore visible onClick={this.handleLoadMoreHashtags} />}
         </div>
       );
     }
diff --git a/app/javascript/mastodon/features/compose/containers/search_results_container.js b/app/javascript/mastodon/features/compose/containers/search_results_container.js
index e4d5f3420..1f714ff83 100644
--- a/app/javascript/mastodon/features/compose/containers/search_results_container.js
+++ b/app/javascript/mastodon/features/compose/containers/search_results_container.js
@@ -1,6 +1,7 @@
 import { connect } from 'react-redux';
 import SearchResults from '../components/search_results';
-import { fetchSuggestions, dismissSuggestion } from '../../../actions/suggestions';
+import { fetchSuggestions, dismissSuggestion } from 'mastodon/actions/suggestions';
+import { expandSearch } from 'mastodon/actions/search';
 
 const mapStateToProps = state => ({
   results: state.getIn(['search', 'results']),
@@ -10,6 +11,7 @@ const mapStateToProps = state => ({
 
 const mapDispatchToProps = dispatch => ({
   fetchSuggestions: () => dispatch(fetchSuggestions()),
+  expandSearch: type => dispatch(expandSearch(type)),
   dismissSuggestion: account => dispatch(dismissSuggestion(account.get('id'))),
 });
 
diff --git a/app/javascript/mastodon/reducers/search.js b/app/javascript/mastodon/reducers/search.js
index 77b7f588c..f437ff217 100644
--- a/app/javascript/mastodon/reducers/search.js
+++ b/app/javascript/mastodon/reducers/search.js
@@ -3,6 +3,7 @@ import {
   SEARCH_CLEAR,
   SEARCH_FETCH_SUCCESS,
   SEARCH_SHOW,
+  SEARCH_EXPAND_SUCCESS,
 } from '../actions/search';
 import {
   COMPOSE_MENTION,
@@ -42,6 +43,8 @@ export default function search(state = initialState, action) {
       statuses: ImmutableList(action.results.statuses.map(item => item.id)),
       hashtags: fromJS(action.results.hashtags),
     })).set('submitted', true).set('searchTerm', action.searchTerm);
+  case SEARCH_EXPAND_SUCCESS:
+    return state.updateIn(['results', action.searchType], list => list.concat(action.results[action.searchType].map(item => item.id)));
   default:
     return state;
   }
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 54442e007..8ed27371e 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -4006,8 +4006,9 @@ a.status-card.compact:hover {
 }
 
 .search-results__info {
-  padding: 10px;
-  color: $secondary-text-color;
+  padding: 20px;
+  color: $darker-text-color;
+  text-align: center;
 }
 
 .modal-root {
-- 
cgit 


From 4cc29eb5ad106c267ff16c9f49f145bc34d1aae0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 27 Jul 2019 19:25:15 +0200
Subject: Fix tabs bar scrolling along with content on mobile (#11418)

---
 app/javascript/styles/mastodon/components.scss | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'app')

diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 8ed27371e..dd6306fb2 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -1936,6 +1936,9 @@ a.account__display-name {
   background: lighten($ui-base-color, 8%);
   flex: 0 0 auto;
   overflow-y: auto;
+  position: sticky;
+  top: 0;
+  z-index: 3;
 }
 
 .tabs-bar__link {
-- 
cgit 


From f371b32137ccd7e74ca29d25af2072fb79654b15 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 05:59:51 +0200
Subject: Change hashtags to preserve first-used casing (#11416)

---
 app/lib/activitypub/activity/create.rb             |  9 ++----
 app/models/tag.rb                                  | 34 +++++++++++++++++++---
 app/services/hashtag_query_service.rb              |  4 +--
 app/services/process_hashtags_service.rb           |  4 +--
 ...726175042_add_case_insensitive_index_to_tags.rb | 15 ++++++++++
 db/schema.rb                                       |  5 ++--
 6 files changed, 53 insertions(+), 18 deletions(-)
 create mode 100644 db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb

(limited to 'app')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 56c24680a..000b77df5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -148,12 +148,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def process_hashtag(tag)
     return if tag['name'].blank?
 
-    hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase
-    hashtag = Tag.where(name: hashtag).first_or_create!(name: hashtag)
-
-    return if @tags.include?(hashtag)
-
-    @tags << hashtag
+    Tag.find_or_create_by_names(tag['name']) do |hashtag|
+      @tags << hashtag unless @tags.include?(hashtag)
+    end
   rescue ActiveRecord::RecordInvalid
     nil
   end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index b371d59c1..972242064 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -20,7 +20,7 @@ class Tag < ApplicationRecord
   HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
-  validates :name, presence: true, uniqueness: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
 
   scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
@@ -64,22 +64,48 @@ class Tag < ApplicationRecord
   end
 
   class << self
+    def find_or_create_by_names(name_or_names)
+      Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name|
+        tag = matching_name(normalized_name).first || create(name: normalized_name)
+
+        yield tag if block_given?
+
+        tag
+      end
+    end
+
     def search_for(term, limit = 5, offset = 0)
-      pattern = sanitize_sql_like(term.strip) + '%'
+      pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
-      Tag.where('lower(name) like lower(?)', pattern)
+      Tag.where(arel_table[:name].lower.matches(pattern.downcase))
          .order(:name)
          .limit(limit)
          .offset(offset)
     end
 
     def find_normalized(name)
-      find_by(name: name.mb_chars.downcase.to_s)
+      matching_name(name).first
     end
 
     def find_normalized!(name)
       find_normalized(name) || raise(ActiveRecord::RecordNotFound)
     end
+
+    def matching_name(name_or_names)
+      names = Array(name_or_names).map { |name| normalize(name).downcase }
+
+      if names.size == 1
+        where(arel_table[:name].lower.eq(names.first))
+      else
+        where(arel_table[:name].lower.in(names))
+      end
+    end
+
+    private
+
+    def normalize(str)
+      str.gsub(/\A#/, '').mb_chars.to_s
+    end
   end
 
   private
diff --git a/app/services/hashtag_query_service.rb b/app/services/hashtag_query_service.rb
index 5773d78c6..282821710 100644
--- a/app/services/hashtag_query_service.rb
+++ b/app/services/hashtag_query_service.rb
@@ -14,7 +14,7 @@ class HashtagQueryService < BaseService
 
   private
 
-  def tags_for(tags)
-    Tag.where(name: tags.map(&:downcase)) if tags.presence
+  def tags_for(names)
+    Tag.matching_name(names) if names.presence
   end
 end
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index b6974e598..e8e139b05 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -5,9 +5,7 @@ class ProcessHashtagsService < BaseService
     tags    = Extractor.extract_hashtags(status.text) if status.local?
     records = []
 
-    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
-      tag = Tag.where(name: name).first_or_create(name: name)
-
+    Tag.find_or_create_by_names(tags) do |tag|
       status.tags << tag
       records << tag
 
diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
new file mode 100644
index 000000000..6fa8c0ec4
--- /dev/null
+++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
@@ -0,0 +1,15 @@
+class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
+    remove_index :tags, name: 'index_tags_on_name'
+    remove_index :tags, name: 'hashtag_search_index'
+  end
+
+  def down
+    add_index :tags, :name, unique: true, algorithm: :concurrently
+    safety_assured { execute 'CREATE INDEX CONCURRENTLY hashtag_search_index ON tags (name text_pattern_ops)' }
+    remove_index :tags, name: 'index_tags_on_name_lower'
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 6319dd932..1847305c7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_15_164535) do
+ActiveRecord::Schema.define(version: 2019_07_26_175042) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -652,8 +652,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index "lower((name)::text) text_pattern_ops", name: "hashtag_search_index"
-    t.index ["name"], name: "index_tags_on_name", unique: true
+    t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
   create_table "tombstones", force: :cascade do |t|
-- 
cgit 


From 10e78ecf57242cc5030b903131bf27e027bb4a97 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 28 Jul 2019 06:00:51 +0200
Subject: Change contrast of status links that are not mentions nor hashtags
 (#11406)

---
 app/javascript/mastodon/components/status_content.js | 1 +
 app/javascript/styles/mastodon/components.scss       | 4 ++++
 2 files changed, 5 insertions(+)

(limited to 'app')

diff --git a/app/javascript/mastodon/components/status_content.js b/app/javascript/mastodon/components/status_content.js
index 8a05415af..871fab2dc 100644
--- a/app/javascript/mastodon/components/status_content.js
+++ b/app/javascript/mastodon/components/status_content.js
@@ -55,6 +55,7 @@ export default class StatusContent extends React.PureComponent {
         link.addEventListener('click', this.onHashtagClick.bind(this, link.text), false);
       } else {
         link.setAttribute('title', link.href);
+        link.classList.add('unhandled-link');
       }
 
       link.setAttribute('target', '_blank');
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index dd6306fb2..293de45fe 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -753,6 +753,10 @@
     }
   }
 
+  a.unhandled-link {
+    color: lighten($ui-highlight-color, 8%);
+  }
+
   .status__content__spoiler-link {
     background: $action-button-color;
 
-- 
cgit 


From 6a1f08d29a1059e7bc101cac5a626b3eeab6e1a8 Mon Sep 17 00:00:00 2001
From: Jeong Arm <kjwonmail@gmail.com>
Date: Sun, 28 Jul 2019 20:46:04 +0900
Subject: Prevent archiving when user set "noindex" (#11421)

---
 app/views/accounts/show.html.haml | 2 +-
 app/views/statuses/show.html.haml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml
index 0dc984dcc..6846abeb6 100644
--- a/app/views/accounts/show.html.haml
+++ b/app/views/accounts/show.html.haml
@@ -5,7 +5,7 @@
   %meta{ name: 'description', content: account_description(@account) }/
 
   - if @account.user&.setting_noindex
-    %meta{ name: 'robots', content: 'noindex' }/
+    %meta{ name: 'robots', content: 'noindex, noarchive' }/
 
   %link{ rel: 'alternate', type: 'application/atom+xml', href: account_url(@account, format: 'atom') }/
   %link{ rel: 'alternate', type: 'application/rss+xml', href: account_url(@account, format: 'rss') }/
diff --git a/app/views/statuses/show.html.haml b/app/views/statuses/show.html.haml
index 704e37a3d..0f22d106b 100644
--- a/app/views/statuses/show.html.haml
+++ b/app/views/statuses/show.html.haml
@@ -3,7 +3,7 @@
 
 - content_for :header_tags do
   - if @account.user&.setting_noindex
-    %meta{ name: 'robots', content: 'noindex' }/
+    %meta{ name: 'robots', content: 'noindex, noarchive' }/
 
   %link{ rel: 'alternate', type: 'application/json+oembed', href: api_oembed_url(url: short_account_status_url(@account, @status), format: 'json') }/
   %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@status) }/
-- 
cgit 


From 9349f1067a99c45648dc66f397b338d458d1a763 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 28 Jul 2019 13:48:05 +0200
Subject: Fix animate on hover in poll options without CW (#11404)

---
 .../mastodon/components/status_content.js          | 49 ++++++----------------
 1 file changed, 13 insertions(+), 36 deletions(-)

(limited to 'app')

diff --git a/app/javascript/mastodon/components/status_content.js b/app/javascript/mastodon/components/status_content.js
index 871fab2dc..549de95fc 100644
--- a/app/javascript/mastodon/components/status_content.js
+++ b/app/javascript/mastodon/components/status_content.js
@@ -234,46 +234,23 @@ export default class StatusContent extends React.PureComponent {
         </div>
       );
     } else if (this.props.onClick) {
-      const output = [
-        <div
-          ref={this.setRef}
-          tabIndex='0'
-          key='content'
-          className={classNames}
-          style={directionStyle}
-          dangerouslySetInnerHTML={content}
-          lang={status.get('language')}
-          onMouseDown={this.handleMouseDown}
-          onMouseUp={this.handleMouseUp}
-        />,
-      ];
-
-      if (this.state.collapsed) {
-        output.push(readMoreButton);
-      }
+      return (
+        <div className={classNames} ref={this.setRef} tabIndex='0' style={directionStyle} onMouseDown={this.handleMouseDown} onMouseUp={this.handleMouseUp}>
+          <div className='status__content__text status__content__text--visible' style={directionStyle} dangerouslySetInnerHTML={content} lang={status.get('language')} />
 
-      if (status.get('poll')) {
-        output.push(<PollContainer pollId={status.get('poll')} />);
-      }
+          {!!this.state.collapsed && readMoreButton}
 
-      return output;
+          {!!status.get('poll') && <PollContainer pollId={status.get('poll')} />}
+        </div>
+      );
     } else {
-      const output = [
-        <div
-          tabIndex='0'
-          ref={this.setRef}
-          className='status__content'
-          style={directionStyle}
-          dangerouslySetInnerHTML={content}
-          lang={status.get('language')}
-        />,
-      ];
-
-      if (status.get('poll')) {
-        output.push(<PollContainer pollId={status.get('poll')} />);
-      }
+      return (
+        <div className={classNames} ref={this.setRef} tabIndex='0' style={directionStyle}>
+          <div className='status__content__text status__content__text--visible' style={directionStyle} dangerouslySetInnerHTML={content} lang={status.get('language')} />
 
-      return output;
+          {!!status.get('poll') && <PollContainer pollId={status.get('poll')} />}
+        </div>
+      );
     }
   }
 
-- 
cgit 


From 0d80f686d81e4a12db1ac54b8f31f69f375dc8dc Mon Sep 17 00:00:00 2001
From: mayaeh <mayaeh@marimo-net.org>
Date: Sun, 28 Jul 2019 20:48:19 +0900
Subject: Add `tootctl preview_cards remove` (#11320)

* Add `tootctl preview_cards remove`

* fix code style

* Remove `Scheduler::PreviewCardsCleanupScheduler` file

* fix code style again
Add exclude case where image_file_name is blank

* Added a function to output confirmation if the specified number of days is less than 2 weeks
---
 .../scheduler/preview_cards_cleanup_scheduler.rb   | 22 -----
 config/sidekiq.yml                                 |  3 -
 lib/cli.rb                                         |  4 +
 lib/mastodon/preview_cards_cli.rb                  | 94 ++++++++++++++++++++++
 4 files changed, 98 insertions(+), 25 deletions(-)
 delete mode 100644 app/workers/scheduler/preview_cards_cleanup_scheduler.rb
 create mode 100644 lib/mastodon/preview_cards_cli.rb

(limited to 'app')

diff --git a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb b/app/workers/scheduler/preview_cards_cleanup_scheduler.rb
deleted file mode 100644
index 2b38792f0..000000000
--- a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-class Scheduler::PreviewCardsCleanupScheduler
-  include Sidekiq::Worker
-
-  sidekiq_options unique: :until_executed, retry: 0
-
-  def perform
-    Maintenance::UncachePreviewWorker.push_bulk(recent_link_preview_cards.pluck(:id))
-    Maintenance::UncachePreviewWorker.push_bulk(older_preview_cards.pluck(:id))
-  end
-
-  private
-
-  def recent_link_preview_cards
-    PreviewCard.where(type: :link).where('updated_at < ?', 1.month.ago)
-  end
-
-  def older_preview_cards
-    PreviewCard.where('updated_at < ?', 6.months.ago)
-  end
-end
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 7f41b6607..6ebe450b0 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -24,9 +24,6 @@
   ip_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
     class: Scheduler::IpCleanupScheduler
-  preview_cards_cleanup_scheduler:
-    cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
-    class: Scheduler::PreviewCardsCleanupScheduler
   email_scheduler:
     cron: '0 10 * * 2'
     class: Scheduler::EmailScheduler
diff --git a/lib/cli.rb b/lib/cli.rb
index be276583d..fbdf49fc3 100644
--- a/lib/cli.rb
+++ b/lib/cli.rb
@@ -9,6 +9,7 @@ require_relative 'mastodon/search_cli'
 require_relative 'mastodon/settings_cli'
 require_relative 'mastodon/statuses_cli'
 require_relative 'mastodon/domains_cli'
+require_relative 'mastodon/preview_cards_cli'
 require_relative 'mastodon/cache_cli'
 require_relative 'mastodon/version'
 
@@ -42,6 +43,9 @@ module Mastodon
     desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains'
     subcommand 'domains', Mastodon::DomainsCLI
 
+    desc 'preview_cards SUBCOMMAND ...ARGS', 'Manage preview cards'
+    subcommand 'preview_cards', Mastodon::PreviewCardsCLI
+
     desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
     subcommand 'cache', Mastodon::CacheCLI
 
diff --git a/lib/mastodon/preview_cards_cli.rb b/lib/mastodon/preview_cards_cli.rb
new file mode 100644
index 000000000..465fe7d0b
--- /dev/null
+++ b/lib/mastodon/preview_cards_cli.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'tty-prompt'
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+module Mastodon
+  class PreviewCardsCLI < Thor
+    include ActionView::Helpers::NumberHelper
+
+    def self.exit_on_failure?
+      true
+    end
+
+    option :days, type: :numeric, default: 180
+    option :background, type: :boolean, default: false
+    option :verbose, type: :boolean, default: false
+    option :dry_run, type: :boolean, default: false
+    option :link, type: :boolean, default: false
+    desc 'remove', 'Remove preview cards'
+    long_desc <<-DESC
+      Removes locally thumbnails for previews.
+
+      The --days option specifies how old preview cards have to be before
+      they are removed. It defaults to 180 days.
+
+      With the --background option, instead of deleting the files sequentially,
+      they will be queued into Sidekiq and the command will exit as soon as
+      possible. In Sidekiq they will be processed with higher concurrency, but
+      it may impact other operations of the Mastodon server, and it may overload
+      the underlying file storage.
+
+      With the --dry-run option, no work will be done.
+
+      With the --verbose option, when preview cards are processed sequentially in the
+      foreground, the IDs of the preview cards will be printed.
+
+      With the --link option, delete only link-type preview cards.
+    DESC
+    def remove
+      prompt    = TTY::Prompt.new
+      time_ago  = options[:days].days.ago
+      queued    = 0
+      processed = 0
+      size      = 0
+      dry_run   = options[:dry_run] ? '(DRY RUN)' : ''
+      link      = options[:link] ? 'link-type ' : ''
+      scope     = PreviewCard.where.not(image_file_name: nil)
+      scope     = scope.where.not(image_file_name: '')
+      scope     = scope.where(type: :link) if options[:link]
+      scope     = scope.where('updated_at < ?', time_ago)
+
+      if time_ago > 2.weeks.ago
+        prompt.say "\n"
+        prompt.say('The preview cards less than the past two weeks will not be re-acquired even when needed.')
+        prompt.say "\n"
+
+        unless prompt.yes?('Are you sure you want to delete the preview cards?', default: false)
+          prompt.say "\n"
+          prompt.warn 'Nothing execute. Bye!'
+          prompt.say "\n"
+          exit(1)
+        end
+      end
+
+      if options[:background]
+        scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards|
+          queued += preview_cards.size
+          size   += preview_cards.reduce(0) { |sum, p| sum + (p.image_file_size || 0) }
+          Maintenance::UncachePreviewWorker.push_bulk(preview_cards.map(&:id)) unless options[:dry_run]
+        end
+
+      else
+        scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards|
+          preview_cards.each do |p|
+            size += p.image_file_size || 0
+            Maintenance::UncachePreviewWorker.new.perform(p.id) unless options[:dry_run]
+            options[:verbose] ? say(p.id) : say('.', :green, false)
+            processed += 1
+          end
+        end
+      end
+
+      say
+
+      if options[:background]
+        say("Scheduled the deletion of #{queued} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
+      else
+        say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
+      end
+    end
+  end
+end
-- 
cgit 


From 15de24a425cb07efd58db5299b043e4110fa138f Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Sun, 28 Jul 2019 13:48:43 +0200
Subject: Bump json-ld-preloaded from 3.0.2 to 3.0.3 (#11316)

* Bump json-ld-preloaded from 3.0.2 to 3.0.3

Bumps [json-ld-preloaded](https://github.com/ruby-rdf/json-ld-preloaded) from 3.0.2 to 3.0.3.
- [Release notes](https://github.com/ruby-rdf/json-ld-preloaded/releases)
- [Commits](https://github.com/ruby-rdf/json-ld-preloaded/compare/3.0.2...3.0.3)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>

* use json-ld edge
---
 Gemfile                      |  2 +-
 Gemfile.lock                 | 23 +++++++++++++++++------
 app/helpers/jsonld_helper.rb |  2 +-
 lib/json_ld/security.rb      |  5 +++--
 4 files changed, 22 insertions(+), 10 deletions(-)

(limited to 'app')

diff --git a/Gemfile b/Gemfile
index 96eb44af7..04ebc4cd8 100644
--- a/Gemfile
+++ b/Gemfile
@@ -92,7 +92,7 @@ gem 'tzinfo-data', '~> 1.2019'
 gem 'webpacker', '~> 4.0'
 gem 'webpush'
 
-gem 'json-ld', '~> 3.0'
+gem 'json-ld', git: 'https://github.com/ruby-rdf/json-ld.git', ref: '345b7a5733308af827e8491d284dbafa9128d7a2'
 gem 'json-ld-preloaded', '~> 3.0'
 gem 'rdf-normalize', '~> 0.3'
 
diff --git a/Gemfile.lock b/Gemfile.lock
index 5adae6f9c..8da261639 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -5,6 +5,19 @@ GIT
   specs:
     posix-spawn (0.3.13)
 
+GIT
+  remote: https://github.com/ruby-rdf/json-ld.git
+  revision: 345b7a5733308af827e8491d284dbafa9128d7a2
+  ref: 345b7a5733308af827e8491d284dbafa9128d7a2
+  specs:
+    json-ld (3.0.2)
+      htmlentities (~> 4.3)
+      json-canonicalization (~> 0.1)
+      link_header (~> 0.0, >= 0.0.8)
+      multi_json (~> 1.13)
+      rack (>= 1.6, < 3.0)
+      rdf (~> 3.0, >= 3.0.8)
+
 GIT
   remote: https://github.com/tmm1/http_parser.rb
   revision: 54b17ba8c7d8d20a16dfc65d1775241833219cf2
@@ -297,10 +310,8 @@ GEM
     jaro_winkler (1.5.3)
     jmespath (1.4.0)
     json (2.2.0)
-    json-ld (3.0.2)
-      multi_json (~> 1.12)
-      rdf (>= 2.2.8, < 4.0)
-    json-ld-preloaded (3.0.2)
+    json-canonicalization (0.1.0)
+    json-ld-preloaded (3.0.3)
       json-ld (~> 3.0)
       multi_json (~> 1.12)
       rdf (~> 3.0)
@@ -479,7 +490,7 @@ GEM
       thor (>= 0.19.0, < 2.0)
     rainbow (3.0.0)
     rake (12.3.2)
-    rdf (3.0.9)
+    rdf (3.0.12)
       hamster (~> 3.0)
       link_header (~> 0.0, >= 0.0.8)
     rdf-normalize (0.3.3)
@@ -700,7 +711,7 @@ DEPENDENCIES
   i18n-tasks (~> 0.9)
   idn-ruby
   iso-639
-  json-ld (~> 3.0)
+  json-ld!
   json-ld-preloaded (~> 3.0)
   kaminari (~> 1.1)
   letter_opener (~> 1.7)
diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index 83a5b2462..1c473efa3 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -130,7 +130,7 @@ module JsonLdHelper
       end
     end
 
-    doc = JSON::LD::API::RemoteDocument.new(url, json)
+    doc = JSON::LD::API::RemoteDocument.new(json, documentUrl: url)
 
     block_given? ? yield(doc) : doc
   end
diff --git a/lib/json_ld/security.rb b/lib/json_ld/security.rb
index 1230206f0..a6fbce95f 100644
--- a/lib/json_ld/security.rb
+++ b/lib/json_ld/security.rb
@@ -1,9 +1,9 @@
 # -*- encoding: utf-8 -*-
 # frozen_string_literal: true
-# This file generated automatically from https://w3id.org/security/v1
+# This file generated automatically from http://w3id.org/security/v1
 require 'json/ld'
 class JSON::LD::Context
-  add_preloaded("https://w3id.org/security/v1") do
+  add_preloaded("http://w3id.org/security/v1") do
     new(processingMode: "json-ld-1.0", term_definitions: {
       "CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true),
       "EcdsaKoblitzSignature2016" => TermDefinition.new("EcdsaKoblitzSignature2016", id: "https://w3id.org/security#EcdsaKoblitzSignature2016", simple: true),
@@ -47,4 +47,5 @@ class JSON::LD::Context
       "xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true)
     })
   end
+  alias_preloaded("https://w3id.org/security/v1", "http://w3id.org/security/v1")
 end
-- 
cgit 


From cfb2ed78231758a79af038a964ab7f7b7b35274e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 14:37:52 +0200
Subject: Add autosuggestions for hashtags (#11422)

---
 app/javascript/mastodon/actions/compose.js         | 41 ++++++++++++++++++----
 .../mastodon/components/autosuggest_hashtag.js     | 28 +++++++++++++++
 .../mastodon/components/autosuggest_input.js       |  9 ++---
 .../mastodon/components/autosuggest_textarea.js    |  9 ++---
 app/javascript/mastodon/reducers/compose.js        | 36 ++++++++++++-------
 app/javascript/styles/mastodon/components.scss     | 11 +++++-
 6 files changed, 107 insertions(+), 27 deletions(-)
 create mode 100644 app/javascript/mastodon/components/autosuggest_hashtag.js

(limited to 'app')

diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js
index fbf97d374..cd9955505 100644
--- a/app/javascript/mastodon/actions/compose.js
+++ b/app/javascript/mastodon/actions/compose.js
@@ -11,7 +11,7 @@ import { showAlertForError } from './alerts';
 import { showAlert } from './alerts';
 import { defineMessages } from 'react-intl';
 
-let cancelFetchComposeSuggestionsAccounts;
+let cancelFetchComposeSuggestionsAccounts, cancelFetchComposeSuggestionsTags;
 
 export const COMPOSE_CHANGE          = 'COMPOSE_CHANGE';
 export const COMPOSE_SUBMIT_REQUEST  = 'COMPOSE_SUBMIT_REQUEST';
@@ -325,10 +325,12 @@ const fetchComposeSuggestionsAccounts = throttle((dispatch, getState, token) =>
   if (cancelFetchComposeSuggestionsAccounts) {
     cancelFetchComposeSuggestionsAccounts();
   }
+
   api(getState).get('/api/v1/accounts/search', {
     cancelToken: new CancelToken(cancel => {
       cancelFetchComposeSuggestionsAccounts = cancel;
     }),
+
     params: {
       q: token.slice(1),
       resolve: false,
@@ -349,9 +351,30 @@ const fetchComposeSuggestionsEmojis = (dispatch, getState, token) => {
   dispatch(readyComposeSuggestionsEmojis(token, results));
 };
 
-const fetchComposeSuggestionsTags = (dispatch, getState, token) => {
-  dispatch(updateSuggestionTags(token));
-};
+const fetchComposeSuggestionsTags = throttle((dispatch, getState, token) => {
+  if (cancelFetchComposeSuggestionsTags) {
+    cancelFetchComposeSuggestionsTags();
+  }
+
+  api(getState).get('/api/v2/search', {
+    cancelToken: new CancelToken(cancel => {
+      cancelFetchComposeSuggestionsTags = cancel;
+    }),
+
+    params: {
+      type: 'hashtags',
+      q: token.slice(1),
+      resolve: false,
+      limit: 4,
+    },
+  }).then(({ data }) => {
+    dispatch(readyComposeSuggestionsTags(token, data.hashtags));
+  }).catch(error => {
+    if (!isCancel(error)) {
+      dispatch(showAlertForError(error));
+    }
+  });
+}, 200, { leading: true, trailing: true });
 
 export function fetchComposeSuggestions(token) {
   return (dispatch, getState) => {
@@ -385,6 +408,12 @@ export function readyComposeSuggestionsAccounts(token, accounts) {
   };
 };
 
+export const readyComposeSuggestionsTags = (token, tags) => ({
+  type: COMPOSE_SUGGESTIONS_READY,
+  token,
+  tags,
+});
+
 export function selectComposeSuggestion(position, token, suggestion, path) {
   return (dispatch, getState) => {
     let completion, startPosition;
@@ -394,8 +423,8 @@ export function selectComposeSuggestion(position, token, suggestion, path) {
       startPosition = position - 1;
 
       dispatch(useEmoji(suggestion));
-    } else if (suggestion[0] === '#') {
-      completion    = suggestion;
+    } else if (typeof suggestion === 'object' && suggestion.name) {
+      completion    = `#${suggestion.name}`;
       startPosition = position - 1;
     } else {
       completion    = getState().getIn(['accounts', suggestion, 'acct']);
diff --git a/app/javascript/mastodon/components/autosuggest_hashtag.js b/app/javascript/mastodon/components/autosuggest_hashtag.js
new file mode 100644
index 000000000..eabb8b178
--- /dev/null
+++ b/app/javascript/mastodon/components/autosuggest_hashtag.js
@@ -0,0 +1,28 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { shortNumberFormat } from 'mastodon/utils/numbers';
+import { FormattedMessage } from 'react-intl';
+
+export default class AutosuggestHashtag extends React.PureComponent {
+
+  static propTypes = {
+    tag: PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      url: PropTypes.string,
+      history: PropTypes.array.isRequired,
+    }).isRequired,
+  };
+
+  render () {
+    const { tag } = this.props;
+    const weeklyUses = shortNumberFormat(tag.history.reduce((total, day) => total + (day.uses * 1), 0));
+
+    return (
+      <div className='autosuggest-hashtag'>
+        <div className='autosuggest-hashtag__name'>#<strong>{tag.name}</strong></div>
+        <div className='autosuggest-hashtag__uses'><FormattedMessage id='autosuggest_hashtag.per_week' defaultMessage='{count} per week' values={{ count: weeklyUses }} /></div>
+      </div>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/components/autosuggest_input.js b/app/javascript/mastodon/components/autosuggest_input.js
index c7d965b53..0df8bf64e 100644
--- a/app/javascript/mastodon/components/autosuggest_input.js
+++ b/app/javascript/mastodon/components/autosuggest_input.js
@@ -1,6 +1,7 @@
 import React from 'react';
 import AutosuggestAccountContainer from '../features/compose/containers/autosuggest_account_container';
 import AutosuggestEmoji from './autosuggest_emoji';
+import AutosuggestHashtag from './autosuggest_hashtag';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import PropTypes from 'prop-types';
 import { isRtl } from '../rtl';
@@ -167,12 +168,12 @@ export default class AutosuggestInput extends ImmutablePureComponent {
     const { selectedSuggestion } = this.state;
     let inner, key;
 
-    if (typeof suggestion === 'object') {
+    if (typeof suggestion === 'object' && suggestion.shortcode) {
       inner = <AutosuggestEmoji emoji={suggestion} />;
       key   = suggestion.id;
-    } else if (suggestion[0] === '#') {
-      inner = suggestion;
-      key   = suggestion;
+    } else if (typeof suggestion === 'object' && suggestion.name) {
+      inner = <AutosuggestHashtag tag={suggestion} />;
+      key   = suggestion.name;
     } else {
       inner = <AutosuggestAccountContainer id={suggestion} />;
       key   = suggestion;
diff --git a/app/javascript/mastodon/components/autosuggest_textarea.js b/app/javascript/mastodon/components/autosuggest_textarea.js
index b070fe3e5..2bd06d28a 100644
--- a/app/javascript/mastodon/components/autosuggest_textarea.js
+++ b/app/javascript/mastodon/components/autosuggest_textarea.js
@@ -1,6 +1,7 @@
 import React from 'react';
 import AutosuggestAccountContainer from '../features/compose/containers/autosuggest_account_container';
 import AutosuggestEmoji from './autosuggest_emoji';
+import AutosuggestHashtag from './autosuggest_hashtag';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import PropTypes from 'prop-types';
 import { isRtl } from '../rtl';
@@ -173,12 +174,12 @@ export default class AutosuggestTextarea extends ImmutablePureComponent {
     const { selectedSuggestion } = this.state;
     let inner, key;
 
-    if (typeof suggestion === 'object') {
+    if (typeof suggestion === 'object' && suggestion.shortcode) {
       inner = <AutosuggestEmoji emoji={suggestion} />;
       key   = suggestion.id;
-    } else if (suggestion[0] === '#') {
-      inner = suggestion;
-      key   = suggestion;
+    } else if (typeof suggestion === 'object' && suggestion.name) {
+      inner = <AutosuggestHashtag tag={suggestion} />;
+      key   = suggestion.name;
     } else {
       inner = <AutosuggestAccountContainer id={suggestion} />;
       key   = suggestion;
diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js
index fae7522b2..fc6d5f30f 100644
--- a/app/javascript/mastodon/reducers/compose.js
+++ b/app/javascript/mastodon/reducers/compose.js
@@ -17,7 +17,6 @@ import {
   COMPOSE_SUGGESTIONS_CLEAR,
   COMPOSE_SUGGESTIONS_READY,
   COMPOSE_SUGGESTION_SELECT,
-  COMPOSE_SUGGESTION_TAGS_UPDATE,
   COMPOSE_TAG_HISTORY_UPDATE,
   COMPOSE_SENSITIVITY_CHANGE,
   COMPOSE_SPOILERNESS_CHANGE,
@@ -144,15 +143,20 @@ const insertSuggestion = (state, position, token, completion, path) => {
   });
 };
 
-const updateSuggestionTags = (state, token) => {
-  const prefix = token.slice(1);
+const sortHashtagsByUse = (state, tags) => {
+  const personalHistory = state.get('tagHistory');
 
-  return state.merge({
-    suggestions: state.get('tagHistory')
-      .filter(tag => tag.toLowerCase().startsWith(prefix.toLowerCase()))
-      .slice(0, 4)
-      .map(tag => '#' + tag),
-    suggestion_token: token,
+  return tags.sort((a, b) => {
+    const usedA = personalHistory.includes(a.name);
+    const usedB = personalHistory.includes(b.name);
+
+    if (usedA === usedB) {
+      return 0;
+    } else if (usedA && !usedB) {
+      return 1;
+    } else {
+      return -1;
+    }
   });
 };
 
@@ -201,6 +205,16 @@ const expiresInFromExpiresAt = expires_at => {
   return [300, 1800, 3600, 21600, 86400, 259200, 604800].find(expires_in => expires_in >= delta) || 24 * 3600;
 };
 
+const normalizeSuggestions = (state, { accounts, emojis, tags }) => {
+  if (accounts) {
+    return accounts.map(item => item.id);
+  } else if (emojis) {
+    return emojis;
+  } else {
+    return sortHashtagsByUse(state, tags);
+  }
+};
+
 export default function compose(state = initialState, action) {
   switch(action.type) {
   case STORE_HYDRATE:
@@ -311,11 +325,9 @@ export default function compose(state = initialState, action) {
   case COMPOSE_SUGGESTIONS_CLEAR:
     return state.update('suggestions', ImmutableList(), list => list.clear()).set('suggestion_token', null);
   case COMPOSE_SUGGESTIONS_READY:
-    return state.set('suggestions', ImmutableList(action.accounts ? action.accounts.map(item => item.id) : action.emojis)).set('suggestion_token', action.token);
+    return state.set('suggestions', ImmutableList(normalizeSuggestions(state, action))).set('suggestion_token', action.token);
   case COMPOSE_SUGGESTION_SELECT:
     return insertSuggestion(state, action.position, action.token, action.completion, action.path);
-  case COMPOSE_SUGGESTION_TAGS_UPDATE:
-    return updateSuggestionTags(state, action.token);
   case COMPOSE_TAG_HISTORY_UPDATE:
     return state.set('tagHistory', fromJS(action.tags));
   case TIMELINE_DELETE:
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 293de45fe..c5d3f2e78 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -445,7 +445,8 @@
   }
 
   .autosuggest-account,
-  .autosuggest-emoji {
+  .autosuggest-emoji,
+  .autosuggest-hashtag {
     display: flex;
     flex-direction: row;
     align-items: center;
@@ -454,6 +455,14 @@
     font-size: 14px;
   }
 
+  .autosuggest-hashtag {
+    justify-content: space-between;
+
+    strong {
+      font-weight: 500;
+    }
+  }
+
   .autosuggest-account-icon,
   .autosuggest-emoji img {
     display: block;
-- 
cgit 


From 31fc3be0a4342dbe91480d058b0afeddb272ed3b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 25 Jul 2019 04:17:35 +0200
Subject: [Glitch] Change account domain block to clear out notifications and
 follows

Port 4eeff26533b75b592395e353c6d4506db9958bcf to glitch-soc

Signed-off-by: Thibaut Girka <thib@sitedethib.com>
---
 app/javascript/flavours/glitch/actions/domain_blocks.js  |  1 +
 app/javascript/flavours/glitch/reducers/conversations.js | 11 +++++++++++
 app/javascript/flavours/glitch/reducers/notifications.js | 11 +++++++----
 app/javascript/flavours/glitch/reducers/suggestions.js   |  7 +++++++
 4 files changed, 26 insertions(+), 4 deletions(-)

(limited to 'app')

diff --git a/app/javascript/flavours/glitch/actions/domain_blocks.js b/app/javascript/flavours/glitch/actions/domain_blocks.js
index 7397f561b..6d3f471fa 100644
--- a/app/javascript/flavours/glitch/actions/domain_blocks.js
+++ b/app/javascript/flavours/glitch/actions/domain_blocks.js
@@ -23,6 +23,7 @@ export function blockDomain(domain) {
     api(getState).post('/api/v1/domain_blocks', { domain }).then(() => {
       const at_domain = '@' + domain;
       const accounts = getState().get('accounts').filter(item => item.get('acct').endsWith(at_domain)).valueSeq().map(item => item.get('id'));
+
       dispatch(blockDomainSuccess(domain, accounts));
     }).catch(err => {
       dispatch(blockDomainFail(domain, err));
diff --git a/app/javascript/flavours/glitch/reducers/conversations.js b/app/javascript/flavours/glitch/reducers/conversations.js
index c01659da5..8fcc2cc79 100644
--- a/app/javascript/flavours/glitch/reducers/conversations.js
+++ b/app/javascript/flavours/glitch/reducers/conversations.js
@@ -8,6 +8,8 @@ import {
   CONVERSATIONS_UPDATE,
   CONVERSATIONS_READ,
 } from '../actions/conversations';
+import { ACCOUNT_BLOCK_SUCCESS, ACCOUNT_MUTE_SUCCESS } from 'flavours/glitch/actions/accounts';
+import { DOMAIN_BLOCK_SUCCESS } from 'flavours/glitch/actions/domain_blocks';
 import compareId from 'flavours/glitch/util/compare_id';
 
 const initialState = ImmutableMap({
@@ -74,6 +76,10 @@ const expandNormalizedConversations = (state, conversations, next, isLoadingRece
   });
 };
 
+const filterConversations = (state, accountIds) => {
+  return state.update('items', list => list.filterNot(item => item.get('accounts').some(accountId => accountIds.includes(accountId))));
+};
+
 export default function conversations(state = initialState, action) {
   switch (action.type) {
   case CONVERSATIONS_FETCH_REQUEST:
@@ -96,6 +102,11 @@ export default function conversations(state = initialState, action) {
 
       return item;
     }));
+  case ACCOUNT_BLOCK_SUCCESS:
+  case ACCOUNT_MUTE_SUCCESS:
+    return filterConversations(state, [action.relationship.id]);
+  case DOMAIN_BLOCK_SUCCESS:
+    return filterConversations(state, action.accounts);
   default:
     return state;
   }
diff --git a/app/javascript/flavours/glitch/reducers/notifications.js b/app/javascript/flavours/glitch/reducers/notifications.js
index d057f8f83..135995da6 100644
--- a/app/javascript/flavours/glitch/reducers/notifications.js
+++ b/app/javascript/flavours/glitch/reducers/notifications.js
@@ -21,6 +21,7 @@ import {
   ACCOUNT_BLOCK_SUCCESS,
   ACCOUNT_MUTE_SUCCESS,
 } from 'flavours/glitch/actions/accounts';
+import { DOMAIN_BLOCK_SUCCESS } from 'flavours/glitch/actions/domain_blocks';
 import { TIMELINE_DELETE, TIMELINE_DISCONNECT } from 'flavours/glitch/actions/timelines';
 import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import compareId from 'flavours/glitch/util/compare_id';
@@ -110,8 +111,8 @@ const expandNormalizedNotifications = (state, notifications, next, usePendingIte
   });
 };
 
-const filterNotifications = (state, relationship) => {
-  const helper = list => list.filterNot(item => item !== null && item.get('account') === relationship.id);
+const filterNotifications = (state, accountIds) => {
+  const helper = list => list.filterNot(item => item !== null && accountIds.includes(item.get('account')));
   return state.update('items', helper).update('pendingItems', helper);
 };
 
@@ -217,9 +218,11 @@ export default function notifications(state = initialState, action) {
   case NOTIFICATIONS_EXPAND_SUCCESS:
     return expandNormalizedNotifications(state, action.notifications, action.next, action.usePendingItems);
   case ACCOUNT_BLOCK_SUCCESS:
-    return filterNotifications(state, action.relationship);
+    return filterNotifications(state, [action.relationship.id]);
   case ACCOUNT_MUTE_SUCCESS:
-    return action.relationship.muting_notifications ? filterNotifications(state, action.relationship) : state;
+    return action.relationship.muting_notifications ? filterNotifications(state, [action.relationship.id]) : state;
+  case DOMAIN_BLOCK_SUCCESS:
+    return filterNotifications(state, action.accounts);
   case NOTIFICATIONS_CLEAR:
     return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', false);
   case TIMELINE_DELETE:
diff --git a/app/javascript/flavours/glitch/reducers/suggestions.js b/app/javascript/flavours/glitch/reducers/suggestions.js
index 9f4b89d58..834be728f 100644
--- a/app/javascript/flavours/glitch/reducers/suggestions.js
+++ b/app/javascript/flavours/glitch/reducers/suggestions.js
@@ -4,6 +4,8 @@ import {
   SUGGESTIONS_FETCH_FAIL,
   SUGGESTIONS_DISMISS,
 } from '../actions/suggestions';
+import { ACCOUNT_BLOCK_SUCCESS, ACCOUNT_MUTE_SUCCESS } from 'mastodon/actions/accounts';
+import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks';
 import { Map as ImmutableMap, List as ImmutableList, fromJS } from 'immutable';
 
 const initialState = ImmutableMap({
@@ -24,6 +26,11 @@ export default function suggestionsReducer(state = initialState, action) {
     return state.set('isLoading', false);
   case SUGGESTIONS_DISMISS:
     return state.update('items', list => list.filterNot(id => id === action.id));
+  case ACCOUNT_BLOCK_SUCCESS:
+  case ACCOUNT_MUTE_SUCCESS:
+    return state.update('items', list => list.filterNot(id => id === action.relationship.id));
+  case DOMAIN_BLOCK_SUCCESS:
+    return state.update('items', list => list.filterNot(id => action.accounts.includes(id)));
   default:
     return state;
   }
-- 
cgit 


From 7f147acea61f1f37928f38eef507b34c98e003fd Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 28 Jul 2019 06:00:51 +0200
Subject: [Glitch] Change contrast of status links that are not mentions nor
 hashtags

Port 10e78ecf57242cc5030b903131bf27e027bb4a97 to glitch-soc

Signed-off-by: Thibaut Girka <thib@sitedethib.com>
---
 app/javascript/flavours/glitch/components/status_content.js  | 1 +
 app/javascript/flavours/glitch/styles/components/status.scss | 4 ++++
 2 files changed, 5 insertions(+)

(limited to 'app')

diff --git a/app/javascript/flavours/glitch/components/status_content.js b/app/javascript/flavours/glitch/components/status_content.js
index befbe340f..602a28064 100644
--- a/app/javascript/flavours/glitch/components/status_content.js
+++ b/app/javascript/flavours/glitch/components/status_content.js
@@ -51,6 +51,7 @@ export default class StatusContent extends React.PureComponent {
       } else {
         link.addEventListener('click', this.onLinkClick.bind(this), false);
         link.setAttribute('title', link.href);
+        link.classList.add('unhandled-link');
       }
 
       link.setAttribute('target', '_blank');
diff --git a/app/javascript/flavours/glitch/styles/components/status.scss b/app/javascript/flavours/glitch/styles/components/status.scss
index ccc6da594..803494df6 100644
--- a/app/javascript/flavours/glitch/styles/components/status.scss
+++ b/app/javascript/flavours/glitch/styles/components/status.scss
@@ -133,6 +133,10 @@
     }
   }
 
+  a.unhandled-link {
+    color: lighten($ui-highlight-color, 8%);
+  }
+
   .status__content__spoiler-link {
     background: lighten($ui-base-color, 30%);
 
-- 
cgit 


From 51411267fda00db576230a270a10e31992378c18 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 27 Jul 2019 05:49:50 +0200
Subject: [Glitch] Add search results pagination to web UI (#11409)

Port 8a4674f2c3d89c998eb5438b96b7977dc2be3167 to glitch-soc

Signed-off-by: Thibaut Girka <thib@sitedethib.com>
---
 app/javascript/flavours/glitch/actions/search.js   | 54 ++++++++++++++++++++--
 .../features/compose/components/search_results.js  | 18 +++++++-
 .../compose/containers/search_results_container.js |  4 +-
 app/javascript/flavours/glitch/reducers/search.js  |  3 ++
 .../flavours/glitch/styles/components/search.scss  |  5 +-
 5 files changed, 76 insertions(+), 8 deletions(-)

(limited to 'app')

diff --git a/app/javascript/flavours/glitch/actions/search.js b/app/javascript/flavours/glitch/actions/search.js
index 9ce77b24b..a025f352a 100644
--- a/app/javascript/flavours/glitch/actions/search.js
+++ b/app/javascript/flavours/glitch/actions/search.js
@@ -10,6 +10,10 @@ export const SEARCH_FETCH_REQUEST = 'SEARCH_FETCH_REQUEST';
 export const SEARCH_FETCH_SUCCESS = 'SEARCH_FETCH_SUCCESS';
 export const SEARCH_FETCH_FAIL    = 'SEARCH_FETCH_FAIL';
 
+export const SEARCH_EXPAND_REQUEST = 'SEARCH_EXPAND_REQUEST';
+export const SEARCH_EXPAND_SUCCESS = 'SEARCH_EXPAND_SUCCESS';
+export const SEARCH_EXPAND_FAIL    = 'SEARCH_EXPAND_FAIL';
+
 export function changeSearch(value) {
   return {
     type: SEARCH_CHANGE,
@@ -77,8 +81,50 @@ export function fetchSearchFail(error) {
   };
 };
 
-export function showSearch() {
-  return {
-    type: SEARCH_SHOW,
-  };
+export const expandSearch = type => (dispatch, getState) => {
+  const value  = getState().getIn(['search', 'value']);
+  const offset = getState().getIn(['search', 'results', type]).size;
+
+  dispatch(expandSearchRequest());
+
+  api(getState).get('/api/v2/search', {
+    params: {
+      q: value,
+      type,
+      offset,
+    },
+  }).then(({ data }) => {
+    if (data.accounts) {
+      dispatch(importFetchedAccounts(data.accounts));
+    }
+
+    if (data.statuses) {
+      dispatch(importFetchedStatuses(data.statuses));
+    }
+
+    dispatch(expandSearchSuccess(data, value, type));
+    dispatch(fetchRelationships(data.accounts.map(item => item.id)));
+  }).catch(error => {
+    dispatch(expandSearchFail(error));
+  });
 };
+
+export const expandSearchRequest = () => ({
+  type: SEARCH_EXPAND_REQUEST,
+});
+
+export const expandSearchSuccess = (results, searchTerm, searchType) => ({
+  type: SEARCH_EXPAND_SUCCESS,
+  results,
+  searchTerm,
+  searchType,
+});
+
+export const expandSearchFail = error => ({
+  type: SEARCH_EXPAND_FAIL,
+  error,
+});
+
+export const showSearch = () => ({
+  type: SEARCH_SHOW,
+});
diff --git a/app/javascript/flavours/glitch/features/compose/components/search_results.js b/app/javascript/flavours/glitch/features/compose/components/search_results.js
index dd99f3430..7220d8529 100644
--- a/app/javascript/flavours/glitch/features/compose/components/search_results.js
+++ b/app/javascript/flavours/glitch/features/compose/components/search_results.js
@@ -8,6 +8,7 @@ import ImmutablePureComponent from 'react-immutable-pure-component';
 import Hashtag from 'flavours/glitch/components/hashtag';
 import Icon from 'flavours/glitch/components/icon';
 import { searchEnabled } from 'flavours/glitch/util/initial_state';
+import LoadMore from 'flavours/glitch/components/load_more';
 
 const messages = defineMessages({
   dismissSuggestion: { id: 'suggestions.dismiss', defaultMessage: 'Dismiss suggestion' },
@@ -20,15 +21,24 @@ class SearchResults extends ImmutablePureComponent {
     results: ImmutablePropTypes.map.isRequired,
     suggestions: ImmutablePropTypes.list.isRequired,
     fetchSuggestions: PropTypes.func.isRequired,
+    expandSearch: PropTypes.func.isRequired,
     dismissSuggestion: PropTypes.func.isRequired,
     searchTerm: PropTypes.string,
     intl: PropTypes.object.isRequired,
   };
 
   componentDidMount () {
-    this.props.fetchSuggestions();
+    if (this.props.searchTerm === '') {
+      this.props.fetchSuggestions();
+    }
   }
 
+  handleLoadMoreAccounts = () => this.props.expandSearch('accounts');
+
+  handleLoadMoreStatuses = () => this.props.expandSearch('statuses');
+
+  handleLoadMoreHashtags = () => this.props.expandSearch('hashtags');
+
   render () {
     const { intl, results, suggestions, dismissSuggestion, searchTerm } = this.props;
 
@@ -75,6 +85,8 @@ class SearchResults extends ImmutablePureComponent {
           <h5><Icon icon='users' fixedWidth /><FormattedMessage id='search_results.accounts' defaultMessage='People' /></h5>
 
           {results.get('accounts').map(accountId => <AccountContainer id={accountId} key={accountId} />)}
+
+          {results.get('accounts').size >= 5 && <LoadMore visible onClick={this.handleLoadMoreAccounts} />}
         </section>
       );
     }
@@ -86,6 +98,8 @@ class SearchResults extends ImmutablePureComponent {
           <h5><Icon icon='quote-right' fixedWidth /><FormattedMessage id='search_results.statuses' defaultMessage='Toots' /></h5>
 
           {results.get('statuses').map(statusId => <StatusContainer id={statusId} key={statusId}/>)}
+
+          {results.get('statuses').size >= 5 && <LoadMore visible onClick={this.handleLoadMoreStatuses} />}
         </section>
       );
     }
@@ -97,6 +111,8 @@ class SearchResults extends ImmutablePureComponent {
           <h5><Icon icon='hashtag' fixedWidth /><FormattedMessage id='search_results.hashtags' defaultMessage='Hashtags' /></h5>
 
           {results.get('hashtags').map(hashtag => <Hashtag key={hashtag.get('name')} hashtag={hashtag} />)}
+
+          {results.get('hashtags').size >= 5 && <LoadMore visible onClick={this.handleLoadMoreHashtags} />}
         </section>
       );
     }
diff --git a/app/javascript/flavours/glitch/features/compose/containers/search_results_container.js b/app/javascript/flavours/glitch/features/compose/containers/search_results_container.js
index e4d5f3420..1f714ff83 100644
--- a/app/javascript/flavours/glitch/features/compose/containers/search_results_container.js
+++ b/app/javascript/flavours/glitch/features/compose/containers/search_results_container.js
@@ -1,6 +1,7 @@
 import { connect } from 'react-redux';
 import SearchResults from '../components/search_results';
-import { fetchSuggestions, dismissSuggestion } from '../../../actions/suggestions';
+import { fetchSuggestions, dismissSuggestion } from 'mastodon/actions/suggestions';
+import { expandSearch } from 'mastodon/actions/search';
 
 const mapStateToProps = state => ({
   results: state.getIn(['search', 'results']),
@@ -10,6 +11,7 @@ const mapStateToProps = state => ({
 
 const mapDispatchToProps = dispatch => ({
   fetchSuggestions: () => dispatch(fetchSuggestions()),
+  expandSearch: type => dispatch(expandSearch(type)),
   dismissSuggestion: account => dispatch(dismissSuggestion(account.get('id'))),
 });
 
diff --git a/app/javascript/flavours/glitch/reducers/search.js b/app/javascript/flavours/glitch/reducers/search.js
index 1c32a5b9f..f4d99a99a 100644
--- a/app/javascript/flavours/glitch/reducers/search.js
+++ b/app/javascript/flavours/glitch/reducers/search.js
@@ -3,6 +3,7 @@ import {
   SEARCH_CLEAR,
   SEARCH_FETCH_SUCCESS,
   SEARCH_SHOW,
+  SEARCH_EXPAND_SUCCESS,
 } from 'flavours/glitch/actions/search';
 import {
   COMPOSE_MENTION,
@@ -42,6 +43,8 @@ export default function search(state = initialState, action) {
       statuses: ImmutableList(action.results.statuses.map(item => item.id)),
       hashtags: fromJS(action.results.hashtags),
     })).set('submitted', true).set('searchTerm', action.searchTerm);
+  case SEARCH_EXPAND_SUCCESS:
+    return state.updateIn(['results', action.searchType], list => list.concat(action.results[action.searchType].map(item => item.id)));
   default:
     return state;
   }
diff --git a/app/javascript/flavours/glitch/styles/components/search.scss b/app/javascript/flavours/glitch/styles/components/search.scss
index 117da362f..0e518997d 100644
--- a/app/javascript/flavours/glitch/styles/components/search.scss
+++ b/app/javascript/flavours/glitch/styles/components/search.scss
@@ -79,8 +79,9 @@
 }
 
 .search-results__info {
-  padding: 10px;
-  color: $secondary-text-color;
+  padding: 20px;
+  color: $darker-text-color;
+  text-align: center;
 }
 
 .trends {
-- 
cgit 


From 75f7f9930eb2a6f5c4041ec44fe0aa795c9ec449 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 17:30:12 +0200
Subject: Remove conversation URI (#11423)

It is not part of ActivityPub and will free up a lot of space
---
 app/lib/activitypub/activity/create.rb                    | 11 -----------
 app/models/conversation.rb                                |  7 -------
 app/serializers/activitypub/note_serializer.rb            | 15 ++-------------
 .../20190519130537_remove_boosts_widening_audience.rb     |  2 ++
 .../20190728084117_remove_uri_from_conversations.rb       | 12 ++++++++++++
 db/schema.rb                                              |  4 +---
 spec/models/conversation_spec.rb                          |  8 --------
 7 files changed, 17 insertions(+), 42 deletions(-)
 create mode 100644 db/post_migrate/20190728084117_remove_uri_from_conversations.rb

(limited to 'app')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 000b77df5..eb7b09e44 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -67,7 +67,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         sensitive: @object['sensitive'] || false,
         visibility: visibility_from_audience,
         thread: replied_to_status,
-        conversation: conversation_from_uri(@object['conversation']),
         media_attachment_ids: process_attachments.take(4).map(&:id),
         poll: process_poll,
       }
@@ -262,16 +261,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
   end
 
-  def conversation_from_uri(uri)
-    return nil if uri.nil?
-    return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
-    begin
-      Conversation.find_or_create_by!(uri: uri)
-    rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
-      retry
-    end
-  end
-
   def visibility_from_audience
     if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public])
       :public
diff --git a/app/models/conversation.rb b/app/models/conversation.rb
index 4dfaea889..37d233f32 100644
--- a/app/models/conversation.rb
+++ b/app/models/conversation.rb
@@ -4,17 +4,10 @@
 # Table name: conversations
 #
 #  id         :bigint(8)        not null, primary key
-#  uri        :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #
 
 class Conversation < ApplicationRecord
-  validates :uri, uniqueness: true, if: :uri?
-
   has_many :statuses
-
-  def local?
-    uri.nil?
-  end
 end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 67f596e78..88b50b3ea 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
-  context_extensions :atom_uri, :conversation, :sensitive,
+  context_extensions :atom_uri, :sensitive,
                      :hashtag, :emoji, :focal_point, :blurhash
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
              :attributed_to, :to, :cc, :sensitive,
-             :atom_uri, :in_reply_to_atom_uri,
-             :conversation
+             :atom_uri, :in_reply_to_atom_uri
 
   attribute :content
   attribute :content_map, if: :language?
@@ -110,16 +109,6 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     OStatus::TagManager.instance.uri_for(object.thread)
   end
 
-  def conversation
-    return if object.conversation.nil?
-
-    if object.conversation.uri?
-      object.conversation.uri
-    else
-      OStatus::TagManager.instance.unique_tag(object.conversation.created_at, object.conversation.id, 'Conversation')
-    end
-  end
-
   def local?
     object.account.local?
   end
diff --git a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
index d2d924239..b89efd989 100644
--- a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
+++ b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class RemoveBoostsWideningAudience < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
new file mode 100644
index 000000000..b2b157ef7
--- /dev/null
+++ b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class RemoveUriFromConversations < ActiveRecord::Migration[5.2]
+  def up
+    safety_assured { remove_column :conversations, :uri, :string }
+  end
+
+  def down
+    add_column :conversations, :uri, :string
+    add_index :conversations, :uri, unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1847305c7..a722bad36 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_26_175042) do
+ActiveRecord::Schema.define(version: 2019_07_28_084117) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -203,10 +203,8 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do
   end
 
   create_table "conversations", force: :cascade do |t|
-    t.string "uri"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index ["uri"], name: "index_conversations_on_uri", unique: true
   end
 
   create_table "custom_emoji_categories", force: :cascade do |t|
diff --git a/spec/models/conversation_spec.rb b/spec/models/conversation_spec.rb
index 8b5e4fdaf..b91d36169 100644
--- a/spec/models/conversation_spec.rb
+++ b/spec/models/conversation_spec.rb
@@ -1,13 +1,5 @@
 require 'rails_helper'
 
 RSpec.describe Conversation, type: :model do
-  describe '#local?' do
-    it 'returns true when URI is nil' do
-      expect(Fabricate(:conversation).local?).to be true
-    end
 
-    it 'returns false when URI is not nil' do
-      expect(Fabricate(:conversation, uri: 'abc').local?).to be false
-    end
-  end
 end
-- 
cgit 


From b9b0313c78ca4efcc8cda089c652be0b333aa16a Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 17:47:37 +0200
Subject: Revert "Remove conversation URI (#11423)" (#11424)

This reverts commit 75f7f9930eb2a6f5c4041ec44fe0aa795c9ec449.
---
 app/lib/activitypub/activity/create.rb                    | 11 +++++++++++
 app/models/conversation.rb                                |  7 +++++++
 app/serializers/activitypub/note_serializer.rb            | 15 +++++++++++++--
 .../20190519130537_remove_boosts_widening_audience.rb     |  2 --
 .../20190728084117_remove_uri_from_conversations.rb       | 12 ------------
 db/schema.rb                                              |  4 +++-
 spec/models/conversation_spec.rb                          |  8 ++++++++
 7 files changed, 42 insertions(+), 17 deletions(-)
 delete mode 100644 db/post_migrate/20190728084117_remove_uri_from_conversations.rb

(limited to 'app')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index eb7b09e44..000b77df5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -67,6 +67,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         sensitive: @object['sensitive'] || false,
         visibility: visibility_from_audience,
         thread: replied_to_status,
+        conversation: conversation_from_uri(@object['conversation']),
         media_attachment_ids: process_attachments.take(4).map(&:id),
         poll: process_poll,
       }
@@ -261,6 +262,16 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
   end
 
+  def conversation_from_uri(uri)
+    return nil if uri.nil?
+    return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
+    begin
+      Conversation.find_or_create_by!(uri: uri)
+    rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
+      retry
+    end
+  end
+
   def visibility_from_audience
     if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public])
       :public
diff --git a/app/models/conversation.rb b/app/models/conversation.rb
index 37d233f32..4dfaea889 100644
--- a/app/models/conversation.rb
+++ b/app/models/conversation.rb
@@ -4,10 +4,17 @@
 # Table name: conversations
 #
 #  id         :bigint(8)        not null, primary key
+#  uri        :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #
 
 class Conversation < ApplicationRecord
+  validates :uri, uniqueness: true, if: :uri?
+
   has_many :statuses
+
+  def local?
+    uri.nil?
+  end
 end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 88b50b3ea..67f596e78 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,13 +1,14 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
-  context_extensions :atom_uri, :sensitive,
+  context_extensions :atom_uri, :conversation, :sensitive,
                      :hashtag, :emoji, :focal_point, :blurhash
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
              :attributed_to, :to, :cc, :sensitive,
-             :atom_uri, :in_reply_to_atom_uri
+             :atom_uri, :in_reply_to_atom_uri,
+             :conversation
 
   attribute :content
   attribute :content_map, if: :language?
@@ -109,6 +110,16 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     OStatus::TagManager.instance.uri_for(object.thread)
   end
 
+  def conversation
+    return if object.conversation.nil?
+
+    if object.conversation.uri?
+      object.conversation.uri
+    else
+      OStatus::TagManager.instance.unique_tag(object.conversation.created_at, object.conversation.id, 'Conversation')
+    end
+  end
+
   def local?
     object.account.local?
   end
diff --git a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
index b89efd989..d2d924239 100644
--- a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
+++ b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 class RemoveBoostsWideningAudience < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
deleted file mode 100644
index b2b157ef7..000000000
--- a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-class RemoveUriFromConversations < ActiveRecord::Migration[5.2]
-  def up
-    safety_assured { remove_column :conversations, :uri, :string }
-  end
-
-  def down
-    add_column :conversations, :uri, :string
-    add_index :conversations, :uri, unique: true
-  end
-end
diff --git a/db/schema.rb b/db/schema.rb
index a722bad36..1847305c7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_28_084117) do
+ActiveRecord::Schema.define(version: 2019_07_26_175042) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -203,8 +203,10 @@ ActiveRecord::Schema.define(version: 2019_07_28_084117) do
   end
 
   create_table "conversations", force: :cascade do |t|
+    t.string "uri"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["uri"], name: "index_conversations_on_uri", unique: true
   end
 
   create_table "custom_emoji_categories", force: :cascade do |t|
diff --git a/spec/models/conversation_spec.rb b/spec/models/conversation_spec.rb
index b91d36169..8b5e4fdaf 100644
--- a/spec/models/conversation_spec.rb
+++ b/spec/models/conversation_spec.rb
@@ -1,5 +1,13 @@
 require 'rails_helper'
 
 RSpec.describe Conversation, type: :model do
+  describe '#local?' do
+    it 'returns true when URI is nil' do
+      expect(Fabricate(:conversation).local?).to be true
+    end
 
+    it 'returns false when URI is not nil' do
+      expect(Fabricate(:conversation, uri: 'abc').local?).to be false
+    end
+  end
 end
-- 
cgit 


From 784c88e16d8e0f75c0d27e34f926569607e02044 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 29 Jul 2019 15:04:49 +0200
Subject: Fix emoji autosuggestions (#11442)

Regression from cfb2ed78231758a79af038a964ab7f7b7b35274e
---
 app/javascript/mastodon/actions/compose.js                 |  8 ++++----
 app/javascript/mastodon/components/autosuggest_input.js    | 10 +++++-----
 app/javascript/mastodon/components/autosuggest_textarea.js | 10 +++++-----
 app/javascript/mastodon/reducers/compose.js                |  6 +++---
 4 files changed, 17 insertions(+), 17 deletions(-)

(limited to 'app')

diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js
index cd9955505..c27c53df0 100644
--- a/app/javascript/mastodon/actions/compose.js
+++ b/app/javascript/mastodon/actions/compose.js
@@ -418,16 +418,16 @@ export function selectComposeSuggestion(position, token, suggestion, path) {
   return (dispatch, getState) => {
     let completion, startPosition;
 
-    if (typeof suggestion === 'object' && suggestion.id) {
+    if (suggestion.type === 'emoji') {
       completion    = suggestion.native || suggestion.colons;
       startPosition = position - 1;
 
       dispatch(useEmoji(suggestion));
-    } else if (typeof suggestion === 'object' && suggestion.name) {
+    } else if (suggestion.type === 'hashtag') {
       completion    = `#${suggestion.name}`;
       startPosition = position - 1;
-    } else {
-      completion    = getState().getIn(['accounts', suggestion, 'acct']);
+    } else if (suggestion.type === 'account') {
+      completion    = getState().getIn(['accounts', suggestion.id, 'acct']);
       startPosition = position;
     }
 
diff --git a/app/javascript/mastodon/components/autosuggest_input.js b/app/javascript/mastodon/components/autosuggest_input.js
index 0df8bf64e..6d2035add 100644
--- a/app/javascript/mastodon/components/autosuggest_input.js
+++ b/app/javascript/mastodon/components/autosuggest_input.js
@@ -168,15 +168,15 @@ export default class AutosuggestInput extends ImmutablePureComponent {
     const { selectedSuggestion } = this.state;
     let inner, key;
 
-    if (typeof suggestion === 'object' && suggestion.shortcode) {
+    if (suggestion.type === 'emoji') {
       inner = <AutosuggestEmoji emoji={suggestion} />;
       key   = suggestion.id;
-    } else if (typeof suggestion === 'object' && suggestion.name) {
+    } else if (suggestion.type ==='hashtag') {
       inner = <AutosuggestHashtag tag={suggestion} />;
       key   = suggestion.name;
-    } else {
-      inner = <AutosuggestAccountContainer id={suggestion} />;
-      key   = suggestion;
+    } else if (suggestion.type === 'account') {
+      inner = <AutosuggestAccountContainer id={suggestion.id} />;
+      key   = suggestion.id;
     }
 
     return (
diff --git a/app/javascript/mastodon/components/autosuggest_textarea.js b/app/javascript/mastodon/components/autosuggest_textarea.js
index 2bd06d28a..ac2a6366a 100644
--- a/app/javascript/mastodon/components/autosuggest_textarea.js
+++ b/app/javascript/mastodon/components/autosuggest_textarea.js
@@ -174,15 +174,15 @@ export default class AutosuggestTextarea extends ImmutablePureComponent {
     const { selectedSuggestion } = this.state;
     let inner, key;
 
-    if (typeof suggestion === 'object' && suggestion.shortcode) {
+    if (suggestion.type === 'emoji') {
       inner = <AutosuggestEmoji emoji={suggestion} />;
       key   = suggestion.id;
-    } else if (typeof suggestion === 'object' && suggestion.name) {
+    } else if (suggestion.type === 'hashtag') {
       inner = <AutosuggestHashtag tag={suggestion} />;
       key   = suggestion.name;
-    } else {
-      inner = <AutosuggestAccountContainer id={suggestion} />;
-      key   = suggestion;
+    } else if (suggestion.type === 'account') {
+      inner = <AutosuggestAccountContainer id={suggestion.id} />;
+      key   = suggestion.id;
     }
 
     return (
diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js
index fc6d5f30f..e683a9c1a 100644
--- a/app/javascript/mastodon/reducers/compose.js
+++ b/app/javascript/mastodon/reducers/compose.js
@@ -207,11 +207,11 @@ const expiresInFromExpiresAt = expires_at => {
 
 const normalizeSuggestions = (state, { accounts, emojis, tags }) => {
   if (accounts) {
-    return accounts.map(item => item.id);
+    return accounts.map(item => ({ id: item.id, type: 'account' }));
   } else if (emojis) {
-    return emojis;
+    return emojis.map(item => ({ ...item, type: 'emoji' }));
   } else {
-    return sortHashtagsByUse(state, tags);
+    return sortHashtagsByUse(state, tags.map(item => ({ ...item, type: 'hashtag' })));
   }
 };
 
-- 
cgit 


From e136112ab76a37cbde5c1bcfeb562c8e0a5b5116 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 29 Jul 2019 20:40:21 +0200
Subject: Fix tag normalization and migration not removing duplicate tags
 (#11441)

Fix #11428
---
 app/models/tag.rb                                  |  8 ++---
 ...726175042_add_case_insensitive_index_to_tags.rb | 13 +++++++++
 spec/models/tag_spec.rb                            | 34 ++++++++++++++++++++++
 3 files changed, 51 insertions(+), 4 deletions(-)

(limited to 'app')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 972242064..46e3a3ec0 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -65,7 +65,7 @@ class Tag < ApplicationRecord
 
   class << self
     def find_or_create_by_names(name_or_names)
-      Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name|
+      Array(name_or_names).map(&method(:normalize)).uniq { |str| str.mb_chars.downcase.to_s }.map do |normalized_name|
         tag = matching_name(normalized_name).first || create(name: normalized_name)
 
         yield tag if block_given?
@@ -77,7 +77,7 @@ class Tag < ApplicationRecord
     def search_for(term, limit = 5, offset = 0)
       pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern.downcase))
+      Tag.where(arel_table[:name].lower.matches(pattern.mb_chars.downcase.to_s))
          .order(:name)
          .limit(limit)
          .offset(offset)
@@ -92,7 +92,7 @@ class Tag < ApplicationRecord
     end
 
     def matching_name(name_or_names)
-      names = Array(name_or_names).map { |name| normalize(name).downcase }
+      names = Array(name_or_names).map { |name| normalize(name).mb_chars.downcase.to_s }
 
       if names.size == 1
         where(arel_table[:name].lower.eq(names.first))
@@ -104,7 +104,7 @@ class Tag < ApplicationRecord
     private
 
     def normalize(str)
-      str.gsub(/\A#/, '').mb_chars.to_s
+      str.gsub(/\A#/, '')
     end
   end
 
diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
index 6fa8c0ec4..057fc86ba 100644
--- a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
+++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
@@ -2,6 +2,19 @@ class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
   def up
+    Tag.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM tags GROUP BY lower(name) HAVING count(*) > 1').to_hash.each do |row|
+      canonical_tag_id  = row['ids'].split(',').first
+      redundant_tag_ids = row['ids'].split(',')[1..-1]
+
+      safety_assured do
+        execute "UPDATE accounts_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')}) AND NOT EXISTS (SELECT t1.tag_id FROM accounts_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.account_id = t0.account_id)"
+        execute "UPDATE statuses_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')}) AND NOT EXISTS (SELECT t1.tag_id FROM statuses_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.status_id = t0.status_id)"
+        execute "UPDATE featured_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')})  AND NOT EXISTS (SELECT t1.tag_id FROM featured_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.account_id = t0.account_id)"
+      end
+
+      Tag.where(id: redundant_tag_ids).in_batches.delete_all
+    end
+
     safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
     remove_index :tags, name: 'index_tags_on_name'
     remove_index :tags, name: 'hashtag_search_index'
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 9a30ceaa5..5f07fd618 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -82,6 +82,40 @@ RSpec.describe Tag, type: :model do
     end
   end
 
+  describe '.find_normalized' do
+    it 'returns tag for a multibyte case-insensitive name' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+
+      tag = Fabricate(:tag, name: downcase_string)
+      expect(Tag.find_normalized(upcase_string)).to eq tag
+    end
+  end
+
+  describe '.matching_name' do
+    it 'returns tags for multibyte case-insensitive names' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+
+      tag = Fabricate(:tag, name: downcase_string)
+      expect(Tag.matching_name(upcase_string)).to eq [tag]
+    end
+  end
+
+  describe '.find_or_create_by_names' do
+    it 'runs a passed block once per tag regardless of duplicates' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+      count           = 0
+
+      Tag.find_or_create_by_names([upcase_string, downcase_string]) do |tag|
+        count += 1
+      end
+
+      expect(count).to eq 1
+    end
+  end
+
   describe '.search_for' do
     it 'finds tag records with matching names' do
       tag = Fabricate(:tag, name: "match")
-- 
cgit 


From 24552b5160a5090e7d6056fb69a209aa48fe4fce Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 30 Jul 2019 11:10:46 +0200
Subject: Add whitelist mode (#11291)

---
 app/controllers/about_controller.rb                |  5 +++
 app/controllers/activitypub/base_controller.rb     |  2 ++
 app/controllers/activitypub/inboxes_controller.rb  |  2 +-
 app/controllers/admin/domain_allows_controller.rb  | 40 ++++++++++++++++++++++
 app/controllers/admin/instances_controller.rb      | 28 +++++++++++++--
 app/controllers/api/base_controller.rb             |  9 +++++
 app/controllers/api/v1/accounts_controller.rb      |  2 ++
 app/controllers/api/v1/apps_controller.rb          |  2 ++
 .../api/v1/instances/activity_controller.rb        |  3 +-
 .../api/v1/instances/peers_controller.rb           |  3 +-
 app/controllers/api/v1/instances_controller.rb     |  1 +
 app/controllers/application_controller.rb          |  4 ++-
 app/controllers/concerns/account_owned_concern.rb  |  1 +
 app/controllers/directories_controller.rb          |  5 +--
 app/controllers/home_controller.rb                 |  2 +-
 app/controllers/media_controller.rb                |  1 +
 app/controllers/media_proxy_controller.rb          |  2 ++
 app/controllers/public_timelines_controller.rb     |  5 +--
 app/controllers/remote_interaction_controller.rb   |  1 +
 app/controllers/tags_controller.rb                 |  1 +
 app/helpers/domain_control_helper.rb               | 10 +++++-
 app/models/domain_allow.rb                         | 33 ++++++++++++++++++
 app/models/instance.rb                             |  3 +-
 app/models/instance_filter.rb                      |  4 +++
 app/policies/domain_allow_policy.rb                | 11 ++++++
 app/services/concerns/payloadable.rb               |  2 +-
 app/services/unallow_domain_service.rb             | 11 ++++++
 app/views/admin/domain_allows/new.html.haml        | 14 ++++++++
 app/views/admin/instances/index.html.haml          | 35 ++++++++++++-------
 app/views/admin/instances/show.html.haml           |  4 ++-
 app/views/admin/settings/edit.html.haml            | 28 ++++++++-------
 app/views/auth/registrations/new.html.haml         |  2 +-
 app/views/layouts/public.html.haml                 |  9 +++--
 config/initializers/2_whitelist_mode.rb            |  5 +++
 config/locales/en.yml                              |  7 ++++
 config/locales/simple_form.en.yml                  |  2 ++
 config/navigation.rb                               |  2 +-
 config/routes.rb                                   |  1 +
 db/migrate/20190705002136_create_domain_allows.rb  |  9 +++++
 db/schema.rb                                       |  9 ++++-
 lib/mastodon/domains_cli.rb                        | 22 ++++++++++--
 spec/fabricators/domain_allow_fabricator.rb        |  3 ++
 spec/models/domain_allow_spec.rb                   |  5 +++
 streaming/index.js                                 |  5 +--
 44 files changed, 302 insertions(+), 53 deletions(-)
 create mode 100644 app/controllers/admin/domain_allows_controller.rb
 create mode 100644 app/models/domain_allow.rb
 create mode 100644 app/policies/domain_allow_policy.rb
 create mode 100644 app/services/unallow_domain_service.rb
 create mode 100644 app/views/admin/domain_allows/new.html.haml
 create mode 100644 config/initializers/2_whitelist_mode.rb
 create mode 100644 db/migrate/20190705002136_create_domain_allows.rb
 create mode 100644 spec/fabricators/domain_allow_fabricator.rb
 create mode 100644 spec/models/domain_allow_spec.rb

(limited to 'app')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 31cf17710..d276e8fe5 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,6 +3,7 @@
 class AboutController < ApplicationController
   layout 'public'
 
+  before_action :require_open_federation!, only: [:show, :more]
   before_action :set_body_classes, only: :show
   before_action :set_instance_presenter
   before_action :set_expires_in
@@ -19,6 +20,10 @@ class AboutController < ApplicationController
 
   private
 
+  def require_open_federation!
+    not_found if whitelist_mode?
+  end
+
   def new_user
     User.new.tap do |user|
       user.build_account
diff --git a/app/controllers/activitypub/base_controller.rb b/app/controllers/activitypub/base_controller.rb
index a3b5c4dfa..0c2591e97 100644
--- a/app/controllers/activitypub/base_controller.rb
+++ b/app/controllers/activitypub/base_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::BaseController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   private
 
   def set_cache_headers
diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index 7cfd9a25e..bcfc1e6d4 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ActivityPub::InboxesController < Api::BaseController
+class ActivityPub::InboxesController < ActivityPub::BaseController
   include SignatureVerification
   include JsonLdHelper
   include AccountOwnedConcern
diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb
new file mode 100644
index 000000000..31be1978b
--- /dev/null
+++ b/app/controllers/admin/domain_allows_controller.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Admin::DomainAllowsController < Admin::BaseController
+  before_action :set_domain_allow, only: [:destroy]
+
+  def new
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(domain: params[:_domain])
+  end
+
+  def create
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(resource_params)
+
+    if @domain_allow.save
+      log_action :create, @domain_allow
+      redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.created_msg')
+    else
+      render :new
+    end
+  end
+
+  def destroy
+    authorize @domain_allow, :destroy?
+    UnallowDomainService.new.call(@domain_allow)
+    redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
+  end
+
+  private
+
+  def set_domain_allow
+    @domain_allow = DomainAllow.find(params[:id])
+  end
+
+  def resource_params
+    params.require(:domain_allow).permit(:domain)
+  end
+end
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 7888e844f..d4f201807 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -2,6 +2,10 @@
 
 module Admin
   class InstancesController < BaseController
+    before_action :set_domain_block, only: :show
+    before_action :set_domain_allow, only: :show
+    before_action :set_instance, only: :show
+
     def index
       authorize :instance, :index?
 
@@ -11,20 +15,38 @@ module Admin
     def show
       authorize :instance, :show?
 
-      @instance        = Instance.new(Account.by_domain_accounts.find_by(domain: params[:id]) || DomainBlock.find_by!(domain: params[:id]))
       @following_count = Follow.where(account: Account.where(domain: params[:id])).count
       @followers_count = Follow.where(target_account: Account.where(domain: params[:id])).count
       @reports_count   = Report.where(target_account: Account.where(domain: params[:id])).count
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
-      @domain_block    = DomainBlock.rule_for(params[:id])
     end
 
     private
 
+    def set_domain_block
+      @domain_block = DomainBlock.rule_for(params[:id])
+    end
+
+    def set_domain_allow
+      @domain_allow = DomainAllow.rule_for(params[:id])
+    end
+
+    def set_instance
+      resource   = Account.by_domain_accounts.find_by(domain: params[:id])
+      resource ||= @domain_block
+      resource ||= @domain_allow
+
+      if resource
+        @instance = Instance.new(resource)
+      else
+        not_found
+      end
+    end
+
     def filtered_instances
-      InstanceFilter.new(filter_params).results
+      InstanceFilter.new(whitelist_mode? ? { allowed: true } : filter_params).results
     end
 
     def paginated_instances
diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
index 6f33a1ea9..109e38ffa 100644
--- a/app/controllers/api/base_controller.rb
+++ b/app/controllers/api/base_controller.rb
@@ -9,6 +9,7 @@ class Api::BaseController < ApplicationController
   skip_before_action :store_current_location
   skip_before_action :require_functional!
 
+  before_action :require_authenticated_user!, if: :disallow_unauthenticated_api_access?
   before_action :set_cache_headers
 
   protect_from_forgery with: :null_session
@@ -69,6 +70,10 @@ class Api::BaseController < ApplicationController
     nil
   end
 
+  def require_authenticated_user!
+    render json: { error: 'This API requires an authenticated user' }, status: 401 unless current_user
+  end
+
   def require_user!
     if !current_user
       render json: { error: 'This method requires an authenticated user' }, status: 422
@@ -94,4 +99,8 @@ class Api::BaseController < ApplicationController
   def set_cache_headers
     response.headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
   end
+
+  def disallow_unauthenticated_api_access?
+    authorized_fetch_mode?
+  end
 end
diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index b0c62778e..b306e8e8c 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -12,6 +12,8 @@ class Api::V1::AccountsController < Api::BaseController
   before_action :check_account_suspension, only: [:show]
   before_action :check_enabled_registrations, only: [:create]
 
+  skip_before_action :require_authenticated_user!, only: :create
+
   respond_to :json
 
   def show
diff --git a/app/controllers/api/v1/apps_controller.rb b/app/controllers/api/v1/apps_controller.rb
index e9f7a7291..97177547a 100644
--- a/app/controllers/api/v1/apps_controller.rb
+++ b/app/controllers/api/v1/apps_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::AppsController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   def create
     @app = Doorkeeper::Application.create!(application_options)
     render json: @app, serializer: REST::ApplicationSerializer
diff --git a/app/controllers/api/v1/instances/activity_controller.rb b/app/controllers/api/v1/instances/activity_controller.rb
index d0080c5c2..4fb5a69d8 100644
--- a/app/controllers/api/v1/instances/activity_controller.rb
+++ b/app/controllers/api/v1/instances/activity_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::ActivityController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -33,6 +34,6 @@ class Api::V1::Instances::ActivityController < Api::BaseController
   end
 
   def require_enabled_api!
-    head 404 unless Setting.activity_api_enabled
+    head 404 unless Setting.activity_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances/peers_controller.rb b/app/controllers/api/v1/instances/peers_controller.rb
index 450e6502f..75c3cb4ba 100644
--- a/app/controllers/api/v1/instances/peers_controller.rb
+++ b/app/controllers/api/v1/instances/peers_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::PeersController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -14,6 +15,6 @@ class Api::V1::Instances::PeersController < Api::BaseController
   private
 
   def require_enabled_api!
-    head 404 unless Setting.peers_api_enabled
+    head 404 unless Setting.peers_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances_controller.rb b/app/controllers/api/v1/instances_controller.rb
index 93e4f0003..8d8231423 100644
--- a/app/controllers/api/v1/instances_controller.rb
+++ b/app/controllers/api/v1/instances_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::InstancesController < Api::BaseController
   respond_to :json
+
   skip_before_action :set_cache_headers
 
   def show
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 41ce1a0ca..0d3913ee0 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -11,12 +11,14 @@ class ApplicationController < ActionController::Base
   include UserTrackingConcern
   include SessionTrackingConcern
   include CacheConcern
+  include DomainControlHelper
 
   helper_method :current_account
   helper_method :current_session
   helper_method :current_theme
   helper_method :single_user_mode?
   helper_method :use_seamless_external_login?
+  helper_method :whitelist_mode?
 
   rescue_from ActionController::RoutingError, with: :not_found
   rescue_from ActiveRecord::RecordNotFound, with: :not_found
@@ -38,7 +40,7 @@ class ApplicationController < ActionController::Base
   end
 
   def authorized_fetch_mode?
-    ENV['AUTHORIZED_FETCH'] == 'true'
+    ENV['AUTHORIZED_FETCH'] == 'true' || Rails.configuration.x.whitelist_mode
   end
 
   def public_fetch_mode?
diff --git a/app/controllers/concerns/account_owned_concern.rb b/app/controllers/concerns/account_owned_concern.rb
index 99c240fe9..460f71f65 100644
--- a/app/controllers/concerns/account_owned_concern.rb
+++ b/app/controllers/concerns/account_owned_concern.rb
@@ -4,6 +4,7 @@ module AccountOwnedConcern
   extend ActiveSupport::Concern
 
   included do
+    before_action :authenticate_user!, if: -> { whitelist_mode? && request.format != :json }
     before_action :set_account, if: :account_required?
     before_action :check_account_approval, if: :account_required?
     before_action :check_account_suspension, if: :account_required?
diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb
index 594907674..d2ef76f06 100644
--- a/app/controllers/directories_controller.rb
+++ b/app/controllers/directories_controller.rb
@@ -3,7 +3,8 @@
 class DirectoriesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_instance_presenter
   before_action :set_tag, only: :show
   before_action :set_tags
@@ -19,7 +20,7 @@ class DirectoriesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     return not_found unless Setting.profile_directory
   end
 
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index 42493cd78..22d507e77 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -55,7 +55,7 @@ class HomeController < ApplicationController
   end
 
   def default_redirect_path
-    if request.path.start_with?('/web')
+    if request.path.start_with?('/web') || whitelist_mode?
       new_user_session_path
     elsif single_user_mode?
       short_account_path(Account.local.without_suspended.where('id > 0').first)
diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb
index b3b7519a1..1f693de32 100644
--- a/app/controllers/media_controller.rb
+++ b/app/controllers/media_controller.rb
@@ -5,6 +5,7 @@ class MediaController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_media_attachment
   before_action :verify_permitted_status!
   before_action :check_playable, only: :player
diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb
index 8fc18dd06..8da6c6fe0 100644
--- a/app/controllers/media_proxy_controller.rb
+++ b/app/controllers/media_proxy_controller.rb
@@ -5,6 +5,8 @@ class MediaProxyController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
+
   def show
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb
index 23506b990..324bdc508 100644
--- a/app/controllers/public_timelines_controller.rb
+++ b/app/controllers/public_timelines_controller.rb
@@ -3,7 +3,8 @@
 class PublicTimelinesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_body_classes
   before_action :set_instance_presenter
 
@@ -16,7 +17,7 @@ class PublicTimelinesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     not_found unless Setting.timeline_preview
   end
 
diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb
index cc6993c52..fa742fb0a 100644
--- a/app/controllers/remote_interaction_controller.rb
+++ b/app/controllers/remote_interaction_controller.rb
@@ -5,6 +5,7 @@ class RemoteInteractionController < ApplicationController
 
   layout 'modal'
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_interaction_type
   before_action :set_status
   before_action :set_body_classes
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index d08e5a61a..3cd2d9e20 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -8,6 +8,7 @@ class TagsController < ApplicationController
   layout 'public'
 
   before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_tag
   before_action :set_body_classes
   before_action :set_instance_presenter
diff --git a/app/helpers/domain_control_helper.rb b/app/helpers/domain_control_helper.rb
index efd328f81..067b2c2cd 100644
--- a/app/helpers/domain_control_helper.rb
+++ b/app/helpers/domain_control_helper.rb
@@ -12,6 +12,14 @@ module DomainControlHelper
       end
     end
 
-    DomainBlock.blocked?(domain)
+    if whitelist_mode?
+      !DomainAllow.allowed?(domain)
+    else
+      DomainBlock.blocked?(domain)
+    end
+  end
+
+  def whitelist_mode?
+    Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb
new file mode 100644
index 000000000..85018b636
--- /dev/null
+++ b/app/models/domain_allow.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: domain_allows
+#
+#  id         :bigint(8)        not null, primary key
+#  domain     :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class DomainAllow < ApplicationRecord
+  include DomainNormalizable
+
+  validates :domain, presence: true, uniqueness: true
+
+  scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
+
+  class << self
+    def allowed?(domain)
+      !rule_for(domain).nil?
+    end
+
+    def rule_for(domain)
+      return if domain.blank?
+
+      uri = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+
+      find_by(domain: uri.normalized_host)
+    end
+  end
+end
diff --git a/app/models/instance.rb b/app/models/instance.rb
index 797a191e0..3c740f8a2 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -7,8 +7,9 @@ class Instance
 
   def initialize(resource)
     @domain         = resource.domain
-    @accounts_count = resource.is_a?(DomainBlock) ? nil : resource.accounts_count
+    @accounts_count = resource.respond_to?(:accounts_count) ? resource.accounts_count : nil
     @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
+    @domain_allow   = resource.is_a?(DomainAllow) ? resource : DomainAllow.rule_for(domain)
   end
 
   def countable?
diff --git a/app/models/instance_filter.rb b/app/models/instance_filter.rb
index 848fff53e..8bfab826d 100644
--- a/app/models/instance_filter.rb
+++ b/app/models/instance_filter.rb
@@ -12,6 +12,10 @@ class InstanceFilter
       scope = DomainBlock
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
       scope.order(id: :desc)
+    elsif params[:allowed].present?
+      scope = DomainAllow
+      scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
+      scope.order(id: :desc)
     else
       scope = Account.remote
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
diff --git a/app/policies/domain_allow_policy.rb b/app/policies/domain_allow_policy.rb
new file mode 100644
index 000000000..5030453bb
--- /dev/null
+++ b/app/policies/domain_allow_policy.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class DomainAllowPolicy < ApplicationPolicy
+  def create?
+    admin?
+  end
+
+  def destroy?
+    admin?
+  end
+end
diff --git a/app/services/concerns/payloadable.rb b/app/services/concerns/payloadable.rb
index 953740faa..7f9f21c4b 100644
--- a/app/services/concerns/payloadable.rb
+++ b/app/services/concerns/payloadable.rb
@@ -14,6 +14,6 @@ module Payloadable
   end
 
   def signing_enabled?
-    ENV['AUTHORIZED_FETCH'] != 'true'
+    ENV['AUTHORIZED_FETCH'] != 'true' && !Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb
new file mode 100644
index 000000000..d4387c1a1
--- /dev/null
+++ b/app/services/unallow_domain_service.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class UnallowDomainService < BaseService
+  def call(domain_allow)
+    Account.where(domain: domain_allow.domain).find_each do |account|
+      SuspendAccountService.new.call(account, destroy: true)
+    end
+
+    domain_allow.destroy
+  end
+end
diff --git a/app/views/admin/domain_allows/new.html.haml b/app/views/admin/domain_allows/new.html.haml
new file mode 100644
index 000000000..52599857a
--- /dev/null
+++ b/app/views/admin/domain_allows/new.html.haml
@@ -0,0 +1,14 @@
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
+- content_for :page_title do
+  = t('admin.domain_allows.add_new')
+
+= simple_form_for @domain_allow, url: admin_domain_allows_path do |f|
+  = render 'shared/error_messages', object: @domain_allow
+
+  .fields-group
+    = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), required: true
+
+  .actions
+    = f.button :button, t('admin.domain_allows.add_new'), type: :submit
diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml
index 61e578409..982dc5035 100644
--- a/app/views/admin/instances/index.html.haml
+++ b/app/views/admin/instances/index.html.haml
@@ -6,24 +6,30 @@
     %strong= t('admin.instances.moderation.title')
     %ul
       %li= filter_link_to t('admin.instances.moderation.all'), limited: nil
-      %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
+
+      - unless whitelist_mode?
+        %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
 
   %div{ style: 'flex: 1 1 auto; text-align: right' }
-    = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
+    - if whitelist_mode?
+      = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button'
+    - else
+      = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
 
-= form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
-  .fields-group
-    - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
-      - if params[key].present?
-        = hidden_field_tag key, params[key]
+- unless whitelist_mode?
+  = form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
+    .fields-group
+      - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
+        - if params[key].present?
+          = hidden_field_tag key, params[key]
 
-    - %i(by_domain).each do |key|
-      .input.string.optional
-        = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
+      - %i(by_domain).each do |key|
+        .input.string.optional
+          = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
 
-    .actions
-      %button= t('admin.accounts.search')
-      = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
+      .actions
+        %button= t('admin.accounts.search')
+        = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
 
 %hr.spacer/
 
@@ -47,8 +53,11 @@
               - unless first_item
                 &bull;
               = t('admin.domain_blocks.rejecting_reports')
+          - elsif whitelist_mode?
+            = t('admin.accounts.whitelisted')
           - else
             = t('admin.accounts.no_limits_imposed')
       - if instance.countable?
         .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= number_to_human instance.accounts_count, strip_insignificant_zeros: true
+
 = paginate paginated_instances
diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml
index c7992a490..fbb49ba02 100644
--- a/app/views/admin/instances/show.html.haml
+++ b/app/views/admin/instances/show.html.haml
@@ -38,7 +38,9 @@
     = link_to t('admin.accounts.title'), admin_accounts_path(remote: '1', by_domain: @instance.domain), class: 'button'
 
   %div{ style: 'float: right' }
-    - if @domain_block
+    - if @domain_allow
+      = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete }
+    - elsif @domain_block
       = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button'
     - else
       = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button'
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index b3bf3849c..1e2ed3f77 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -42,11 +42,12 @@
 
   %hr.spacer/
 
-  .fields-group
-    = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
 
-  .fields-group
-    = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
+    .fields-group
+      = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
 
   .fields-group
     = f.input :show_staff_badge, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_staff_badge.title'), hint: t('admin.settings.show_staff_badge.desc_html')
@@ -54,17 +55,18 @@
   .fields-group
     = f.input :open_deletion, as: :boolean, wrapper: :with_label, label: t('admin.settings.registrations.deletion.title'), hint: t('admin.settings.registrations.deletion.desc_html')
 
-  .fields-group
-    = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
+    .fields-group
+      = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
+    .fields-group
+      = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
 
-  .fields-group
-    = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
+    .fields-group
+      = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
 
   .fields-group
     = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
@@ -76,7 +78,7 @@
 
   .fields-group
     = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 }
-    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 }
+    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } unless whitelist_mode?
     = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 }
     = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html')
 
diff --git a/app/views/auth/registrations/new.html.haml b/app/views/auth/registrations/new.html.haml
index b4a7cced5..83384d737 100644
--- a/app/views/auth/registrations/new.html.haml
+++ b/app/views/auth/registrations/new.html.haml
@@ -33,7 +33,7 @@
   = f.input :invite_code, as: :hidden
 
   .fields-group
-    = f.input :agreement, as: :boolean, wrapper: :with_label, label: t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
+    = f.input :agreement, as: :boolean, wrapper: :with_label, label: whitelist_mode? ? t('auth.checkbox_agreement_without_rules_html', terms_path: terms_path) : t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
 
   .actions
     = f.button :button, @invite.present? ? t('auth.register') : sign_up_message, type: :submit
diff --git a/app/views/layouts/public.html.haml b/app/views/layouts/public.html.haml
index 2929ac599..69738a2f7 100644
--- a/app/views/layouts/public.html.haml
+++ b/app/views/layouts/public.html.haml
@@ -10,10 +10,13 @@
             = link_to root_url, class: 'brand' do
               = svg_logo_full
 
-            = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
-            = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
-            = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+            - unless whitelist_mode?
+              = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
+              = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
+              = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+
           .nav-center
+
           .nav-right
             - if user_signed_in?
               = link_to t('settings.back'), root_url, class: 'nav-link nav-button webapp-btn'
diff --git a/config/initializers/2_whitelist_mode.rb b/config/initializers/2_whitelist_mode.rb
new file mode 100644
index 000000000..a17ad07a2
--- /dev/null
+++ b/config/initializers/2_whitelist_mode.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.x.whitelist_mode = ENV['WHITELIST_MODE'] == 'true'
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 9e1be87be..6c1a34300 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -186,6 +186,7 @@ en:
       username: Username
       warn: Warn
       web: Web
+      whitelisted: Whitelisted
     action_logs:
       actions:
         assigned_to_self_report: "%{name} assigned report %{target} to themselves"
@@ -269,6 +270,11 @@ en:
       week_interactions: interactions this week
       week_users_active: active this week
       week_users_new: users this week
+    domain_allows:
+      add_new: Whitelist domain
+      created_msg: Domain has been successfully whitelisted
+      destroyed_msg: Domain has been removed from the whitelist
+      undo: Remove from whitelist
     domain_blocks:
       add_new: Add new domain block
       created_msg: Domain block is now being processed
@@ -524,6 +530,7 @@ en:
     apply_for_account: Request an invite
     change_password: Password
     checkbox_agreement_html: I agree to the <a href="%{rules_path}" target="_blank">server rules</a> and <a href="%{terms_path}" target="_blank">terms of service</a>
+    checkbox_agreement_without_rules_html: I agree to the <a href="%{terms_path}" target="_blank">terms of service</a>
     delete_account: Delete account
     delete_account_html: If you wish to delete your account, you can <a href="%{path}">proceed here</a>. You will be asked for confirmation.
     didnt_get_confirmation: Didn't receive confirmation instructions?
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 12a7ec2b3..10b30e627 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -38,6 +38,8 @@ en:
         setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed
         username: Your username will be unique on %{domain}
         whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word
+      domain_allow:
+        domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored
       featured_tag:
         name: 'You might want to use one of these:'
       imports:
diff --git a/config/navigation.rb b/config/navigation.rb
index 5ab2e4399..9b46da603 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -39,7 +39,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
       s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
-      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks}, if: -> { current_user.admin? }
+      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
     end
 
diff --git a/config/routes.rb b/config/routes.rb
index b6c215888..04424bbbd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -154,6 +154,7 @@ Rails.application.routes.draw do
   namespace :admin do
     get '/dashboard', to: 'dashboard#index'
 
+    resources :domain_allows, only: [:new, :create, :show, :destroy]
     resources :domain_blocks, only: [:new, :create, :show, :destroy]
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
diff --git a/db/migrate/20190705002136_create_domain_allows.rb b/db/migrate/20190705002136_create_domain_allows.rb
new file mode 100644
index 000000000..83b0728d9
--- /dev/null
+++ b/db/migrate/20190705002136_create_domain_allows.rb
@@ -0,0 +1,9 @@
+class CreateDomainAllows < ActiveRecord::Migration[5.2]
+  def change
+    create_table :domain_allows do |t|
+      t.string :domain, default: '', null: false, index: { unique: true }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1847305c7..2d83d8b76 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_26_175042) do
+ActiveRecord::Schema.define(version: 2019_07_28_084117) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -245,6 +245,13 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do
     t.index ["account_id"], name: "index_custom_filters_on_account_id"
   end
 
+  create_table "domain_allows", force: :cascade do |t|
+    t.string "domain", default: "", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["domain"], name: "index_domain_allows_on_domain", unique: true
+  end
+
   create_table "domain_blocks", force: :cascade do |t|
     t.string "domain", default: "", null: false
     t.datetime "created_at", null: false
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index b081581fe..f30062363 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -12,17 +12,33 @@ module Mastodon
     end
 
     option :dry_run, type: :boolean
-    desc 'purge DOMAIN', 'Remove accounts from a DOMAIN without a trace'
+    option :whitelist_mode, type: :boolean
+    desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace'
     long_desc <<-LONG_DESC
       Remove all accounts from a given DOMAIN without leaving behind any
       records. Unlike a suspension, if the DOMAIN still exists in the wild,
       it means the accounts could return if they are resolved again.
+
+      When the --whitelist-mode option is given, instead of purging accounts
+      from a single domain, all accounts from domains that are not whitelisted
+      are removed from the database.
     LONG_DESC
-    def purge(domain)
+    def purge(domain = nil)
       removed = 0
       dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
 
-      Account.where(domain: domain).find_each do |account|
+      scope = begin
+        if options[:whitelist_mode]
+          Account.remote.where.not(domain: DomainAllow.pluck(:domain))
+        elsif domain.present?
+          Account.remote.where(domain: domain)
+        else
+          say('No domain given', :red)
+          exit(1)
+        end
+      end
+
+      scope.find_each do |account|
         SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
         removed += 1
         say('.', :green, false)
diff --git a/spec/fabricators/domain_allow_fabricator.rb b/spec/fabricators/domain_allow_fabricator.rb
new file mode 100644
index 000000000..6226b1e20
--- /dev/null
+++ b/spec/fabricators/domain_allow_fabricator.rb
@@ -0,0 +1,3 @@
+Fabricator(:domain_allow) do
+  domain "MyString"
+end
diff --git a/spec/models/domain_allow_spec.rb b/spec/models/domain_allow_spec.rb
new file mode 100644
index 000000000..e65435127
--- /dev/null
+++ b/spec/models/domain_allow_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe DomainAllow, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
diff --git a/streaming/index.js b/streaming/index.js
index 0529804b1..304e7e046 100644
--- a/streaming/index.js
+++ b/streaming/index.js
@@ -12,6 +12,7 @@ const uuid = require('uuid');
 const fs = require('fs');
 
 const env = process.env.NODE_ENV || 'development';
+const alwaysRequireAuth = process.env.WHITELIST_MODE === 'true' || process.env.AUTHORIZED_FETCH === 'true';
 
 dotenv.config({
   path: env === 'production' ? '.env.production' : '.env',
@@ -271,7 +272,7 @@ const startWorker = (workerId) => {
 
   const wsVerifyClient = (info, cb) => {
     const location = url.parse(info.req.url, true);
-    const authRequired = !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
+    const authRequired = alwaysRequireAuth || !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
     const allowedScopes = [];
 
     if (authRequired) {
@@ -306,7 +307,7 @@ const startWorker = (workerId) => {
       return;
     }
 
-    const authRequired = !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
+    const authRequired = alwaysRequireAuth || !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
     const allowedScopes = [];
 
     if (authRequired) {
-- 
cgit 


From 78144f4c7923d502cc86b322f044e740e4a8fa8a Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 30 Jul 2019 12:06:21 +0200
Subject: Fix crash when expanding search results for hashtags (#11447)

---
 app/javascript/mastodon/reducers/search.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/javascript/mastodon/reducers/search.js b/app/javascript/mastodon/reducers/search.js
index f437ff217..875b2d92b 100644
--- a/app/javascript/mastodon/reducers/search.js
+++ b/app/javascript/mastodon/reducers/search.js
@@ -44,7 +44,8 @@ export default function search(state = initialState, action) {
       hashtags: fromJS(action.results.hashtags),
     })).set('submitted', true).set('searchTerm', action.searchTerm);
   case SEARCH_EXPAND_SUCCESS:
-    return state.updateIn(['results', action.searchType], list => list.concat(action.results[action.searchType].map(item => item.id)));
+    const results = action.searchType === 'hashtags' ? fromJS(action.results.hashtags) : action.results[action.searchType].map(item => item.id);
+    return state.updateIn(['results', action.searchType], list => list.concat(results));
   default:
     return state;
   }
-- 
cgit 


From b31b232edfcc7f04acf828bf6829ab716b290692 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 30 Jul 2019 12:13:29 +0200
Subject: Change links in webUI to rewrite misleading links (#11426)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [WiP] Show host for “misleading” links

* Disallow misleading targets which domain names are prefixes of link text

* Move decodeIDNA to app/javascript/mastodon/utils

* Add support for international domain names

* Change link origin tag color to darker text color

* Handle links to domains starting with www. as shortened by Mastodon

* [WiP] Ignore links that cannot be misread as URLs, rewrite other links
---
 .../mastodon/components/status_content.js          | 90 ++++++++++++++++++++++
 .../mastodon/features/status/components/card.js    | 11 +--
 app/javascript/mastodon/utils/idna.js              | 10 +++
 3 files changed, 101 insertions(+), 10 deletions(-)
 create mode 100644 app/javascript/mastodon/utils/idna.js

(limited to 'app')

diff --git a/app/javascript/mastodon/components/status_content.js b/app/javascript/mastodon/components/status_content.js
index 549de95fc..e717934fa 100644
--- a/app/javascript/mastodon/components/status_content.js
+++ b/app/javascript/mastodon/components/status_content.js
@@ -8,9 +8,71 @@ import classnames from 'classnames';
 import PollContainer from 'mastodon/containers/poll_container';
 import Icon from 'mastodon/components/icon';
 import { autoPlayGif } from 'mastodon/initial_state';
+import { decode as decodeIDNA } from 'mastodon/utils/idna';
 
 const MAX_HEIGHT = 642; // 20px * 32 (+ 2px padding at the top)
 
+// Regex matching what "looks like a link", that is, something that starts with
+// an optional "http://" or "https://" scheme and then what could look like a
+// domain main, that is, at least two sequences of characters not including spaces
+// and separated by "." or an homoglyph. The idea is not to match valid URLs or
+// domain names, but what could be confused for a valid URL or domain name,
+// especially to the untrained eye.
+
+const h_confusables = 'h\u13c2\u1d58d\u1d4f1\u1d691\u0068\uff48\u1d525\u210e\u1d489\u1d629\u0570\u1d4bd\u1d65d\u1d421\u1d5c1\u1d5f5\u04bb\u1d559';
+const t_confusables = 't\u1d42d\u1d5cd\u1d531\u1d565\u1d4c9\u1d669\u1d4fd\u1d69d\u0074\u1d461\u1d601\u1d495\u1d635\u1d599';
+const p_confusables = 'p\u0440\u03c1\u1d52d\u1d631\u1d665\u1d429\uff50\u1d6e0\u1d45d\u1d561\u1d595\u1d71a\u1d699\u1d78e\u2ca3\u1d754\u1d6d2\u1d491\u1d7c8\u1d746\u1d4c5\u1d70c\u1d5c9\u0070\u1d780\u03f1\u1d5fd\u2374\u1d7ba\u1d4f9';
+const s_confusables = 's\u1d530\u118c1\u1d494\u1d634\u1d4c8\u1d668\uabaa\u1d42c\u1d5cc\u1d460\u1d600\ua731\u0073\uff53\u1d564\u0455\u1d598\u1d4fc\u1d69c\u10448\u01bd';
+const column_confusables = ':\u0903\u0a83\u0703\u1803\u05c3\u0704\u0589\u1809\ua789\u16ec\ufe30\u02d0\u2236\u02f8\u003a\uff1a\u205a\ua4fd';
+const slash_confusables = '/\u2041\u2f03\u2044\u2cc6\u27cb\u30ce\u002f\u2571\u31d3\u3033\u1735\u2215\u29f8\u1d23a\u4e3f';
+const dot_confusables = '.\u002e\u0660\u06f0\u0701\u0702\u2024\ua4f8\ua60e\u10a50\u1d16d';
+
+const linkRegex = new RegExp(`^\\s*(([${h_confusables}][${t_confusables}][${t_confusables}][${p_confusables}][${s_confusables}]?[${column_confusables}][${slash_confusables}][${slash_confusables}]))?[^:/\\n ]+([${dot_confusables}][^:/\\n ]+)+`);
+
+const isLinkMisleading = (link) => {
+  let linkTextParts = [];
+
+  // Reconstruct visible text, as we do not have much control over how links
+  // from remote software look, and we can't rely on `innerText` because the
+  // `invisible` class does not set `display` to `none`.
+
+  const walk = (node) => {
+    switch (node.nodeType) {
+    case Node.TEXT_NODE:
+      linkTextParts.push(node.textContent);
+      break;
+    case Node.ELEMENT_NODE:
+      if (node.classList.contains('invisible')) return;
+      const children = node.childNodes;
+      for (let i = 0; i < children.length; i++) {
+        walk(children[i]);
+      }
+      break;
+    }
+  };
+
+  walk(link);
+
+  const linkText = linkTextParts.join('');
+  const targetURL = new URL(link.href);
+
+  // The following may not work with international domain names
+  if (linkText === targetURL.origin || linkText === targetURL.host || 'www.' + linkText === targetURL.host || linkText.startsWith(targetURL.origin + '/') || linkText.startsWith(targetURL.host + '/')) {
+    return false;
+  }
+
+  // The link hasn't been recognized, maybe it features an international domain name
+  const hostname = decodeIDNA(targetURL.hostname);
+  const host = targetURL.host.replace(targetURL.hostname, hostname);
+  const origin = targetURL.origin.replace(targetURL.host, host);
+  if (linkText === origin || linkText === host || linkText.startsWith(origin + '/') || linkText.startsWith(host + '/')) {
+    return false;
+  }
+
+  // If the link text looks like an URL or auto-generated link, it is misleading
+  return linkRegex.test(linkText);
+};
+
 export default class StatusContent extends React.PureComponent {
 
   static contextTypes = {
@@ -56,6 +118,34 @@ export default class StatusContent extends React.PureComponent {
       } else {
         link.setAttribute('title', link.href);
         link.classList.add('unhandled-link');
+
+        if (isLinkMisleading(link)) {
+          while (link.firstChild) {
+            link.removeChild(link.firstChild);
+          }
+
+          const prefix = (link.href.match(/https?:\/\/(www\.)?/) || [''])[0];
+          const text   = link.href.substr(prefix.length, 30);
+          const suffix = link.href.substr(prefix.length + 30);
+          const cutoff = !!suffix;
+
+          const prefixTag = document.createElement('span');
+          prefixTag.classList.add('invisible');
+          prefixTag.textContent = prefix;
+          link.appendChild(prefixTag);
+
+          const textTag = document.createElement('span');
+          if (cutoff) {
+            textTag.classList.add('ellipsis');
+          }
+          textTag.textContent = text;
+          link.appendChild(textTag);
+
+          const suffixTag = document.createElement('span');
+          suffixTag.classList.add('invisible');
+          suffixTag.textContent = suffix;
+          link.appendChild(suffixTag);
+        }
       }
 
       link.setAttribute('target', '_blank');
diff --git a/app/javascript/mastodon/features/status/components/card.js b/app/javascript/mastodon/features/status/components/card.js
index 0eff54411..012542843 100644
--- a/app/javascript/mastodon/features/status/components/card.js
+++ b/app/javascript/mastodon/features/status/components/card.js
@@ -2,18 +2,9 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import Immutable from 'immutable';
 import ImmutablePropTypes from 'react-immutable-proptypes';
-import punycode from 'punycode';
 import classnames from 'classnames';
 import Icon from 'mastodon/components/icon';
-
-const IDNA_PREFIX = 'xn--';
-
-const decodeIDNA = domain => {
-  return domain
-    .split('.')
-    .map(part => part.indexOf(IDNA_PREFIX) === 0 ? punycode.decode(part.slice(IDNA_PREFIX.length)) : part)
-    .join('.');
-};
+import { decode as decodeIDNA } from 'mastodon/utils/idna';
 
 const getHostname = url => {
   const parser = document.createElement('a');
diff --git a/app/javascript/mastodon/utils/idna.js b/app/javascript/mastodon/utils/idna.js
new file mode 100644
index 000000000..efab5bacf
--- /dev/null
+++ b/app/javascript/mastodon/utils/idna.js
@@ -0,0 +1,10 @@
+import punycode from 'punycode';
+
+const IDNA_PREFIX = 'xn--';
+
+export const decode = domain => {
+  return domain
+    .split('.')
+    .map(part => part.indexOf(IDNA_PREFIX) === 0 ? punycode.decode(part.slice(IDNA_PREFIX.length)) : part)
+    .join('.');
+};
-- 
cgit 


From d8097ecd2f787aa0a065a753146151074eef55c6 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 30 Jul 2019 12:06:21 +0200
Subject: [Glitch] Fix crash when expanding search results for hashtags

Port 78144f4c7923d502cc86b322f044e740e4a8fa8a to glitch-soc
---
 app/javascript/flavours/glitch/reducers/search.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/javascript/flavours/glitch/reducers/search.js b/app/javascript/flavours/glitch/reducers/search.js
index f4d99a99a..c346e958b 100644
--- a/app/javascript/flavours/glitch/reducers/search.js
+++ b/app/javascript/flavours/glitch/reducers/search.js
@@ -44,7 +44,8 @@ export default function search(state = initialState, action) {
       hashtags: fromJS(action.results.hashtags),
     })).set('submitted', true).set('searchTerm', action.searchTerm);
   case SEARCH_EXPAND_SUCCESS:
-    return state.updateIn(['results', action.searchType], list => list.concat(action.results[action.searchType].map(item => item.id)));
+    const results = action.searchType === 'hashtags' ? fromJS(action.results.hashtags) : action.results[action.searchType].map(item => item.id);
+    return state.updateIn(['results', action.searchType], list => list.concat(results));
   default:
     return state;
   }
-- 
cgit