From d7bdddbeef3df2fa61d2eee42b544b6553d2d2ea Mon Sep 17 00:00:00 2001
From: Arnout Engelen <github@bzzt.net>
Date: Thu, 15 Aug 2019 20:20:20 +0200
Subject: Include max image dimensions in error (#11552)

---
 app/models/concerns/attachmentable.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index 7c78bb456..246c2c27c 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -43,7 +43,7 @@ module Attachmentable
 
       width, height = FastImage.size(attachment.queued_for_write[:original].path)
 
-      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height >= MAX_MATRIX_LIMIT)
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported, must be below #{MAX_MATRIX_LIMIT} sqpx" if width.present? && height.present? && (width * height >= MAX_MATRIX_LIMIT)
     end
   end
 
-- 
cgit 


From 8fdff2748ffbc4ce3d36c75f2bd33182b641e895 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 16 Aug 2019 01:24:03 +0200
Subject: Add more accurate account search (#11537)

* Add more accurate account search

When ElasticSearch is available, a more accurate search is implemented:

- Using edge n-gram index for acct and display name
- Using asciifolding and cjk width normalization on display names
- Using Gaussian decay on account activity for additional scoring (recency)
- Using followers/friends ratio for additional scoring (spamminess)
- Using followers number for additional scoring (size)

The exact match precedence only takes effect when the input conforms
to the username format and the username part of it is complete, i.e.
when the user started typing the domain part.

* Support single-letter usernames

* Fix tests

* Fix not picking up account updates

* Add weights and normalization for scores, skip zero terms queries

* Use local counts for accounts index, adjust search parameters

* Fix mistakes

* Using updated_at of accounts is inadequate for remote accounts
---
 app/chewy/accounts_index.rb                  |  36 +++++
 app/models/account.rb                        |   6 +
 app/models/account_stat.rb                   |   2 +
 app/services/account_search_service.rb       | 189 ++++++++++++++++++---------
 spec/services/account_search_service_spec.rb | 122 ++++-------------
 5 files changed, 194 insertions(+), 161 deletions(-)
 create mode 100644 app/chewy/accounts_index.rb

(limited to 'app/models')

diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb
new file mode 100644
index 000000000..e11b80039
--- /dev/null
+++ b/app/chewy/accounts_index.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class AccountsIndex < Chewy::Index
+  settings index: { refresh_interval: '5m' }, analysis: {
+    analyzer: {
+      content: {
+        tokenizer: 'whitespace',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+
+      edge_ngram: {
+        tokenizer: 'edge_ngram',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+    },
+
+    tokenizer: {
+      edge_ngram: {
+        type: 'edge_ngram',
+        min_gram: 1,
+        max_gram: 15,
+      },
+    },
+  }
+
+  define_type ::Account.searchable.includes(:account_stat), delete_if: ->(account) { account.destroyed? || !account.searchable? } do
+    root date_detection: false do
+      field :id, type: 'long'
+      field :display_name, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
+      field :acct, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content', value: ->(account) { [account.username, account.domain].compact.join('@') }
+      field :following_count, type: 'long', value: ->(account) { account.active_relationships.count }
+      field :followers_count, type: 'long', value: ->(account) { account.passive_relationships.count }
+      field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
+    end
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index 60c06aaf0..392cc625f 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -127,6 +127,8 @@ class Account < ApplicationRecord
 
   delegate :chosen_languages, to: :user, prefix: false, allow_nil: true
 
+  update_index('accounts#account', :self) if Chewy.enabled?
+
   def local?
     domain.nil?
   end
@@ -169,6 +171,10 @@ class Account < ApplicationRecord
     subscription_expires_at.present?
   end
 
+  def searchable?
+    !(suspended? || moved?)
+  end
+
   def possibly_stale?
     last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
   end
diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
index 9813aa84f..6d1097cec 100644
--- a/app/models/account_stat.rb
+++ b/app/models/account_stat.rb
@@ -16,6 +16,8 @@
 class AccountStat < ApplicationRecord
   belongs_to :account, inverse_of: :account_stat
 
+  update_index('accounts#account', :account) if Chewy.enabled?
+
   def increment_count!(key)
     update(attributes_for_increment(key))
   end
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index e1874d045..2d602a31d 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -4,105 +4,134 @@ class AccountSearchService < BaseService
   attr_reader :query, :limit, :offset, :options, :account
 
   def call(query, account = nil, options = {})
-    @query   = query.strip
-    @limit   = options[:limit].to_i
-    @offset  = options[:offset].to_i
-    @options = options
-    @account = account
+    @acct_hint = query.start_with?('@')
+    @query     = query.strip.gsub(/\A@/, '')
+    @limit     = options[:limit].to_i
+    @offset    = options[:offset].to_i
+    @options   = options
+    @account   = account
 
-    search_service_results
+    search_service_results.compact.uniq
   end
 
   private
 
   def search_service_results
-    return [] if query_blank_or_hashtag? || limit < 1
+    return [] if query.blank? || limit < 1
 
-    if resolving_non_matching_remote_account?
-      [ResolveAccountService.new.call("#{query_username}@#{query_domain}")].compact
-    else
-      search_results_and_exact_match.compact.uniq
-    end
+    [exact_match] + search_results
   end
 
-  def resolving_non_matching_remote_account?
-    offset.zero? && options[:resolve] && !exact_match? && !domain_is_local?
-  end
+  def exact_match
+    return unless offset.zero? && username_complete?
 
-  def search_results_and_exact_match
-    return search_results.to_a unless offset.zero?
+    return @exact_match if defined?(@exact_match)
 
-    results = [exact_match]
+    @exact_match = begin
+      if options[:resolve]
+        ResolveAccountService.new.call(query)
+      elsif domain_is_local?
+        Account.find_local(query_username)
+      else
+        Account.find_remote(query_username, query_domain)
+      end
+    end
+  end
 
-    return results if exact_match? && limit == 1
+  def search_results
+    return [] if limit_for_non_exact_results.zero?
 
-    results + search_results.to_a
+    @search_results ||= begin
+      if Chewy.enabled?
+        from_elasticsearch
+      else
+        from_database
+      end
+    end
   end
 
-  def query_blank_or_hashtag?
-    query.blank? || query.start_with?('#')
+  def from_database
+    if account
+      advanced_search_results
+    else
+      simple_search_results
+    end
   end
 
-  def split_query_string
-    @split_query_string ||= query.gsub(/\A@/, '').split('@')
+  def advanced_search_results
+    Account.advanced_search_for(terms_for_query, account, limit_for_non_exact_results, options[:following], offset)
   end
 
-  def query_username
-    @query_username ||= split_query_string.first || ''
+  def simple_search_results
+    Account.search_for(terms_for_query, limit_for_non_exact_results, offset)
   end
 
-  def query_domain
-    @query_domain ||= query_without_split? ? nil : split_query_string.last
-  end
+  def from_elasticsearch
+    must_clauses   = [{ multi_match: { query: terms_for_query, fields: likely_acct? ? %w(acct) : %w(acct^2 display_name), type: 'best_fields' } }]
+    should_clauses = []
 
-  def query_without_split?
-    split_query_string.size == 1
-  end
+    if account
+      return [] if options[:following] && following_ids.empty?
 
-  def domain_is_local?
-    @domain_is_local ||= TagManager.instance.local_domain?(query_domain)
-  end
+      if options[:following]
+        must_clauses << { terms: { id: following_ids } }
+      elsif following_ids.any?
+        should_clauses << { terms: { id: following_ids, boost: 100 } }
+      end
+    end
 
-  def search_from
-    options[:following] && account ? account.following : Account
-  end
+    query     = { bool: { must: must_clauses, should: should_clauses } }
+    functions = [reputation_score_function, followers_score_function, time_distance_function]
 
-  def exact_match?
-    exact_match.present?
-  end
+    records = AccountsIndex.query(function_score: { query: query, functions: functions, boost_mode: 'multiply', score_mode: 'avg' })
+                           .limit(limit_for_non_exact_results)
+                           .offset(offset)
+                           .objects
+                           .compact
 
-  def exact_match
-    return @exact_match if defined?(@exact_match)
+    ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
 
-    @exact_match = begin
-      if domain_is_local?
-        search_from.without_suspended.find_local(query_username)
-      else
-        search_from.without_suspended.find_remote(query_username, query_domain)
-      end
-    end
+    records
   end
 
-  def search_results
-    @search_results ||= begin
-      if account
-        advanced_search_results
-      else
-        simple_search_results
-      end
-    end
+  def reputation_score_function
+    {
+      script_score: {
+        script: {
+          source: "(doc['followers_count'].value + 0.0) / (doc['followers_count'].value + doc['following_count'].value + 1)",
+        },
+      },
+    }
   end
 
-  def advanced_search_results
-    Account.advanced_search_for(terms_for_query, account, limit_for_non_exact_results, options[:following], offset)
+  def followers_score_function
+    {
+      field_value_factor: {
+        field: 'followers_count',
+        modifier: 'log2p',
+        missing: 1,
+      },
+    }
   end
 
-  def simple_search_results
-    Account.search_for(terms_for_query, limit_for_non_exact_results, offset)
+  def time_distance_function
+    {
+      gauss: {
+        last_status_at: {
+          scale: '30d',
+          offset: '30d',
+          decay: 0.3,
+        },
+      },
+    }
+  end
+
+  def following_ids
+    @following_ids ||= account.active_relationships.pluck(:target_account_id)
   end
 
   def limit_for_non_exact_results
-    if offset.zero? && exact_match?
+    if exact_match?
       limit - 1
     else
       limit
@@ -113,7 +142,39 @@ class AccountSearchService < BaseService
     if domain_is_local?
       query_username
     else
-      "#{query_username} #{query_domain}"
+      query
     end
   end
+
+  def split_query_string
+    @split_query_string ||= query.split('@')
+  end
+
+  def query_username
+    @query_username ||= split_query_string.first || ''
+  end
+
+  def query_domain
+    @query_domain ||= query_without_split? ? nil : split_query_string.last
+  end
+
+  def query_without_split?
+    split_query_string.size == 1
+  end
+
+  def domain_is_local?
+    @domain_is_local ||= TagManager.instance.local_domain?(query_domain)
+  end
+
+  def exact_match?
+    exact_match.present?
+  end
+
+  def username_complete?
+    query.include?('@') && "@#{query}" =~ Account::MENTION_RE
+  end
+
+  def likely_acct?
+    @acct_hint || username_complete?
+  end
 end
diff --git a/spec/services/account_search_service_spec.rb b/spec/services/account_search_service_spec.rb
index 7b071b378..5b7182586 100644
--- a/spec/services/account_search_service_spec.rb
+++ b/spec/services/account_search_service_spec.rb
@@ -1,126 +1,56 @@
 require 'rails_helper'
 
 describe AccountSearchService, type: :service do
-  describe '.call' do
-    describe 'with a query to ignore' do
+  describe '#call' do
+    context 'with a query to ignore' do
       it 'returns empty array for missing query' do
         results = subject.call('', nil, limit: 10)
 
         expect(results).to eq []
       end
-      it 'returns empty array for hashtag query' do
-        results = subject.call('#tag', nil, limit: 10)
 
-        expect(results).to eq []
-      end
       it 'returns empty array for limit zero' do
         Fabricate(:account, username: 'match')
+
         results = subject.call('match', nil, limit: 0)
 
         expect(results).to eq []
       end
     end
 
-    describe 'searching for a simple term that is not an exact match' do
+    context 'searching for a simple term that is not an exact match' do
       it 'does not return a nil entry in the array for the exact match' do
-        match = Fabricate(:account, username: 'matchingusername')
-
+        account = Fabricate(:account, username: 'matchingusername')
         results = subject.call('match', nil, limit: 5)
-        expect(results).to eq [match]
-      end
-    end
 
-    describe 'searching local and remote users' do
-      describe "when only '@'" do
-        before do
-          allow(Account).to receive(:find_local)
-          allow(Account).to receive(:search_for)
-          subject.call('@', nil, limit: 10)
-        end
-
-        it 'uses find_local with empty query to look for local accounts' do
-          expect(Account).to have_received(:find_local).with('')
-        end
-      end
-
-      describe 'when no domain' do
-        before do
-          allow(Account).to receive(:find_local)
-          allow(Account).to receive(:search_for)
-          subject.call('one', nil, limit: 10)
-        end
-
-        it 'uses find_local to look for local accounts' do
-          expect(Account).to have_received(:find_local).with('one')
-        end
-
-        it 'uses search_for to find matches' do
-          expect(Account).to have_received(:search_for).with('one', 10, 0)
-        end
-      end
-
-      describe 'when there is a domain' do
-        before do
-          allow(Account).to receive(:find_remote)
-        end
-
-        it 'uses find_remote to look for remote accounts' do
-          subject.call('two@example.com', nil, limit: 10)
-          expect(Account).to have_received(:find_remote).with('two', 'example.com')
-        end
-
-        describe 'and there is no account provided' do
-          it 'uses search_for to find matches' do
-            allow(Account).to receive(:search_for)
-            subject.call('two@example.com', nil, limit: 10, resolve: false)
-
-            expect(Account).to have_received(:search_for).with('two example.com', 10, 0)
-          end
-        end
-
-        describe 'and there is an account provided' do
-          it 'uses advanced_search_for to find matches' do
-            account = Fabricate(:account)
-            allow(Account).to receive(:advanced_search_for)
-            subject.call('two@example.com', account, limit: 10, resolve: false)
-
-            expect(Account).to have_received(:advanced_search_for).with('two example.com', account, 10, nil, 0)
-          end
-        end
+        expect(results).to eq [account]
       end
     end
 
-    describe 'with an exact match' do
-      it 'returns exact match first, and does not return duplicates' do
-        partial = Fabricate(:account, username: 'exactness')
-        exact = Fabricate(:account, username: 'exact')
-
-        results = subject.call('exact', nil, limit: 10)
-        expect(results.size).to eq 2
-        expect(results).to eq [exact, partial]
-      end
-    end
-
-    describe 'when there is a local domain' do
+    context 'when there is a local domain' do
       around do |example|
         before = Rails.configuration.x.local_domain
+
         example.run
+
         Rails.configuration.x.local_domain = before
       end
 
       it 'returns exact match first' do
         remote     = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e')
         remote_too = Fabricate(:account, username: 'b', domain: 'remote', display_name: 'e')
-        exact = Fabricate(:account, username: 'e')
+        exact      = Fabricate(:account, username: 'e')
+
         Rails.configuration.x.local_domain = 'example.com'
 
         results = subject.call('e@example.com', nil, limit: 2)
+
         expect(results.size).to eq 2
         expect(results).to eq([exact, remote]).or eq([exact, remote_too])
       end
     end
 
-    describe 'when there is a domain but no exact match' do
+    context 'when there is a domain but no exact match' do
       it 'follows the remote account when resolve is true' do
         service = double(call: nil)
         allow(ResolveAccountService).to receive(:new).and_return(service)
@@ -138,23 +68,21 @@ describe AccountSearchService, type: :service do
       end
     end
 
-    describe 'should not include suspended accounts' do
-      it 'returns the fuzzy match first, and does not return suspended exacts' do
-        partial = Fabricate(:account, username: 'exactness')
-        exact = Fabricate(:account, username: 'exact', suspended: true)
+    it 'returns the fuzzy match first, and does not return suspended exacts' do
+      partial = Fabricate(:account, username: 'exactness')
+      exact   = Fabricate(:account, username: 'exact', suspended: true)
+      results = subject.call('exact', nil, limit: 10)
 
-        results = subject.call('exact', nil, limit: 10)
-        expect(results.size).to eq 1
-        expect(results).to eq [partial]
-      end
+      expect(results.size).to eq 1
+      expect(results).to eq [partial]
+    end
 
-      it "does not return suspended remote accounts" do
-        remote = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e', suspended: true)
+    it "does not return suspended remote accounts" do
+      remote  = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e', suspended: true)
+      results = subject.call('a@example.com', nil, limit: 2)
 
-        results = subject.call('a@example.com', nil, limit: 2)
-        expect(results.size).to eq 0
-        expect(results).to eq []
-      end
+      expect(results.size).to eq 0
+      expect(results).to eq []
     end
   end
 end
-- 
cgit 


From 5d8ee24cd5d82d41a6914ee9982c38cc18d859b4 Mon Sep 17 00:00:00 2001
From: Stanislas <angristan@pm.me>
Date: Sat, 17 Aug 2019 22:04:15 +0200
Subject: Remove WebP support (#11589)

---
 app/javascript/mastodon/utils/resize_image.js | 2 +-
 app/models/concerns/account_avatar.rb         | 2 +-
 app/models/concerns/account_header.rb         | 2 +-
 app/models/custom_emoji.rb                    | 2 +-
 app/models/media_attachment.rb                | 4 ++--
 app/models/preview_card.rb                    | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'app/models')

diff --git a/app/javascript/mastodon/utils/resize_image.js b/app/javascript/mastodon/utils/resize_image.js
index d566edb03..7196dc96b 100644
--- a/app/javascript/mastodon/utils/resize_image.js
+++ b/app/javascript/mastodon/utils/resize_image.js
@@ -31,7 +31,7 @@ const loadImage = inputFile => new Promise((resolve, reject) => {
 });
 
 const getOrientation = (img, type = 'image/png') => new Promise(resolve => {
-  if (!['image/jpeg', 'image/webp'].includes(type)) {
+  if (type !== 'image/jpeg') {
     resolve(1);
     return;
   }
diff --git a/app/models/concerns/account_avatar.rb b/app/models/concerns/account_avatar.rb
index 5fff3ef5d..2d5ebfca3 100644
--- a/app/models/concerns/account_avatar.rb
+++ b/app/models/concerns/account_avatar.rb
@@ -3,7 +3,7 @@
 module AccountAvatar
   extend ActiveSupport::Concern
 
-  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
+  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
   LIMIT = 2.megabytes
 
   class_methods do
diff --git a/app/models/concerns/account_header.rb b/app/models/concerns/account_header.rb
index a748fdff7..067e166eb 100644
--- a/app/models/concerns/account_header.rb
+++ b/app/models/concerns/account_header.rb
@@ -3,7 +3,7 @@
 module AccountHeader
   extend ActiveSupport::Concern
 
-  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
+  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
   LIMIT = 2.megabytes
   MAX_PIXELS = 750_000 # 1500x500px
 
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index 643a7e46a..b21ad9042 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -28,7 +28,7 @@ class CustomEmoji < ApplicationRecord
     :(#{SHORTCODE_RE_FRAGMENT}):
     (?=[^[:alnum:]:]|$)/x
 
-  IMAGE_MIME_TYPES = %w(image/png image/gif image/webp).freeze
+  IMAGE_MIME_TYPES = %w(image/png image/gif).freeze
 
   belongs_to :category, class_name: 'CustomEmojiCategory', optional: true
   has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 2fc5ccf27..fe76b36aa 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -26,11 +26,11 @@ class MediaAttachment < ApplicationRecord
 
   enum type: [:image, :gifv, :video, :unknown, :audio]
 
-  IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif .webp).freeze
+  IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif).freeze
   VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
   AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp).freeze
 
-  IMAGE_MIME_TYPES             = %w(image/jpeg image/png image/gif image/webp).freeze
+  IMAGE_MIME_TYPES             = %w(image/jpeg image/png image/gif).freeze
   VIDEO_MIME_TYPES             = %w(video/webm video/mp4 video/quicktime video/ogg).freeze
   VIDEO_CONVERTIBLE_MIME_TYPES = %w(video/webm video/quicktime).freeze
   AUDIO_MIME_TYPES             = %w(audio/wave audio/wav audio/x-wav audio/x-pn-wave audio/ogg audio/mpeg audio/mp3 audio/webm audio/flac audio/aac audio/m4a audio/3gpp).freeze
diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb
index f26ea0c74..a792b352b 100644
--- a/app/models/preview_card.rb
+++ b/app/models/preview_card.rb
@@ -25,7 +25,7 @@
 #
 
 class PreviewCard < ApplicationRecord
-  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
+  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
   LIMIT = 1.megabytes
 
   self.inheritance_column = false
-- 
cgit 


From cc0a55cf9aead00e1cb044649f84c2187e0e4a35 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 18 Aug 2019 03:45:51 +0200
Subject: Add more accurate hashtag search (#11579)

* Add more accurate hashtag search

Using ElasticSearch to index hashtags with edge n-grams and score
them by usage within the last 7 days since last activity. Only
hashtags that have been reviewed and are listable can appear in
searches, unless they match the query exactly

* Fix search analyzer dropping non-ascii characters
---
 app/chewy/tags_index.rb                            | 37 ++++++++++
 app/models/tag.rb                                  | 14 ++--
 app/models/trending_tags.rb                        |  3 +
 app/services/account_search_service.rb             |  2 +-
 app/services/search_service.rb                     |  8 +--
 app/services/tag_search_service.rb                 | 82 ++++++++++++++++++++++
 config/locales/simple_form.en.yml                  |  2 +-
 .../20190815225426_add_last_status_at_to_tags.rb   |  6 ++
 db/schema.rb                                       |  4 +-
 spec/models/tag_spec.rb                            |  4 +-
 10 files changed, 149 insertions(+), 13 deletions(-)
 create mode 100644 app/chewy/tags_index.rb
 create mode 100644 app/services/tag_search_service.rb
 create mode 100644 db/migrate/20190815225426_add_last_status_at_to_tags.rb

(limited to 'app/models')

diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb
new file mode 100644
index 000000000..300fc128f
--- /dev/null
+++ b/app/chewy/tags_index.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class TagsIndex < Chewy::Index
+  settings index: { refresh_interval: '15m' }, analysis: {
+    analyzer: {
+      content: {
+        tokenizer: 'keyword',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+
+      edge_ngram: {
+        tokenizer: 'edge_ngram',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+    },
+
+    tokenizer: {
+      edge_ngram: {
+        type: 'edge_ngram',
+        min_gram: 2,
+        max_gram: 15,
+      },
+    },
+  }
+
+  define_type ::Tag.listable, delete_if: ->(tag) { tag.destroyed? || !tag.listable? } do
+    root date_detection: false do
+      field :name, type: 'text', analyzer: 'content' do
+        field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
+      end
+
+      field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
+      field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
+      field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
+    end
+  end
+end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 1364d1dba..5094d973d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -13,6 +13,8 @@
 #  listable            :boolean
 #  reviewed_at         :datetime
 #  requested_review_at :datetime
+#  last_status_at      :datetime
+#  last_trend_at       :datetime
 #
 
 class Tag < ApplicationRecord
@@ -33,7 +35,8 @@ class Tag < ApplicationRecord
   scope :unreviewed, -> { where(reviewed_at: nil) }
   scope :pending_review, -> { unreviewed.where.not(requested_review_at: nil) }
   scope :usable, -> { where(usable: [true, nil]) }
-  scope :discoverable, -> { where(listable: [true, nil]).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
+  scope :listable, -> { where(listable: [true, nil]) }
+  scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
@@ -44,6 +47,8 @@ class Tag < ApplicationRecord
 
   after_save :save_account_tag_stat
 
+  update_index('tags#tag', :self) if Chewy.enabled?
+
   def account_tag_stat
     super || build_account_tag_stat
   end
@@ -121,9 +126,10 @@ class Tag < ApplicationRecord
       normalized_term = normalize(term.strip).mb_chars.downcase.to_s
       pattern         = sanitize_sql_like(normalized_term) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern))
-         .where(arel_table[:score].gt(0).or(arel_table[:name].lower.eq(normalized_term)))
-         .order(Arel.sql('length(name) ASC, score DESC, name ASC'))
+      Tag.listable
+         .where(arel_table[:name].lower.matches(pattern))
+         .where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil)))
+         .order(Arel.sql('length(name) ASC, name ASC'))
          .limit(limit)
          .offset(offset)
     end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 3d60a7fea..e4ce988c1 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -17,6 +17,9 @@ class TrendingTags
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
       increment_vote!(tag, at_time)
+
+      tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
+      tag.update(last_trend_at: Time.now.utc)  if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago)
     end
 
     def get(limit, filtered: true)
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index d7bccdfe0..01caaefa9 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -109,7 +109,7 @@ class AccountSearchService < BaseService
       field_value_factor: {
         field: 'followers_count',
         modifier: 'log2p',
-        missing: 1,
+        missing: 0,
       },
     }
   end
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index 786d34b15..fe601bbf4 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -57,10 +57,10 @@ class SearchService < BaseService
   end
 
   def perform_hashtags_search!
-    Tag.search_for(
-      @query.gsub(/\A#/, ''),
-      @limit,
-      @offset
+    TagSearchService.new.call(
+      @query,
+      limit: @limit,
+      offset: @offset
     )
   end
 
diff --git a/app/services/tag_search_service.rb b/app/services/tag_search_service.rb
new file mode 100644
index 000000000..64dd76bb7
--- /dev/null
+++ b/app/services/tag_search_service.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+class TagSearchService < BaseService
+  def call(query, options = {})
+    @query  = query.strip.gsub(/\A#/, '')
+    @offset = options[:offset].to_i
+    @limit  = options[:limit].to_i
+
+    if Chewy.enabled?
+      from_elasticsearch
+    else
+      from_database
+    end
+  end
+
+  private
+
+  def from_elasticsearch
+    query = {
+      function_score: {
+        query: {
+          multi_match: {
+            query: @query,
+            fields: %w(name.edge_ngram name),
+            type: 'most_fields',
+            operator: 'and',
+          },
+        },
+
+        functions: [
+          {
+            field_value_factor: {
+              field: 'usage',
+              modifier: 'log2p',
+              missing: 0,
+            },
+          },
+
+          {
+            gauss: {
+              last_status_at: {
+                scale: '7d',
+                offset: '14d',
+                decay: 0.5,
+              },
+            },
+          },
+        ],
+
+        boost_mode: 'multiply',
+      },
+    }
+
+    filter = {
+      bool: {
+        should: [
+          {
+            term: {
+              reviewed: {
+                value: true,
+              },
+            },
+          },
+
+          {
+            term: {
+              name: {
+                value: @query,
+              },
+            },
+          },
+        ],
+      },
+    }
+
+    TagsIndex.query(query).filter(filter).limit(@limit).offset(@offset).objects.compact
+  end
+
+  def from_database
+    Tag.search_for(@query, @limit, @offset)
+  end
+end
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index e15d5904f..98f0843d0 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -142,7 +142,7 @@ en:
         report: Send e-mail when a new report is submitted
         trending_tag: Send e-mail when an unreviewed hashtag is trending
       tag:
-        listable: Allow this hashtag to appear on the profile directory
+        listable: Allow this hashtag to appear in searches and on the profile directory
         trendable: Allow this hashtag to appear under trends
         usable: Allow toots to use this hashtag
     'no': 'No'
diff --git a/db/migrate/20190815225426_add_last_status_at_to_tags.rb b/db/migrate/20190815225426_add_last_status_at_to_tags.rb
new file mode 100644
index 000000000..d83537c47
--- /dev/null
+++ b/db/migrate/20190815225426_add_last_status_at_to_tags.rb
@@ -0,0 +1,6 @@
+class AddLastStatusAtToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :last_status_at, :datetime
+    add_column :tags, :last_trend_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f8fc6a821..0da20d62f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_07_135426) do
+ActiveRecord::Schema.define(version: 2019_08_15_225426) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -667,6 +667,8 @@ ActiveRecord::Schema.define(version: 2019_08_07_135426) do
     t.boolean "listable"
     t.datetime "reviewed_at"
     t.datetime "requested_review_at"
+    t.datetime "last_status_at"
+    t.datetime "last_trend_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 9d700849b..2bb30fb57 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -136,8 +136,8 @@ RSpec.describe Tag, type: :model do
     end
 
     it 'finds the exact matching tag as the first item' do
-      similar_tag = Fabricate(:tag, name: "matchlater", score: 1)
-      tag = Fabricate(:tag, name: "match", score: 1)
+      similar_tag = Fabricate(:tag, name: "matchlater", reviewed_at: Time.now.utc)
+      tag = Fabricate(:tag, name: "match", reviewed_at: Time.now.utc)
 
       results = Tag.search_for("match")
 
-- 
cgit 


From 96702e7f6727d9c688b2c6e2527c4a5bfefff886 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 18 Aug 2019 14:55:03 +0200
Subject: Add `tootctl cache recount` command (#11597)

---
 app/models/concerns/account_counters.rb |  3 ++-
 app/models/status.rb                    |  7 +++--
 lib/mastodon/cache_cli.rb               | 45 +++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 3 deletions(-)

(limited to 'app/models')

diff --git a/app/models/concerns/account_counters.rb b/app/models/concerns/account_counters.rb
index 3581df8dd..6e25e1905 100644
--- a/app/models/concerns/account_counters.rb
+++ b/app/models/concerns/account_counters.rb
@@ -26,7 +26,8 @@ module AccountCounters
   private
 
   def save_account_stat
-    return unless account_stat&.changed?
+    return unless association(:account_stat).loaded? && account_stat&.changed?
+
     account_stat.save
   end
 end
diff --git a/app/models/status.rb b/app/models/status.rb
index 23682c84b..0538c4e9e 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -388,13 +388,16 @@ class Status < ApplicationRecord
     end
   end
 
+  def status_stat
+    super || build_status_stat
+  end
+
   private
 
   def update_status_stat!(attrs)
     return if marked_for_destruction? || destroyed?
 
-    record = status_stat || build_status_stat
-    record.update(attrs)
+    status_stat.update(attrs)
   end
 
   def store_uri
diff --git a/lib/mastodon/cache_cli.rb b/lib/mastodon/cache_cli.rb
index e9b6667b3..5b0eea91b 100644
--- a/lib/mastodon/cache_cli.rb
+++ b/lib/mastodon/cache_cli.rb
@@ -15,5 +15,50 @@ module Mastodon
       Rails.cache.clear
       say('OK', :green)
     end
+
+    desc 'recount TYPE', 'Update hard-cached counters'
+    long_desc <<~LONG_DESC
+      Update hard-cached counters of TYPE by counting referenced
+      records from scratch. TYPE can be "accounts" or "statuses".
+
+      It may take a very long time to finish, depending on the
+      size of the database.
+    LONG_DESC
+    def recount(type)
+      processed = 0
+
+      case type
+      when 'accounts'
+        Account.local.includes(:account_stat).find_each do |account|
+          account_stat                 = account.account_stat
+          account_stat.following_count = account.active_relationships.count
+          account_stat.followers_count = account.passive_relationships.count
+          account_stat.statuses_count  = account.statuses.where.not(visibility: :direct).count
+
+          account_stat.save if account_stat.changed?
+
+          processed += 1
+          say('.', :green, false)
+        end
+      when 'statuses'
+        Status.includes(:status_stat).find_each do |status|
+          status_stat                  = status.status_stat
+          status_stat.replies_count    = status.replies.where.not(visibility: :direct).count
+          status_stat.reblogs_count    = status.reblogs.count
+          status_stat.favourites_count = status.favourites.count
+
+          status_stat.save if status_stat.changed?
+
+          processed += 1
+          say('.', :green, false)
+        end
+      else
+        say("Unknown type: #{type}", :red)
+        exit(1)
+      end
+
+      say
+      say("OK, recounted #{processed} records", :green)
+    end
   end
 end
-- 
cgit 


From 9b6a5ed109e1986149c1f15a41d4f442ae8ae39c Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 19 Aug 2019 11:35:48 +0200
Subject: Add public blocks to /about/blocks (#11298)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add automatic blocklist display in /about/blocks

Inspired by https://github.com/Gargron/mastodon.social-misc

* Add admin option to set who can see instance blocks

* Normalize locales files

* Rename “Sandbox” to “Silence” for consistency

* Disable /about/blocks when in whitelist mode

* Optionally display rationale for domain blocks

* Only display domain blocks that have user-facing limitations, and order them

* Redesign table of blocked domains to better handle long domain names and rationales

* Change domain blocks ordering now that rationales aren't displayed right away

* Only show explanation for block severities actually in use

* Reword instance block explanations and add disclaimer for public fetch mode
---
 app/controllers/about_controller.rb        | 34 ++++++++++++++-
 app/javascript/packs/public.js             |  9 ++++
 app/javascript/styles/mastodon/tables.scss | 67 ++++++++++++++++++++++++++++++
 app/models/domain_block.rb                 |  1 +
 app/models/form/admin_settings.rb          |  4 ++
 app/views/about/blocks.html.haml           | 48 +++++++++++++++++++++
 app/views/admin/settings/edit.html.haml    |  6 +++
 config/locales/en.yml                      | 24 +++++++++++
 config/routes.rb                           |  7 ++--
 config/settings.yml                        |  2 +
 10 files changed, 197 insertions(+), 5 deletions(-)
 create mode 100644 app/views/about/blocks.html.haml

(limited to 'app/models')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index d276e8fe5..5e942e5c0 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,10 +3,12 @@
 class AboutController < ApplicationController
   layout 'public'
 
-  before_action :require_open_federation!, only: [:show, :more]
+  before_action :require_open_federation!, only: [:show, :more, :blocks]
+  before_action :check_blocklist_enabled, only: [:blocks]
+  before_action :authenticate_user!, only: [:blocks], if: :blocklist_account_required?
   before_action :set_body_classes, only: :show
   before_action :set_instance_presenter
-  before_action :set_expires_in
+  before_action :set_expires_in, only: [:show, :more, :terms]
 
   skip_before_action :require_functional!, only: [:more, :terms]
 
@@ -18,12 +20,40 @@ class AboutController < ApplicationController
 
   def terms; end
 
+  def blocks
+    @show_rationale = Setting.show_domain_blocks_rationale == 'all'
+    @show_rationale |= Setting.show_domain_blocks_rationale == 'users' && !current_user.nil? && current_user.functional?
+    @blocks = DomainBlock.with_user_facing_limitations.order('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), reject_media, domain').to_a
+  end
+
   private
 
   def require_open_federation!
     not_found if whitelist_mode?
   end
 
+  def check_blocklist_enabled
+    not_found if Setting.show_domain_blocks == 'disabled'
+  end
+
+  def blocklist_account_required?
+    Setting.show_domain_blocks == 'users'
+  end
+
+  def block_severity_text(block)
+    if block.severity == 'suspend'
+      I18n.t('domain_blocks.suspension')
+    else
+      limitations = []
+      limitations << I18n.t('domain_blocks.media_block') if block.reject_media?
+      limitations << I18n.t('domain_blocks.silence') if block.severity == 'silence'
+      limitations.join(', ')
+    end
+  end
+
+  helper_method :block_severity_text
+  helper_method :public_fetch_mode?
+
   def new_user
     User.new.tap do |user|
       user.build_account
diff --git a/app/javascript/packs/public.js b/app/javascript/packs/public.js
index b58622a8d..c5cd7129f 100644
--- a/app/javascript/packs/public.js
+++ b/app/javascript/packs/public.js
@@ -141,6 +141,15 @@ function main() {
     return false;
   });
 
+  delegate(document, '.blocks-table button.icon-button', 'click', function(e) {
+    e.preventDefault();
+
+    const classList = this.firstElementChild.classList;
+    classList.toggle('fa-chevron-down');
+    classList.toggle('fa-chevron-up');
+    this.parentElement.parentElement.nextElementSibling.classList.toggle('hidden');
+  });
+
   delegate(document, '.modal-button', 'click', e => {
     e.preventDefault();
 
diff --git a/app/javascript/styles/mastodon/tables.scss b/app/javascript/styles/mastodon/tables.scss
index 11ac6dfeb..fe6beba5d 100644
--- a/app/javascript/styles/mastodon/tables.scss
+++ b/app/javascript/styles/mastodon/tables.scss
@@ -241,3 +241,70 @@ a.table-action-link {
     }
   }
 }
+
+.blocks-table {
+  width: 100%;
+  max-width: 100%;
+  border-spacing: 0;
+  border-collapse: collapse;
+  table-layout: fixed;
+  border: 1px solid darken($ui-base-color, 8%);
+
+  thead {
+    border: 1px solid darken($ui-base-color, 8%);
+    background: darken($ui-base-color, 4%);
+    font-weight: 500;
+
+    th.severity-column {
+      width: 120px;
+    }
+
+    th.button-column {
+      width: 23px;
+    }
+  }
+
+  tbody > tr {
+    border: 1px solid darken($ui-base-color, 8%);
+    border-bottom: 0;
+    background: darken($ui-base-color, 4%);
+
+    &:hover {
+      background: darken($ui-base-color, 2%);
+    }
+
+    &.even {
+      background: $ui-base-color;
+
+      &:hover {
+        background: lighten($ui-base-color, 2%);
+      }
+    }
+
+    &.rationale {
+      background: lighten($ui-base-color, 4%);
+      border-top: 0;
+
+      &:hover {
+        background: lighten($ui-base-color, 6%);
+      }
+
+      &.hidden {
+        display: none;
+      }
+    }
+
+    td:first-child {
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
+  }
+
+  th,
+  td {
+    padding: 8px;
+    line-height: 18px;
+    vertical-align: top;
+    text-align: left;
+  }
+}
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 37b8d98c6..4383cbd05 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -25,6 +25,7 @@ class DomainBlock < ApplicationRecord
   delegate :count, to: :accounts, prefix: true
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
+  scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) }
 
   class << self
     def suspend?(domain)
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 051268375..6bc3ca9f5 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -30,6 +30,8 @@ class Form::AdminSettings
     mascot
     spam_check_enabled
     trends
+    show_domain_blocks
+    show_domain_blocks_rationale
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -60,6 +62,8 @@ class Form::AdminSettings
   validates :site_contact_email, :site_contact_username, presence: true
   validates :site_contact_username, existing_username: true
   validates :bootstrap_timeline_accounts, existing_username: { multiple: true }
+  validates :show_domain_blocks, inclusion: { in: %w(disabled users all) }
+  validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) }
 
   def initialize(_attributes = {})
     super
diff --git a/app/views/about/blocks.html.haml b/app/views/about/blocks.html.haml
new file mode 100644
index 000000000..a81a4d1eb
--- /dev/null
+++ b/app/views/about/blocks.html.haml
@@ -0,0 +1,48 @@
+- content_for :page_title do
+  = t('domain_blocks.title', instance: site_hostname)
+
+.grid
+  .column-0
+    .box-widget.rich-formatting
+      %h2= t('domain_blocks.blocked_domains')
+      %p= t('domain_blocks.description', instance: site_hostname)
+      .table-wrapper
+        %table.blocks-table
+          %thead
+            %tr
+              %th= t('domain_blocks.domain')
+              %th.severity-column= t('domain_blocks.severity')
+              - if @show_rationale
+                %th.button-column
+          %tbody
+            - if @blocks.empty?
+              %tr
+                %td{ colspan: @show_rationale ? 3 : 2 }= t('domain_blocks.no_domain_blocks')
+            - else
+              - @blocks.each_with_index do |block, i|
+                %tr{ class: i % 2 == 0 ? 'even': nil }
+                  %td{ title: block.domain }= block.domain
+                  %td= block_severity_text(block)
+                  - if @show_rationale
+                    %td
+                      - if block.public_comment.present?
+                        %button.icon-button{ title: t('domain_blocks.show_rationale'), 'aria-label' => t('domain_blocks.show_rationale') }
+                          = fa_icon 'chevron-down fw', 'aria-hidden' => true
+                - if @show_rationale
+                  - if block.public_comment.present?
+                    %tr.rationale.hidden
+                      %td{ colspan: 3 }= block.public_comment.presence
+      %h2= t('domain_blocks.severity_legend.title')
+      - if @blocks.any? { |block| block.reject_media? }
+        %h3= t('domain_blocks.media_block')
+        %p= t('domain_blocks.severity_legend.media_block')
+      - if @blocks.any? { |block| block.severity == 'silence' }
+        %h3= t('domain_blocks.silence')
+        %p= t('domain_blocks.severity_legend.silence')
+      - if @blocks.any? { |block| block.severity == 'suspend' }
+        %h3= t('domain_blocks.suspension')
+        %p= t('domain_blocks.severity_legend.suspension')
+        - if public_fetch_mode?
+          %p= t('domain_blocks.severity_legend.suspension_disclaimer')
+  .column-1
+    = render 'application/sidebar'
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 28c0ece15..28880c087 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -79,6 +79,12 @@
   .fields-group
     = f.input :min_invite_role, wrapper: :with_label, collection: %i(disabled user moderator admin), label: t('admin.settings.registrations.min_invite_role.title'), label_method: lambda { |role| role == :disabled ? t('admin.settings.registrations.min_invite_role.disabled') : t("admin.accounts.roles.#{role}") }, include_blank: false, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
 
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :show_domain_blocks, wrapper: :with_label, collection: %i(disabled users all), label: t('admin.settings.domain_blocks.title'), label_method: lambda { |value| t("admin.settings.domain_blocks.#{value}") }, include_blank: false, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :show_domain_blocks_rationale, wrapper: :with_label, collection: %i(disabled users all), label: t('admin.settings.domain_blocks_rationale.title'), label_method: lambda { |value| t("admin.settings.domain_blocks.#{value}") }, include_blank: false, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
+
   .fields-group
     = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 }
     = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } unless whitelist_mode?
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 4696dc11b..8d267065c 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -423,6 +423,13 @@ en:
       custom_css:
         desc_html: Modify the look with CSS loaded on every page
         title: Custom CSS
+      domain_blocks:
+        all: To everyone
+        disabled: To no one
+        title: Show domain blocks
+        users: To logged-in local users
+      domain_blocks_rationale:
+        title: Show rationale
       hero:
         desc_html: Displayed on the frontpage. At least 600x100px recommended. When not set, falls back to server thumbnail
         title: Hero image
@@ -630,6 +637,23 @@ en:
     people:
       one: "%{count} person"
       other: "%{count} people"
+  domain_blocks:
+    blocked_domains: List of limited and blocked domains
+    description: This is the list of servers that %{instance} limits or reject federation with.
+    domain: Domain
+    media_block: Media block
+    no_domain_blocks: "(No domain blocks)"
+    severity: Severity
+    severity_legend:
+      media_block: Media files coming from the server are neither fetched, stored, or displayed to the user.
+      silence: Accounts from silenced servers can be found, followed and interacted with, but their toots will not appear in the public timelines, and notifications from them will not reach local users who are not following them.
+      suspension: No content from suspended servers is stored or displayed, nor is any content sent to them. Interactions from suspended servers are ignored.
+      suspension_disclaimer: Suspended servers may occasionally retrieve public content from this server.
+      title: Severities
+    show_rationale: Show rationale
+    silence: Silence
+    suspension: Suspension
+    title: "%{instance} List of blocked instances"
   domain_validator:
     invalid_domain: is not a valid domain name
   errors:
diff --git a/config/routes.rb b/config/routes.rb
index 9c33b8190..9ae24b0cd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -423,9 +423,10 @@ Rails.application.routes.draw do
 
   get '/web/(*any)', to: 'home#index', as: :web
 
-  get '/about',      to: 'about#show'
-  get '/about/more', to: 'about#more'
-  get '/terms',      to: 'about#terms'
+  get '/about',        to: 'about#show'
+  get '/about/more',   to: 'about#more'
+  get '/about/blocks', to: 'about#blocks'
+  get '/terms',        to: 'about#terms'
 
   root 'home#index'
 
diff --git a/config/settings.yml b/config/settings.yml
index 4e5eefb59..6dbc46706 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -64,6 +64,8 @@ defaults: &defaults
   peers_api_enabled: true
   show_known_fediverse_at_about_page: true
   spam_check_enabled: true
+  show_domain_blocks: 'disabled'
+  show_domain_blocks_rationale: 'disabled'
 
 development:
   <<: *defaults
-- 
cgit 


From cb62a83a71a9679061f8daa468119124b5da5e76 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 19 Aug 2019 11:40:42 +0200
Subject: Add invite comments (#10465)

---
 app/controllers/invites_controller.rb               | 2 +-
 app/models/invite.rb                                | 3 +++
 app/views/invites/_form.html.haml                   | 3 +++
 app/views/invites/_invite.html.haml                 | 3 +++
 app/views/invites/index.html.haml                   | 1 +
 db/migrate/20190403141604_add_comment_to_invites.rb | 5 +++++
 db/schema.rb                                        | 1 +
 7 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20190403141604_add_comment_to_invites.rb

(limited to 'app/models')

diff --git a/app/controllers/invites_controller.rb b/app/controllers/invites_controller.rb
index de5280305..8d92147e2 100644
--- a/app/controllers/invites_controller.rb
+++ b/app/controllers/invites_controller.rb
@@ -43,7 +43,7 @@ class InvitesController < ApplicationController
   end
 
   def resource_params
-    params.require(:invite).permit(:max_uses, :expires_in, :autofollow)
+    params.require(:invite).permit(:max_uses, :expires_in, :autofollow, :comment)
   end
 
   def set_body_classes
diff --git a/app/models/invite.rb b/app/models/invite.rb
index 02ab8e0b2..29d25eae8 100644
--- a/app/models/invite.rb
+++ b/app/models/invite.rb
@@ -12,6 +12,7 @@
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #  autofollow :boolean          default(FALSE), not null
+#  comment    :text
 #
 
 class Invite < ApplicationRecord
@@ -22,6 +23,8 @@ class Invite < ApplicationRecord
 
   scope :available, -> { where(expires_at: nil).or(where('expires_at >= ?', Time.now.utc)) }
 
+  validates :comment, length: { maximum: 420 }
+
   before_validation :set_code
 
   def valid_for_use?
diff --git a/app/views/invites/_form.html.haml b/app/views/invites/_form.html.haml
index 3a2a5ef0e..b19f70539 100644
--- a/app/views/invites/_form.html.haml
+++ b/app/views/invites/_form.html.haml
@@ -10,5 +10,8 @@
   .fields-group
     = f.input :autofollow, wrapper: :with_label
 
+  .fields-group
+    = f.input :comment, wrapper: :with_label, input_html: { maxlength: 420 }
+
   .actions
     = f.button :button, t('invites.generate'), type: :submit
diff --git a/app/views/invites/_invite.html.haml b/app/views/invites/_invite.html.haml
index 62799ca5b..03050c868 100644
--- a/app/views/invites/_invite.html.haml
+++ b/app/views/invites/_invite.html.haml
@@ -20,6 +20,9 @@
     %td{ colspan: 2 }
       = t('invites.expired')
 
+  %td
+    = invite.comment
+
   %td
     - if invite.valid_for_use? && policy(invite).destroy?
       = table_link_to 'times', t('invites.delete'), invite_path(invite), method: :delete
diff --git a/app/views/invites/index.html.haml b/app/views/invites/index.html.haml
index 61420ab1e..62065d6ae 100644
--- a/app/views/invites/index.html.haml
+++ b/app/views/invites/index.html.haml
@@ -15,6 +15,7 @@
         %th
         %th= t('invites.table.uses')
         %th= t('invites.table.expires_at')
+        %th= t('invites.table.comment')
         %th
     %tbody
       = render @invites
diff --git a/db/migrate/20190403141604_add_comment_to_invites.rb b/db/migrate/20190403141604_add_comment_to_invites.rb
new file mode 100644
index 000000000..f0d7b1dcd
--- /dev/null
+++ b/db/migrate/20190403141604_add_comment_to_invites.rb
@@ -0,0 +1,5 @@
+class AddCommentToInvites < ActiveRecord::Migration[5.2]
+  def change
+    add_column :invites, :comment, :text
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 0da20d62f..18f615d61 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -344,6 +344,7 @@ ActiveRecord::Schema.define(version: 2019_08_15_225426) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.boolean "autofollow", default: false, null: false
+    t.text "comment"
     t.index ["code"], name: "index_invites_on_code", unique: true
     t.index ["user_id"], name: "index_invites_on_user_id"
   end
-- 
cgit 


From dff46b260b2f7d765d254c84a4b89105c7de5e97 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 19 Aug 2019 20:36:44 +0200
Subject: Fix ignoring whole status because of one invalid hashtag (#11621)

Fix #11618
---
 app/models/tag.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 5094d973d..945e3a3c6 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -114,7 +114,7 @@ class Tag < ApplicationRecord
   class << self
     def find_or_create_by_names(name_or_names)
       Array(name_or_names).map(&method(:normalize)).uniq { |str| str.mb_chars.downcase.to_s }.map do |normalized_name|
-        tag = matching_name(normalized_name).first || create(name: normalized_name)
+        tag = matching_name(normalized_name).first || create!(name: normalized_name)
 
         yield tag if block_given?
 
-- 
cgit