diff options
40 files changed, 409 insertions, 162 deletions
diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 957a2cbc9..dfb8fcb8b 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -61,7 +61,7 @@ module JsonLdHelper def fetch_resource_without_id_validation(uri) build_request(uri).perform do |response| - response.code == 200 ? body_to_json(response.to_s) : nil + response.code == 200 ? body_to_json(response.body_with_limit) : nil end end diff --git a/app/javascript/mastodon/actions/accounts.js b/app/javascript/mastodon/actions/accounts.js index 1d1947aca..7cacff909 100644 --- a/app/javascript/mastodon/actions/accounts.js +++ b/app/javascript/mastodon/actions/accounts.js @@ -1,5 +1,5 @@ import api, { getLinks } from '../api'; -import asyncDB from '../db/async'; +import asyncDB from '../storage/db'; import { importAccount, importFetchedAccount, importFetchedAccounts } from './importer'; export const ACCOUNT_FETCH_REQUEST = 'ACCOUNT_FETCH_REQUEST'; diff --git a/app/javascript/mastodon/actions/importer/index.js b/app/javascript/mastodon/actions/importer/index.js index d1ea40c36..e671d417c 100644 --- a/app/javascript/mastodon/actions/importer/index.js +++ b/app/javascript/mastodon/actions/importer/index.js @@ -1,4 +1,4 @@ -import { putAccounts, putStatuses } from '../../db/modifier'; +import { putAccounts, putStatuses } from '../../storage/modifier'; import { normalizeAccount, normalizeStatus } from './normalizer'; export const ACCOUNT_IMPORT = 'ACCOUNT_IMPORT'; @@ -39,7 +39,7 @@ export function importFetchedAccounts(accounts) { pushUnique(normalAccounts, normalizeAccount(account)); if (account.moved) { - processAccount(account); + processAccount(account.moved); } } diff --git a/app/javascript/mastodon/actions/importer/normalizer.js b/app/javascript/mastodon/actions/importer/normalizer.js index c88f6946f..1b09f319f 100644 --- a/app/javascript/mastodon/actions/importer/normalizer.js +++ b/app/javascript/mastodon/actions/importer/normalizer.js @@ -10,6 +10,10 @@ export function normalizeAccount(account) { account.display_name_html = emojify(escapeTextContentForBrowser(displayName)); account.note_emojified = emojify(account.note); + if (account.moved) { + account.moved = account.moved.id; + } + return account; } diff --git a/app/javascript/mastodon/actions/statuses.js b/app/javascript/mastodon/actions/statuses.js index dcd813dd9..d28aef880 100644 --- a/app/javascript/mastodon/actions/statuses.js +++ b/app/javascript/mastodon/actions/statuses.js @@ -1,6 +1,6 @@ import api from '../api'; -import asyncDB from '../db/async'; -import { evictStatus } from '../db/modifier'; +import asyncDB from '../storage/db'; +import { evictStatus } from '../storage/modifier'; import { deleteFromTimelines } from './timelines'; import { fetchStatusCard } from './cards'; diff --git a/app/javascript/mastodon/components/status_content.js b/app/javascript/mastodon/components/status_content.js index b6082f008..9b86592f6 100644 --- a/app/javascript/mastodon/components/status_content.js +++ b/app/javascript/mastodon/components/status_content.js @@ -158,7 +158,7 @@ export default class StatusContent extends React.PureComponent { <p style={{ marginBottom: hidden && status.get('mentions').isEmpty() ? '0px' : null }}> <span dangerouslySetInnerHTML={spoilerContent} /> {' '} - <button tabIndex='0' className='status__content__spoiler-link' onClick={this.handleSpoilerClick}>{toggleText}</button> + <button tabIndex='0' className={`status__content__spoiler-link ${hidden ? 'status__content__spoiler-link--show-more' : 'status__content__spoiler-link--show-less'}`} onClick={this.handleSpoilerClick}>{toggleText}</button> </p> {mentionsPlaceholder} diff --git a/app/javascript/mastodon/db/modifier.js b/app/javascript/mastodon/db/modifier.js deleted file mode 100644 index eb951905a..000000000 --- a/app/javascript/mastodon/db/modifier.js +++ /dev/null @@ -1,93 +0,0 @@ -import asyncDB from './async'; - -const limit = 1024; - -function put(name, objects, callback) { - asyncDB.then(db => { - const putTransaction = db.transaction(name, 'readwrite'); - const putStore = putTransaction.objectStore(name); - const putIndex = putStore.index('id'); - - objects.forEach(object => { - function add() { - putStore.add(object); - } - - putIndex.getKey(object.id).onsuccess = retrieval => { - if (retrieval.target.result) { - putStore.delete(retrieval.target.result).onsuccess = add; - } else { - add(); - } - }; - }); - - putTransaction.oncomplete = () => { - const readTransaction = db.transaction(name, 'readonly'); - const readStore = readTransaction.objectStore(name); - - readStore.count().onsuccess = count => { - const excess = count.target.result - limit; - - if (excess > 0) { - readStore.getAll(null, excess).onsuccess = - retrieval => callback(retrieval.target.result.map(({ id }) => id)); - } - }; - }; - }); -} - -export function evictAccounts(ids) { - asyncDB.then(db => { - const transaction = db.transaction(['accounts', 'statuses'], 'readwrite'); - const accounts = transaction.objectStore('accounts'); - const accountsIdIndex = accounts.index('id'); - const accountsMovedIndex = accounts.index('moved'); - const statuses = transaction.objectStore('statuses'); - const statusesIndex = statuses.index('account'); - - function evict(toEvict) { - toEvict.forEach(id => { - accountsMovedIndex.getAllKeys(id).onsuccess = - ({ target }) => evict(target.result); - - statusesIndex.getAll(id).onsuccess = - ({ target }) => evictStatuses(target.result.map(({ id }) => id)); - - accountsIdIndex.getKey(id).onsuccess = - ({ target }) => target.result && accounts.delete(target.result); - }); - } - - evict(ids); - }); -} - -export function evictStatus(id) { - return evictStatuses([id]); -} - -export function evictStatuses(ids) { - asyncDB.then(db => { - const store = db.transaction('statuses', 'readwrite').objectStore('statuses'); - const idIndex = store.index('id'); - const reblogIndex = store.index('reblog'); - - ids.forEach(id => { - reblogIndex.getAllKeys(id).onsuccess = - ({ target }) => target.result.forEach(reblogKey => store.delete(reblogKey)); - - idIndex.getKey(id).onsuccess = - ({ target }) => target.result && store.delete(target.result); - }); - }); -} - -export function putAccounts(records) { - put('accounts', records, evictAccounts); -} - -export function putStatuses(records) { - put('statuses', records, evictStatuses); -} diff --git a/app/javascript/mastodon/features/account_gallery/index.js b/app/javascript/mastodon/features/account_gallery/index.js index 9a40d139c..5f564d3a9 100644 --- a/app/javascript/mastodon/features/account_gallery/index.js +++ b/app/javascript/mastodon/features/account_gallery/index.js @@ -67,7 +67,7 @@ export default class AccountGallery extends ImmutablePureComponent { handleScrollToBottom = () => { if (this.props.hasMore) { - this.handleLoadMore(this.props.medias.last().get('id')); + this.handleLoadMore(this.props.medias.last().getIn(['status', 'id'])); } } diff --git a/app/javascript/mastodon/reducers/notifications.js b/app/javascript/mastodon/reducers/notifications.js index f023984b8..1ac7eb706 100644 --- a/app/javascript/mastodon/reducers/notifications.js +++ b/app/javascript/mastodon/reducers/notifications.js @@ -82,7 +82,7 @@ const expandNormalizedNotifications = (state, notifications, next) => { }; const filterNotifications = (state, relationship) => { - return state.update('items', list => list.filterNot(item => item.get('account') === relationship.id)); + return state.update('items', list => list.filterNot(item => item !== null && item.get('account') === relationship.id)); }; const updateTop = (state, top) => { @@ -94,7 +94,7 @@ const updateTop = (state, top) => { }; const deleteByStatus = (state, statusId) => { - return state.update('items', list => list.filterNot(item => item.get('status') === statusId)); + return state.update('items', list => list.filterNot(item => item !== null && item.get('status') === statusId)); }; export default function notifications(state = initialState, action) { diff --git a/app/javascript/mastodon/service_worker/entry.js b/app/javascript/mastodon/service_worker/entry.js index 8b65f27a3..160c3fbf2 100644 --- a/app/javascript/mastodon/service_worker/entry.js +++ b/app/javascript/mastodon/service_worker/entry.js @@ -1,6 +1,10 @@ import './web_push_notifications'; -function openCache() { +function openSystemCache() { + return caches.open('mastodon-system'); +} + +function openWebCache() { return caches.open('mastodon-web'); } @@ -11,7 +15,7 @@ function fetchRoot() { // Cause a new version of a registered Service Worker to replace an existing one // that is already installed, and replace the currently active worker on open pages. self.addEventListener('install', function(event) { - event.waitUntil(Promise.all([openCache(), fetchRoot()]).then(([cache, root]) => cache.put('/', root))); + event.waitUntil(Promise.all([openWebCache(), fetchRoot()]).then(([cache, root]) => cache.put('/', root))); }); self.addEventListener('activate', function(event) { event.waitUntil(self.clients.claim()); @@ -21,7 +25,7 @@ self.addEventListener('fetch', function(event) { if (url.pathname.startsWith('/web/')) { const asyncResponse = fetchRoot(); - const asyncCache = openCache(); + const asyncCache = openWebCache(); event.respondWith(asyncResponse.then(async response => { if (response.ok) { @@ -31,10 +35,10 @@ self.addEventListener('fetch', function(event) { } throw null; - }).catch(() => caches.match('/'))); + }).catch(() => asyncCache.then(cache => cache.match('/')))); } else if (url.pathname === '/auth/sign_out') { const asyncResponse = fetch(event.request); - const asyncCache = openCache(); + const asyncCache = openWebCache(); event.respondWith(asyncResponse.then(async response => { if (response.ok || response.type === 'opaqueredirect') { @@ -44,5 +48,21 @@ self.addEventListener('fetch', function(event) { return response; })); + } else if (process.env.CDN_HOST ? url.host === process.env.CDN_HOST : url.pathname.startsWith('/system/')) { + event.respondWith(openSystemCache().then(async cache => { + const cached = await cache.match(event.request.url); + + if (cached === undefined) { + const fetched = await fetch(event.request); + + if (fetched.ok) { + await cache.put(event.request.url, fetched.clone()); + } + + return fetched; + } + + return cached; + })); } }); diff --git a/app/javascript/mastodon/db/async.js b/app/javascript/mastodon/storage/db.js index e08fc3f3d..e08fc3f3d 100644 --- a/app/javascript/mastodon/db/async.js +++ b/app/javascript/mastodon/storage/db.js diff --git a/app/javascript/mastodon/storage/modifier.js b/app/javascript/mastodon/storage/modifier.js new file mode 100644 index 000000000..63e49fe6e --- /dev/null +++ b/app/javascript/mastodon/storage/modifier.js @@ -0,0 +1,151 @@ +import asyncDB from './db'; +import { autoPlayGif } from '../initial_state'; + +const accountAssetKeys = ['avatar', 'avatar_static', 'header', 'header_static']; +const avatarKey = autoPlayGif ? 'avatar' : 'avatar_static'; +const limit = 1024; +const asyncCache = caches.open('mastodon-system'); + +function put(name, objects, onupdate, oncreate) { + return asyncDB.then(db => new Promise((resolve, reject) => { + const putTransaction = db.transaction(name, 'readwrite'); + const putStore = putTransaction.objectStore(name); + const putIndex = putStore.index('id'); + + objects.forEach(object => { + putIndex.getKey(object.id).onsuccess = retrieval => { + function addObject() { + putStore.add(object); + } + + function deleteObject() { + putStore.delete(retrieval.target.result).onsuccess = addObject; + } + + if (retrieval.target.result) { + if (onupdate) { + onupdate(object, retrieval.target.result, putStore, deleteObject); + } else { + deleteObject(); + } + } else { + if (oncreate) { + oncreate(object, addObject); + } else { + addObject(); + } + } + }; + }); + + putTransaction.oncomplete = () => { + const readTransaction = db.transaction(name, 'readonly'); + const readStore = readTransaction.objectStore(name); + const count = readStore.count(); + + count.onsuccess = () => { + const excess = count.result - limit; + + if (excess > 0) { + const retrieval = readStore.getAll(null, excess); + + retrieval.onsuccess = () => resolve(retrieval.result); + retrieval.onerror = reject; + } else { + resolve([]); + } + }; + + count.onerror = reject; + }; + + putTransaction.onerror = reject; + })); +} + +function evictAccountsByRecords(records) { + asyncDB.then(db => { + const transaction = db.transaction(['accounts', 'statuses'], 'readwrite'); + const accounts = transaction.objectStore('accounts'); + const accountsIdIndex = accounts.index('id'); + const accountsMovedIndex = accounts.index('moved'); + const statuses = transaction.objectStore('statuses'); + const statusesIndex = statuses.index('account'); + + function evict(toEvict) { + toEvict.forEach(record => { + asyncCache.then(cache => accountAssetKeys.forEach(key => cache.delete(records[key]))); + + accountsMovedIndex.getAll(record.id).onsuccess = ({ target }) => evict(target.result); + + statusesIndex.getAll(record.id).onsuccess = + ({ target }) => evictStatusesByRecords(target.result); + + accountsIdIndex.getKey(record.id).onsuccess = + ({ target }) => target.result && accounts.delete(target.result); + }); + } + + evict(records); + }); +} + +export function evictStatus(id) { + return evictStatuses([id]); +} + +export function evictStatuses(ids) { + asyncDB.then(db => { + const store = db.transaction('statuses', 'readwrite').objectStore('statuses'); + const idIndex = store.index('id'); + const reblogIndex = store.index('reblog'); + + ids.forEach(id => { + reblogIndex.getAllKeys(id).onsuccess = + ({ target }) => target.result.forEach(reblogKey => store.delete(reblogKey)); + + idIndex.getKey(id).onsuccess = + ({ target }) => target.result && store.delete(target.result); + }); + }); +} + +function evictStatusesByRecords(records) { + evictStatuses(records.map(({ id }) => id)); +} + +export function putAccounts(records) { + const newURLs = []; + + put('accounts', records, (newRecord, oldKey, store, oncomplete) => { + store.get(oldKey).onsuccess = ({ target }) => { + accountAssetKeys.forEach(key => { + const newURL = newRecord[key]; + const oldURL = target.result[key]; + + if (newURL !== oldURL) { + asyncCache.then(cache => cache.delete(oldURL)); + } + }); + + const newURL = newRecord[avatarKey]; + const oldURL = target.result[avatarKey]; + + if (newURL !== oldURL) { + newURLs.push(newURL); + } + + oncomplete(); + }; + }, (newRecord, oncomplete) => { + newURLs.push(newRecord[avatarKey]); + oncomplete(); + }).then(records => { + evictAccountsByRecords(records); + asyncCache.then(cache => cache.addAll(newURLs)); + }); +} + +export function putStatuses(records) { + put('statuses', records).then(evictStatusesByRecords); +} diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 5c6189bae..042a84742 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -1435,14 +1435,19 @@ position: relative; width: 100%; height: 100%; + display: flex; + align-items: center; + justify-content: center; - &.image-loader--loading { - display: flex; - align-content: center; + .image-loader__preview-canvas { + max-width: $media-modal-media-max-width; + max-height: $media-modal-media-max-height; + background: url('~images/void.png') repeat; + object-fit: contain; + } - .image-loader__preview-canvas { - filter: blur(2px); - } + &.image-loader--loading .image-loader__preview-canvas { + filter: blur(2px); } &.image-loader--amorphous .image-loader__preview-canvas { @@ -1455,7 +1460,16 @@ width: 100%; height: 100%; display: flex; - align-content: center; + align-items: center; + justify-content: center; + + img { + max-width: $media-modal-media-max-width; + max-height: $media-modal-media-max-height; + width: auto; + height: auto; + object-fit: contain; + } } .navigation-bar { @@ -3423,25 +3437,17 @@ a.status-card { height: 100%; position: relative; - img, - canvas, - video { - max-width: 100%; - /* - put margins on top and bottom of image to avoid the screen coverd by - image. - */ - max-height: 80%; - width: auto; - height: auto; - margin: auto; - } + .extended-video-player { + width: 100%; + height: 100%; + display: flex; + align-items: center; + justify-content: center; - img, - canvas { - display: block; - background: url('~images/void.png') repeat; - object-fit: contain; + video { + max-width: $media-modal-media-max-width; + max-height: $media-modal-media-max-height; + } } } @@ -4418,8 +4424,8 @@ a.status-card { border-radius: 4px; video { - height: 100%; - width: 100%; + max-width: 100vw; + max-height: 80vh; z-index: 1; } diff --git a/app/javascript/styles/mastodon/variables.scss b/app/javascript/styles/mastodon/variables.scss index dcc2857ff..e456c27ee 100644 --- a/app/javascript/styles/mastodon/variables.scss +++ b/app/javascript/styles/mastodon/variables.scss @@ -30,3 +30,8 @@ $ui-highlight-color: $classic-highlight-color !default; // Vibrant // Language codes that uses CJK fonts $cjk-langs: ja, ko, zh-CN, zh-HK, zh-TW; + +// Variables for components +$media-modal-media-max-width: 100%; +// put margins on top and bottom of image to avoid the screen covered by image. +$media-modal-media-max-height: 80%; diff --git a/app/lib/exceptions.rb b/app/lib/exceptions.rb index 95e3365c2..e88e98eae 100644 --- a/app/lib/exceptions.rb +++ b/app/lib/exceptions.rb @@ -5,6 +5,7 @@ module Mastodon class NotPermittedError < Error; end class ValidationError < Error; end class HostValidationError < ValidationError; end + class LengthValidationError < ValidationError; end class RaceConditionError < Error; end class UnexpectedResponseError < Error diff --git a/app/lib/provider_discovery.rb b/app/lib/provider_discovery.rb index bbd3a2d43..3bec7211b 100644 --- a/app/lib/provider_discovery.rb +++ b/app/lib/provider_discovery.rb @@ -18,7 +18,7 @@ class ProviderDiscovery < OEmbed::ProviderDiscovery else Request.new(:get, url).perform do |res| raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html' - Nokogiri::HTML(res.to_s) + Nokogiri::HTML(res.body_with_limit) end end diff --git a/app/lib/request.rb b/app/lib/request.rb index 8a127c65f..dca93a6e9 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -40,7 +40,7 @@ class Request end begin - yield response + yield response.extend(ClientLimit) ensure http_client.close end @@ -99,6 +99,33 @@ class Request @http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2) end + module ClientLimit + def body_with_limit(limit = 1.megabyte) + raise Mastodon::LengthValidationError if content_length.present? && content_length > limit + + if charset.nil? + encoding = Encoding::BINARY + else + begin + encoding = Encoding.find(charset) + rescue ArgumentError + encoding = Encoding::BINARY + end + end + + contents = String.new(encoding: encoding) + + while (chunk = readpartial) + contents << chunk + chunk.clear + + raise Mastodon::LengthValidationError if contents.bytesize > limit + end + + contents + end + end + class Socket < TCPSocket class << self def open(host, *args) @@ -118,5 +145,5 @@ class Request end end - private_constant :Socket + private_constant :ClientLimit, :Socket end diff --git a/app/models/account.rb b/app/models/account.rb index 821f1e64c..95ecc3c1c 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -55,7 +55,6 @@ class Account < ApplicationRecord include AccountHeader include AccountInteractions include Attachmentable - include Remotable include Paginable MAX_NOTE_LENGTH = 500 diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 71fbba5b3..83134d41a 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -2,4 +2,5 @@ class ApplicationRecord < ActiveRecord::Base self.abstract_class = true + include Remotable end diff --git a/app/models/concerns/account_avatar.rb b/app/models/concerns/account_avatar.rb index 9e34a9461..2d5ebfca3 100644 --- a/app/models/concerns/account_avatar.rb +++ b/app/models/concerns/account_avatar.rb @@ -4,6 +4,7 @@ module AccountAvatar extend ActiveSupport::Concern IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze + LIMIT = 2.megabytes class_methods do def avatar_styles(file) @@ -19,7 +20,8 @@ module AccountAvatar # Avatar upload has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail] validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES - validates_attachment_size :avatar, less_than: 2.megabytes + validates_attachment_size :avatar, less_than: LIMIT + remotable_attachment :avatar, LIMIT end def avatar_original_url diff --git a/app/models/concerns/account_finder_concern.rb b/app/models/concerns/account_finder_concern.rb index 2e8a7fb37..6b7237e89 100644 --- a/app/models/concerns/account_finder_concern.rb +++ b/app/models/concerns/account_finder_concern.rb @@ -30,7 +30,7 @@ module AccountFinderConcern end def account - scoped_accounts.take + scoped_accounts.order(id: :asc).take end private diff --git a/app/models/concerns/account_header.rb b/app/models/concerns/account_header.rb index 04c576b28..ef40b8126 100644 --- a/app/models/concerns/account_header.rb +++ b/app/models/concerns/account_header.rb @@ -4,6 +4,7 @@ module AccountHeader extend ActiveSupport::Concern IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze + LIMIT = 2.megabytes class_methods do def header_styles(file) @@ -19,7 +20,8 @@ module AccountHeader # Header upload has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail] validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES - validates_attachment_size :header, less_than: 2.megabytes + validates_attachment_size :header, less_than: LIMIT + remotable_attachment :header, LIMIT end def header_original_url diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb index 0f18c5d96..3b8c507c3 100644 --- a/app/models/concerns/remotable.rb +++ b/app/models/concerns/remotable.rb @@ -3,8 +3,8 @@ module Remotable extend ActiveSupport::Concern - included do - attachment_definitions.each_key do |attachment_name| + class_methods do + def remotable_attachment(attachment_name, limit) attribute_name = "#{attachment_name}_remote_url".to_sym method_name = "#{attribute_name}=".to_sym alt_method_name = "reset_#{attachment_name}!".to_sym @@ -33,7 +33,7 @@ module Remotable File.extname(filename) end - send("#{attachment_name}=", StringIO.new(response.to_s)) + send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit))) send("#{attachment_name}_file_name=", basename + extname) self[attribute_name] = url if has_attribute?(attribute_name) diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index a77b53c98..476178e86 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -19,6 +19,8 @@ # class CustomEmoji < ApplicationRecord + LIMIT = 50.kilobytes + SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}' SCAN_RE = /(?<=[^[:alnum:]:]|\n|^) @@ -29,14 +31,14 @@ class CustomEmoji < ApplicationRecord has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } } - validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { in: 0..50.kilobytes } + validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { less_than: LIMIT } validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 } scope :local, -> { where(domain: nil) } scope :remote, -> { where.not(domain: nil) } scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) } - include Remotable + remotable_attachment :image, LIMIT def local? domain.nil? diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 283d0e714..49c24ac01 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -74,6 +74,8 @@ class MediaAttachment < ApplicationRecord }, }.freeze + LIMIT = 8.megabytes + belongs_to :account, inverse_of: :media_attachments, optional: true belongs_to :status, inverse_of: :media_attachments, optional: true @@ -85,7 +87,8 @@ class MediaAttachment < ApplicationRecord include Remotable validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES - validates_attachment_size :file, less_than: 8.megabytes + validates_attachment_size :file, less_than: LIMIT + remotable_attachment :file, LIMIT validates :account, presence: true validates :description, length: { maximum: 420 }, if: :local? diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index 86eecdfe5..0c82f06ce 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -26,6 +26,7 @@ class PreviewCard < ApplicationRecord IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze + LIMIT = 1.megabytes self.inheritance_column = false @@ -36,11 +37,11 @@ class PreviewCard < ApplicationRecord has_attached_file :image, styles: { original: { geometry: '400x400>', file_geometry_parser: FastGeometryParser } }, convert_options: { all: '-quality 80 -strip' } include Attachmentable - include Remotable validates :url, presence: true, uniqueness: true validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES - validates_attachment_size :image, less_than: 1.megabytes + validates_attachment_size :image, less_than: LIMIT + remotable_attachment :image, LIMIT before_save :extract_dimensions, if: :link? diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb index 48ad5dcd3..62dea8298 100644 --- a/app/services/fetch_atom_service.rb +++ b/app/services/fetch_atom_service.rb @@ -38,13 +38,14 @@ class FetchAtomService < BaseService return nil if response.code != 200 if response.mime_type == 'application/atom+xml' - [@url, { prefetched_body: response.to_s }, :ostatus] + [@url, { prefetched_body: response.body_with_limit }, :ostatus] elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type) - json = body_to_json(response.to_s) + body = response.body_with_limit + json = body_to_json(body) if supported_context?(json) && json['type'] == 'Person' && json['inbox'].present? - [json['id'], { prefetched_body: response.to_s, id: true }, :activitypub] + [json['id'], { prefetched_body: body, id: true }, :activitypub] elsif supported_context?(json) && json['type'] == 'Note' - [json['id'], { prefetched_body: response.to_s, id: true }, :activitypub] + [json['id'], { prefetched_body: body, id: true }, :activitypub] else @unsupported_activity = true nil @@ -61,7 +62,7 @@ class FetchAtomService < BaseService end def process_html(response) - page = Nokogiri::HTML(response.to_s) + page = Nokogiri::HTML(response.body_with_limit) json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' } diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 26deb5ecc..d5920a417 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -45,7 +45,7 @@ class FetchLinkCardService < BaseService Request.new(:get, @url).perform do |res| if res.code == 200 && res.mime_type == 'text/html' - @html = res.to_s + @html = res.body_with_limit @html_charset = res.charset else @html = nil diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 034821dc0..744ea24f4 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -181,7 +181,7 @@ class ResolveAccountService < BaseService @atom_body = Request.new(:get, atom_url).perform do |response| raise Mastodon::UnexpectedResponseError, response unless response.code == 200 - response.to_s + response.body_with_limit end end diff --git a/app/validators/unique_username_validator.rb b/app/validators/unique_username_validator.rb index c76407b16..fb67105dd 100644 --- a/app/validators/unique_username_validator.rb +++ b/app/validators/unique_username_validator.rb @@ -6,7 +6,7 @@ class UniqueUsernameValidator < ActiveModel::Validator normalized_username = account.username.downcase.delete('.') - scope = Account.where(domain: nil, username: normalized_username) + scope = Account.where(domain: nil).where('lower(username) = ?', normalized_username) scope = scope.where.not(id: account.id) if account.persisted? account.errors.add(:username, :taken) if scope.exists? diff --git a/app/workers/pubsubhubbub/confirmation_worker.rb b/app/workers/pubsubhubbub/confirmation_worker.rb index cc2d1225b..c0e7b677e 100644 --- a/app/workers/pubsubhubbub/confirmation_worker.rb +++ b/app/workers/pubsubhubbub/confirmation_worker.rb @@ -57,7 +57,7 @@ class Pubsubhubbub::ConfirmationWorker def callback_get_with_params Request.new(:get, subscription.callback_url, params: callback_params).perform do |response| - @callback_response_body = response.body.to_s + @callback_response_body = response.body_with_limit end end diff --git a/config/webpack/production.js b/config/webpack/production.js index e2d7f11dc..e1c681232 100644 --- a/config/webpack/production.js +++ b/config/webpack/production.js @@ -90,7 +90,7 @@ module.exports = merge(sharedConfig, { '**/*.woff', ], ServiceWorker: { - entry: path.join(__dirname, '../../app/javascript/mastodon/service_worker/entry.js'), + entry: `imports-loader?process.env=>${encodeURIComponent(JSON.stringify(process.env))}!${encodeURI(path.join(__dirname, '../../app/javascript/mastodon/service_worker/entry.js'))}`, cacheName: 'mastodon', output: '../assets/sw.js', publicPath: '/sw.js', diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 121c5c693..a6927eec3 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 2 + 3 end def pre diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 0972e4367..cfd6a1d25 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -740,6 +740,24 @@ namespace :mastodon do LinkCrawlWorker.push_bulk status_ids end + desc 'Find case-insensitive username duplicates of local users' + task find_duplicate_usernames: :environment do + include RoutingHelper + + disable_log_stdout! + + duplicate_masters = Account.find_by_sql('SELECT * FROM accounts WHERE id IN (SELECT min(id) FROM accounts WHERE domain IS NULL GROUP BY lower(username) HAVING count(*) > 1)') + pastel = Pastel.new + + duplicate_masters.each do |account| + puts pastel.yellow("First of their name: ") + pastel.bold(account.username) + " (#{admin_account_url(account.id)})" + + Account.where('lower(username) = ?', account.username.downcase).where.not(id: account.id).each do |duplicate| + puts " " + pastel.red("Duplicate: ") + admin_account_url(duplicate.id) + end + end + end + desc 'Remove all home feed regeneration markers' task remove_regeneration_markers: :environment do keys = Redis.current.keys('account:*:regeneration') diff --git a/package.json b/package.json index d84bfb06f..a3aaaf0c8 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "glob": "^7.1.1", "http-link-header": "^0.8.0", "immutable": "^3.8.2", + "imports-loader": "^0.8.0", "intersection-observer": "^0.5.0", "intl": "^1.2.5", "intl-messageformat": "^2.2.0", diff --git a/scalingo.json b/scalingo.json index 426698b9c..0cc648f02 100644 --- a/scalingo.json +++ b/scalingo.json @@ -21,6 +21,10 @@ "description": "The secret key base", "generator": "secret" }, + "OTP_SECRET": { + "description": "One-time password secret", + "generator": "secret" + }, "SINGLE_USER_MODE": { "description": "Should the instance run in single user mode? (Disable registrations, redirect to front page)", "value": "false", diff --git a/spec/lib/request_spec.rb b/spec/lib/request_spec.rb index 4d6b20dd5..939ac006a 100644 --- a/spec/lib/request_spec.rb +++ b/spec/lib/request_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'rails_helper' +require 'securerandom' describe Request do subject { Request.new(:get, 'http://example.com') } @@ -64,6 +65,12 @@ describe Request do expect_any_instance_of(HTTP::Client).to receive(:close) expect { |block| subject.perform &block }.to yield_control end + + it 'returns response which implements body_with_limit' do + subject.perform do |response| + expect(response).to respond_to :body_with_limit + end + end end context 'with private host' do @@ -81,4 +88,46 @@ describe Request do end end end + + describe "response's body_with_limit method" do + it 'rejects body more than 1 megabyte by default' do + stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes)) + expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError + end + + it 'accepts body less than 1 megabyte by default' do + stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes)) + expect { subject.perform { |response| response.body_with_limit } }.not_to raise_error + end + + it 'rejects body by given size' do + stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes)) + expect { subject.perform { |response| response.body_with_limit(1.kilobyte) } }.to raise_error Mastodon::LengthValidationError + end + + it 'rejects too large chunked body' do + stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Transfer-Encoding' => 'chunked' }) + expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError + end + + it 'rejects too large monolithic body' do + stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Content-Length' => 2.megabytes }) + expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError + end + + it 'uses binary encoding if Content-Type does not tell encoding' do + stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html' }) + expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY + end + + it 'uses binary encoding if Content-Type tells unknown encoding' do + stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=unknown' }) + expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY + end + + it 'uses encoding specified by Content-Type' do + stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=UTF-8' }) + expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::UTF_8 + end + end end diff --git a/spec/models/concerns/remotable_spec.rb b/spec/models/concerns/remotable_spec.rb index 0b2dad23f..b39233739 100644 --- a/spec/models/concerns/remotable_spec.rb +++ b/spec/models/concerns/remotable_spec.rb @@ -29,7 +29,10 @@ RSpec.describe Remotable do context 'Remotable module is included' do before do - class Foo; include Remotable; end + class Foo + include Remotable + remotable_attachment :hoge, 1.kilobyte + end end let(:attribute_name) { "#{hoge}_remote_url".to_sym } diff --git a/spec/validators/unique_username_validator_spec.rb b/spec/validators/unique_username_validator_spec.rb new file mode 100644 index 000000000..b9d773bed --- /dev/null +++ b/spec/validators/unique_username_validator_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe UniqueUsernameValidator do + describe '#validate' do + it 'does not add errors if username is nil' do + account = double(username: nil, persisted?: false, errors: double(add: nil)) + subject.validate(account) + expect(account.errors).to_not have_received(:add) + end + + it 'does not add errors when existing one is subject itself' do + account = Fabricate(:account, username: 'abcdef') + expect(account).to be_valid + end + + it 'adds an error when the username is already used with ignoring dots' do + pending 'allowing dots in username is still in development' + Fabricate(:account, username: 'abcd.ef') + account = double(username: 'ab.cdef', persisted?: false, errors: double(add: nil)) + subject.validate(account) + expect(account.errors).to have_received(:add) + end + + it 'adds an error when the username is already used with ignoring cases' do + Fabricate(:account, username: 'ABCdef') + account = double(username: 'abcDEF', persisted?: false, errors: double(add: nil)) + subject.validate(account) + expect(account.errors).to have_received(:add) + end + end +end diff --git a/yarn.lock b/yarn.lock index c97015fd8..a4465c02e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3368,6 +3368,13 @@ import-local@^0.1.1: pkg-dir "^2.0.0" resolve-cwd "^2.0.0" +imports-loader@^0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/imports-loader/-/imports-loader-0.8.0.tgz#030ea51b8ca05977c40a3abfd9b4088fe0be9a69" + dependencies: + loader-utils "^1.0.2" + source-map "^0.6.1" + imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" |