From 06424786a140e03fcb899232c3c418b7eaf26d5e Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 30 Jun 2019 16:10:43 +0200 Subject: Fix support for MP4 files that are actually M4V files (#11210) Resolve #11187 --- app/models/concerns/attachmentable.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb index 24f5968de..7c78bb456 100644 --- a/app/models/concerns/attachmentable.rb +++ b/app/models/concerns/attachmentable.rb @@ -60,7 +60,9 @@ module Attachmentable end def calculated_content_type(attachment) - Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp + content_type = Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp + content_type = 'video/mp4' if content_type == 'video/x-m4v' + content_type rescue Terrapin::CommandLineError '' end -- cgit From 082b3b282f7863e4b10e9cb34624b4b167b75588 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 30 Jun 2019 16:11:21 +0200 Subject: Fix expiration date of filters being set to “Never” when editing them (#11204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When editing a custom filter, select the shortest preset duration that still covers the remaining time of that filter. Fixes #9506 --- app/models/custom_filter.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb index 342207a55..382562fb8 100644 --- a/app/models/custom_filter.rb +++ b/app/models/custom_filter.rb @@ -35,6 +35,13 @@ class CustomFilter < ApplicationRecord before_validation :clean_up_contexts after_commit :remove_cache + def expires_in + return @expires_in if defined?(@expires_in) + return nil if expires_at.nil? + + [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].find { |expires_in| expires_in.from_now >= expires_at } + end + private def clean_up_contexts -- cgit From d33a30377c4f470433c4ad4881de48357c469783 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 21:23:40 +0900 Subject: Bump webpack-cli from 3.3.4 to 3.3.5 (#11221) Bumps [webpack-cli](https://github.com/webpack/webpack-cli) from 3.3.4 to 3.3.5. - [Release notes](https://github.com/webpack/webpack-cli/releases) - [Changelog](https://github.com/webpack/webpack-cli/blob/master/CHANGELOG.md) - [Commits](https://github.com/webpack/webpack-cli/compare/v3.3.4...v3.3.5) Signed-off-by: dependabot-preview[bot] --- package.json | 2 +- yarn.lock | 179 ++++++++++++++++++++++++++++++++++++++++------------------- 2 files changed, 122 insertions(+), 59 deletions(-) diff --git a/package.json b/package.json index b45fad637..a87ec891d 100644 --- a/package.json +++ b/package.json @@ -161,7 +161,7 @@ "webpack": "^4.34.0", "webpack-assets-manifest": "^3.1.1", "webpack-bundle-analyzer": "^3.3.2", - "webpack-cli": "^3.3.4", + "webpack-cli": "^3.3.5", "webpack-merge": "^4.2.1", "websocket.js": "^0.1.12" }, diff --git a/yarn.lock b/yarn.lock index 2c402782c..f48f484c6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2270,6 +2270,15 @@ caseless@~0.12.0: resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= +chalk@2.4.2, chalk@^2.0, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -2281,15 +2290,6 @@ chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" -chalk@^2.0, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" - integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== - dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" - chardet@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" @@ -2404,6 +2404,15 @@ cliui@^4.0.0: strip-ansi "^4.0.0" wrap-ansi "^2.0.0" +cliui@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-5.0.0.tgz#deefcfdb2e800784aa34f46fa08e06851c7bbbc5" + integrity sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA== + dependencies: + string-width "^3.1.0" + strip-ansi "^5.2.0" + wrap-ansi "^5.1.0" + clone-deep@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-2.0.2.tgz#00db3a1e173656730d1188c3d6aced6d7ea97713" @@ -2737,7 +2746,7 @@ cross-env@^5.1.4: cross-spawn "^6.0.5" is-windows "^1.0.0" -cross-spawn@^6.0.0, cross-spawn@^6.0.5: +cross-spawn@6.0.5, cross-spawn@^6.0.0, cross-spawn@^6.0.5: version "6.0.5" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== @@ -3424,7 +3433,7 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0: dependencies: once "^1.4.0" -enhanced-resolve@^4.1.0: +enhanced-resolve@4.1.0, enhanced-resolve@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.1.0.tgz#41c7e0bfdfe74ac1ffe1e57ad6a5c6c9f3742a7f" integrity sha512-F/7vkyTtyc/llOIn8oWclcB25KdRaiPBpZYDgJHgh/UHtpgT2p2eldQgtQnLtUvfMKPKxbRaQM/hHkvLHt1Vng== @@ -4200,13 +4209,13 @@ find-up@^3.0.0: dependencies: locate-path "^3.0.0" -findup-sync@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-2.0.0.tgz#9326b1488c22d1a6088650a86901b2d9a90a2cbc" - integrity sha1-kyaxSIwi0aYIhlCoaQGy2akKLLw= +findup-sync@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1" + integrity sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg== dependencies: detect-file "^1.0.0" - is-glob "^3.1.0" + is-glob "^4.0.0" micromatch "^3.0.4" resolve-dir "^1.0.1" @@ -4429,6 +4438,11 @@ get-caller-file@^1.0.1: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a" integrity sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w== +get-caller-file@^2.0.1: + version "2.0.5" + resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" + integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== + get-stream@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" @@ -4468,6 +4482,13 @@ glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@~7.1.1: once "^1.3.0" path-is-absolute "^1.0.0" +global-modules@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" + integrity sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A== + dependencies: + global-prefix "^3.0.0" + global-modules@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-1.0.0.tgz#6d770f0eb523ac78164d72b5e71a8877265cc3ea" @@ -4488,6 +4509,15 @@ global-prefix@^1.0.1: is-windows "^1.0.1" which "^1.2.14" +global-prefix@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-3.0.0.tgz#fc85f73064df69f50421f47f883fe5b913ba9b97" + integrity sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg== + dependencies: + ini "^1.3.5" + kind-of "^6.0.2" + which "^1.3.1" + globals@^11.1.0, globals@^11.7.0: version "11.12.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" @@ -4913,7 +4943,7 @@ import-from@^2.1.0: dependencies: resolve-from "^3.0.0" -import-local@^2.0.0: +import-local@2.0.0, import-local@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/import-local/-/import-local-2.0.0.tgz#55070be38a5993cf18ef6db7e961f5bee5c5a09d" integrity sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ== @@ -4962,7 +4992,7 @@ inherits@2.0.1: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.1.tgz#b17d08d326b4423e568eff719f91b0b1cbdf69f1" integrity sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE= -ini@^1.3.4, ini@~1.3.0: +ini@^1.3.4, ini@^1.3.5, ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw== @@ -5013,7 +5043,7 @@ internal-ip@^4.3.0: default-gateway "^4.2.0" ipaddr.js "^1.9.0" -interpret@^1.1.0: +interpret@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.2.0.tgz#d5061a6224be58e8083985f5014d844359576296" integrity sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw== @@ -6111,7 +6141,7 @@ loader-utils@0.2.x: json5 "^0.5.0" object-assign "^4.0.1" -loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.2, loader-utils@^1.2.3: +loader-utils@1.2.3, loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.2, loader-utils@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7" integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA== @@ -7026,7 +7056,7 @@ os-homedir@^1.0.0: resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= -os-locale@^3.0.0: +os-locale@^3.0.0, os-locale@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-3.1.0.tgz#a802a6ee17f24c10483ab9935719cef4ed16bf1a" integrity sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q== @@ -7823,11 +7853,6 @@ prepend-http@^1.0.0: resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-1.0.4.tgz#d4f4562b0ce3696e41ac52d0e002e57a635dc6dc" integrity sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw= -prettier@^1.17.0: - version "1.18.2" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.18.2.tgz#6823e7c5900017b4bd3acf46fe9ac4b4d7bda9ea" - integrity sha512-OeHeMc0JhFE9idD4ZdtNibzY0+TPHSpSSb9h8FqtP+YnoZZ1sl8Vc9b1sasjfymH3SonAF4QcA2+mzHPhMvIiw== - pretty-format@^24.8.0: version "24.8.0" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-24.8.0.tgz#8dae7044f58db7cb8be245383b565a963e3c27f2" @@ -8642,6 +8667,11 @@ require-main-filename@^1.0.1: resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" integrity sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE= +require-main-filename@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" + integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== + require-package-name@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/require-package-name/-/require-package-name-2.0.1.tgz#c11e97276b65b8e2923f75dabf5fb2ef0c3841b9" @@ -9403,7 +9433,7 @@ string-width@^1.0.1: is-fullwidth-code-point "^2.0.0" strip-ansi "^4.0.0" -string-width@^3.0.0: +string-width@^3.0.0, string-width@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961" integrity sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w== @@ -9456,7 +9486,7 @@ strip-ansi@^4.0.0: dependencies: ansi-regex "^3.0.0" -strip-ansi@^5.0.0, strip-ansi@^5.1.0: +strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae" integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA== @@ -9507,6 +9537,13 @@ substring-trie@^1.0.2: resolved "https://registry.yarnpkg.com/substring-trie/-/substring-trie-1.0.2.tgz#7b42592391628b4f2cb17365c6cce4257c7b7af5" integrity sha1-e0JZI5Fii08ssXNlxszkJXx7evU= +supports-color@6.1.0, supports-color@^6.0.0, supports-color@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" + integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== + dependencies: + has-flag "^3.0.0" + supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" @@ -9519,20 +9556,13 @@ supports-color@^3.2.3: dependencies: has-flag "^1.0.0" -supports-color@^5.3.0, supports-color@^5.5.0: +supports-color@^5.3.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== dependencies: has-flag "^3.0.0" -supports-color@^6.0.0, supports-color@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" - integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== - dependencies: - has-flag "^3.0.0" - svgo@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/svgo/-/svgo-1.1.1.tgz#12384b03335bcecd85cfa5f4e3375fed671cb985" @@ -10009,10 +10039,10 @@ uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== -v8-compile-cache@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.2.tgz#a428b28bb26790734c4fc8bc9fa106fccebf6a6c" - integrity sha512-1wFuMUIM16MDJRCrpbpuEPTUGmM5QMUg0cr3KFwra2XgOgFcPGDQHDh3CszSCD2Zewc/dh/pamNEW8CbfDebUw== +v8-compile-cache@2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.3.tgz#00f7494d2ae2b688cfe2899df6ed2c54bef91dbe" + integrity sha512-CNmdbwQMBjwr9Gsmohvm0pbL954tJrNzf6gWL3K+QMQf00PF7ERGrEiLgjuU3mKreLC2MeGhUsNV9ybTbLgd3w== validate-npm-package-license@^3.0.1: version "3.0.4" @@ -10134,23 +10164,22 @@ webpack-bundle-analyzer@^3.3.2: opener "^1.5.1" ws "^6.0.0" -webpack-cli@^3.3.4: - version "3.3.4" - resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-3.3.4.tgz#de27e281c48a897b8c219cb093e261d5f6afe44a" - integrity sha512-ubJGQEKMtBSpT+LiL5hXvn2GIOWiRWItR1DGUqJRhwRBeGhpRXjvF5f0erqdRJLErkfqS5/Ldkkedh4AL5Q1ZQ== - dependencies: - chalk "^2.4.1" - cross-spawn "^6.0.5" - enhanced-resolve "^4.1.0" - findup-sync "^2.0.0" - global-modules "^1.0.0" - import-local "^2.0.0" - interpret "^1.1.0" - loader-utils "^1.1.0" - prettier "^1.17.0" - supports-color "^5.5.0" - v8-compile-cache "^2.0.2" - yargs "^12.0.5" +webpack-cli@^3.3.5: + version "3.3.5" + resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-3.3.5.tgz#f4d1238a66a2843d9cebf189835ea22142e72767" + integrity sha512-w0j/s42c5UhchwTmV/45MLQnTVwRoaUTu9fM5LuyOd/8lFoCNCELDogFoecx5NzRUndO0yD/gF2b02XKMnmAWQ== + dependencies: + chalk "2.4.2" + cross-spawn "6.0.5" + enhanced-resolve "4.1.0" + findup-sync "3.0.0" + global-modules "2.0.0" + import-local "2.0.0" + interpret "1.2.0" + loader-utils "1.2.3" + supports-color "6.1.0" + v8-compile-cache "2.0.3" + yargs "13.2.4" webpack-dev-middleware@^3.7.0: version "3.7.0" @@ -10312,7 +10341,7 @@ which-module@^2.0.0: resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho= -which@^1.2.14, which@^1.2.9, which@^1.3.0: +which@^1.2.14, which@^1.2.9, which@^1.3.0, which@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== @@ -10351,6 +10380,15 @@ wrap-ansi@^2.0.0: string-width "^1.0.1" strip-ansi "^3.0.1" +wrap-ansi@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-5.1.0.tgz#1fd1f67235d5b6d0fee781056001bfb694c03b09" + integrity sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q== + dependencies: + ansi-styles "^3.2.0" + string-width "^3.0.0" + strip-ansi "^5.0.0" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" @@ -10421,6 +10459,14 @@ yargs-parser@^11.1.1: camelcase "^5.0.0" decamelize "^1.2.0" +yargs-parser@^13.1.0: + version "13.1.1" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-13.1.1.tgz#d26058532aa06d365fe091f6a1fc06b2f7e5eca0" + integrity sha512-oVAVsHz6uFrg3XQheFII8ESO2ssAf9luWuAd6Wexsu4F3OtIW0o8IribPXYrD4WC24LWtPrJlGy87y5udK+dxQ== + dependencies: + camelcase "^5.0.0" + decamelize "^1.2.0" + yargs@12.0.5, yargs@^12.0.2, yargs@^12.0.5: version "12.0.5" resolved "https://registry.yarnpkg.com/yargs/-/yargs-12.0.5.tgz#05f5997b609647b64f66b81e3b4b10a368e7ad13" @@ -10438,3 +10484,20 @@ yargs@12.0.5, yargs@^12.0.2, yargs@^12.0.5: which-module "^2.0.0" y18n "^3.2.1 || ^4.0.0" yargs-parser "^11.1.1" + +yargs@13.2.4: + version "13.2.4" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.2.4.tgz#0b562b794016eb9651b98bd37acf364aa5d6dc83" + integrity sha512-HG/DWAJa1PAnHT9JAhNa8AbAv3FPaiLzioSjCcmuXXhP8MlpHO5vwls4g4j6n30Z74GVQj8Xa62dWVx1QCGklg== + dependencies: + cliui "^5.0.0" + find-up "^3.0.0" + get-caller-file "^2.0.1" + os-locale "^3.1.0" + require-directory "^2.1.1" + require-main-filename "^2.0.0" + set-blocking "^2.0.0" + string-width "^3.0.0" + which-module "^2.0.0" + y18n "^4.0.0" + yargs-parser "^13.1.0" -- cgit From 39364049118238f4ba0ce70b16059ec2dea890bc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 21:29:56 +0900 Subject: Bump eslint-plugin-promise from 4.1.1 to 4.2.1 (#11223) Bumps [eslint-plugin-promise](https://github.com/xjamundx/eslint-plugin-promise) from 4.1.1 to 4.2.1. - [Release notes](https://github.com/xjamundx/eslint-plugin-promise/releases) - [Changelog](https://github.com/xjamundx/eslint-plugin-promise/blob/master/CHANGELOG.md) - [Commits](https://github.com/xjamundx/eslint-plugin-promise/commits) Signed-off-by: dependabot-preview[bot] --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index a87ec891d..358f5b156 100644 --- a/package.json +++ b/package.json @@ -173,7 +173,7 @@ "eslint": "^5.16.0", "eslint-plugin-import": "~2.17.3", "eslint-plugin-jsx-a11y": "~6.2.1", - "eslint-plugin-promise": "~4.1.1", + "eslint-plugin-promise": "~4.2.1", "eslint-plugin-react": "~7.12.1", "jest": "^24.8.0", "raf": "^3.4.1", diff --git a/yarn.lock b/yarn.lock index f48f484c6..6698f6a19 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3693,10 +3693,10 @@ eslint-plugin-jsx-a11y@~6.2.1: has "^1.0.3" jsx-ast-utils "^2.0.1" -eslint-plugin-promise@~4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-4.1.1.tgz#1e08cb68b5b2cd8839f8d5864c796f56d82746db" - integrity sha512-faAHw7uzlNPy7b45J1guyjazw28M+7gJokKUjC5JSFoYfUEyy6Gw/i7YQvmv2Yk00sUjWcmzXQLpU1Ki/C2IZQ== +eslint-plugin-promise@~4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz#845fd8b2260ad8f82564c1222fce44ad71d9418a" + integrity sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw== eslint-plugin-react@~7.12.1: version "7.12.1" -- cgit From 47b4cd0fff947a45f149378c9c01a69a8a16985c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:33:54 +0900 Subject: Bump bullet from 6.0.0 to 6.0.1 (#11228) Bumps [bullet](https://github.com/flyerhzm/bullet) from 6.0.0 to 6.0.1. - [Release notes](https://github.com/flyerhzm/bullet/releases) - [Changelog](https://github.com/flyerhzm/bullet/blob/master/CHANGELOG.md) - [Commits](https://github.com/flyerhzm/bullet/compare/6.0.0...6.0.1) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8b0c93b1e..5034c1411 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,7 +106,7 @@ GEM brakeman (4.5.1) browser (2.5.3) builder (3.2.3) - bullet (6.0.0) + bullet (6.0.1) activesupport (>= 3.0.0) uniform_notifier (~> 1.11) bundler-audit (0.6.1) -- cgit From 8c4287c239172e02082c7fec170fffc8b75cab4a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:34:25 +0900 Subject: Bump memory_profiler from 0.9.13 to 0.9.14 (#11227) Bumps [memory_profiler](https://github.com/SamSaffron/memory_profiler) from 0.9.13 to 0.9.14. - [Release notes](https://github.com/SamSaffron/memory_profiler/releases) - [Changelog](https://github.com/SamSaffron/memory_profiler/blob/master/CHANGELOG.md) - [Commits](https://github.com/SamSaffron/memory_profiler/compare/v0.9.13...v0.9.14) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5034c1411..8854b5fce 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -336,7 +336,7 @@ GEM mimemagic (~> 0.3.2) mario-redis-lock (1.2.1) redis (>= 3.0.5) - memory_profiler (0.9.13) + memory_profiler (0.9.14) method_source (0.9.2) microformats (4.1.0) json (~> 2.1) -- cgit From ffb6da8b782455f3d07750e1faa79e955d72fd54 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:34:51 +0900 Subject: Bump capybara from 3.24.0 to 3.25.0 (#11225) Bumps [capybara](https://github.com/teamcapybara/capybara) from 3.24.0 to 3.25.0. - [Release notes](https://github.com/teamcapybara/capybara/releases) - [Changelog](https://github.com/teamcapybara/capybara/blob/master/History.md) - [Commits](https://github.com/teamcapybara/capybara/compare/3.24.0...3.25.0) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 4d63cca37..42499cd2c 100644 --- a/Gemfile +++ b/Gemfile @@ -108,7 +108,7 @@ group :production, :test do end group :test do - gem 'capybara', '~> 3.24' + gem 'capybara', '~> 3.25' gem 'climate_control', '~> 0.2' gem 'faker', '~> 1.9' gem 'microformats', '~> 4.1' diff --git a/Gemfile.lock b/Gemfile.lock index 8854b5fce..4a4e86e72 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -129,7 +129,7 @@ GEM sshkit (~> 1.3) capistrano-yarn (2.0.2) capistrano (~> 3.0) - capybara (3.24.0) + capybara (3.25.0) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) @@ -420,7 +420,7 @@ GEM pry (~> 0.10) pry-rails (0.3.9) pry (>= 0.10.4) - public_suffix (3.1.0) + public_suffix (3.1.1) puma (3.12.1) pundit (2.0.1) activesupport (>= 3.0.0) @@ -660,7 +660,7 @@ DEPENDENCIES capistrano-rails (~> 1.4) capistrano-rbenv (~> 2.1) capistrano-yarn (~> 2.0) - capybara (~> 3.24) + capybara (~> 3.25) charlock_holmes (~> 0.7.6) chewy (~> 5.0) cld3 (~> 3.2.4) -- cgit From 13f54f41ade5abfa561c72264481afed5f1cd8a7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:36:45 +0900 Subject: Bump rubocop from 0.71.0 to 0.72.0 (#11229) Bumps [rubocop](https://github.com/rubocop-hq/rubocop) from 0.71.0 to 0.72.0. - [Release notes](https://github.com/rubocop-hq/rubocop/releases) - [Changelog](https://github.com/rubocop-hq/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop-hq/rubocop/compare/v0.71.0...v0.72.0) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 42499cd2c..e99a808d4 100644 --- a/Gemfile +++ b/Gemfile @@ -128,7 +128,7 @@ group :development do gem 'letter_opener', '~> 1.7' gem 'letter_opener_web', '~> 1.3' gem 'memory_profiler' - gem 'rubocop', '~> 0.71', require: false + gem 'rubocop', '~> 0.72', require: false gem 'rubocop-rails', '~> 2.0', require: false gem 'brakeman', '~> 4.5', require: false gem 'bundler-audit', '~> 0.6', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 4a4e86e72..c8856ccc5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -287,7 +287,7 @@ GEM idn-ruby (0.1.0) ipaddress (0.8.3) iso-639 (0.2.8) - jaro_winkler (1.5.2) + jaro_winkler (1.5.3) jmespath (1.4.0) json (2.1.0) json-ld (3.0.2) @@ -524,7 +524,7 @@ GEM rspec-core (~> 3.0, >= 3.0.0) sidekiq (>= 2.4.0) rspec-support (3.8.0) - rubocop (0.71.0) + rubocop (0.72.0) jaro_winkler (~> 1.5.1) parallel (~> 1.10) parser (>= 2.6) @@ -737,7 +737,7 @@ DEPENDENCIES rqrcode (~> 0.10) rspec-rails (~> 3.8) rspec-sidekiq (~> 3.0) - rubocop (~> 0.71) + rubocop (~> 0.72) rubocop-rails (~> 2.0) sanitize (~> 5.0) sidekiq (~> 5.2) -- cgit From 2cfa427ea7c08abc3fa52fb2e8bfd569146e9c98 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 1 Jul 2019 19:13:30 +0200 Subject: Change domain block behaviour to prevent creation of accounts from suspended domains (#11219) --- app/services/activitypub/process_account_service.rb | 4 +++- app/services/resolve_account_service.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 05c017bdf..3857e7c16 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -15,6 +15,8 @@ class ActivityPub::ProcessAccountService < BaseService @domain = domain @collections = {} + return if auto_suspend? + RedisLock.acquire(lock_options) do |lock| if lock.acquired? @account = Account.find_remote(@username, @domain) @@ -55,7 +57,7 @@ class ActivityPub::ProcessAccountService < BaseService @account.domain = @domain @account.private_key = nil @account.suspended_at = domain_block.created_at if auto_suspend? - @account.silenced_at = domain_block.created_at if auto_silence? + @account.silenced_at = domain_block.created_at if auto_silence? end def update_account diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 57c9ccfe1..e557706da 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -48,7 +48,7 @@ class ResolveAccountService < BaseService return end - return if links_missing? + return if links_missing? || auto_suspend? return Account.find_local(@username) if TagManager.instance.local_domain?(@domain) RedisLock.acquire(lock_options) do |lock| -- cgit From 0d9ffe56fb59e0d1fce91265f44140d874c0bfba Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 00:34:38 +0200 Subject: Add request pool to improve delivery performance (#10353) * Add request pool to improve delivery performance Fix #7909 * Ensure connection is closed when exception interrupts execution * Remove Timeout#timeout from socket connection * Fix infinite retrial loop on HTTP::ConnectionError * Close sockets on failure, reduce idle time to 90 seconds * Add MAX_REQUEST_POOL_SIZE option to limit concurrent connections to the same server * Use a shared pool size, 512 by default, to stay below open file limit * Add some tests * Add more tests * Reduce MAX_IDLE_TIME from 90 to 30 seconds, reap every 30 seconds * Use a shared pool that returns preferred connection but re-purposes other ones when needed * Fix wrong connection being returned on subsequent calls within the same thread * Reduce mutex calls on flushes from 2 to 1 and add test for reaping --- Gemfile | 1 + Gemfile.lock | 1 + app/lib/connection_pool/shared_connection_pool.rb | 63 ++++++++++++ app/lib/connection_pool/shared_timed_stack.rb | 95 +++++++++++++++++ app/lib/request.rb | 70 +++++++++---- app/lib/request_pool.rb | 114 +++++++++++++++++++++ app/workers/activitypub/delivery_worker.rb | 15 ++- .../connection_pool/shared_connection_pool_spec.rb | 28 +++++ .../lib/connection_pool/shared_timed_stack_spec.rb | 61 +++++++++++ spec/lib/request_pool_spec.rb | 63 ++++++++++++ 10 files changed, 488 insertions(+), 23 deletions(-) create mode 100644 app/lib/connection_pool/shared_connection_pool.rb create mode 100644 app/lib/connection_pool/shared_timed_stack.rb create mode 100644 app/lib/request_pool.rb create mode 100644 spec/lib/connection_pool/shared_connection_pool_spec.rb create mode 100644 spec/lib/connection_pool/shared_timed_stack_spec.rb create mode 100644 spec/lib/request_pool_spec.rb diff --git a/Gemfile b/Gemfile index e99a808d4..2fcb603c3 100644 --- a/Gemfile +++ b/Gemfile @@ -148,3 +148,4 @@ group :production do end gem 'concurrent-ruby', require: false +gem 'connection_pool', require: false diff --git a/Gemfile.lock b/Gemfile.lock index c8856ccc5..6043a2fa9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -666,6 +666,7 @@ DEPENDENCIES cld3 (~> 3.2.4) climate_control (~> 0.2) concurrent-ruby + connection_pool derailed_benchmarks devise (~> 4.6) devise-two-factor (~> 3.0) diff --git a/app/lib/connection_pool/shared_connection_pool.rb b/app/lib/connection_pool/shared_connection_pool.rb new file mode 100644 index 000000000..2865a4108 --- /dev/null +++ b/app/lib/connection_pool/shared_connection_pool.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'connection_pool' +require_relative './shared_timed_stack' + +class ConnectionPool::SharedConnectionPool < ConnectionPool + def initialize(options = {}, &block) + super(options, &block) + + @available = ConnectionPool::SharedTimedStack.new(@size, &block) + end + + delegate :size, :flush, to: :@available + + def with(preferred_tag, options = {}) + Thread.handle_interrupt(Exception => :never) do + conn = checkout(preferred_tag, options) + + begin + Thread.handle_interrupt(Exception => :immediate) do + yield conn + end + ensure + checkin(preferred_tag) + end + end + end + + def checkout(preferred_tag, options = {}) + if ::Thread.current[key(preferred_tag)] + ::Thread.current[key_count(preferred_tag)] += 1 + ::Thread.current[key(preferred_tag)] + else + ::Thread.current[key_count(preferred_tag)] = 1 + ::Thread.current[key(preferred_tag)] = @available.pop(preferred_tag, options[:timeout] || @timeout) + end + end + + def checkin(preferred_tag) + if ::Thread.current[key(preferred_tag)] + if ::Thread.current[key_count(preferred_tag)] == 1 + @available.push(::Thread.current[key(preferred_tag)]) + ::Thread.current[key(preferred_tag)] = nil + else + ::Thread.current[key_count(preferred_tag)] -= 1 + end + else + raise ConnectionPool::Error, 'no connections are checked out' + end + + nil + end + + private + + def key(tag) + :"#{@key}-#{tag}" + end + + def key_count(tag) + :"#{@key_count}-#{tag}" + end +end diff --git a/app/lib/connection_pool/shared_timed_stack.rb b/app/lib/connection_pool/shared_timed_stack.rb new file mode 100644 index 000000000..14a5285c4 --- /dev/null +++ b/app/lib/connection_pool/shared_timed_stack.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +class ConnectionPool::SharedTimedStack + def initialize(max = 0, &block) + @create_block = block + @max = max + @created = 0 + @queue = [] + @tagged_queue = Hash.new { |hash, key| hash[key] = [] } + @mutex = Mutex.new + @resource = ConditionVariable.new + end + + def push(connection) + @mutex.synchronize do + store_connection(connection) + @resource.broadcast + end + end + + alias << push + + def pop(preferred_tag, timeout = 5.0) + deadline = current_time + timeout + + @mutex.synchronize do + loop do + return fetch_preferred_connection(preferred_tag) unless @tagged_queue[preferred_tag].empty? + + connection = try_create(preferred_tag) + return connection if connection + + to_wait = deadline - current_time + raise Timeout::Error, "Waited #{timeout} sec" if to_wait <= 0 + + @resource.wait(@mutex, to_wait) + end + end + end + + def empty? + size.zero? + end + + def size + @mutex.synchronize do + @queue.size + end + end + + def flush + @mutex.synchronize do + @queue.delete_if do |connection| + delete = !connection.in_use && (connection.dead || connection.seconds_idle >= RequestPool::MAX_IDLE_TIME) + + if delete + @tagged_queue[connection.site].delete(connection) + connection.close + @created -= 1 + end + + delete + end + end + end + + private + + def try_create(preferred_tag) + if @created == @max && !@queue.empty? + throw_away_connection = @queue.pop + @tagged_queue[throw_away_connection.site].delete(throw_away_connection) + @create_block.call(preferred_tag) + elsif @created != @max + connection = @create_block.call(preferred_tag) + @created += 1 + connection + end + end + + def fetch_preferred_connection(preferred_tag) + connection = @tagged_queue[preferred_tag].pop + @queue.delete(connection) + connection + end + + def current_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def store_connection(connection) + @tagged_queue[connection.site].push(connection) + @queue.push(connection) + end +end diff --git a/app/lib/request.rb b/app/lib/request.rb index e555ae6a1..af49d6c77 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -17,15 +17,21 @@ end class Request REQUEST_TARGET = '(request-target)' + # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening + # and 5s timeout on the TLS handshake, meaning the worst case should take + # about 15s in total + TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze + include RoutingHelper def initialize(verb, url, **options) raise ArgumentError if url.blank? - @verb = verb - @url = Addressable::URI.parse(url).normalize - @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) - @headers = {} + @verb = verb + @url = Addressable::URI.parse(url).normalize + @http_client = options.delete(:http_client) + @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) + @headers = {} raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? @@ -50,15 +56,24 @@ class Request def perform begin - response = http_client.headers(headers).public_send(@verb, @url.to_s, @options) + response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers)) rescue => e raise e.class, "#{e.message} on #{@url}", e.backtrace[0] end begin - yield response.extend(ClientLimit) if block_given? + response = response.extend(ClientLimit) + + # If we are using a persistent connection, we have to + # read every response to be able to move forward at all. + # However, simply calling #to_s or #flush may not be safe, + # as the response body, if malicious, could be too big + # for our memory. So we use the #body_with_limit method + response.body_with_limit if http_client.persistent? + + yield response if block_given? ensure - http_client.close + http_client.close unless http_client.persistent? end end @@ -76,6 +91,10 @@ class Request %w(http https).include?(parsed_url.scheme) && parsed_url.host.present? end + + def http_client + HTTP.use(:auto_inflate).timeout(:per_operation, TIMEOUT.dup).follow(max_hops: 2) + end end private @@ -116,16 +135,8 @@ class Request end end - def timeout - # We enforce a 1s timeout on DNS resolving, 10s timeout on socket opening - # and 5s timeout on the TLS handshake, meaning the worst case should take - # about 16s in total - - { connect: 5, read: 10, write: 10 } - end - def http_client - @http_client ||= HTTP.use(:auto_inflate).timeout(:per_operation, timeout).follow(max_hops: 2) + @http_client ||= Request.http_client end def use_proxy? @@ -169,20 +180,41 @@ class Request return super(host, *args) if thru_hidden_service?(host) outer_e = nil + port = args.first Resolv::DNS.open do |dns| dns.timeouts = 5 addresses = dns.getaddresses(host).take(2) - time_slot = 10.0 / addresses.size addresses.each do |address| begin raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s)) - ::Timeout.timeout(time_slot, HTTP::TimeoutError) do - return super(address.to_s, *args) + sock = ::Socket.new(::Socket::AF_INET, ::Socket::SOCK_STREAM, 0) + sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s) + + sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1) + + begin + sock.connect_nonblock(sockaddr) + rescue IO::WaitWritable + if IO.select(nil, [sock], nil, Request::TIMEOUT[:connect]) + begin + sock.connect_nonblock(sockaddr) + rescue Errno::EISCONN + # Yippee! + rescue + sock.close + raise + end + else + sock.close + raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds" + end end + + return sock rescue => e outer_e = e end diff --git a/app/lib/request_pool.rb b/app/lib/request_pool.rb new file mode 100644 index 000000000..e5899a79a --- /dev/null +++ b/app/lib/request_pool.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require_relative './connection_pool/shared_connection_pool' + +class RequestPool + def self.current + @current ||= RequestPool.new + end + + class Reaper + attr_reader :pool, :frequency + + def initialize(pool, frequency) + @pool = pool + @frequency = frequency + end + + def run + return unless frequency&.positive? + + Thread.new(frequency, pool) do |t, p| + loop do + sleep t + p.flush + end + end + end + end + + MAX_IDLE_TIME = 30 + WAIT_TIMEOUT = 5 + MAX_POOL_SIZE = ENV.fetch('MAX_REQUEST_POOL_SIZE', 512).to_i + + class Connection + attr_reader :site, :last_used_at, :created_at, :in_use, :dead, :fresh + + def initialize(site) + @site = site + @http_client = http_client + @last_used_at = nil + @created_at = current_time + @dead = false + @fresh = true + end + + def use + @last_used_at = current_time + @in_use = true + + retries = 0 + + begin + yield @http_client + rescue HTTP::ConnectionError + # It's possible the connection was closed, so let's + # try re-opening it once + + close + + if @fresh || retries.positive? + raise + else + @http_client = http_client + retries += 1 + retry + end + rescue StandardError + # If this connection raises errors of any kind, it's + # better if it gets reaped as soon as possible + + close + @dead = true + raise + end + ensure + @fresh = false + @in_use = false + end + + def seconds_idle + current_time - (@last_used_at || @created_at) + end + + def close + @http_client.close + end + + private + + def http_client + Request.http_client.persistent(@site, timeout: MAX_IDLE_TIME) + end + + def current_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + end + + def initialize + @pool = ConnectionPool::SharedConnectionPool.new(size: MAX_POOL_SIZE, timeout: WAIT_TIMEOUT) { |site| Connection.new(site) } + @reaper = Reaper.new(self, 30) + @reaper.run + end + + def with(site, &block) + @pool.with(site) do |connection| + ActiveSupport::Notifications.instrument('with.request_pool', miss: connection.fresh, host: connection.site) do + connection.use(&block) + end + end + end + + delegate :size, :flush, to: :@pool +end diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 5e4c391f0..79f1e8153 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -17,6 +17,7 @@ class ActivityPub::DeliveryWorker @json = json @source_account = Account.find(source_account_id) @inbox_url = inbox_url + @host = Addressable::URI.parse(inbox_url).normalized_site perform_request @@ -28,16 +29,18 @@ class ActivityPub::DeliveryWorker private - def build_request - request = Request.new(:post, @inbox_url, body: @json) + def build_request(http_client) + request = Request.new(:post, @inbox_url, body: @json, http_client: http_client) request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) request.add_headers(HEADERS) end def perform_request light = Stoplight(@inbox_url) do - build_request.perform do |response| - raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + request_pool.with(@host) do |http_client| + build_request(http_client).perform do |response| + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + end end end @@ -57,4 +60,8 @@ class ActivityPub::DeliveryWorker def failure_tracker @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url) end + + def request_pool + RequestPool.current + end end diff --git a/spec/lib/connection_pool/shared_connection_pool_spec.rb b/spec/lib/connection_pool/shared_connection_pool_spec.rb new file mode 100644 index 000000000..114464558 --- /dev/null +++ b/spec/lib/connection_pool/shared_connection_pool_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe ConnectionPool::SharedConnectionPool do + class MiniConnection + attr_reader :site + + def initialize(site) + @site = site + end + end + + subject { described_class.new(size: 5, timeout: 5) { |site| MiniConnection.new(site) } } + + describe '#with' do + it 'runs a block with a connection' do + block_run = false + + subject.with('foo') do |connection| + expect(connection).to be_a MiniConnection + block_run = true + end + + expect(block_run).to be true + end + end +end diff --git a/spec/lib/connection_pool/shared_timed_stack_spec.rb b/spec/lib/connection_pool/shared_timed_stack_spec.rb new file mode 100644 index 000000000..f680c5966 --- /dev/null +++ b/spec/lib/connection_pool/shared_timed_stack_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe ConnectionPool::SharedTimedStack do + class MiniConnection + attr_reader :site + + def initialize(site) + @site = site + end + end + + subject { described_class.new(5) { |site| MiniConnection.new(site) } } + + describe '#push' do + it 'keeps the connection in the stack' do + subject.push(MiniConnection.new('foo')) + expect(subject.size).to eq 1 + end + end + + describe '#pop' do + it 'returns a connection' do + expect(subject.pop('foo')).to be_a MiniConnection + end + + it 'returns the same connection that was pushed in' do + connection = MiniConnection.new('foo') + subject.push(connection) + expect(subject.pop('foo')).to be connection + end + + it 'does not create more than maximum amount of connections' do + expect { 6.times { subject.pop('foo', 0) } }.to raise_error Timeout::Error + end + + it 'repurposes a connection for a different site when maximum amount is reached' do + 5.times { subject.push(MiniConnection.new('foo')) } + expect(subject.pop('bar')).to be_a MiniConnection + end + end + + describe '#empty?' do + it 'returns true when no connections on the stack' do + expect(subject.empty?).to be true + end + + it 'returns false when there are connections on the stack' do + subject.push(MiniConnection.new('foo')) + expect(subject.empty?).to be false + end + end + + describe '#size' do + it 'returns the number of connections on the stack' do + 2.times { subject.push(MiniConnection.new('foo')) } + expect(subject.size).to eq 2 + end + end +end diff --git a/spec/lib/request_pool_spec.rb b/spec/lib/request_pool_spec.rb new file mode 100644 index 000000000..4a144d7c7 --- /dev/null +++ b/spec/lib/request_pool_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe RequestPool do + subject { described_class.new } + + describe '#with' do + it 'returns a HTTP client for a host' do + subject.with('http://example.com') do |http_client| + expect(http_client).to be_a HTTP::Client + end + end + + it 'returns the same instance of HTTP client within the same thread for the same host' do + test_client = nil + + subject.with('http://example.com') { |http_client| test_client = http_client } + expect(test_client).to_not be_nil + subject.with('http://example.com') { |http_client| expect(http_client).to be test_client } + end + + it 'returns different HTTP clients for different hosts' do + test_client = nil + + subject.with('http://example.com') { |http_client| test_client = http_client } + expect(test_client).to_not be_nil + subject.with('http://example.org') { |http_client| expect(http_client).to_not be test_client } + end + + it 'grows to the number of threads accessing it' do + stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') + + subject + + threads = 20.times.map do |i| + Thread.new do + 20.times do + subject.with('http://example.com') do |http_client| + http_client.get('/').flush + end + end + end + end + + threads.map(&:join) + + expect(subject.size).to be > 1 + end + + it 'closes idle connections' do + stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') + + subject.with('http://example.com') do |http_client| + http_client.get('/').flush + end + + expect(subject.size).to eq 1 + sleep RequestPool::MAX_IDLE_TIME + 30 + 1 + expect(subject.size).to eq 0 + end + end +end -- cgit From dc88d226e1fdb20499d6e81838b81894b2d0be2d Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 2 Jul 2019 00:36:16 +0200 Subject: When deleting & redrafting a poll, fill in closest expires_in (#11203) Use the smallest preset expires_in such that the new poll would not expire before the old one. In the typical case of a quick delete & redraft, this results in using the same poll duration. Fixes #10567 --- app/javascript/mastodon/reducers/compose.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js index 8cdd29bfe..fae7522b2 100644 --- a/app/javascript/mastodon/reducers/compose.js +++ b/app/javascript/mastodon/reducers/compose.js @@ -195,6 +195,12 @@ const expandMentions = status => { return fragment.innerHTML; }; +const expiresInFromExpiresAt = expires_at => { + if (!expires_at) return 24 * 3600; + const delta = (new Date(expires_at).getTime() - Date.now()) / 1000; + return [300, 1800, 3600, 21600, 86400, 259200, 604800].find(expires_in => expires_in >= delta) || 24 * 3600; +}; + export default function compose(state = initialState, action) { switch(action.type) { case STORE_HYDRATE: @@ -353,7 +359,7 @@ export default function compose(state = initialState, action) { map.set('poll', ImmutableMap({ options: action.status.getIn(['poll', 'options']).map(x => x.get('title')), multiple: action.status.getIn(['poll', 'multiple']), - expires_in: 24 * 3600, + expires_in: expiresInFromExpiresAt(action.status.getIn(['poll', 'expires_at'])), })); } }); -- cgit From bc60d794f8411daa3c4c1541fb373309e4c029b6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 00:59:53 +0200 Subject: Change ActivityPub::DeliveryWorker to not retry HTTP 501 errors (#11233) --- app/workers/activitypub/delivery_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 79f1e8153..818fd8f5d 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -54,7 +54,7 @@ class ActivityPub::DeliveryWorker end def response_error_unsalvageable?(response) - (400...500).cover?(response.code) && ![401, 408, 429].include?(response.code) + response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code)) end def failure_tracker -- cgit From 1a747f7020b6888f84ef54d1631f85bf748da868 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 01:01:17 +0200 Subject: Fix statsd UDP sockets not being cleaned up in Sidekiq (#11230) --- app/lib/sidekiq_error_handler.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/lib/sidekiq_error_handler.rb b/app/lib/sidekiq_error_handler.rb index 23785cf05..8eb6b942d 100644 --- a/app/lib/sidekiq_error_handler.rb +++ b/app/lib/sidekiq_error_handler.rb @@ -3,9 +3,11 @@ class SidekiqErrorHandler def call(*) yield - rescue Mastodon::HostValidationError => e - Rails.logger.error "#{e.class}: #{e.message}" - Rails.logger.error e.backtrace.join("\n") + rescue Mastodon::HostValidationError # Do not retry + ensure + socket = Thread.current[:statsd_socket] + socket&.close + Thread.current[:statsd_socket] = nil end end -- cgit From 3bc0c4a884ef054e0f715be1f3fd8566288fa514 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 11:34:39 +0200 Subject: Remove unused StatsD code and expose StatsD as a global variable (#11232) The instrumentation code was used for StatsD metrics collection prior to the switch to the nsa gem and should have been removed at that point as it no longer does anything at all --- config/initializers/instrumentation.rb | 18 ------------------ config/initializers/statsd.rb | 6 +++--- 2 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 config/initializers/instrumentation.rb diff --git a/config/initializers/instrumentation.rb b/config/initializers/instrumentation.rb deleted file mode 100644 index 8483f2be2..000000000 --- a/config/initializers/instrumentation.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -instrumentation_hostname = ENV.fetch('INSTRUMENTATION_HOSTNAME') { 'localhost' } - -ActiveSupport::Notifications.subscribe(/process_action.action_controller/) do |*args| - event = ActiveSupport::Notifications::Event.new(*args) - controller = event.payload[:controller] - action = event.payload[:action] - format = event.payload[:format] || 'all' - format = 'all' if format == '*/*' - status = event.payload[:status] - key = "#{controller}.#{action}.#{format}.#{instrumentation_hostname}" - - ActiveSupport::Notifications.instrument :performance, action: :measure, measurement: "#{key}.total_duration", value: event.duration - ActiveSupport::Notifications.instrument :performance, action: :measure, measurement: "#{key}.db_time", value: event.payload[:db_runtime] - ActiveSupport::Notifications.instrument :performance, action: :measure, measurement: "#{key}.view_time", value: event.payload[:view_runtime] - ActiveSupport::Notifications.instrument :performance, measurement: "#{key}.status.#{status}" -end diff --git a/config/initializers/statsd.rb b/config/initializers/statsd.rb index ce83fd9de..93ea1d1e4 100644 --- a/config/initializers/statsd.rb +++ b/config/initializers/statsd.rb @@ -3,10 +3,10 @@ if ENV['STATSD_ADDR'].present? host, port = ENV['STATSD_ADDR'].split(':') - statsd = ::Statsd.new(host, port) - statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } + $statsd = ::Statsd.new(host, port) + $statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } - ::NSA.inform_statsd(statsd) do |informant| + ::NSA.inform_statsd($statsd) do |informant| informant.collect(:action_controller, :web) informant.collect(:active_record, :db) informant.collect(:active_support_cache, :cache) -- cgit From 99924f282f53593e670c70a38450a1c0e2d24c20 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 2 Jul 2019 16:03:54 +0200 Subject: Memoize ancestorIds and descendantIds in detailed status view (#11234) --- app/javascript/mastodon/features/status/index.js | 68 +++++++++++++++--------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/app/javascript/mastodon/features/status/index.js b/app/javascript/mastodon/features/status/index.js index 981eb9d58..0422111ae 100644 --- a/app/javascript/mastodon/features/status/index.js +++ b/app/javascript/mastodon/features/status/index.js @@ -4,6 +4,7 @@ import { connect } from 'react-redux'; import PropTypes from 'prop-types'; import classNames from 'classnames'; import ImmutablePropTypes from 'react-immutable-proptypes'; +import { createSelector } from 'reselect'; import { fetchStatus } from '../../actions/statuses'; import MissingIndicator from '../../components/missing_indicator'; import DetailedStatus from './components/detailed_status'; @@ -63,39 +64,58 @@ const messages = defineMessages({ const makeMapStateToProps = () => { const getStatus = makeGetStatus(); - const mapStateToProps = (state, props) => { - const status = getStatus(state, { id: props.params.statusId }); + const getAncestorsIds = createSelector([ + (_, { id }) => id, + state => state.getIn(['contexts', 'inReplyTos']), + ], (statusId, inReplyTos) => { let ancestorsIds = Immutable.List(); + ancestorsIds = ancestorsIds.withMutations(mutable => { + let id = statusId; + + while (id) { + mutable.unshift(id); + id = inReplyTos.get(id); + } + }); + + return ancestorsIds; + }); + + const getDescendantsIds = createSelector([ + (_, { id }) => id, + state => state.getIn(['contexts', 'replies']), + ], (statusId, contextReplies) => { let descendantsIds = Immutable.List(); + descendantsIds = descendantsIds.withMutations(mutable => { + const ids = [statusId]; - if (status) { - ancestorsIds = ancestorsIds.withMutations(mutable => { - let id = status.get('in_reply_to_id'); + while (ids.length > 0) { + let id = ids.shift(); + const replies = contextReplies.get(id); - while (id) { - mutable.unshift(id); - id = state.getIn(['contexts', 'inReplyTos', id]); + if (statusId !== id) { + mutable.push(id); } - }); - descendantsIds = descendantsIds.withMutations(mutable => { - const ids = [status.get('id')]; + if (replies) { + replies.reverse().forEach(reply => { + ids.unshift(reply); + }); + } + } + }); - while (ids.length > 0) { - let id = ids.shift(); - const replies = state.getIn(['contexts', 'replies', id]); + return descendantsIds; + }); - if (status.get('id') !== id) { - mutable.push(id); - } + const mapStateToProps = (state, props) => { + const status = getStatus(state, { id: props.params.statusId }); + let ancestorsIds = Immutable.List(); + let descendantsIds = Immutable.List(); - if (replies) { - replies.reverse().forEach(reply => { - ids.unshift(reply); - }); - } - } - }); + if (status) { + ancestorsIds = getAncestorsIds(state, { id: status.get('in_reply_to_id') }); + descendantsIds = getDescendantsIds(state, { id: status.get('id') }); } return { -- cgit From 894d82a3258c763adafad6640b71072d3071d411 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 2 Jul 2019 00:36:16 +0200 Subject: [Glitch] When deleting & redrafting a poll, fill in closest expires_in Port dc88d226e1fdb20499d6e81838b81894b2d0be2d to glitch-soc --- app/javascript/flavours/glitch/reducers/compose.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/javascript/flavours/glitch/reducers/compose.js b/app/javascript/flavours/glitch/reducers/compose.js index 36dfb8f15..a47b8b7bd 100644 --- a/app/javascript/flavours/glitch/reducers/compose.js +++ b/app/javascript/flavours/glitch/reducers/compose.js @@ -275,6 +275,12 @@ const expandMentions = status => { return fragment.innerHTML; }; +const expiresInFromExpiresAt = expires_at => { + if (!expires_at) return 24 * 3600; + const delta = (new Date(expires_at).getTime() - Date.now()) / 1000; + return [300, 1800, 3600, 21600, 86400, 259200, 604800].find(expires_in => expires_in >= delta) || 24 * 3600; +}; + export default function compose(state = initialState, action) { switch(action.type) { case STORE_HYDRATE: @@ -456,7 +462,7 @@ export default function compose(state = initialState, action) { map.set('poll', ImmutableMap({ options: action.status.getIn(['poll', 'options']).map(x => x.get('title')), multiple: action.status.getIn(['poll', 'multiple']), - expires_in: 24 * 3600, + expires_in: expiresInFromExpiresAt(action.status.getIn(['poll', 'expires_at'])), })); } }); -- cgit From 47c30be8d810fd6d5d27285a097d01843f4dd9f6 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 2 Jul 2019 16:03:54 +0200 Subject: [Glitch] Memoize ancestorIds and descendantIds in detailed status view Port 99924f282f53593e670c70a38450a1c0e2d24c20 to glitch-soc --- .../flavours/glitch/features/status/index.js | 68 ++++++++++++++-------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/app/javascript/flavours/glitch/features/status/index.js b/app/javascript/flavours/glitch/features/status/index.js index 76bfaaffa..40e100fd5 100644 --- a/app/javascript/flavours/glitch/features/status/index.js +++ b/app/javascript/flavours/glitch/features/status/index.js @@ -4,6 +4,7 @@ import { connect } from 'react-redux'; import PropTypes from 'prop-types'; import classNames from 'classnames'; import ImmutablePropTypes from 'react-immutable-proptypes'; +import { createSelector } from 'reselect'; import { fetchStatus } from 'flavours/glitch/actions/statuses'; import MissingIndicator from 'flavours/glitch/components/missing_indicator'; import DetailedStatus from './components/detailed_status'; @@ -61,39 +62,58 @@ const messages = defineMessages({ const makeMapStateToProps = () => { const getStatus = makeGetStatus(); - const mapStateToProps = (state, props) => { - const status = getStatus(state, { id: props.params.statusId }); + const getAncestorsIds = createSelector([ + (_, { id }) => id, + state => state.getIn(['contexts', 'inReplyTos']), + ], (statusId, inReplyTos) => { let ancestorsIds = Immutable.List(); + ancestorsIds = ancestorsIds.withMutations(mutable => { + let id = statusId; + + while (id) { + mutable.unshift(id); + id = inReplyTos.get(id); + } + }); + + return ancestorsIds; + }); + + const getDescendantsIds = createSelector([ + (_, { id }) => id, + state => state.getIn(['contexts', 'replies']), + ], (statusId, contextReplies) => { let descendantsIds = Immutable.List(); + descendantsIds = descendantsIds.withMutations(mutable => { + const ids = [statusId]; - if (status) { - ancestorsIds = ancestorsIds.withMutations(mutable => { - let id = status.get('in_reply_to_id'); + while (ids.length > 0) { + let id = ids.shift(); + const replies = contextReplies.get(id); - while (id) { - mutable.unshift(id); - id = state.getIn(['contexts', 'inReplyTos', id]); + if (statusId !== id) { + mutable.push(id); } - }); - descendantsIds = descendantsIds.withMutations(mutable => { - const ids = [status.get('id')]; + if (replies) { + replies.reverse().forEach(reply => { + ids.unshift(reply); + }); + } + } + }); - while (ids.length > 0) { - let id = ids.shift(); - const replies = state.getIn(['contexts', 'replies', id]); + return descendantsIds; + }); - if (status.get('id') !== id) { - mutable.push(id); - } + const mapStateToProps = (state, props) => { + const status = getStatus(state, { id: props.params.statusId }); + let ancestorsIds = Immutable.List(); + let descendantsIds = Immutable.List(); - if (replies) { - replies.reverse().forEach(reply => { - ids.unshift(reply); - }); - } - } - }); + if (status) { + ancestorsIds = getAncestorsIds(state, { id: status.get('in_reply_to_id') }); + descendantsIds = getDescendantsIds(state, { id: status.get('id') }); } return { -- cgit From 3fd6ab99e69e736e10c7022f07dfa34b60a4e54d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 5 Jul 2019 02:14:56 +0200 Subject: Remove deprecated REST API `GET /api/v1/timelines/direct` (#11212) --- .../api/v1/timelines/direct_controller.rb | 63 ---------------------- app/models/status.rb | 41 -------------- config/routes.rb | 1 - .../api/v1/timelines/direct_controller_spec.rb | 17 ------ spec/models/status_spec.rb | 50 ----------------- 5 files changed, 172 deletions(-) delete mode 100644 app/controllers/api/v1/timelines/direct_controller.rb delete mode 100644 spec/controllers/api/v1/timelines/direct_controller_spec.rb diff --git a/app/controllers/api/v1/timelines/direct_controller.rb b/app/controllers/api/v1/timelines/direct_controller.rb deleted file mode 100644 index d8a76d153..000000000 --- a/app/controllers/api/v1/timelines/direct_controller.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -class Api::V1::Timelines::DirectController < Api::BaseController - before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: [:show] - before_action :require_user!, only: [:show] - after_action :insert_pagination_headers, unless: -> { @statuses.empty? } - - respond_to :json - - def show - @statuses = load_statuses - render json: @statuses, each_serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id) - end - - private - - def load_statuses - cached_direct_statuses - end - - def cached_direct_statuses - cache_collection direct_statuses, Status - end - - def direct_statuses - direct_timeline_statuses - end - - def direct_timeline_statuses - # this query requires built in pagination. - Status.as_direct_timeline( - current_account, - limit_param(DEFAULT_STATUSES_LIMIT), - params[:max_id], - params[:since_id], - true # returns array of cache_ids object - ) - end - - def insert_pagination_headers - set_pagination_headers(next_path, prev_path) - end - - def pagination_params(core_params) - params.permit(:local, :limit).merge(core_params) - end - - def next_path - api_v1_timelines_direct_url pagination_params(max_id: pagination_max_id) - end - - def prev_path - api_v1_timelines_direct_url pagination_params(since_id: pagination_since_id) - end - - def pagination_max_id - @statuses.last.id - end - - def pagination_since_id - @statuses.first.id - end -end diff --git a/app/models/status.rb b/app/models/status.rb index fb9bbc9a9..2258e2d07 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -280,47 +280,6 @@ class Status < ApplicationRecord where(account: [account] + account.following).where(visibility: [:public, :unlisted, :private]) end - def as_direct_timeline(account, limit = 20, max_id = nil, since_id = nil, cache_ids = false) - # direct timeline is mix of direct message from_me and to_me. - # 2 queries are executed with pagination. - # constant expression using arel_table is required for partial index - - # _from_me part does not require any timeline filters - query_from_me = where(account_id: account.id) - .where(Status.arel_table[:visibility].eq(3)) - .limit(limit) - .order('statuses.id DESC') - - # _to_me part requires mute and block filter. - # FIXME: may we check mutes.hide_notifications? - query_to_me = Status - .joins(:mentions) - .merge(Mention.where(account_id: account.id)) - .where(Status.arel_table[:visibility].eq(3)) - .limit(limit) - .order('mentions.status_id DESC') - .not_excluded_by_account(account) - - if max_id.present? - query_from_me = query_from_me.where('statuses.id < ?', max_id) - query_to_me = query_to_me.where('mentions.status_id < ?', max_id) - end - - if since_id.present? - query_from_me = query_from_me.where('statuses.id > ?', since_id) - query_to_me = query_to_me.where('mentions.status_id > ?', since_id) - end - - if cache_ids - # returns array of cache_ids object that have id and updated_at - (query_from_me.cache_ids.to_a + query_to_me.cache_ids.to_a).uniq(&:id).sort_by(&:id).reverse.take(limit) - else - # returns ActiveRecord.Relation - items = (query_from_me.select(:id).to_a + query_to_me.select(:id).to_a).uniq(&:id).sort_by(&:id).reverse.take(limit) - Status.where(id: items.map(&:id)) - end - end - def as_public_timeline(account = nil, local_only = false) query = timeline_scope(local_only).without_replies diff --git a/config/routes.rb b/config/routes.rb index 764db8db2..29db54a32 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -299,7 +299,6 @@ Rails.application.routes.draw do end namespace :timelines do - resource :direct, only: :show, controller: :direct resource :home, only: :show, controller: :home resource :public, only: :show, controller: :public resources :tag, only: :show diff --git a/spec/controllers/api/v1/timelines/direct_controller_spec.rb b/spec/controllers/api/v1/timelines/direct_controller_spec.rb deleted file mode 100644 index a22c2cbea..000000000 --- a/spec/controllers/api/v1/timelines/direct_controller_spec.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Api::V1::Timelines::DirectController, type: :controller do - let(:user) { Fabricate(:user) } - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') } - - describe 'GET #show' do - it 'returns 200' do - allow(controller).to receive(:doorkeeper_token) { token } - get :show - - expect(response).to have_http_status(200) - end - end -end diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb index e8cf18af9..a8c567414 100644 --- a/spec/models/status_spec.rb +++ b/spec/models/status_spec.rb @@ -339,56 +339,6 @@ RSpec.describe Status, type: :model do end end - describe '.as_direct_timeline' do - let(:account) { Fabricate(:account) } - let(:followed) { Fabricate(:account) } - let(:not_followed) { Fabricate(:account) } - - before do - Fabricate(:follow, account: account, target_account: followed) - - @self_public_status = Fabricate(:status, account: account, visibility: :public) - @self_direct_status = Fabricate(:status, account: account, visibility: :direct) - @followed_public_status = Fabricate(:status, account: followed, visibility: :public) - @followed_direct_status = Fabricate(:status, account: followed, visibility: :direct) - @not_followed_direct_status = Fabricate(:status, account: not_followed, visibility: :direct) - - @results = Status.as_direct_timeline(account) - end - - it 'does not include public statuses from self' do - expect(@results).to_not include(@self_public_status) - end - - it 'includes direct statuses from self' do - expect(@results).to include(@self_direct_status) - end - - it 'does not include public statuses from followed' do - expect(@results).to_not include(@followed_public_status) - end - - it 'does not include direct statuses not mentioning recipient from followed' do - expect(@results).to_not include(@followed_direct_status) - end - - it 'does not include direct statuses not mentioning recipient from non-followed' do - expect(@results).to_not include(@not_followed_direct_status) - end - - it 'includes direct statuses mentioning recipient from followed' do - Fabricate(:mention, account: account, status: @followed_direct_status) - results2 = Status.as_direct_timeline(account) - expect(results2).to include(@followed_direct_status) - end - - it 'includes direct statuses mentioning recipient from non-followed' do - Fabricate(:mention, account: account, status: @not_followed_direct_status) - results2 = Status.as_direct_timeline(account) - expect(results2).to include(@not_followed_direct_status) - end - end - describe '.as_public_timeline' do it 'only includes statuses with public visibility' do public_status = Fabricate(:status, visibility: :public) -- cgit From 30f5bcf3e749be9651ed39a07b893f70605f8a39 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 5 Jul 2019 02:15:13 +0200 Subject: Remove deprecated REST API `POST /api/v1/notifications/dismiss?id=:id` (#11214) --- config/routes.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/config/routes.rb b/config/routes.rb index 29db54a32..1f97dea4c 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -351,7 +351,6 @@ Rails.application.routes.draw do resources :notifications, only: [:index, :show] do collection do post :clear - post :dismiss # Deprecated end member do -- cgit From 0c1b1069c96d3452d43d922a737c522d10e4747b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 5 Jul 2019 02:15:24 +0200 Subject: Remove deprecated REST API `GET /api/v1/statuses/:id/card` (#11213) --- app/controllers/api/v1/statuses_controller.rb | 14 ++------------ config/routes.rb | 1 - spec/controllers/api/v1/statuses_controller_spec.rb | 14 -------------- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index b0e134554..71a505c26 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -5,8 +5,8 @@ class Api::V1::StatusesController < Api::BaseController before_action -> { authorize_if_got_token! :read, :'read:statuses' }, except: [:create, :destroy] before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, only: [:create, :destroy] - before_action :require_user!, except: [:show, :context, :card] - before_action :set_status, only: [:show, :context, :card] + before_action :require_user!, except: [:show, :context] + before_action :set_status, only: [:show, :context] respond_to :json @@ -33,16 +33,6 @@ class Api::V1::StatusesController < Api::BaseController render json: @context, serializer: REST::ContextSerializer, relationships: StatusRelationshipsPresenter.new(statuses, current_user&.account_id) end - def card - @card = @status.preview_cards.first - - if @card.nil? - render_empty - else - render json: @card, serializer: REST::PreviewCardSerializer - end - end - def create @status = PostStatusService.new.call(current_user.account, text: status_params[:status], diff --git a/config/routes.rb b/config/routes.rb index 1f97dea4c..9ab5ba7f0 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -294,7 +294,6 @@ Rails.application.routes.draw do member do get :context - get :card end end diff --git a/spec/controllers/api/v1/statuses_controller_spec.rb b/spec/controllers/api/v1/statuses_controller_spec.rb index 8bc3b0c67..9ff5fcd3b 100644 --- a/spec/controllers/api/v1/statuses_controller_spec.rb +++ b/spec/controllers/api/v1/statuses_controller_spec.rb @@ -91,13 +91,6 @@ RSpec.describe Api::V1::StatusesController, type: :controller do expect(response).to have_http_status(404) end end - - describe 'GET #card' do - it 'returns http unautharized' do - get :card, params: { id: status.id } - expect(response).to have_http_status(404) - end - end end context 'with a public status' do @@ -120,13 +113,6 @@ RSpec.describe Api::V1::StatusesController, type: :controller do expect(response).to have_http_status(200) end end - - describe 'GET #card' do - it 'returns http success' do - get :card, params: { id: status.id } - expect(response).to have_http_status(200) - end - end end end end -- cgit From e5f4d83e60305e2f502f18f507ad79d8fbccfe77 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Fri, 5 Jul 2019 21:34:03 +0200 Subject: Allow serializing local-only toots in backup service Fixes #1153 --- app/serializers/activitypub/note_serializer.rb | 2 +- app/services/backup_service.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb index 7225b2319..61fa77852 100644 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@ -25,7 +25,7 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer attribute :closed, if: :poll_and_expired? def id - raise Mastodon::NotPermittedError, 'Local-only statuses should not be serialized' if object.local_only? + raise Mastodon::NotPermittedError, 'Local-only statuses should not be serialized' if object.local_only? && !instance_options[:allow_local_only] ActivityPub::TagManager.instance.uri_for(object) end diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb index 5fcc98057..bd9e77223 100644 --- a/app/services/backup_service.rb +++ b/app/services/backup_service.rb @@ -149,7 +149,8 @@ class BackupService < BaseService ActiveModelSerializers::SerializableResource.new( object, serializer: serializer, - adapter: ActivityPub::Adapter + adapter: ActivityPub::Adapter, + allow_local_only: true, ).as_json end -- cgit From ed544f3166a2a726a22736ee53d43c29d0f44773 Mon Sep 17 00:00:00 2001 From: ThibG Date: Fri, 5 Jul 2019 22:03:00 +0200 Subject: Fix HTTP requests to IPv6 hosts (#11240) --- app/lib/request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/request.rb b/app/lib/request.rb index af49d6c77..e25b9026c 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -191,7 +191,7 @@ class Request begin raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s)) - sock = ::Socket.new(::Socket::AF_INET, ::Socket::SOCK_STREAM, 0) + sock = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0) sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s) sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1) -- cgit From ae003d54f956fa2cbb422b04519b3a3e602652c3 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 6 Jul 2019 13:54:32 +0200 Subject: Fix option to send e-mail notification about account action always being true (#11242) --- app/models/admin/account_action.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb index 84c3f880d..bdbd342fb 100644 --- a/app/models/admin/account_action.rb +++ b/app/models/admin/account_action.rb @@ -17,10 +17,13 @@ class Admin::AccountAction :type, :text, :report_id, - :warning_preset_id, - :send_email_notification + :warning_preset_id - attr_reader :warning + attr_reader :warning, :send_email_notification + + def send_email_notification=(value) + @send_email_notification = ActiveModel::Type::Boolean.new.cast(value) + end def save! ApplicationRecord.transaction do -- cgit From c07cca4727041ea5a5721acbc603d4bfb45a15a6 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sat, 6 Jul 2019 18:18:08 +0200 Subject: Only scroll to the compose form if it's not horizontally in the viewport (#11246) Avoids jumping the scroll around vertically when giving it focus and editing long toots. --- app/javascript/mastodon/features/compose/components/compose_form.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/javascript/mastodon/features/compose/components/compose_form.js b/app/javascript/mastodon/features/compose/components/compose_form.js index 1f476397f..47e189251 100644 --- a/app/javascript/mastodon/features/compose/components/compose_form.js +++ b/app/javascript/mastodon/features/compose/components/compose_form.js @@ -117,7 +117,10 @@ class ComposeForm extends ImmutablePureComponent { handleFocus = () => { if (this.composeForm && !this.props.singleColumn) { - this.composeForm.scrollIntoView(); + const { left, right } = this.composeForm.getBoundingClientRect(); + if (left < 0 || right > (window.innerWidth || document.documentElement.clientWidth)) { + this.composeForm.scrollIntoView(); + } } } -- cgit From a0b614f10acef03ade34cdae498b254059e1ba1d Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 7 Jul 2019 02:05:38 +0200 Subject: Fix support for HTTP proxies (#11245) * Disable incorrect check for hidden services in Socket Hidden services can only be accessed with an HTTP proxy, in which case the host seen by the Socket class will be the proxy, not the target host. Hidden services are already filtered in `Request#initialize`. * Use our Socket class to connect to HTTP proxies Avoid the timeout logic being bypassed * Add support for IP addresses in Request::Socket * Refactor a bit, no need to keep the DNS resolver around --- app/lib/request.rb | 83 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/app/lib/request.rb b/app/lib/request.rb index e25b9026c..5f7075a3c 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -30,7 +30,8 @@ class Request @verb = verb @url = Addressable::URI.parse(url).normalize @http_client = options.delete(:http_client) - @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) + @options = options.merge(socket_class: use_proxy? ? ProxySocket : Socket) + @options = @options.merge(Rails.configuration.x.http_client_proxy) if use_proxy? @headers = {} raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? @@ -177,47 +178,49 @@ class Request class Socket < TCPSocket class << self def open(host, *args) - return super(host, *args) if thru_hidden_service?(host) - outer_e = nil port = args.first - Resolv::DNS.open do |dns| - dns.timeouts = 5 + addresses = [] + begin + addresses = [IPAddr.new(host)] + rescue IPAddr::InvalidAddressError + Resolv::DNS.open do |dns| + dns.timeouts = 5 + addresses = dns.getaddresses(host).take(2) + end + end - addresses = dns.getaddresses(host).take(2) + addresses.each do |address| + begin + check_private_address(address) + + sock = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0) + sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s) + + sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1) - addresses.each do |address| begin - raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s)) - - sock = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0) - sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s) - - sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1) - - begin - sock.connect_nonblock(sockaddr) - rescue IO::WaitWritable - if IO.select(nil, [sock], nil, Request::TIMEOUT[:connect]) - begin - sock.connect_nonblock(sockaddr) - rescue Errno::EISCONN - # Yippee! - rescue - sock.close - raise - end - else + sock.connect_nonblock(sockaddr) + rescue IO::WaitWritable + if IO.select(nil, [sock], nil, Request::TIMEOUT[:connect]) + begin + sock.connect_nonblock(sockaddr) + rescue Errno::EISCONN + # Yippee! + rescue sock.close - raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds" + raise end + else + sock.close + raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds" end - - return sock - rescue => e - outer_e = e end + + return sock + rescue => e + outer_e = e end end @@ -230,11 +233,21 @@ class Request alias new open - def thru_hidden_service?(host) - Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match(host) + def check_private_address(address) + raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s)) + end + end + end + + class ProxySocket < Socket + class << self + def check_private_address(_address) + # Accept connections to private addresses as HTTP proxies will usually + # be on local addresses + nil end end end - private_constant :ClientLimit, :Socket + private_constant :ClientLimit, :Socket, :ProxySocket end -- cgit From 64f3bc77acf2b992a189769b09427f93e7e5eb60 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sat, 6 Jul 2019 18:18:08 +0200 Subject: [Glitch] Only scroll to the compose form if it's not horizontally in the viewport Port c07cca4727041ea5a5721acbc603d4bfb45a15a6 to glitch-soc --- .../flavours/glitch/features/compose/components/compose_form.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/javascript/flavours/glitch/features/compose/components/compose_form.js b/app/javascript/flavours/glitch/features/compose/components/compose_form.js index 822cfa95d..3d9002fe4 100644 --- a/app/javascript/flavours/glitch/features/compose/components/compose_form.js +++ b/app/javascript/flavours/glitch/features/compose/components/compose_form.js @@ -197,7 +197,10 @@ class ComposeForm extends ImmutablePureComponent { handleFocus = () => { if (this.composeForm && !this.props.singleColumn) { - this.composeForm.scrollIntoView(); + const { left, right } = this.composeForm.getBoundingClientRect(); + if (left < 0 || right > (window.innerWidth || document.documentElement.clientWidth)) { + this.composeForm.scrollIntoView(); + } } } -- cgit From 5ff9970c8d4acfb69b4696a3b5ce8e2a9580e370 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Sun, 7 Jul 2019 14:12:41 +0200 Subject: Re-add route for direct timeline --- config/routes.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/routes.rb b/config/routes.rb index 62374d0df..1b88fe5e3 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -303,6 +303,7 @@ Rails.application.routes.draw do end namespace :timelines do + resource :direct, only: :show, controller: :direct resource :home, only: :show, controller: :home resource :public, only: :show, controller: :public resources :tag, only: :show -- cgit