Add customizable thumbnails for audio and video attachments (#14145)

- Change audio files to not be stripped of metadata
- Automatically extract cover art from audio if it exists
- Add `thumbnail` parameter to `POST /api/v1/media`, `POST /api/v2/media` and `PUT /api/v1/media/:id`
- Add `icon` to represent it in attachments in ActivityPub
- Fix `preview_url` containing URL of missing missing image when there is no thumbnail instead of null
- Fix duration of audio not being displayed on public pages until the file is loaded

3 files changed, 33 insertions, 15 deletions

diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js
index 2dc961936..827b69500 100644
--- a/app/javascript/mastodon/components/status.js
+++ b/app/javascript/mastodon/components/status.js
@@ -352,7 +352,8 @@ class Status extends ImmutablePureComponent {
               <Component
                 src={attachment.get('url')}
                 alt={attachment.get('description')}
-                poster={status.getIn(['account', 'avatar_static'])}
+                poster={attachment.get('preview_url') || status.getIn(['account', 'avatar_static'])}
+                blurhash={attachment.get('blurhash')}
                 duration={attachment.getIn(['meta', 'original', 'duration'], 0)}
                 width={this.props.cachedMediaWidth}
                 height={110}
diff --git a/app/javascript/mastodon/features/audio/index.js b/app/javascript/mastodon/features/audio/index.js
index 5f6132f12..99926e52a 100644
--- a/app/javascript/mastodon/features/audio/index.js
+++ b/app/javascript/mastodon/features/audio/index.js
@@ -157,6 +157,7 @@ class Audio extends React.PureComponent {
     fullscreen: PropTypes.bool,
     intl: PropTypes.object.isRequired,
     cacheWidth: PropTypes.func,
+    blurhash: PropTypes.string,
   };
 
   state = {
@@ -222,32 +223,42 @@ class Audio extends React.PureComponent {
     window.addEventListener('scroll', this.handleScroll);
     window.addEventListener('resize', this.handleResize, { passive: true });
 
-    const img = new Image();
-    img.crossOrigin = 'anonymous';
-    img.onload = () => this.handlePosterLoad(img);
-    img.src = this.props.poster;
+    if (!this.props.blurhash) {
+      const img = new Image();
+      img.crossOrigin = 'anonymous';
+      img.onload = () => this.handlePosterLoad(img);
+      img.src = this.props.poster;
+    } else {
+      this._setColorScheme();
+      this._decodeBlurhash();
+    }
   }
 
   componentDidUpdate (prevProps, prevState) {
-    if (prevProps.poster !== this.props.poster) {
+    if (prevProps.poster !== this.props.poster && !this.props.blurhash) {
       const img = new Image();
       img.crossOrigin = 'anonymous';
       img.onload = () => this.handlePosterLoad(img);
       img.src = this.props.poster;
     }
 
-    if (prevState.blurhash !== this.state.blurhash) {
-      const context = this.blurhashCanvas.getContext('2d');
-      const pixels = decode(this.state.blurhash, 32, 32);
-      const outputImageData = new ImageData(pixels, 32, 32);
-
-      context.putImageData(outputImageData, 0, 0);
+    if (prevState.blurhash !== this.state.blurhash || prevProps.blurhash !== this.props.blurhash) {
+      this._setColorScheme();
+      this._decodeBlurhash();
     }
 
     this._clear();
     this._draw();
   }
 
+  _decodeBlurhash () {
+    const context = this.blurhashCanvas.getContext('2d');
+    const pixels = decode(this.props.blurhash || this.state.blurhash, 32, 32);
+    const outputImageData = new ImageData(pixels, 32, 32);
+
+    context.putImageData(outputImageData, 0, 0);
+  }
+
   componentWillUnmount () {
     window.removeEventListener('scroll', this.handleScroll);
     window.removeEventListener('resize', this.handleResize);
@@ -415,7 +426,7 @@ class Audio extends React.PureComponent {
   }
 
   handlePosterLoad = image => {
-    const canvas = document.createElement('canvas');
+    const canvas  = document.createElement('canvas');
     const context = canvas.getContext('2d');
 
     canvas.width  = image.width;
@@ -425,10 +436,15 @@ class Audio extends React.PureComponent {
 
     const inputImageData = context.getImageData(0, 0, image.width, image.height);
     const blurhash = encode(inputImageData.data, image.width, image.height, 4, 4);
+
+    this.setState({ blurhash });
+  }
+
+  _setColorScheme () {
+    const blurhash     = this.props.blurhash || this.state.blurhash;
     const averageColor = decodeRGB(decode83(blurhash.slice(2, 6)));
 
     this.setState({
-      blurhash,
       color: adjustColor(averageColor),
       darkText: luma(averageColor) >= 165,
     });
diff --git a/app/javascript/mastodon/features/status/components/detailed_status.js b/app/javascript/mastodon/features/status/components/detailed_status.js
index 935e4207e..f7d0c9bd4 100644
--- a/app/javascript/mastodon/features/status/components/detailed_status.js
+++ b/app/javascript/mastodon/features/status/components/detailed_status.js
@@ -125,7 +125,8 @@ class DetailedStatus extends ImmutablePureComponent {
             src={attachment.get('url')}
             alt={attachment.get('description')}
             duration={attachment.getIn(['meta', 'original', 'duration'], 0)}
-            poster={status.getIn(['account', 'avatar_static'])}
+            poster={attachment.get('preview_url') || status.getIn(['account', 'avatar_static'])}
+            blurhash={attachment.get('blurhash')}
             height={150}
           />
         );