Keyword/phrase filtering (#7905)

* Add keyword filtering

    GET|POST       /api/v1/filters
    GET|PUT|DELETE /api/v1/filters/:id

- Irreversible filters can drop toots from home or notifications
- Other filters can hide toots through the client app
- Filters use a phrase valid in particular contexts, expiration

* Make sure expired filters don't get applied client-side

* Add missing API methods

* Remove "regex filter" from column settings

* Add tests

* Add test for FeedManager

* Add CustomFilter test

* Add UI for managing filters

* Add streaming API event to allow syncing filters

* Fix tests

16 files changed, 124 insertions, 55 deletions

diff --git a/app/javascript/mastodon/actions/filters.js b/app/javascript/mastodon/actions/filters.js
new file mode 100644
index 000000000..7fa1c9a70
--- /dev/null
+++ b/app/javascript/mastodon/actions/filters.js
@@ -0,0 +1,26 @@
+import api from '../api';
+
+export const FILTERS_FETCH_REQUEST = 'FILTERS_FETCH_REQUEST';
+export const FILTERS_FETCH_SUCCESS = 'FILTERS_FETCH_SUCCESS';
+export const FILTERS_FETCH_FAIL    = 'FILTERS_FETCH_FAIL';
+
+export const fetchFilters = () => (dispatch, getState) => {
+  dispatch({
+    type: FILTERS_FETCH_REQUEST,
+    skipLoading: true,
+  });
+
+  api(getState)
+    .get('/api/v1/filters')
+    .then(({ data }) => dispatch({
+      type: FILTERS_FETCH_SUCCESS,
+      filters: data,
+      skipLoading: true,
+    }))
+    .catch(err => dispatch({
+      type: FILTERS_FETCH_FAIL,
+      err,
+      skipLoading: true,
+      skipAlert: true,
+    }));
+};
diff --git a/app/javascript/mastodon/actions/streaming.js b/app/javascript/mastodon/actions/streaming.js
index f56853bff..32fc67e67 100644
--- a/app/javascript/mastodon/actions/streaming.js
+++ b/app/javascript/mastodon/actions/streaming.js
@@ -6,6 +6,7 @@ import {
   disconnectTimeline,
 } from './timelines';
 import { updateNotifications, expandNotifications } from './notifications';
+import { fetchFilters } from './filters';
 import { getLocale } from '../locales';
 
 const { messages } = getLocale();
@@ -30,6 +31,9 @@ export function connectTimelineStream (timelineId, path, pollingRefresh = null)
         case 'notification':
           dispatch(updateNotifications(JSON.parse(data.payload), messages, locale));
           break;
+        case 'filters_changed':
+          dispatch(fetchFilters());
+          break;
         }
       },
     };
diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js
index fd08ff3b7..922b609ec 100644
--- a/app/javascript/mastodon/components/status.js
+++ b/app/javascript/mastodon/components/status.js
@@ -157,6 +157,21 @@ export default class Status extends ImmutablePureComponent {
       );
     }
 
+    if (status.get('filtered') || status.getIn(['reblog', 'filtered'])) {
+      const minHandlers = this.props.muted ? {} : {
+        moveUp: this.handleHotkeyMoveUp,
+        moveDown: this.handleHotkeyMoveDown,
+      };
+
+      return (
+        <HotKeys handlers={minHandlers}>
+          <div className='status__wrapper status__wrapper--filtered focusable' tabIndex='0'>
+            <FormattedMessage id='status.filtered' defaultMessage='Filtered' />
+          </div>
+        </HotKeys>
+      );
+    }
+
     if (featured) {
       prepend = (
         <div className='status__prepend'>
diff --git a/app/javascript/mastodon/components/status_list.js b/app/javascript/mastodon/components/status_list.js
index 1c34d0640..68c9eef54 100644
--- a/app/javascript/mastodon/components/status_list.js
+++ b/app/javascript/mastodon/components/status_list.js
@@ -25,6 +25,7 @@ export default class StatusList extends ImmutablePureComponent {
     prepend: PropTypes.node,
     emptyMessage: PropTypes.node,
     alwaysPrepend: PropTypes.bool,
+    timelineId: PropTypes.string.isRequired,
   };
 
   static defaultProps = {
@@ -70,7 +71,7 @@ export default class StatusList extends ImmutablePureComponent {
   }
 
   render () {
-    const { statusIds, featuredStatusIds, onLoadMore, ...other }  = this.props;
+    const { statusIds, featuredStatusIds, onLoadMore, timelineId, ...other }  = this.props;
     const { isLoading, isPartial } = other;
 
     if (isPartial) {
@@ -102,6 +103,7 @@ export default class StatusList extends ImmutablePureComponent {
           id={statusId}
           onMoveUp={this.handleMoveUp}
           onMoveDown={this.handleMoveDown}
+          contextType={timelineId}
         />
       ))
     ) : null;
@@ -114,6 +116,7 @@ export default class StatusList extends ImmutablePureComponent {
           featured
           onMoveUp={this.handleMoveUp}
           onMoveDown={this.handleMoveDown}
+          contextType={timelineId}
         />
       )).concat(scrollableContent);
     }
diff --git a/app/javascript/mastodon/containers/status_container.js b/app/javascript/mastodon/containers/status_container.js
index 3e7b5215b..eb6329fdc 100644
--- a/app/javascript/mastodon/containers/status_container.js
+++ b/app/javascript/mastodon/containers/status_container.js
@@ -42,7 +42,7 @@ const makeMapStateToProps = () => {
   const getStatus = makeGetStatus();
 
   const mapStateToProps = (state, props) => ({
-    status: getStatus(state, props.id),
+    status: getStatus(state, props),
   });
 
   return mapStateToProps;
diff --git a/app/javascript/mastodon/features/community_timeline/components/column_settings.js b/app/javascript/mastodon/features/community_timeline/components/column_settings.js
index 3a1d19aa8..f4325f58d 100644
--- a/app/javascript/mastodon/features/community_timeline/components/column_settings.js
+++ b/app/javascript/mastodon/features/community_timeline/components/column_settings.js
@@ -1,15 +1,9 @@
 import React from 'react';
 import PropTypes from 'prop-types';
 import ImmutablePropTypes from 'react-immutable-proptypes';
-import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
-import SettingText from '../../../components/setting_text';
+import { injectIntl, FormattedMessage } from 'react-intl';
 import SettingToggle from '../../notifications/components/setting_toggle';
 
-const messages = defineMessages({
-  filter_regex: { id: 'home.column_settings.filter_regex', defaultMessage: 'Filter out by regular expressions' },
-  settings: { id: 'home.settings', defaultMessage: 'Column settings' },
-});
-
 @injectIntl
 export default class ColumnSettings extends React.PureComponent {
 
@@ -21,19 +15,13 @@ export default class ColumnSettings extends React.PureComponent {
   };
 
   render () {
-    const { settings, onChange, intl } = this.props;
+    const { settings, onChange } = this.props;
 
     return (
       <div>
         <div className='column-settings__row'>
           <SettingToggle settings={settings} settingPath={['other', 'onlyMedia']} onChange={onChange} label={<FormattedMessage id='community.column_settings.media_only' defaultMessage='Media Only' />} />
         </div>
-
-        <span className='column-settings__section'><FormattedMessage id='home.column_settings.advanced' defaultMessage='Advanced' /></span>
-
-        <div className='column-settings__row'>
-          <SettingText settings={settings} settingKey={['regex', 'body']} onChange={onChange} label={intl.formatMessage(messages.filter_regex)} />
-        </div>
       </div>
     );
   }
diff --git a/app/javascript/mastodon/features/compose/containers/reply_indicator_container.js b/app/javascript/mastodon/features/compose/containers/reply_indicator_container.js
index 73f394c1a..5eb1eb72a 100644
--- a/app/javascript/mastodon/features/compose/containers/reply_indicator_container.js
+++ b/app/javascript/mastodon/features/compose/containers/reply_indicator_container.js
@@ -7,7 +7,7 @@ const makeMapStateToProps = () => {
   const getStatus = makeGetStatus();
 
   const mapStateToProps = state => ({
-    status: getStatus(state, state.getIn(['compose', 'in_reply_to'])),
+    status: getStatus(state, { id: state.getIn(['compose', 'in_reply_to']) }),
   });
 
   return mapStateToProps;
diff --git a/app/javascript/mastodon/features/direct_timeline/index.js b/app/javascript/mastodon/features/direct_timeline/index.js
index fda57f69a..63dc41d9e 100644
--- a/app/javascript/mastodon/features/direct_timeline/index.js
+++ b/app/javascript/mastodon/features/direct_timeline/index.js
@@ -7,7 +7,6 @@ import ColumnHeader from '../../components/column_header';
 import { expandDirectTimeline } from '../../actions/timelines';
 import { addColumn, removeColumn, moveColumn } from '../../actions/columns';
 import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
-import ColumnSettingsContainer from './containers/column_settings_container';
 import { connectDirectStream } from '../../actions/streaming';
 
 const messages = defineMessages({
@@ -86,9 +85,7 @@ export default class DirectTimeline extends React.PureComponent {
           onClick={this.handleHeaderClick}
           pinned={pinned}
           multiColumn={multiColumn}
-        >
-          <ColumnSettingsContainer />
-        </ColumnHeader>
+        />
 
         <StatusListContainer
           trackScroll={!pinned}
diff --git a/app/javascript/mastodon/features/home_timeline/components/column_settings.js b/app/javascript/mastodon/features/home_timeline/components/column_settings.js
index 0c0c4fa36..932ac2049 100644
--- a/app/javascript/mastodon/features/home_timeline/components/column_settings.js
+++ b/app/javascript/mastodon/features/home_timeline/components/column_settings.js
@@ -1,14 +1,8 @@
 import React from 'react';
 import PropTypes from 'prop-types';
 import ImmutablePropTypes from 'react-immutable-proptypes';
-import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
+import { injectIntl, FormattedMessage } from 'react-intl';
 import SettingToggle from '../../notifications/components/setting_toggle';
-import SettingText from '../../../components/setting_text';
-
-const messages = defineMessages({
-  filter_regex: { id: 'home.column_settings.filter_regex', defaultMessage: 'Filter out by regular expressions' },
-  settings: { id: 'home.settings', defaultMessage: 'Column settings' },
-});
 
 @injectIntl
 export default class ColumnSettings extends React.PureComponent {
@@ -20,7 +14,7 @@ export default class ColumnSettings extends React.PureComponent {
   };
 
   render () {
-    const { settings, onChange, intl } = this.props;
+    const { settings, onChange } = this.props;
 
     return (
       <div>
@@ -33,12 +27,6 @@ export default class ColumnSettings extends React.PureComponent {
         <div className='column-settings__row'>
           <SettingToggle prefix='home_timeline' settings={settings} settingPath={['shows', 'reply']} onChange={onChange} label={<FormattedMessage id='home.column_settings.show_replies' defaultMessage='Show replies' />} />
         </div>
-
-        <span className='column-settings__section'><FormattedMessage id='home.column_settings.advanced' defaultMessage='Advanced' /></span>
-
-        <div className='column-settings__row'>
-          <SettingText prefix='home_timeline' settings={settings} settingKey={['regex', 'body']} onChange={onChange} label={intl.formatMessage(messages.filter_regex)} />
-        </div>
       </div>
     );
   }
diff --git a/app/javascript/mastodon/features/status/index.js b/app/javascript/mastodon/features/status/index.js
index ca792043f..3c66536d4 100644
--- a/app/javascript/mastodon/features/status/index.js
+++ b/app/javascript/mastodon/features/status/index.js
@@ -58,7 +58,7 @@ const makeMapStateToProps = () => {
   const getStatus = makeGetStatus();
 
   const mapStateToProps = (state, props) => {
-    const status = getStatus(state, props.params.statusId);
+    const status = getStatus(state, { id: props.params.statusId });
     let ancestorsIds = Immutable.List();
     let descendantsIds = Immutable.List();
 
@@ -336,6 +336,7 @@ export default class Status extends ImmutablePureComponent {
         id={id}
         onMoveUp={this.handleMoveUp}
         onMoveDown={this.handleMoveDown}
+        contextType='thread'
       />
     ));
   }
diff --git a/app/javascript/mastodon/features/ui/containers/status_list_container.js b/app/javascript/mastodon/features/ui/containers/status_list_container.js
index e5b1edc4a..3df5b7bea 100644
--- a/app/javascript/mastodon/features/ui/containers/status_list_container.js
+++ b/app/javascript/mastodon/features/ui/containers/status_list_container.js
@@ -11,15 +11,6 @@ const makeGetStatusIds = () => createSelector([
   (state, { type }) => state.getIn(['timelines', type, 'items'], ImmutableList()),
   (state)           => state.get('statuses'),
 ], (columnSettings, statusIds, statuses) => {
-  const rawRegex = columnSettings.getIn(['regex', 'body'], '').trim();
-  let regex      = null;
-
-  try {
-    regex = rawRegex && new RegExp(rawRegex, 'i');
-  } catch (e) {
-    // Bad regex, don't affect filters
-  }
-
   return statusIds.filter(id => {
     if (id === null) return true;
 
@@ -34,11 +25,6 @@ const makeGetStatusIds = () => createSelector([
       showStatus = showStatus && (statusForId.get('in_reply_to_id') === null || statusForId.get('in_reply_to_account_id') === me);
     }
 
-    if (showStatus && regex && statusForId.get('account') !== me) {
-      const searchIndex = statusForId.get('reblog') ? statuses.getIn([statusForId.get('reblog'), 'search_index']) : statusForId.get('search_index');
-      showStatus = !regex.test(searchIndex);
-    }
-
     return showStatus;
   });
 });
diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js
index 3c1a266e3..56a856230 100644
--- a/app/javascript/mastodon/features/ui/index.js
+++ b/app/javascript/mastodon/features/ui/index.js
@@ -12,6 +12,7 @@ import { debounce } from 'lodash';
 import { uploadCompose, resetCompose } from '../../actions/compose';
 import { expandHomeTimeline } from '../../actions/timelines';
 import { expandNotifications } from '../../actions/notifications';
+import { fetchFilters } from '../../actions/filters';
 import { clearHeight } from '../../actions/height_cache';
 import { WrappedSwitch, WrappedRoute } from './util/react_router_helpers';
 import UploadArea from './components/upload_area';
@@ -297,6 +298,7 @@ export default class UI extends React.PureComponent {
 
     this.props.dispatch(expandHomeTimeline());
     this.props.dispatch(expandNotifications());
+    setTimeout(() => this.props.dispatch(fetchFilters()), 500);
   }
 
   componentDidMount () {
diff --git a/app/javascript/mastodon/reducers/filters.js b/app/javascript/mastodon/reducers/filters.js
new file mode 100644
index 000000000..33f0c6732
--- /dev/null
+++ b/app/javascript/mastodon/reducers/filters.js
@@ -0,0 +1,11 @@
+import { FILTERS_FETCH_SUCCESS } from '../actions/filters';
+import { List as ImmutableList, fromJS } from 'immutable';
+
+export default function filters(state = ImmutableList(), action) {
+  switch(action.type) {
+  case FILTERS_FETCH_SUCCESS:
+    return fromJS(action.filters);
+  default:
+    return state;
+  }
+};
diff --git a/app/javascript/mastodon/reducers/index.js b/app/javascript/mastodon/reducers/index.js
index 3d9a6a132..4a981fada 100644
--- a/app/javascript/mastodon/reducers/index.js
+++ b/app/javascript/mastodon/reducers/index.js
@@ -26,6 +26,7 @@ import height_cache from './height_cache';
 import custom_emojis from './custom_emojis';
 import lists from './lists';
 import listEditor from './list_editor';
+import filters from './filters';
 
 const reducers = {
   dropdown_menu,
@@ -55,6 +56,7 @@ const reducers = {
   custom_emojis,
   lists,
   listEditor,
+  filters,
 };
 
 export default combineReducers(reducers);
diff --git a/app/javascript/mastodon/selectors/index.js b/app/javascript/mastodon/selectors/index.js
index e47ec5183..56eca1f02 100644
--- a/app/javascript/mastodon/selectors/index.js
+++ b/app/javascript/mastodon/selectors/index.js
@@ -19,16 +19,44 @@ export const makeGetAccount = () => {
   });
 };
 
+const toServerSideType = columnType => {
+  switch (columnType) {
+  case 'home':
+  case 'notifications':
+  case 'public':
+  case 'thread':
+    return columnType;
+  default:
+    if (columnType.indexOf('list:') > -1) {
+      return 'home';
+    } else {
+      return 'public'; // community, account, hashtag
+    }
+  }
+};
+
+const escapeRegExp = string =>
+  string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+
+const regexFromFilters = filters => {
+  if (filters.size === 0) {
+    return null;
+  }
+
+  return new RegExp(filters.map(filter => escapeRegExp(filter.get('phrase'))).join('|'), 'i');
+};
+
 export const makeGetStatus = () => {
   return createSelector(
     [
-      (state, id) => state.getIn(['statuses', id]),
-      (state, id) => state.getIn(['statuses', state.getIn(['statuses', id, 'reblog'])]),
-      (state, id) => state.getIn(['accounts', state.getIn(['statuses', id, 'account'])]),
-      (state, id) => state.getIn(['accounts', state.getIn(['statuses', state.getIn(['statuses', id, 'reblog']), 'account'])]),
+      (state, { id }) => state.getIn(['statuses', id]),
+      (state, { id }) => state.getIn(['statuses', state.getIn(['statuses', id, 'reblog'])]),
+      (state, { id }) => state.getIn(['accounts', state.getIn(['statuses', id, 'account'])]),
+      (state, { id }) => state.getIn(['accounts', state.getIn(['statuses', state.getIn(['statuses', id, 'reblog']), 'account'])]),
+      (state, { contextType }) => state.get('filters', ImmutableList()).filter(filter => contextType && filter.get('context').includes(toServerSideType(contextType)) && (filter.get('expires_at') === null || Date.parse(filter.get('expires_at')) > (new Date()))),
     ],
 
-    (statusBase, statusReblog, accountBase, accountReblog) => {
+    (statusBase, statusReblog, accountBase, accountReblog, filters) => {
       if (!statusBase) {
         return null;
       }
@@ -39,9 +67,13 @@ export const makeGetStatus = () => {
         statusReblog = null;
       }
 
+      const regex    = regexFromFilters(filters);
+      const filtered = regex && regex.test(statusBase.get('reblog') ? statusReblog.get('search_index') : statusBase.get('search_index'));
+
       return statusBase.withMutations(map => {
         map.set('reblog', statusReblog);
         map.set('account', accountBase);
+        map.set('filtered', filtered);
       });
     }
   );
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index c16cf3437..5fa73d58a 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -725,6 +725,20 @@
   vertical-align: middle;
 }
 
+.status__wrapper--filtered {
+  color: $dark-text-color;
+  border: 0;
+  font-size: inherit;
+  text-align: center;
+  line-height: inherit;
+  margin: 0;
+  padding: 15px;
+  box-sizing: border-box;
+  width: 100%;
+  clear: both;
+  border-bottom: 1px solid lighten($ui-base-color, 8%);
+}
+
 .status__prepend-icon-wrapper {
   left: -26px;
   position: absolute;