From 97192d9a77c0b4b68afe50d6a94d87110a8adbcd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 22 Aug 2019 04:17:12 +0200 Subject: Fix remote and staff-removed statuses leaving media behind for a day (#11638) The reason for unattaching media instead of removing it is to support delete & redraft functionality, but remote or staff-removed statuses will never be redrafted, so the media should be deleted immediately --- app/lib/activitypub/activity/delete.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 1f2b40c15..345060462 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -70,7 +70,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity end def delete_now! - RemoveStatusService.new.call(@status) + RemoveStatusService.new.call(@status, redraft: false) end def payload -- cgit From cb447b28c403c7db32e3e3d7c2510004287edfda Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 30 Aug 2019 00:14:36 +0200 Subject: Add profile directory to web UI (#11688) * Add profile directory to web UI * Add a line of bio to the directory --- app/controllers/api/v1/directories_controller.rb | 30 +++ app/controllers/directories_controller.rb | 8 +- app/javascript/mastodon/actions/directory.js | 61 +++++ app/javascript/mastodon/components/radio_button.js | 35 +++ .../features/directory/components/account_card.js | 149 +++++++++++ .../mastodon/features/directory/index.js | 171 +++++++++++++ .../mastodon/features/getting_started/index.js | 4 +- .../containers/column_settings_container.js | 2 +- .../features/ui/components/columns_area.js | 14 +- .../features/ui/components/navigation_panel.js | 2 +- app/javascript/mastodon/features/ui/index.js | 2 + .../mastodon/features/ui/util/async-components.js | 4 + app/javascript/mastodon/reducers/user_lists.js | 18 ++ app/javascript/styles/mastodon/components.scss | 284 +++++++++++++++------ app/lib/activitypub/adapter.rb | 1 + app/models/account.rb | 7 +- app/serializers/activitypub/actor_serializer.rb | 6 +- app/serializers/rest/account_serializer.rb | 2 +- .../activitypub/process_account_service.rb | 1 + app/views/application/_card.html.haml | 2 +- app/views/directories/index.html.haml | 60 +---- app/views/settings/profiles/show.html.haml | 2 +- config/locales/en.yml | 6 - config/locales/simple_form.en.yml | 2 +- config/routes.rb | 1 + 25 files changed, 715 insertions(+), 159 deletions(-) create mode 100644 app/controllers/api/v1/directories_controller.rb create mode 100644 app/javascript/mastodon/actions/directory.js create mode 100644 app/javascript/mastodon/components/radio_button.js create mode 100644 app/javascript/mastodon/features/directory/components/account_card.js create mode 100644 app/javascript/mastodon/features/directory/index.js (limited to 'app/lib') diff --git a/app/controllers/api/v1/directories_controller.rb b/app/controllers/api/v1/directories_controller.rb new file mode 100644 index 000000000..c91543e3a --- /dev/null +++ b/app/controllers/api/v1/directories_controller.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class Api::V1::DirectoriesController < Api::BaseController + before_action :require_enabled! + before_action :set_accounts + + def show + render json: @accounts, each_serializer: REST::AccountSerializer + end + + private + + def require_enabled! + return not_found unless Setting.profile_directory + end + + def set_accounts + @accounts = accounts_scope.offset(params[:offset]).limit(limit_param(DEFAULT_ACCOUNTS_LIMIT)) + end + + def accounts_scope + Account.discoverable.tap do |scope| + scope.merge!(Account.local) if truthy_param?(:local) + scope.merge!(Account.by_recent_status) if params[:order].blank? || params[:order] == 'active' + scope.merge!(Account.order(id: :desc)) if params[:order] == 'new' + scope.merge!(Account.not_excluded_by_account(current_account)) if current_account + scope.merge!(Account.not_domain_blocked_by_account(current_account)) if current_account && !truthy_param?(:local) + end + end +end diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb index a5c47b515..7244f02f0 100644 --- a/app/controllers/directories_controller.rb +++ b/app/controllers/directories_controller.rb @@ -7,7 +7,6 @@ class DirectoriesController < ApplicationController before_action :require_enabled! before_action :set_instance_presenter before_action :set_tag, only: :show - before_action :set_tags before_action :set_accounts def index @@ -28,13 +27,10 @@ class DirectoriesController < ApplicationController @tag = Tag.discoverable.find_normalized!(params[:id]) end - def set_tags - @tags = Tag.discoverable.limit(30).reject { |tag| tag.cached_sample_accounts.empty? } - end - def set_accounts - @accounts = Account.discoverable.by_recent_status.page(params[:page]).per(40).tap do |query| + @accounts = Account.local.discoverable.by_recent_status.page(params[:page]).per(15).tap do |query| query.merge!(Account.tagged_with(@tag.id)) if @tag + query.merge!(Account.not_excluded_by_account(current_account)) if current_account end end diff --git a/app/javascript/mastodon/actions/directory.js b/app/javascript/mastodon/actions/directory.js new file mode 100644 index 000000000..4b2b6dd56 --- /dev/null +++ b/app/javascript/mastodon/actions/directory.js @@ -0,0 +1,61 @@ +import api from '../api'; +import { importFetchedAccounts } from './importer'; +import { fetchRelationships } from './accounts'; + +export const DIRECTORY_FETCH_REQUEST = 'DIRECTORY_FETCH_REQUEST'; +export const DIRECTORY_FETCH_SUCCESS = 'DIRECTORY_FETCH_SUCCESS'; +export const DIRECTORY_FETCH_FAIL = 'DIRECTORY_FETCH_FAIL'; + +export const DIRECTORY_EXPAND_REQUEST = 'DIRECTORY_EXPAND_REQUEST'; +export const DIRECTORY_EXPAND_SUCCESS = 'DIRECTORY_EXPAND_SUCCESS'; +export const DIRECTORY_EXPAND_FAIL = 'DIRECTORY_EXPAND_FAIL'; + +export const fetchDirectory = params => (dispatch, getState) => { + dispatch(fetchDirectoryRequest()); + + api(getState).get('/api/v1/directory', { params: { ...params, limit: 20 } }).then(({ data }) => { + dispatch(importFetchedAccounts(data)); + dispatch(fetchDirectorySuccess(data)); + dispatch(fetchRelationships(data.map(x => x.id))); + }).catch(error => dispatch(fetchDirectoryFail(error))); +}; + +export const fetchDirectoryRequest = () => ({ + type: DIRECTORY_FETCH_REQUEST, +}); + +export const fetchDirectorySuccess = accounts => ({ + type: DIRECTORY_FETCH_SUCCESS, + accounts, +}); + +export const fetchDirectoryFail = error => ({ + type: DIRECTORY_FETCH_FAIL, + error, +}); + +export const expandDirectory = params => (dispatch, getState) => { + dispatch(expandDirectoryRequest()); + + const loadedItems = getState().getIn(['user_lists', 'directory', 'items']).size; + + api(getState).get('/api/v1/directory', { params: { ...params, offset: loadedItems, limit: 20 } }).then(({ data }) => { + dispatch(importFetchedAccounts(data)); + dispatch(expandDirectorySuccess(data)); + dispatch(fetchRelationships(data.map(x => x.id))); + }).catch(error => dispatch(expandDirectoryFail(error))); +}; + +export const expandDirectoryRequest = () => ({ + type: DIRECTORY_EXPAND_REQUEST, +}); + +export const expandDirectorySuccess = accounts => ({ + type: DIRECTORY_EXPAND_SUCCESS, + accounts, +}); + +export const expandDirectoryFail = error => ({ + type: DIRECTORY_EXPAND_FAIL, + error, +}); diff --git a/app/javascript/mastodon/components/radio_button.js b/app/javascript/mastodon/components/radio_button.js new file mode 100644 index 000000000..0496fa286 --- /dev/null +++ b/app/javascript/mastodon/components/radio_button.js @@ -0,0 +1,35 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import classNames from 'classnames'; + +export default class RadioButton extends React.PureComponent { + + static propTypes = { + value: PropTypes.string.isRequired, + checked: PropTypes.bool, + name: PropTypes.string.isRequired, + onChange: PropTypes.func.isRequired, + label: PropTypes.node.isRequired, + }; + + render () { + const { name, value, checked, onChange, label } = this.props; + + return ( + + ); + } + +} diff --git a/app/javascript/mastodon/features/directory/components/account_card.js b/app/javascript/mastodon/features/directory/components/account_card.js new file mode 100644 index 000000000..a9c9976be --- /dev/null +++ b/app/javascript/mastodon/features/directory/components/account_card.js @@ -0,0 +1,149 @@ +import React from 'react'; +import ImmutablePureComponent from 'react-immutable-pure-component'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import PropTypes from 'prop-types'; +import { connect } from 'react-redux'; +import { makeGetAccount } from 'mastodon/selectors'; +import Avatar from 'mastodon/components/avatar'; +import DisplayName from 'mastodon/components/display_name'; +import Permalink from 'mastodon/components/permalink'; +import RelativeTimestamp from 'mastodon/components/relative_timestamp'; +import IconButton from 'mastodon/components/icon_button'; +import { FormattedMessage, injectIntl, defineMessages } from 'react-intl'; +import { autoPlayGif, me, unfollowModal } from 'mastodon/initial_state'; +import { shortNumberFormat } from 'mastodon/utils/numbers'; +import { followAccount, unfollowAccount, blockAccount, unblockAccount, unmuteAccount } from 'mastodon/actions/accounts'; +import { openModal } from 'mastodon/actions/modal'; +import { initMuteModal } from 'mastodon/actions/mutes'; + +const messages = defineMessages({ + follow: { id: 'account.follow', defaultMessage: 'Follow' }, + unfollow: { id: 'account.unfollow', defaultMessage: 'Unfollow' }, + requested: { id: 'account.requested', defaultMessage: 'Awaiting approval' }, + unblock: { id: 'account.unblock', defaultMessage: 'Unblock @{name}' }, + unmute: { id: 'account.unmute', defaultMessage: 'Unmute @{name}' }, +}); + +const makeMapStateToProps = () => { + const getAccount = makeGetAccount(); + + const mapStateToProps = (state, { id }) => ({ + account: getAccount(state, id), + }); + + return mapStateToProps; +}; + +const mapDispatchToProps = (dispatch, { intl }) => ({ + + onFollow (account) { + if (account.getIn(['relationship', 'following']) || account.getIn(['relationship', 'requested'])) { + if (unfollowModal) { + dispatch(openModal('CONFIRM', { + message: @{account.get('acct')} }} />, + confirm: intl.formatMessage(messages.unfollowConfirm), + onConfirm: () => dispatch(unfollowAccount(account.get('id'))), + })); + } else { + dispatch(unfollowAccount(account.get('id'))); + } + } else { + dispatch(followAccount(account.get('id'))); + } + }, + + onBlock (account) { + if (account.getIn(['relationship', 'blocking'])) { + dispatch(unblockAccount(account.get('id'))); + } else { + dispatch(blockAccount(account.get('id'))); + } + }, + + onMute (account) { + if (account.getIn(['relationship', 'muting'])) { + dispatch(unmuteAccount(account.get('id'))); + } else { + dispatch(initMuteModal(account)); + } + }, + +}); + +export default @injectIntl +@connect(makeMapStateToProps, mapDispatchToProps) +class AccountCard extends ImmutablePureComponent { + + static propTypes = { + account: ImmutablePropTypes.map.isRequired, + intl: PropTypes.object.isRequired, + onFollow: PropTypes.func.isRequired, + onBlock: PropTypes.func.isRequired, + onMute: PropTypes.func.isRequired, + }; + + handleFollow = () => { + this.props.onFollow(this.props.account); + } + + handleBlock = () => { + this.props.onBlock(this.props.account); + } + + handleMute = () => { + this.props.onMute(this.props.account); + } + + render () { + const { account, intl } = this.props; + + let buttons; + + if (account.get('id') !== me && account.get('relationship', null) !== null) { + const following = account.getIn(['relationship', 'following']); + const requested = account.getIn(['relationship', 'requested']); + const blocking = account.getIn(['relationship', 'blocking']); + const muting = account.getIn(['relationship', 'muting']); + + if (requested) { + buttons = ; + } else if (blocking) { + buttons = ; + } else if (muting) { + buttons = ; + } else if (!account.get('moved') || following) { + buttons = ; + } + } + + return ( +
+
+ +
+ +
+ + + + + +
+ {buttons} +
+
+ +
+ {account.get('note').length > 0 && account.get('note') !== '

' &&
} +
+ +
+
{shortNumberFormat(account.get('statuses_count'))}
+
{shortNumberFormat(account.get('followers_count'))}
+
{account.get('last_status_at') === null ? : }
+
+
+ ); + } + +} diff --git a/app/javascript/mastodon/features/directory/index.js b/app/javascript/mastodon/features/directory/index.js new file mode 100644 index 000000000..2f91e759b --- /dev/null +++ b/app/javascript/mastodon/features/directory/index.js @@ -0,0 +1,171 @@ +import React from 'react'; +import { connect } from 'react-redux'; +import { defineMessages, injectIntl } from 'react-intl'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import Column from 'mastodon/components/column'; +import ColumnHeader from 'mastodon/components/column_header'; +import { addColumn, removeColumn, moveColumn, changeColumnParams } from 'mastodon/actions/columns'; +import { fetchDirectory, expandDirectory } from 'mastodon/actions/directory'; +import { List as ImmutableList } from 'immutable'; +import AccountCard from './components/account_card'; +import RadioButton from 'mastodon/components/radio_button'; +import classNames from 'classnames'; +import LoadMore from 'mastodon/components/load_more'; +import { ScrollContainer } from 'react-router-scroll-4'; + +const messages = defineMessages({ + title: { id: 'column.directory', defaultMessage: 'Browse profiles' }, + recentlyActive: { id: 'directory.recently_active', defaultMessage: 'Recently active' }, + newArrivals: { id: 'directory.new_arrivals', defaultMessage: 'New arrivals' }, + local: { id: 'directory.local', defaultMessage: 'From {domain} only' }, + federated: { id: 'directory.federated', defaultMessage: 'From known fediverse' }, +}); + +const mapStateToProps = state => ({ + accountIds: state.getIn(['user_lists', 'directory', 'items'], ImmutableList()), + isLoading: state.getIn(['user_lists', 'directory', 'isLoading'], true), + domain: state.getIn(['meta', 'domain']), +}); + +export default @connect(mapStateToProps) +@injectIntl +class Directory extends React.PureComponent { + + static contextTypes = { + router: PropTypes.object, + }; + + static propTypes = { + isLoading: PropTypes.bool, + accountIds: ImmutablePropTypes.list.isRequired, + dispatch: PropTypes.func.isRequired, + shouldUpdateScroll: PropTypes.func, + columnId: PropTypes.string, + intl: PropTypes.object.isRequired, + multiColumn: PropTypes.bool, + domain: PropTypes.string.isRequired, + params: PropTypes.shape({ + order: PropTypes.string, + local: PropTypes.bool, + }), + }; + + state = { + order: null, + local: null, + }; + + handlePin = () => { + const { columnId, dispatch } = this.props; + + if (columnId) { + dispatch(removeColumn(columnId)); + } else { + dispatch(addColumn('DIRECTORY', this.getParams(this.props, this.state))); + } + } + + getParams = (props, state) => ({ + order: state.order === null ? (props.params.order || 'active') : state.order, + local: state.local === null ? (props.params.local || false) : state.local, + }); + + handleMove = dir => { + const { columnId, dispatch } = this.props; + dispatch(moveColumn(columnId, dir)); + } + + handleHeaderClick = () => { + this.column.scrollTop(); + } + + componentDidMount () { + const { dispatch } = this.props; + dispatch(fetchDirectory(this.getParams(this.props, this.state))); + } + + componentDidUpdate (prevProps, prevState) { + const { dispatch } = this.props; + const paramsOld = this.getParams(prevProps, prevState); + const paramsNew = this.getParams(this.props, this.state); + + if (paramsOld.order !== paramsNew.order || paramsOld.local !== paramsNew.local) { + dispatch(fetchDirectory(paramsNew)); + } + } + + setRef = c => { + this.column = c; + } + + handleChangeOrder = e => { + const { dispatch, columnId } = this.props; + + if (columnId) { + dispatch(changeColumnParams(columnId, ['order'], e.target.value)); + } else { + this.setState({ order: e.target.value }); + } + } + + handleChangeLocal = e => { + const { dispatch, columnId } = this.props; + + if (columnId) { + dispatch(changeColumnParams(columnId, ['local'], e.target.value === '1')); + } else { + this.setState({ local: e.target.value === '1' }); + } + } + + handleLoadMore = () => { + const { dispatch } = this.props; + dispatch(expandDirectory(this.getParams(this.props, this.state))); + } + + render () { + const { isLoading, accountIds, intl, columnId, multiColumn, domain, shouldUpdateScroll } = this.props; + const { order, local } = this.getParams(this.props, this.state); + const pinned = !!columnId; + + const scrollableArea = ( +
+
+
+ + +
+ +
+ + +
+
+ +
+ {accountIds.map(accountId => )} +
+ + +
+ ); + + return ( + + + + {multiColumn && !pinned ? {scrollableArea} : scrollableArea} + + ); + } + +} diff --git a/app/javascript/mastodon/features/getting_started/index.js b/app/javascript/mastodon/features/getting_started/index.js index 6a122a750..f6d90580b 100644 --- a/app/javascript/mastodon/features/getting_started/index.js +++ b/app/javascript/mastodon/features/getting_started/index.js @@ -107,7 +107,7 @@ class GettingStarted extends ImmutablePureComponent { if (profile_directory) { navItems.push( - + ); height += 48; @@ -120,7 +120,7 @@ class GettingStarted extends ImmutablePureComponent { height += 34; } else if (profile_directory) { navItems.push( - + ); height += 48; diff --git a/app/javascript/mastodon/features/hashtag_timeline/containers/column_settings_container.js b/app/javascript/mastodon/features/hashtag_timeline/containers/column_settings_container.js index c5098052c..5914bbeaf 100644 --- a/app/javascript/mastodon/features/hashtag_timeline/containers/column_settings_container.js +++ b/app/javascript/mastodon/features/hashtag_timeline/containers/column_settings_container.js @@ -20,7 +20,7 @@ const mapDispatchToProps = (dispatch, { columnId }) => ({ }, onLoad (value) { - return api().get('/api/v2/search', { params: { q: value } }).then(response => { + return api().get('/api/v2/search', { params: { q: value, type: 'hashtags' } }).then(response => { return (response.data.hashtags || []).map((tag) => { return { value: tag.name, label: `#${tag.name}` }; }); diff --git a/app/javascript/mastodon/features/ui/components/columns_area.js b/app/javascript/mastodon/features/ui/components/columns_area.js index 042e44e43..8a4e89b3d 100644 --- a/app/javascript/mastodon/features/ui/components/columns_area.js +++ b/app/javascript/mastodon/features/ui/components/columns_area.js @@ -12,7 +12,18 @@ import BundleContainer from '../containers/bundle_container'; import ColumnLoading from './column_loading'; import DrawerLoading from './drawer_loading'; import BundleColumnError from './bundle_column_error'; -import { Compose, Notifications, HomeTimeline, CommunityTimeline, PublicTimeline, HashtagTimeline, DirectTimeline, FavouritedStatuses, ListTimeline } from '../../ui/util/async-components'; +import { + Compose, + Notifications, + HomeTimeline, + CommunityTimeline, + PublicTimeline, + HashtagTimeline, + DirectTimeline, + FavouritedStatuses, + ListTimeline, + Directory, +} from '../../ui/util/async-components'; import Icon from 'mastodon/components/icon'; import ComposePanel from './compose_panel'; import NavigationPanel from './navigation_panel'; @@ -30,6 +41,7 @@ const componentMap = { 'DIRECT': DirectTimeline, 'FAVOURITES': FavouritedStatuses, 'LIST': ListTimeline, + 'DIRECTORY': Directory, }; const messages = defineMessages({ diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.js b/app/javascript/mastodon/features/ui/components/navigation_panel.js index 64a40a9da..6f07778f2 100644 --- a/app/javascript/mastodon/features/ui/components/navigation_panel.js +++ b/app/javascript/mastodon/features/ui/components/navigation_panel.js @@ -18,6 +18,7 @@ const NavigationPanel = () => ( + {profile_directory && } @@ -25,7 +26,6 @@ const NavigationPanel = () => ( - {!!profile_directory && } {showTrends &&
} {showTrends && } diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js index 9d284c221..49c5c8d0e 100644 --- a/app/javascript/mastodon/features/ui/index.js +++ b/app/javascript/mastodon/features/ui/index.js @@ -47,6 +47,7 @@ import { PinnedStatuses, Lists, Search, + Directory, } from './util/async-components'; import { me, forceSingleColumn } from '../../initial_state'; import { previewState as previewMediaState } from './components/media_modal'; @@ -188,6 +189,7 @@ class SwitchingColumnsArea extends React.PureComponent { + diff --git a/app/javascript/mastodon/features/ui/util/async-components.js b/app/javascript/mastodon/features/ui/util/async-components.js index a9b95c7b8..0084c1510 100644 --- a/app/javascript/mastodon/features/ui/util/async-components.js +++ b/app/javascript/mastodon/features/ui/util/async-components.js @@ -141,3 +141,7 @@ export function Tesseract () { export function Audio () { return import(/* webpackChunkName: "features/audio" */'../../audio'); } + +export function Directory () { + return import(/* webpackChunkName: "features/directory" */'../../directory'); +} diff --git a/app/javascript/mastodon/reducers/user_lists.js b/app/javascript/mastodon/reducers/user_lists.js index 8db18c5dc..08e94022f 100644 --- a/app/javascript/mastodon/reducers/user_lists.js +++ b/app/javascript/mastodon/reducers/user_lists.js @@ -20,6 +20,14 @@ import { MUTES_FETCH_SUCCESS, MUTES_EXPAND_SUCCESS, } from '../actions/mutes'; +import { + DIRECTORY_FETCH_REQUEST, + DIRECTORY_FETCH_SUCCESS, + DIRECTORY_FETCH_FAIL, + DIRECTORY_EXPAND_REQUEST, + DIRECTORY_EXPAND_SUCCESS, + DIRECTORY_EXPAND_FAIL, +} from 'mastodon/actions/directory'; import { Map as ImmutableMap, List as ImmutableList } from 'immutable'; const initialState = ImmutableMap({ @@ -74,6 +82,16 @@ export default function userLists(state = initialState, action) { return state.setIn(['mutes', 'items'], ImmutableList(action.accounts.map(item => item.id))).setIn(['mutes', 'next'], action.next); case MUTES_EXPAND_SUCCESS: return state.updateIn(['mutes', 'items'], list => list.concat(action.accounts.map(item => item.id))).setIn(['mutes', 'next'], action.next); + case DIRECTORY_FETCH_SUCCESS: + return state.setIn(['directory', 'items'], ImmutableList(action.accounts.map(item => item.id))).setIn(['directory', 'isLoading'], false); + case DIRECTORY_EXPAND_SUCCESS: + return state.updateIn(['directory', 'items'], list => list.concat(action.accounts.map(item => item.id))).setIn(['directory', 'isLoading'], false); + case DIRECTORY_FETCH_REQUEST: + case DIRECTORY_EXPAND_REQUEST: + return state.setIn(['directory', 'isLoading'], true); + case DIRECTORY_FETCH_FAIL: + case DIRECTORY_EXPAND_FAIL: + return state.setIn(['directory', 'isLoading'], false); default: return state; } diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 36466d5c1..1129680f1 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -2092,13 +2092,23 @@ a.account__display-name { padding: 0; } - //.column { - // margin-top: 0; + .directory__list { + display: grid; + grid-gap: 10px; + grid-template-columns: minmax(0, 50%) minmax(0, 50%); - // @media screen and (min-width: $no-gap-breakpoint) { - // margin-top: 10px; - // } - //} + @media screen and (max-width: $no-gap-breakpoint) { + display: block; + } + } + + .directory__card { + margin-bottom: 0; + } + + .filter-form { + display: flex; + } .autosuggest-textarea__textarea { font-size: 16px; @@ -4982,59 +4992,6 @@ a.status-card.compact:hover { } /* End Media Gallery */ -/* Status Video Player */ -.status__video-player { - background: $base-overlay-background; - box-sizing: border-box; - cursor: default; /* May not be needed */ - margin-top: 8px; - overflow: hidden; - position: relative; -} - -.status__video-player-video { - height: 100%; - object-fit: cover; - position: relative; - top: 50%; - transform: translateY(-50%); - width: 100%; - z-index: 1; -} - -.status__video-player-expand, -.status__video-player-mute { - color: $primary-text-color; - opacity: 0.8; - position: absolute; - right: 4px; - text-shadow: 0 1px 1px $base-shadow-color, 1px 0 1px $base-shadow-color; -} - -.status__video-player-spoiler { - display: none; - color: $primary-text-color; - left: 4px; - position: absolute; - text-shadow: 0 1px 1px $base-shadow-color, 1px 0 1px $base-shadow-color; - top: 4px; - z-index: 100; - - &.status__video-player-spoiler--visible { - display: block; - } -} - -.status__video-player-expand { - bottom: 4px; - z-index: 100; -} - -.status__video-player-mute { - top: 4px; - z-index: 5; -} - .detailed, .fullscreen { .video-player__volume__current, @@ -5387,28 +5344,130 @@ a.status-card.compact:hover { } } -.media-spoiler-video { - background-size: cover; - background-repeat: no-repeat; - background-position: center; - cursor: pointer; - margin-top: 8px; - position: relative; - border: 0; - display: block; -} +.directory { + &__list { + width: 100%; + margin: 10px 0; + transition: opacity 100ms ease-in; -.media-spoiler-video-play-icon { - border-radius: 100px; - color: rgba($primary-text-color, 0.8); - font-size: 36px; - left: 50%; - padding: 5px; - position: absolute; - top: 50%; - transform: translate(-50%, -50%); + &.loading { + opacity: 0.7; + } + + @media screen and (max-width: $no-gap-breakpoint) { + margin: 0; + } + } + + &__card { + box-sizing: border-box; + margin-bottom: 10px; + + &__img { + height: 125px; + position: relative; + background: darken($ui-base-color, 12%); + + img { + display: block; + width: 100%; + height: 100%; + margin: 0; + object-fit: cover; + } + } + + &__bar { + display: flex; + align-items: center; + background: lighten($ui-base-color, 4%); + padding: 10px; + + &__name { + flex: 1 1 auto; + display: flex; + align-items: center; + text-decoration: none; + } + + &__relationship { + width: 23px; + min-height: 1px; + flex: 0 0 auto; + } + + .avatar { + flex: 0 0 auto; + width: 48px; + height: 48px; + padding-top: 2px; + + img { + width: 100%; + height: 100%; + display: block; + margin: 0; + border-radius: 4px; + background: darken($ui-base-color, 8%); + object-fit: cover; + } + } + + .display-name { + margin-left: 15px; + text-align: left; + + strong { + font-size: 15px; + color: $primary-text-color; + font-weight: 500; + overflow: hidden; + text-overflow: ellipsis; + } + + span { + display: block; + font-size: 14px; + color: $darker-text-color; + font-weight: 400; + overflow: hidden; + text-overflow: ellipsis; + } + } + } + + &__extra { + background: $ui-base-color; + display: flex; + align-items: center; + justify-content: center; + + .accounts-table__count { + width: 33.33%; + flex: 0 0 auto; + padding: 15px 0; + } + + .account__header__content { + box-sizing: border-box; + padding: 15px 10px; + border-bottom: 1px solid lighten($ui-base-color, 8%); + width: 100%; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + + p { + display: none; + + &:first-child { + display: inline; + } + } + } + } + } } -/* End Video Player */ .account-gallery__container { display: flex; @@ -5484,6 +5543,73 @@ a.status-card.compact:hover { } } } + + &.directory__section-headline { + background: darken($ui-base-color, 2%); + border-bottom-color: transparent; + + a, + button { + &.active { + &::before { + display: none; + } + + &::after { + border-color: transparent transparent darken($ui-base-color, 7%); + } + } + } + } +} + +.filter-form { + background: $ui-base-color; + + &__column { + padding: 10px 15px; + } + + .radio-button { + display: block; + } +} + +.radio-button { + font-size: 14px; + position: relative; + display: inline-block; + padding: 6px 0; + line-height: 18px; + cursor: default; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + cursor: pointer; + + input[type=radio], + input[type=checkbox] { + display: none; + } + + &__input { + display: inline-block; + position: relative; + border: 1px solid $ui-primary-color; + box-sizing: border-box; + width: 18px; + height: 18px; + flex: 0 0 auto; + margin-right: 10px; + top: -1px; + border-radius: 50%; + vertical-align: middle; + + &.checked { + border-color: lighten($ui-highlight-color, 8%); + background: lighten($ui-highlight-color, 8%); + } + } } ::-webkit-scrollbar-thumb { diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index a1d84de2f..1c58be8c0 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -20,6 +20,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } }, identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' }, blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' }, + discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' }, }.freeze def self.default_key_transform diff --git a/app/models/account.rb b/app/models/account.rb index 392cc625f..8c9388b95 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -51,7 +51,6 @@ class Account < ApplicationRecord USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i - MIN_FOLLOWERS_DISCOVERY = 10 include AccountAssociations include AccountAvatar @@ -100,11 +99,13 @@ class Account < ApplicationRecord scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) } scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) } - scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).joins(:account_stat).where(AccountStat.arel_table[:followers_count].gteq(MIN_FOLLOWERS_DISCOVERY)) } + scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) } scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) } - scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc')) } + scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) } scope :popular, -> { order('account_stats.followers_count desc') } scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) } + scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) } + scope :not_domain_blocked_by_account, ->(account) { where(arel_table[:domain].eq(nil).or(arel_table[:domain].not_in(account.excluded_from_timeline_domains))) } delegate :email, :unconfirmed_email, diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb index 0bd7aed2e..222e17c99 100644 --- a/app/serializers/activitypub/actor_serializer.rb +++ b/app/serializers/activitypub/actor_serializer.rb @@ -6,12 +6,14 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer context :security context_extensions :manually_approves_followers, :featured, :also_known_as, - :moved_to, :property_value, :hashtag, :emoji, :identity_proof + :moved_to, :property_value, :hashtag, :emoji, :identity_proof, + :discoverable attributes :id, :type, :following, :followers, :inbox, :outbox, :featured, :preferred_username, :name, :summary, - :url, :manually_approves_followers + :url, :manually_approves_followers, + :discoverable has_one :public_key, serializer: ActivityPub::PublicKeySerializer diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb index 272e3eb9c..75b6cf13b 100644 --- a/app/serializers/rest/account_serializer.rb +++ b/app/serializers/rest/account_serializer.rb @@ -5,7 +5,7 @@ class REST::AccountSerializer < ActiveModel::Serializer attributes :id, :username, :acct, :display_name, :locked, :bot, :created_at, :note, :url, :avatar, :avatar_static, :header, :header_static, - :followers_count, :following_count, :statuses_count + :followers_count, :following_count, :statuses_count, :last_status_at has_one :moved_to_account, key: :moved, serializer: REST::AccountSerializer, if: :moved_and_not_nested? has_many :emojis, serializer: REST::CustomEmojiSerializer diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 603e27ed9..cef658e19 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -83,6 +83,7 @@ class ActivityPub::ProcessAccountService < BaseService @account.fields = property_values || {} @account.also_known_as = as_array(@json['alsoKnownAs'] || []).map { |item| value_or_id(item) } @account.actor_type = actor_type + @account.discoverable = @json['discoverable'] || false end def set_fetchable_attributes! diff --git a/app/views/application/_card.html.haml b/app/views/application/_card.html.haml index 00254c40c..8719ce484 100644 --- a/app/views/application/_card.html.haml +++ b/app/views/application/_card.html.haml @@ -9,7 +9,7 @@ = image_tag account.avatar.url, alt: '', width: 48, height: 48, class: 'u-photo' .display-name - %span{id: "default_account_display_name", style: "display:none;"}= account.username + %span{ id: "default_account_display_name", style: "display: none" }= account.username %bdi %strong.emojify.p-name= display_name(account, custom_emojify: true) %span diff --git a/app/views/directories/index.html.haml b/app/views/directories/index.html.haml index a8aa68cc4..54b27114c 100644 --- a/app/views/directories/index.html.haml +++ b/app/views/directories/index.html.haml @@ -14,58 +14,10 @@ %h1= t('directories.explore_mastodon', title: site_title) %p= t('directories.explanation') -.grid - .column-0 - - if @accounts.empty? - = nothing_here - - else - .directory - %table.accounts-table - %tbody - - @accounts.each do |account| - %tr - %td= account_link_to account - %td.accounts-table__count.optional - = number_to_human account.statuses_count, strip_insignificant_zeros: true - %small= t('accounts.posts', count: account.statuses_count).downcase - %td.accounts-table__count.optional - = number_to_human account.followers_count, strip_insignificant_zeros: true - %small= t('accounts.followers', count: account.followers_count).downcase - %td.accounts-table__count - - if account.last_status_at.present? - %time.time-ago{ datetime: account.last_status_at.iso8601, title: l(account.last_status_at) }= l account.last_status_at - - else - \- - %small= t('accounts.last_active') +- if @accounts.empty? + = nothing_here +- else + .card-grid + = render partial: 'application/card', collection: @accounts, as: :account - = paginate @accounts - - .column-1 - - if user_signed_in? - .box-widget.notice-widget - - if current_account.discoverable? - - if current_account.followers_count < Account::MIN_FOLLOWERS_DISCOVERY - %p= t('directories.enabled_but_waiting', min_followers: Account::MIN_FOLLOWERS_DISCOVERY) - - else - %p= t('directories.enabled') - - else - %p= t('directories.how_to_enable') - - = link_to settings_profile_path do - = t('settings.edit_profile') - = fa_icon 'chevron-right fw' - - - if @tags.empty? && !user_signed_in? - .nothing-here - - else - - @tags.each do |tag| - .directory__tag{ class: tag.id == @tag&.id ? 'active' : nil } - = link_to explore_hashtag_path(tag) do - %h4 - = fa_icon 'hashtag' - = tag.name - %small= t('directories.people', count: tag.accounts_count) - - .avatar-stack - - tag.cached_sample_accounts.each do |account| - = image_tag current_account&.user&.setting_auto_play_gif ? account.avatar_original_url : account.avatar_static_url, width: 48, height: 48, alt: '', class: 'account__avatar' + = paginate @accounts diff --git a/app/views/settings/profiles/show.html.haml b/app/views/settings/profiles/show.html.haml index f8a8fddd3..f042011d6 100644 --- a/app/views/settings/profiles/show.html.haml +++ b/app/views/settings/profiles/show.html.haml @@ -28,7 +28,7 @@ - if Setting.profile_directory .fields-group - = f.input :discoverable, as: :boolean, wrapper: :with_label, hint: t('simple_form.hints.defaults.discoverable_html', min_followers: Account::MIN_FOLLOWERS_DISCOVERY, path: explore_path), recommended: true + = f.input :discoverable, as: :boolean, wrapper: :with_label, hint: t('simple_form.hints.defaults.discoverable'), recommended: true %hr.spacer/ diff --git a/config/locales/en.yml b/config/locales/en.yml index ee78e4720..2f601f274 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -630,14 +630,8 @@ en: warning_title: Disseminated content availability directories: directory: Profile directory - enabled: You are currently listed in the directory. - enabled_but_waiting: You have opted-in to be listed in the directory, but you do not have the minimum number of followers (%{min_followers}) to be listed yet. explanation: Discover users based on their interests explore_mastodon: Explore %{title} - how_to_enable: You are not currently opted-in to the directory. You can opt-in below. Use hashtags in your bio text to be listed under specific hashtags! - people: - one: "%{count} person" - other: "%{count} people" domain_blocks: blocked_domains: List of limited and blocked domains description: This is the list of servers that %{instance} limits or reject federation with. diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index 2d5ada0a4..2e5982de9 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -16,7 +16,7 @@ en: bot: This account mainly performs automated actions and might not be monitored context: One or multiple contexts where the filter should apply digest: Only sent after a long period of inactivity and only if you have received any personal messages in your absence - discoverable_html: The directory lets people find accounts based on interests and activity. Requires at least %{min_followers} followers + discoverable: The profile directory is another way by which your account can reach a wider audience email: You will be sent a confirmation e-mail fields: You can have up to 4 items displayed as a table on your profile header: PNG, GIF or JPG. At most %{size}. Will be downscaled to %{dimensions}px diff --git a/config/routes.rb b/config/routes.rb index 9ae24b0cd..92f272ff5 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -325,6 +325,7 @@ Rails.application.routes.draw do end resource :domain_blocks, only: [:show, :create, :destroy] + resource :directory, only: [:show] resources :follow_requests, only: [:index] do member do -- cgit From 70ddef2654a931827ce5e4323e3042365f6078f2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 2 Sep 2019 18:11:13 +0200 Subject: Change trending hashtags to not disappear instantly after midnight (#11712) --- app/controllers/admin/tags_controller.rb | 2 +- app/lib/feed_manager.rb | 2 +- app/models/tag.rb | 4 +- app/models/trending_tags.rb | 102 +++++++++++++++------ app/workers/scheduler/trending_tags_scheduler.rb | 11 +++ config/sidekiq.yml | 3 + db/migrate/20190901035623_add_max_score_to_tags.rb | 6 ++ .../20190901040524_remove_score_from_tags.rb | 12 +++ db/schema.rb | 6 +- spec/models/trending_tags_spec.rb | 68 ++++++++++++++ 10 files changed, 179 insertions(+), 37 deletions(-) create mode 100644 app/workers/scheduler/trending_tags_scheduler.rb create mode 100644 db/migrate/20190901035623_add_max_score_to_tags.rb create mode 100644 db/post_migrate/20190901040524_remove_score_from_tags.rb create mode 100644 spec/models/trending_tags_spec.rb (limited to 'app/lib') diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb index 25d9b7d3d..8bd4e5f8b 100644 --- a/app/controllers/admin/tags_controller.rb +++ b/app/controllers/admin/tags_controller.rb @@ -57,7 +57,7 @@ module Admin scope = scope.unreviewed if filter_params[:review] == 'unreviewed' scope = scope.reviewed.order(reviewed_at: :desc) if filter_params[:review] == 'reviewed' scope = scope.pending_review.order(requested_review_at: :desc) if filter_params[:review] == 'pending_review' - scope.order(score: :desc) + scope.order(max_score: :desc) end def filter_params diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index ca3d890a8..871ec5c19 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -63,7 +63,7 @@ class FeedManager reblog_key = key(type, account_id, 'reblogs') # Remove any items past the MAX_ITEMS'th entry in our feed - redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s) + redis.zremrangebyrank(timeline_key, 0, -(FeedManager::MAX_ITEMS + 1)) # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop # tracking anything after it for deduplication purposes. diff --git a/app/models/tag.rb b/app/models/tag.rb index 945e3a3c6..135e0a030 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -7,14 +7,14 @@ # name :string default(""), not null # created_at :datetime not null # updated_at :datetime not null -# score :integer # usable :boolean # trendable :boolean # listable :boolean # reviewed_at :datetime # requested_review_at :datetime # last_status_at :datetime -# last_trend_at :datetime +# max_score :float +# max_score_at :datetime # class Tag < ApplicationRecord diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb index e4ce988c1..e1b92b175 100644 --- a/app/models/trending_tags.rb +++ b/app/models/trending_tags.rb @@ -7,6 +7,8 @@ class TrendingTags THRESHOLD = 5 LIMIT = 10 REVIEW_THRESHOLD = 3 + MAX_SCORE_COOLDOWN = 3.days.freeze + MAX_SCORE_HALFLIFE = 6.hours.freeze class << self include Redisable @@ -16,14 +18,75 @@ class TrendingTags increment_historical_use!(tag.id, at_time) increment_unique_use!(tag.id, account.id, at_time) - increment_vote!(tag, at_time) + increment_use!(tag.id, at_time) tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago - tag.update(last_trend_at: Time.now.utc) if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago) + end + + def update!(at_time = Time.now.utc) + tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1) + tags = Tag.where(id: tag_ids.uniq) + + # First pass to calculate scores and update the set + + tags.each do |tag| + expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f + expected = 1.0 if expected.zero? + observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f + max_time = tag.max_score_at + max_score = tag.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) + + score = begin + if expected > observed || observed < THRESHOLD + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + tag.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) + + if decaying_score.zero? + redis.zrem(KEY, tag.id) + else + redis.zadd(KEY, decaying_score, tag.id) + end + end + + users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?) + + # Second pass to notify about previously unreviewed trends + + tags.each do |tag| + current_rank = redis.zrevrank(KEY, tag.id) + needs_review_notification = tag.requires_review? && !tag.requested_review? + rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD + + next unless !tag.trendable? && rank_passes_threshold && needs_review_notification + + tag.touch(:requested_review_at) + + users_for_review.each do |user| + AdminMailer.new_trending_tag(user.account, tag).deliver_later! + end + end + + # Trim older items + + redis.zremrangebyrank(KEY, 0, -(LIMIT + 1)) end def get(limit, filtered: true) - tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, LIMIT - 1).map(&:to_i) + tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i) tags = Tag.where(id: tag_ids) tags = tags.where(trendable: true) if filtered @@ -33,8 +96,8 @@ class TrendingTags end def trending?(tag) - rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id) - rank.present? && rank <= LIMIT + rank = redis.zrevrank(KEY, tag.id) + rank.present? && rank < LIMIT end private @@ -51,31 +114,10 @@ class TrendingTags redis.expire(key, EXPIRE_HISTORY_AFTER) end - def increment_vote!(tag, at_time) - key = "#{KEY}:#{at_time.beginning_of_day.to_i}" - expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f - expected = 1.0 if expected.zero? - observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f - - if expected > observed || observed < THRESHOLD - redis.zrem(key, tag.id) - else - score = ((observed - expected)**2) / expected - old_rank = redis.zrevrank(key, tag.id) - - redis.zadd(key, score, tag.id) - request_review!(tag) if (old_rank.nil? || old_rank > REVIEW_THRESHOLD) && redis.zrevrank(key, tag.id) <= REVIEW_THRESHOLD && !tag.trendable? && tag.requires_review? && !tag.requested_review? - end - - redis.expire(key, EXPIRE_TRENDS_AFTER) - end - - def request_review!(tag) - return unless Setting.trends - - tag.touch(:requested_review_at) - - User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? } + def increment_use!(tag_id, at_time) + key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}" + redis.sadd(key, tag_id) + redis.expire(key, EXPIRE_HISTORY_AFTER) end end end diff --git a/app/workers/scheduler/trending_tags_scheduler.rb b/app/workers/scheduler/trending_tags_scheduler.rb new file mode 100644 index 000000000..77f0d5747 --- /dev/null +++ b/app/workers/scheduler/trending_tags_scheduler.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class Scheduler::TrendingTagsScheduler + include Sidekiq::Worker + + sidekiq_options unique: :until_executed, retry: 0 + + def perform + TrendingTags.update! if Setting.trends + end +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 6ebe450b0..5de25de23 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -9,6 +9,9 @@ scheduled_statuses_scheduler: every: '5m' class: Scheduler::ScheduledStatusesScheduler + trending_tags_scheduler: + every: '5m' + class: Scheduler::TrendingTagsScheduler media_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::MediaCleanupScheduler diff --git a/db/migrate/20190901035623_add_max_score_to_tags.rb b/db/migrate/20190901035623_add_max_score_to_tags.rb new file mode 100644 index 000000000..f936e9871 --- /dev/null +++ b/db/migrate/20190901035623_add_max_score_to_tags.rb @@ -0,0 +1,6 @@ +class AddMaxScoreToTags < ActiveRecord::Migration[5.2] + def change + add_column :tags, :max_score, :float + add_column :tags, :max_score_at, :datetime + end +end diff --git a/db/post_migrate/20190901040524_remove_score_from_tags.rb b/db/post_migrate/20190901040524_remove_score_from_tags.rb new file mode 100644 index 000000000..a1112700b --- /dev/null +++ b/db/post_migrate/20190901040524_remove_score_from_tags.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class RemoveScoreFromTags < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + safety_assured do + remove_column :tags, :score, :int + remove_column :tags, :last_trend_at, :datetime + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 482bca367..5576f70bf 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_08_23_221802) do +ActiveRecord::Schema.define(version: 2019_09_01_040524) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -664,14 +664,14 @@ ActiveRecord::Schema.define(version: 2019_08_23_221802) do t.string "name", default: "", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "score" t.boolean "usable" t.boolean "trendable" t.boolean "listable" t.datetime "reviewed_at" t.datetime "requested_review_at" t.datetime "last_status_at" - t.datetime "last_trend_at" + t.float "max_score" + t.datetime "max_score_at" t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true end diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb new file mode 100644 index 000000000..b6122c994 --- /dev/null +++ b/spec/models/trending_tags_spec.rb @@ -0,0 +1,68 @@ +require 'rails_helper' + +RSpec.describe TrendingTags do + describe '.record_use!' do + pending + end + + describe '.update!' do + let!(:at_time) { Time.now.utc } + let!(:tag1) { Fabricate(:tag, name: 'Catstodon') } + let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon') } + let!(:tag3) { Fabricate(:tag, name: 'OCs') } + + before do + allow(Redis.current).to receive(:pfcount) do |key| + case key + when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" + 2 + when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts" + 16 + when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" + 0 + when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts" + 4 + when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" + 13 + end + end + + Redis.current.zadd('trending_tags', 0.9, tag3.id) + Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id]) + + tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours) + + described_class.update!(at_time) + end + + it 'calculates and re-calculates scores' do + expect(described_class.get(10, filtered: false)).to eq [tag1, tag3] + end + + it 'omits hashtags below threshold' do + expect(described_class.get(10, filtered: false)).to_not include(tag2) + end + + it 'decays scores' do + expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9 + end + end + + describe '.trending?' do + let(:tag) { Fabricate(:tag) } + + before do + 10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) } + end + + it 'returns true if the hashtag is within limit' do + Redis.current.zadd('trending_tags', 11, tag.id) + expect(described_class.trending?(tag)).to be true + end + + it 'returns false if the hashtag is outside the limit' do + Redis.current.zadd('trending_tags', 0, tag.id) + expect(described_class.trending?(tag)).to be false + end + end +end -- cgit From 692c5b439ae8659e459da692cf9e6b8e6f29d2a1 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 3 Sep 2019 22:52:32 +0200 Subject: Fix ActivityPub context not being dynamically computed (#11746) * Fix contexts not being dynamically included Fixes #11649 * Refactor Note context in serializer * Refactor Actor serializer --- app/lib/activitypub/adapter.rb | 13 +++++++------ app/lib/activitypub/serializer.rb | 8 ++++++++ app/serializers/activitypub/actor_serializer.rb | 4 +++- app/serializers/activitypub/note_serializer.rb | 7 +++++-- config/initializers/active_model_serializers.rb | 19 ------------------- spec/lib/activitypub/activity/update_spec.rb | 2 +- 6 files changed, 24 insertions(+), 29 deletions(-) (limited to 'app/lib') diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index 1c58be8c0..cb2ac72d4 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -32,22 +32,23 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base end def serializable_hash(options = nil) + named_contexts = {} + context_extensions = {} options = serialization_options(options) - serialized_hash = serializer.serializable_hash(options) + serialized_hash = serializer.serializable_hash(options.merge(named_contexts: named_contexts, context_extensions: context_extensions)) serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields] serialized_hash = self.class.transform_key_casing!(serialized_hash, instance_options) - { '@context' => serialized_context }.merge(serialized_hash) + { '@context' => serialized_context(named_contexts, context_extensions) }.merge(serialized_hash) end private - def serialized_context + def serialized_context(named_contexts_map, context_extensions_map) context_array = [] - serializer_options = serializer.send(:instance_options) || {} - named_contexts = [:activitystreams] + serializer._named_contexts.keys + serializer_options.fetch(:named_contexts, {}).keys - context_extensions = serializer._context_extensions.keys + serializer_options.fetch(:context_extensions, {}).keys + named_contexts = [:activitystreams] + named_contexts_map.keys + context_extensions = context_extensions_map.keys named_contexts.each do |key| context_array << NAMED_CONTEXT_MAP[key] diff --git a/app/lib/activitypub/serializer.rb b/app/lib/activitypub/serializer.rb index 07bd8c494..1fdc79310 100644 --- a/app/lib/activitypub/serializer.rb +++ b/app/lib/activitypub/serializer.rb @@ -27,4 +27,12 @@ class ActivityPub::Serializer < ActiveModel::Serializer _context_extensions[extension_name] = true end end + + def serializable_hash(adapter_options = nil, options = {}, adapter_instance = self.class.serialization_adapter_instance) + unless adapter_options&.fetch(:named_contexts, nil).nil? + adapter_options[:named_contexts].merge!(_named_contexts) + adapter_options[:context_extensions].merge!(_context_extensions) + end + super(adapter_options, options, adapter_instance) + end end diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb index 222e17c99..17df85de3 100644 --- a/app/serializers/activitypub/actor_serializer.rb +++ b/app/serializers/activitypub/actor_serializer.rb @@ -6,7 +6,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer context :security context_extensions :manually_approves_followers, :featured, :also_known_as, - :moved_to, :property_value, :hashtag, :emoji, :identity_proof, + :moved_to, :property_value, :identity_proof, :discoverable attributes :id, :type, :following, :followers, @@ -138,6 +138,8 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer end class TagSerializer < ActivityPub::Serializer + context_extensions :hashtag + include RoutingHelper attributes :type, :href, :name diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb index 7592e0b1a..364d3eda5 100644 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@ -1,8 +1,7 @@ # frozen_string_literal: true class ActivityPub::NoteSerializer < ActivityPub::Serializer - context_extensions :atom_uri, :conversation, :sensitive, - :hashtag, :emoji, :focal_point, :blurhash + context_extensions :atom_uri, :conversation, :sensitive attributes :id, :type, :summary, :in_reply_to, :published, :url, @@ -151,6 +150,8 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer end class MediaAttachmentSerializer < ActivityPub::Serializer + context_extensions :blurhash, :focal_point + include RoutingHelper attributes :type, :media_type, :url, :name, :blurhash @@ -198,6 +199,8 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer end class TagSerializer < ActivityPub::Serializer + context_extensions :hashtag + include RoutingHelper attributes :type, :href, :name diff --git a/config/initializers/active_model_serializers.rb b/config/initializers/active_model_serializers.rb index 329a5fb2c..0e69e1d96 100644 --- a/config/initializers/active_model_serializers.rb +++ b/config/initializers/active_model_serializers.rb @@ -3,22 +3,3 @@ ActiveModelSerializers.config.tap do |config| end ActiveSupport::Notifications.unsubscribe(ActiveModelSerializers::Logging::RENDER_EVENT) - -class ActiveModel::Serializer::Reflection - # We monkey-patch this method so that when we include associations in a serializer, - # the nested serializers can send information about used contexts upwards back to - # the root. We do this via instance_options because the nesting can be dynamic. - def build_association(parent_serializer, parent_serializer_options, include_slice = {}) - serializer = options[:serializer] - - parent_serializer_options.merge!(named_contexts: serializer._named_contexts, context_extensions: serializer._context_extensions) if serializer.respond_to?(:_named_contexts) - - association_options = { - parent_serializer: parent_serializer, - parent_serializer_options: parent_serializer_options, - include_slice: include_slice, - } - - ActiveModel::Serializer::Association.new(self, association_options) - end -end diff --git a/spec/lib/activitypub/activity/update_spec.rb b/spec/lib/activitypub/activity/update_spec.rb index fbfc585cf..42da29860 100644 --- a/spec/lib/activitypub/activity/update_spec.rb +++ b/spec/lib/activitypub/activity/update_spec.rb @@ -19,7 +19,7 @@ RSpec.describe ActivityPub::Activity::Update do end let(:actor_json) do - ActiveModelSerializers::SerializableResource.new(modified_sender, serializer: ActivityPub::ActorSerializer, key_transform: :camel_lower).as_json + ActiveModelSerializers::SerializableResource.new(modified_sender, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter).as_json end let(:json) do -- cgit From 1653b587778db0df201507115d19c0530096a5e3 Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 5 Sep 2019 05:32:53 +0200 Subject: Attempt to concurrently connect to remote IP addresses (#11757) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Attempt to concurrently connect to remote IP addresses * Reduce code length to please CodeClimate 🤷 --- app/lib/request.rb | 52 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 17 deletions(-) (limited to 'app/lib') diff --git a/app/lib/request.rb b/app/lib/request.rb index 9d874fe2c..42ccc6513 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -191,6 +191,9 @@ class Request end end + socks = [] + addr_by_socket = {} + addresses.each do |address| begin check_private_address(address) @@ -200,30 +203,45 @@ class Request sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1) - begin - sock.connect_nonblock(sockaddr) - rescue IO::WaitWritable - if IO.select(nil, [sock], nil, Request::TIMEOUT[:connect]) - begin - sock.connect_nonblock(sockaddr) - rescue Errno::EISCONN - # Yippee! - rescue - sock.close - raise - end - else - sock.close - raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds" - end - end + sock.connect_nonblock(sockaddr) + # If that hasn't raised an exception, we somehow managed to connect + # immediately, close pending sockets and return immediately + socks.each(&:close) return sock + rescue IO::WaitWritable + socks << sock + addr_by_socket[sock] = sockaddr rescue => e outer_e = e end end + until socks.empty? + _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect]) + + if available_socks.nil? + socks.each(&:close) + raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds" + end + + available_socks.each do |sock| + socks.delete(sock) + + begin + sock.connect_nonblock(addr_by_socket[sock]) + rescue Errno::EISCONN + rescue => e + sock.close + outer_e = e + next + end + + socks.each(&:close) + return sock + end + end + if outer_e raise outer_e else -- cgit