From 1268277a8c203bcae515e0ccc8b3432119bafed2 Mon Sep 17 00:00:00 2001 From: multiple creatures Date: Fri, 10 Jan 2020 20:07:15 -0600 Subject: add custom filter master toggle, add media gallery mode, & fix various filter logic + caching bugs --- streaming/index.js | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) (limited to 'streaming/index.js') diff --git a/streaming/index.js b/streaming/index.js index f429bbcc1..015546658 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -202,7 +202,7 @@ const startWorker = (workerId) => { return; } - client.query('SELECT oauth_access_tokens.resource_owner_id, users.account_id, users.chosen_languages, users.hide_boosts, users.only_known, users.invert_filters, oauth_access_tokens.scopes FROM oauth_access_tokens INNER JOIN users ON oauth_access_tokens.resource_owner_id = users.id WHERE oauth_access_tokens.token = $1 AND oauth_access_tokens.revoked_at IS NULL LIMIT 1', [token], (err, result) => { + client.query('SELECT oauth_access_tokens.resource_owner_id, users.account_id, users.chosen_languages, users.filters_enabled, users.hide_boosts, users.only_known, users.invert_filters, users.media_only, users.filter_undescribed, oauth_access_tokens.scopes FROM oauth_access_tokens INNER JOIN users ON oauth_access_tokens.resource_owner_id = users.id WHERE oauth_access_tokens.token = $1 AND oauth_access_tokens.revoked_at IS NULL LIMIT 1', [token], (err, result) => { done(); if (err) { @@ -230,9 +230,12 @@ const startWorker = (workerId) => { req.accountId = result.rows[0].account_id; req.chosenLanguages = result.rows[0].chosen_languages; + req.filtersEnabled = result.rows[0].filters_enabled; req.hideBoosts = result.rows[0].hide_boosts; req.onlyKnown = result.rows[0].only_known; req.invertFilters = result.rows[0].invert_filters; + req.mediaOnly = result.rows[0].media_only; + req.filterUndescribed = result.rows[0].filter_undescribed; req.allowNotifications = scopes.some(scope => ['read', 'read:notifications'].includes(scope)); next(); @@ -387,35 +390,44 @@ const startWorker = (workerId) => { // Only messages that may require filtering are statuses, since notifications // are already personalized and deletes do not matter - if (!needsFiltering || event !== 'update') { + if (event !== 'update') { transmit(); return; } const unpackedPayload = payload; - const targetAccountIds = [unpackedPayload.account.id].concat(unpackedPayload.mentions.map(item => item.id)); - const accountDomain = unpackedPayload.account.acct.split('@')[1]; + + // Don't filter user's own events. + if (req.accountId === unpackedPayload.account.id) { + transmit(); + return; + } if (req.hideBoosts && (unpackedPayload.in_reply_to !== undefined || unpackedPayload.in_reply_to !== null)) { return; } + if (req.mediaOnly && (!unpackedPayload.media_attachments || unpackedPayload.media_attachments.length === 0)) { + return; + } + + if (req.filterUndescribed && unpackedPayload.media_attachments && unpackedPayload.media_attachments.every(m => !m.description || m.description.length === 0)) { + return; + } + if (Array.isArray(req.chosenLanguages) && unpackedPayload.language !== null && req.chosenLanguages.indexOf(unpackedPayload.language) === -1) { log.silly(req.requestId, `Message ${unpackedPayload.id} filtered by language (${unpackedPayload.language})`); return; } // When the account is not logged in, it is not necessary to confirm the block or mute - if (!req.accountId) { + if (!needsFiltering || !req.accountId) { transmit(); return; } - // Don't filter user's own events. - if (req.accountId === unpackedPayload.account.id) { - transmit(); - return - } + const targetAccountIds = [unpackedPayload.account.id].concat(unpackedPayload.mentions.map(item => item.id)); + const accountDomain = unpackedPayload.account.acct.split('@')[1]; pgPool.connect((err, client, done) => { if (err) { @@ -424,13 +436,17 @@ const startWorker = (workerId) => { } const queries = [ - client.query(`SELECT 1 FROM blocks WHERE (account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 3)})) OR (account_id = $2 AND target_account_id = $1) UNION SELECT 1 FROM mutes WHERE account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 3)}) UNION SELECT 1 FROM statuses WHERE id = $3 ${req.invertFilters ? 'AND NOT' : 'AND'} tsv @@ (SELECT tsquery_union(websearch_to_tsquery(phrase)) FROM custom_filters WHERE account_id = $1 AND is_enabled) UNION SELECT 1 FROM media_attachments WHERE (1 = (SELECT 1 FROM accounts WHERE id = $1 AND filter_undescribed)) AND status_id = $3 AND description IS NULL LIMIT 1`, [req.accountId, unpackedPayload.account.id, unpackedPayload.id].concat(targetAccountIds)), + client.query(`SELECT 1 FROM blocks WHERE (account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)})) OR (account_id = $2 AND target_account_id = $1) UNION SELECT 1 FROM mutes WHERE account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)})`, [req.accountId, unpackedPayload.account.id].concat(targetAccountIds)), ]; if (accountDomain) { queries.push(client.query('SELECT 1 FROM account_domain_blocks WHERE account_id = $1 AND domain = $2', [req.accountId, accountDomain])); } + if (req.filtersEnabled) { + queries.push(client.query(`SELECT 1 FROM statuses WHERE id = $2 ${req.invertFilters ? 'AND NOT' : 'AND'} tsv @@ (SELECT tsquery_union(websearch_to_tsquery('fedi', phrase)) FROM custom_filters WHERE account_id = $1 AND is_enabled) LIMIT 1`, [req.accountId, unpackedPayload.id])); + } + if (req.onlyKnown) { queries.push(client.query('SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM follows WHERE account_id = $1 AND target_account_id = $2)', [req.accountId, unpackedPayload.account.id])); } @@ -534,7 +550,7 @@ const startWorker = (workerId) => { app.get('/api/v1/streaming/user', (req, res) => { const channel = `timeline:${req.accountId}`; - streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel))); + streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel)), true); }); app.get('/api/v1/streaming/user/notification', (req, res) => { @@ -592,7 +608,7 @@ const startWorker = (workerId) => { } const channel = `timeline:list:${listId}`; - streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel))); + streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel)), true); }); }); -- cgit