From 01584f03e8159cbb7b411f33bd4f66f7131ce993 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 3 Feb 2023 21:29:23 +0100 Subject: Add metrics endpoint to streaming API (#23388) --- streaming/index.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'streaming/index.js') diff --git a/streaming/index.js b/streaming/index.js index 0350c488d..e207411f2 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -850,6 +850,18 @@ const startWorker = async (workerId) => { res.end('OK'); }); + app.get('/metrics', (req, res) => server.getConnections((err, count) => { + res.writeHeader(200, { 'Content-Type': 'application/openmetrics-text; version=1.0.0; charset=utf-8' }); + res.write('# TYPE connected_clients gauge\n'); + res.write('# HELP connected_clients The number of clients connected to the streaming server\n'); + res.write(`connected_clients ${count}.0\n`); + res.write('# TYPE connected_channels gauge\n'); + res.write('# HELP connected_channels The number of Redis channels the streaming server is subscribed to\n'); + res.write(`connected_channels ${Object.keys(subs).length}.0\n`); + res.write('# EOF\n'); + res.end(); + })); + app.use(authenticationMiddleware); app.use(errorMiddleware); -- cgit From 4cb76475167750ef03b747aded962db7f1702f88 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 9 Feb 2023 11:20:59 +0100 Subject: Add database pool configuration over `DB_POOL` to streaming API (#23470) Add connection timeout of 15s --- streaming/index.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'streaming/index.js') diff --git a/streaming/index.js b/streaming/index.js index e207411f2..1fbbdea3b 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -127,7 +127,6 @@ const startWorker = async (workerId) => { database: process.env.DB_NAME || 'mastodon_development', host: process.env.DB_HOST || pg.defaults.host, port: process.env.DB_PORT || pg.defaults.port, - max: 10, }, production: { @@ -136,20 +135,19 @@ const startWorker = async (workerId) => { database: process.env.DB_NAME || 'mastodon_production', host: process.env.DB_HOST || 'localhost', port: process.env.DB_PORT || 5432, - max: 10, }, }; - if (!!process.env.DB_SSLMODE && process.env.DB_SSLMODE !== 'disable') { - pgConfigs.development.ssl = true; - pgConfigs.production.ssl = true; - } - const app = express(); app.set('trust proxy', process.env.TRUSTED_PROXY_IP ? process.env.TRUSTED_PROXY_IP.split(/(?:\s*,\s*|\s+)/) : 'loopback,uniquelocal'); - const pgPool = new pg.Pool(Object.assign(pgConfigs[env], dbUrlToConfig(process.env.DATABASE_URL))); + const pgPool = new pg.Pool(Object.assign(pgConfigs[env], dbUrlToConfig(process.env.DATABASE_URL), { + max: process.env.DB_POOL || 10, + connectionTimeoutMillis: 15000, + ssl: !!process.env.DB_SSLMODE && process.env.DB_SSLMODE !== 'disable', + })); + const server = http.createServer(app); const redisNamespace = process.env.REDIS_NAMESPACE || null; -- cgit From c5a4d8c82ddd5265abc830959a58cdaf1f061a43 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 9 Feb 2023 11:24:16 +0100 Subject: Add information about database pool to metrics endpoint in streaming API (#23469) --- streaming/index.js | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'streaming/index.js') diff --git a/streaming/index.js b/streaming/index.js index 1fbbdea3b..a8708ec78 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -856,6 +856,15 @@ const startWorker = async (workerId) => { res.write('# TYPE connected_channels gauge\n'); res.write('# HELP connected_channels The number of Redis channels the streaming server is subscribed to\n'); res.write(`connected_channels ${Object.keys(subs).length}.0\n`); + res.write('# TYPE pg.pool.total_connections gauge \n'); + res.write('# HELP pg.pool.total_connections The total number of clients existing within the pool\n'); + res.write(`pg.pool.total_connections ${pgPool.totalCount}.0\n`); + res.write('# TYPE pg.pool.idle_connections gauge \n'); + res.write('# HELP pg.pool.idle_connections The number of clients which are not checked out but are currently idle in the pool\n'); + res.write(`pg.pool.idle_connections ${pgPool.idleCount}.0\n`); + res.write('# TYPE pg.pool.waiting_queries gauge \n'); + res.write('# HELP pg.pool.waiting_queries The number of queued requests waiting on a client when all clients are checked out\n'); + res.write(`pg.pool.waiting_queries ${pgPool.waitingCount}.0\n`); res.write('# EOF\n'); res.end(); })); -- cgit From c1b823368c894c80b586d37c85012ee766a32c0e Mon Sep 17 00:00:00 2001 From: emilweth <7402764+emilweth@users.noreply.github.com> Date: Sat, 11 Feb 2023 02:15:07 +0100 Subject: dot is not allowed (#23519) --- streaming/index.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'streaming/index.js') diff --git a/streaming/index.js b/streaming/index.js index a8708ec78..1121c77bf 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -856,15 +856,15 @@ const startWorker = async (workerId) => { res.write('# TYPE connected_channels gauge\n'); res.write('# HELP connected_channels The number of Redis channels the streaming server is subscribed to\n'); res.write(`connected_channels ${Object.keys(subs).length}.0\n`); - res.write('# TYPE pg.pool.total_connections gauge \n'); - res.write('# HELP pg.pool.total_connections The total number of clients existing within the pool\n'); - res.write(`pg.pool.total_connections ${pgPool.totalCount}.0\n`); - res.write('# TYPE pg.pool.idle_connections gauge \n'); - res.write('# HELP pg.pool.idle_connections The number of clients which are not checked out but are currently idle in the pool\n'); - res.write(`pg.pool.idle_connections ${pgPool.idleCount}.0\n`); - res.write('# TYPE pg.pool.waiting_queries gauge \n'); - res.write('# HELP pg.pool.waiting_queries The number of queued requests waiting on a client when all clients are checked out\n'); - res.write(`pg.pool.waiting_queries ${pgPool.waitingCount}.0\n`); + res.write('# TYPE pg_pool_total_connections gauge \n'); + res.write('# HELP pg_pool_total_connections The total number of clients existing within the pool\n'); + res.write(`pg_pool_total_connections ${pgPool.totalCount}.0\n`); + res.write('# TYPE pg_pool_idle_connections gauge \n'); + res.write('# HELP pg_pool_idle_connections The number of clients which are not checked out but are currently idle in the pool\n'); + res.write(`pg_pool_idle_connections ${pgPool.idleCount}.0\n`); + res.write('# TYPE pg_pool_waiting_queries gauge \n'); + res.write('# HELP pg_pool_waiting_queries The number of queued requests waiting on a client when all clients are checked out\n'); + res.write(`pg_pool_waiting_queries ${pgPool.waitingCount}.0\n`); res.write('# EOF\n'); res.end(); })); -- cgit From 7c45145430d7f61c01de6a8c6a55b4c8588a67c4 Mon Sep 17 00:00:00 2001 From: emilweth <7402764+emilweth@users.noreply.github.com> Date: Sat, 11 Feb 2023 03:00:44 +0100 Subject: fix metrics format (#23520) --- streaming/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'streaming/index.js') diff --git a/streaming/index.js b/streaming/index.js index 1121c77bf..32e3babaa 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -856,13 +856,13 @@ const startWorker = async (workerId) => { res.write('# TYPE connected_channels gauge\n'); res.write('# HELP connected_channels The number of Redis channels the streaming server is subscribed to\n'); res.write(`connected_channels ${Object.keys(subs).length}.0\n`); - res.write('# TYPE pg_pool_total_connections gauge \n'); + res.write('# TYPE pg_pool_total_connections gauge\n'); res.write('# HELP pg_pool_total_connections The total number of clients existing within the pool\n'); res.write(`pg_pool_total_connections ${pgPool.totalCount}.0\n`); - res.write('# TYPE pg_pool_idle_connections gauge \n'); + res.write('# TYPE pg_pool_idle_connections gauge\n'); res.write('# HELP pg_pool_idle_connections The number of clients which are not checked out but are currently idle in the pool\n'); res.write(`pg_pool_idle_connections ${pgPool.idleCount}.0\n`); - res.write('# TYPE pg_pool_waiting_queries gauge \n'); + res.write('# TYPE pg_pool_waiting_queries gauge\n'); res.write('# HELP pg_pool_waiting_queries The number of queued requests waiting on a client when all clients are checked out\n'); res.write(`pg_pool_waiting_queries ${pgPool.waitingCount}.0\n`); res.write('# EOF\n'); -- cgit From 0872f3e3d743a16533ac4fad4cd83b103047808c Mon Sep 17 00:00:00 2001 From: Ramūns Usovs Date: Fri, 3 Mar 2023 21:01:18 +0100 Subject: Allow streaming to connect to postgress with self-signed certs (#21431) --- package.json | 1 + streaming/index.js | 38 +------------------------------------- yarn.lock | 5 +++++ 3 files changed, 7 insertions(+), 37 deletions(-) (limited to 'streaming/index.js') diff --git a/package.json b/package.json index 1ab297d04..15bf30925 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,7 @@ "object.values": "^1.1.6", "path-complete-extname": "^1.0.0", "pg": "^8.5.0", + "pg-connection-string": "^2.5.0", "postcss": "^8.4.21", "postcss-loader": "^3.0.0", "promise.prototype.finally": "^3.1.4", diff --git a/streaming/index.js b/streaming/index.js index 32e3babaa..ba7cfea19 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -7,6 +7,7 @@ const express = require('express'); const http = require('http'); const redis = require('redis'); const pg = require('pg'); +const dbUrlToConfig = require('pg-connection-string').parse; const log = require('npmlog'); const url = require('url'); const uuid = require('uuid'); @@ -23,43 +24,6 @@ dotenv.config({ log.level = process.env.LOG_LEVEL || 'verbose'; -/** - * @param {string} dbUrl - * @return {Object.} - */ -const dbUrlToConfig = (dbUrl) => { - if (!dbUrl) { - return {}; - } - - const params = url.parse(dbUrl, true); - const config = {}; - - if (params.auth) { - [config.user, config.password] = params.auth.split(':'); - } - - if (params.hostname) { - config.host = params.hostname; - } - - if (params.port) { - config.port = params.port; - } - - if (params.pathname) { - config.database = params.pathname.split('/')[1]; - } - - const ssl = params.query && params.query.ssl; - - if (ssl && ssl === 'true' || ssl === '1') { - config.ssl = true; - } - - return config; -}; - /** * @param {Object.} defaultConfig * @param {string} redisUrl diff --git a/yarn.lock b/yarn.lock index a856cf91d..68cef7965 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8322,6 +8322,11 @@ pg-connection-string@^2.4.0: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.4.0.tgz#c979922eb47832999a204da5dbe1ebf2341b6a10" integrity sha512-3iBXuv7XKvxeMrIgym7njT+HlZkwZqqGX4Bu9cci8xHZNT+Um1gWKqCsAzcC0d95rcKMU5WBg6YRUcHyV0HZKQ== +pg-connection-string@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" + integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== + pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" -- cgit From be1792e1d45fd32c037195d3d03f09fbd4dbf00e Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 5 Mar 2023 01:52:12 +0100 Subject: Fix streaming API not being usable without `DATABASE_URL` (#23960) --- streaming/index.js | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) (limited to 'streaming/index.js') diff --git a/streaming/index.js b/streaming/index.js index ba7cfea19..2cc4934ee 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -81,9 +81,10 @@ const startMaster = () => { log.warn(`Starting streaming API server master with ${numWorkers} workers`); }; -const startWorker = async (workerId) => { - log.warn(`Starting worker ${workerId}`); - +/** + * @return {Object.} + */ +const pgConfigFromEnv = () => { const pgConfigs = { development: { user: process.env.DB_USER || pg.defaults.user, @@ -102,16 +103,45 @@ const startWorker = async (workerId) => { }, }; - const app = express(); + let baseConfig; - app.set('trust proxy', process.env.TRUSTED_PROXY_IP ? process.env.TRUSTED_PROXY_IP.split(/(?:\s*,\s*|\s+)/) : 'loopback,uniquelocal'); + if (process.env.DATABASE_URL) { + baseConfig = dbUrlToConfig(process.env.DATABASE_URL); + } else { + baseConfig = pgConfigs[env]; + + if (process.env.DB_SSLMODE) { + switch(process.env.DB_SSLMODE) { + case 'disable': + case '': + baseConfig.ssl = false; + break; + case 'no-verify': + baseConfig.ssl = { rejectUnauthorized: false }; + break; + default: + baseConfig.ssl = {}; + break; + } + } + } - const pgPool = new pg.Pool(Object.assign(pgConfigs[env], dbUrlToConfig(process.env.DATABASE_URL), { + return { + ...baseConfig, max: process.env.DB_POOL || 10, connectionTimeoutMillis: 15000, - ssl: !!process.env.DB_SSLMODE && process.env.DB_SSLMODE !== 'disable', - })); + application_name: '', + }; +}; + +const startWorker = async (workerId) => { + log.warn(`Starting worker ${workerId}`); + + const app = express(); + + app.set('trust proxy', process.env.TRUSTED_PROXY_IP ? process.env.TRUSTED_PROXY_IP.split(/(?:\s*,\s*|\s+)/) : 'loopback,uniquelocal'); + const pgPool = new pg.Pool(pgConfigFromEnv()); const server = http.createServer(app); const redisNamespace = process.env.REDIS_NAMESPACE || null; -- cgit From 174352716ada7c8da451f1a2db57f18b66cd3fdb Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 6 Mar 2023 21:00:10 +0100 Subject: Disable anonymous access to the streaming API (#23989) --- streaming/index.js | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) (limited to 'streaming/index.js') diff --git a/streaming/index.js b/streaming/index.js index 2cc4934ee..91e86fdbc 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -16,7 +16,6 @@ const WebSocket = require('ws'); const { JSDOM } = require('jsdom'); const env = process.env.NODE_ENV || 'development'; -const alwaysRequireAuth = process.env.LIMITED_FEDERATION_MODE === 'true' || process.env.WHITELIST_MODE === 'true' || process.env.AUTHORIZED_FETCH === 'true'; dotenv.config({ path: env === 'production' ? '.env.production' : '.env', @@ -347,22 +346,17 @@ const startWorker = async (workerId) => { * @param {boolean=} required * @return {Promise.} */ - const accountFromRequest = (req, required = true) => new Promise((resolve, reject) => { + const accountFromRequest = (req) => new Promise((resolve, reject) => { const authorization = req.headers.authorization; const location = url.parse(req.url, true); const accessToken = location.query.access_token || req.headers['sec-websocket-protocol']; if (!authorization && !accessToken) { - if (required) { - const err = new Error('Missing access token'); - err.status = 401; + const err = new Error('Missing access token'); + err.status = 401; - reject(err); - return; - } else { - resolve(); - return; - } + reject(err); + return; } const token = authorization ? authorization.replace(/^Bearer /, '') : accessToken; @@ -465,7 +459,7 @@ const startWorker = async (workerId) => { // variables. OAuth scope checks are moved to the point of subscription // to a specific stream. - accountFromRequest(info.req, alwaysRequireAuth).then(() => { + accountFromRequest(info.req).then(() => { callback(true, undefined, undefined); }).catch(err => { log.error(info.req.requestId, err.toString()); @@ -539,7 +533,7 @@ const startWorker = async (workerId) => { return; } - accountFromRequest(req, alwaysRequireAuth).then(() => checkScopes(req, channelNameFromPath(req))).then(() => { + accountFromRequest(req).then(() => checkScopes(req, channelNameFromPath(req))).then(() => { subscribeHttpToSystemChannel(req, res); }).then(() => { next(); -- cgit