From 7515716e7f3950a3a6de04aac1b88215aa40795e Mon Sep 17 00:00:00 2001 From: Starfall Date: Tue, 17 Jan 2023 13:59:40 -0600 Subject: update almost everything except the setup script --- deploy/conf/.env.ambassador | 18 +++ deploy/conf/.env.production | 54 +++++++++ deploy/conf/common-ssl.conf | 28 +++++ deploy/conf/nginx.conf | 146 ++++++----------------- deploy/conf/robots.txt | 3 + deploy/docker-compose.yml | 225 ++++++++++++++++++------------------ deploy/scripts/backup.sh | 2 +- deploy/scripts/update-containers.sh | 19 --- deploy/setup-mastodon.sh | 12 ++ 9 files changed, 266 insertions(+), 241 deletions(-) create mode 100644 deploy/conf/.env.ambassador create mode 100644 deploy/conf/.env.production create mode 100644 deploy/conf/common-ssl.conf create mode 100644 deploy/conf/robots.txt delete mode 100644 deploy/scripts/update-containers.sh (limited to 'deploy') diff --git a/deploy/conf/.env.ambassador b/deploy/conf/.env.ambassador new file mode 100644 index 0000000..6643bc5 --- /dev/null +++ b/deploy/conf/.env.ambassador @@ -0,0 +1,18 @@ +DB_HOST=db +DB_USER=ambassador +DB_NAME=postgres +DB_PASS= +DB_PORT=5432 +AMBASSADOR_TOKEN= +INSTANCE_HOST=https://plural.cafe +BOOSTS_PER_CYCLE=1 +THRESHOLD_INTERVAL_DAYS=30 +BOOST_MIN_DAYS=1 +BOOST_MAX_DAYS=3 +THRESHOLD_CHECK_INTERVAL=15 +CYCLE_INTERVAL=15 +BOOST_MIN_HOURS=12 +THRESHOLD_NUMERATOR=1 +THRESHOLD_DENOMINATOR=1 +USER_BOOST_LIMIT=1 +USER_BOOST_INTERVAL_HOURS=72 diff --git a/deploy/conf/.env.production b/deploy/conf/.env.production new file mode 100644 index 0000000..3d1181c --- /dev/null +++ b/deploy/conf/.env.production @@ -0,0 +1,54 @@ +REDIS_HOST=redis +REDIS_PORT=6379 + +DB_HOST=db +DB_USER=postgres +DB_NAME=postgres +DB_PASS= +DB_PORT=5432 + +LOCAL_DOMAIN=plural.cafe +DEFAULT_LOCALE=en-cafe + +# Generate with `docker-compose run --rm web rake secret` +SECRET_KEY_BASE= +OTP_SECRET= + +# Instance keys for push notifications +# Generate with `docker-compose run --rm web rake mastodon:webpush:generate_vapid_key` +VAPID_PRIVATE_KEY= +VAPID_PUBLIC_KEY= + +# email +SMTP_SERVER=smtp.mailgun.org +SMTP_PORT=2525 +SMTP_LOGIN=postmaster@plural.cafe +SMTP_PASSWORD= +SMTP_FROM_ADDRESS=noreply@plural.cafe + +# jortage +S3_ENABLED=true +S3_BUCKET=pluralcafe +AWS_ACCESS_KEY_ID=pluralcafe +AWS_SECRET_ACCESS_KEY= +S3_REGION=jort +S3_PROTOCOL=https +S3_HOSTNAME=pool-api.jortage.com +S3_ENDPOINT=https://pool-api.jortage.com +S3_SIGNATURE_VERSION=v4 +S3_ALIAS_HOST=pool.jortage.com/pluralcafe + +# necessary for docker +UID=991 +GID=991 + +# configuration +MAX_TOOT_CHARS=5000 +MAX_PINNED_TOOTS=5 +MAX_BIO_CHARS=2000 +MAX_PROFILE_FIELDS=10 +MAX_DISPLAY_NAME_CHARS=100 + +# should be the fucking default +AUTHORIZED_FETCH=true +DISALLOW_UNAUTHENTICATED_API_ACCESS=true diff --git a/deploy/conf/common-ssl.conf b/deploy/conf/common-ssl.conf new file mode 100644 index 0000000..e30b7b8 --- /dev/null +++ b/deploy/conf/common-ssl.conf @@ -0,0 +1,28 @@ +server_tokens off; + +ssl_certificate /srv/plural.cafe/.acme.sh/plural.cafe/fullchain.cer; +ssl_certificate_key /srv/plural.cafe/.acme.sh/plural.cafe/plural.cafe.key; +ssl_trusted_certificate /srv/plural.cafe/.acme.sh/plural.cafe/plural.cafe.cer; + +ssl_protocols TLSv1.2 TLSv1.3; +ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA; +ssl_ecdh_curve X25519:secp384r1:prime256v1; +ssl_prefer_server_ciphers on; +ssl_session_cache shared:TLS:2m; +ssl_session_timeout 10m; +ssl_session_tickets off; +ssl_stapling on; +ssl_stapling_verify on; + +keepalive_timeout 70; +sendfile on; +client_max_body_size 0; + +add_header X-Frame-Options DENY; +add_header X-Content-Type-Options nosniff; +add_header X-XSS-Protection "1; mode=block"; +add_header Referrer-Policy "same-origin"; +add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + +resolver 1.1.1.1 1.0.0.1 [2606:4700:4700::1111] [2606:4700:4700::1001] valid=300s; +resolver_timeout 5s; diff --git a/deploy/conf/nginx.conf b/deploy/conf/nginx.conf index 7804345..d8ccfce 100644 --- a/deploy/conf/nginx.conf +++ b/deploy/conf/nginx.conf @@ -3,21 +3,14 @@ map $http_upgrade $connection_upgrade { '' close; } -upstream dockernetdata { - server netdata:19999; - keepalive 64; -} - server { listen 80; listen [::]:80; - server_name $NGINX_HOST; - root /var/www/html; + server_name plural.cafe *.plural.cafe; + server_tokens off; - location /.well-known/acme-challenge/ { - allow all; - } + root /srv/plural.cafe/html; location / { return 301 https://$host$request_uri; @@ -29,34 +22,8 @@ server { listen [::]:443 ssl http2; server_name ~^(?\w+)\.plural\.cafe$; - server_tokens off; - - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers '[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305|ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]:ECDHE+AES128:RSA+AES128:ECDHE+AES256:RSA+AES256:ECDHE+3DES:RSA+3DES'; - ssl_ecdh_curve X25519:secp384r1; - ssl_prefer_server_ciphers on; - ssl_session_cache shared:TLS:2m; - ssl_session_timeout 10m; - ssl_session_tickets off; - ssl_stapling on; - ssl_stapling_verify on; - - keepalive_timeout 70; - sendfile on; - client_max_body_size 0; - - add_header X-Frame-Options DENY; - add_header X-Content-Type-Options nosniff; - add_header X-XSS-Protection "1; mode=block"; - add_header Referrer-Policy "same-origin"; - add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; - - ssl_certificate /home/mastodon/.acme.sh/certs/fullchain.pem; - ssl_certificate_key /home/mastodon/.acme.sh/certs/privkey.pem; - ssl_trusted_certificate /home/mastodon/.acme.sh/certs/cert.pem; - - resolver 1.1.1.1 1.0.0.1 [2606:4700:4700::1111] [2606:4700:4700::1001] valid=300s; - resolver_timeout 5s; + + include /etc/nginx/snippets/common-ssl.conf; return 301 "https://plural.cafe/@${subdomain}"; } @@ -65,44 +32,25 @@ server { listen 443 ssl http2; listen [::]:443 ssl http2; - server_name $NGINX_HOST; - server_tokens off; + server_name plural.cafe; + + include /etc/nginx/snippets/common-ssl.conf; + root /srv/plural.cafe/html; + + gzip on; + gzip_disable "msie6"; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers '[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305|ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]:ECDHE+AES128:RSA+AES128:ECDHE+AES256:RSA+AES256:ECDHE+3DES:RSA+3DES'; - ssl_ecdh_curve X25519:secp384r1; - ssl_prefer_server_ciphers on; - ssl_session_cache shared:TLS:2m; - ssl_session_timeout 10m; - ssl_session_tickets off; - ssl_stapling on; - ssl_stapling_verify on; - - keepalive_timeout 70; - sendfile on; - client_max_body_size 0; - - add_header X-Frame-Options DENY; - add_header X-Content-Type-Options nosniff; - add_header X-XSS-Protection "1; mode=block"; - add_header Referrer-Policy "same-origin"; - add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; - - ssl_certificate /etc/ssl/fullchain.pem; - ssl_certificate_key /etc/ssl/privkey.pem; - ssl_trusted_certificate /etc/ssl/cert.pem; - - resolver 1.1.1.1 1.0.0.1 [2606:4700:4700::1111] [2606:4700:4700::1001] valid=300s; - resolver_timeout 5s; - - root /var/www/html; - - #add_header Content-Security-Policy "Content-Security-Policy: frame-ancestors 'none'; object-src 'none'; script-src 'self'; base-uri 'none';"; - add_header Access-Control-Allow-Origin "https://$host"; add_header X-Cache-Status $upstream_cache_status; - location / { - try_files $uri @proxy; + # hang up on scrubs + if ($http_user_agent ~* (fedsearch|gabsocial|fedichive|seekport|dotbot|semrush|fasthttp|^\d+$|wdestiny|megaindex|webmeup|fedi-block)) { + return 444; } location /sw.js { @@ -110,45 +58,13 @@ server { try_files $uri @proxy; } - location = /sysinfo { - return 301 /sysinfo/; - } - - location ~ /sysinfo/(?.*) { - proxy_redirect off; - proxy_set_header Host $host; - - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Server $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_http_version 1.1; - proxy_pass_request_headers on; - proxy_set_header Connection "keep-alive"; - proxy_store off; - proxy_pass http://dockernetdata/$ndpath$is_args$args; - - gzip on; - gzip_proxied any; - gzip_types *; - } - location ~ ^/(emoji|packs|sounds) { - add_header Cache-Control "public, max-age=31536000, immutable"; - - gzip on; - gzip_disable "msie6"; - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_buffers 16 8k; - gzip_http_version 1.1; - gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; - + expires max; try_files $uri @proxy; } - location ~ ^/system/(?(media_attachments|accounts|preview_cards)/.+) { - return 301 "https://d2rm2wyqhf92ej.cloudfront.net/$req"; + location / { + try_files $uri @proxy; } location @proxy { @@ -157,9 +73,14 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; proxy_set_header Proxy ""; + proxy_hide_header X-Frame-Options; + proxy_hide_header X-Content-Type-Options; + proxy_hide_header X-XSS-Protection; + proxy_hide_header Referrer-Policy; + proxy_hide_header Strict-Transport-Security; proxy_pass_header Server; - proxy_pass http://mstweb:3000; + proxy_pass http://127.0.0.1:3010; proxy_buffering on; proxy_redirect off; proxy_http_version 1.1; @@ -182,7 +103,7 @@ server { proxy_set_header X-Forwarded-Proto https; proxy_set_header Proxy ""; - proxy_pass http://mststreaming:4000; + proxy_pass http://127.0.0.1:3011; proxy_buffering off; proxy_redirect off; proxy_http_version 1.1; @@ -192,6 +113,11 @@ server { tcp_nodelay on; } + location /api/v2/search { + access_log off; + try_files $uri @proxy; + } + error_page 403 /assets/403.html; error_page 404 /assets/404.html; error_page 410 /assets/410.html; diff --git a/deploy/conf/robots.txt b/deploy/conf/robots.txt new file mode 100644 index 0000000..e70ee85 --- /dev/null +++ b/deploy/conf/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /media_proxy/ +Disallow: /interact/ diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index ed09d0c..c8a2cba 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,167 +1,170 @@ version: '2.4' services: - nginx: - restart: always - image: nginx:mainline-alpine - ports: - - 80:80 - - 443:443 - environment: - - NGINX_HOST=plural.cafe - volumes: - - /etc/localtime:/etc/localtime:ro - - ./.docker/nginx/nginx.conf:/etc/nginx/conf.d/web.template:ro - - ./.acme.sh/${NGINX_HOST}_ecc/${NGINX_HOST}.cer:/etc/ssl/cert.pem:ro - - ./.acme.sh/${NGINX_HOST}_ecc/${NGINX_HOST}.key:/etc/ssl/privkey.pem:ro - - ./.acme.sh/${NGINX_HOST}_ecc/fullchain.cer:/etc/ssl/fullchain.pem:ro - - ./public:/var/www/html:ro - command: sh -c "envsubst \"`env | awk -F = '{printf \" $$%s\", $$1}'`\" < /etc/nginx/conf.d/web.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'" - networks: - - external_network - - mstweb_network - - mststreaming_network - - netdata_network - - netdata: - restart: always - image: titpetric/netdata - cap_add: - - SYS_PTRACE + # automatically update images + watchtower: + image: containrrr/watchtower + restart: unless-stopped volumes: - - /etc/localtime:/etc/localtime:ro - - ./.docker/netdata:/etc/netdata - - /proc:/host/proc:ro - - /sys:/host/sys:ro - /var/run/docker.sock:/var/run/docker.sock networks: - - netdata_network + - external + # may not be necessary with recent docker, need to investigate ipv6nat: + image: robbertkl/ipv6nat restart: always + privileged: true + network_mode: host volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - /lib/modules:/lib/modules:ro - privileged: true - network_mode: host - image: robbertkl/ipv6nat - mstdb: - restart: always - image: postgres:9.6-alpine - networks: - - mstdb_network + logrotate: + image: blacklabelops/logrotate + restart: unless-stopped + environment: + - LOGS_DIRECTORIES=/var/lib/docker/containers + - LOGROTATE_INTERVAL=daily + - LOGROTATE_COPIES=90 volumes: - /etc/localtime:/etc/localtime:ro - - ./.docker/mastodon/db:/var/lib/postgresql/data + - /var/lib/docker/containers:/var/lib/docker/containers + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" - mstredis: + db: restart: always - image: redis:4-alpine + image: postgres:11-alpine + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] networks: - - mstredis_network + - db_network volumes: - /etc/localtime:/etc/localtime:ro - - ./.docker/mastodon/redis:/data + - /etc/timezone:/etc/timezone:ro + - ./mastodon/db:/var/lib/postgresql/data - mstes: + redis: restart: always - image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.3 - environment: - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 + image: redis:alpine + healthcheck: + test: ["CMD", "redis-cli", "ping"] networks: - - mstes_network + - redis_network volumes: - /etc/localtime:/etc/localtime:ro - - ./.docker/mastodon/es:/usr/share/elasticsearch/data + - /etc/timezone:/etc/timezone:ro + - ./mastodon/redis:/data - mstweb: - image: pluralcafe/mastodon:stable + mastodon-web: + image: pluralcafe/mastodon:edge + healthcheck: + test: ["CMD-SHELL", "wget -q --spider --header 'x-forwarded-proto: https' --proxy=off localhost:3000/health || exit 1"] restart: always - env_file: ./.docker/mastodon/.env.production - environment: - - WEB_CONCURRENCY=1 - - MAX_THREADS=15 - command: sh -c "rm -f /mastodon/tmp/pids/server.pid; rake db:migrate; bundle exec rails s -p 3000 -b '0.0.0.0'" + env_file: ./mastodon/.env.production + command: sh -c "rm -f /mastodon/tmp/pids/server.pid; RAILS_ENV=production bundle exec rails db:migrate; bundle exec rails s -p 3000 -b '0.0.0.0'" networks: - - mstdb_network - - mstes_network - - mstredis_network - - mstweb_network + - db_network + - redis_network + - external depends_on: - - mstdb - - mstredis - - mstes + - ipv6nat + - db + - redis + ports: + - "127.0.0.1:3010:3000" volumes: - /etc/localtime:/etc/localtime:ro - - ./public/system:/mastodon/public/system + - /etc/timezone:/etc/timezone:ro + - ./html/system:/mastodon/public/system - mststreaming: - image: pluralcafe/mastodon:stable + mastodon-streaming: + image: pluralcafe/mastodon:edge + healthcheck: + test: ["CMD-SHELL", "wget -q --spider --header 'x-forwarded-proto: https' --proxy=off localhost:4000/api/v1/streaming/health || exit 1"] restart: always - env_file: .docker/mastodon/.env.production + env_file: ./mastodon/.env.production command: yarn start + volumes: + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro networks: - - mstdb_network - - mstredis_network - - mststreaming_network + - db_network + - redis_network + - external depends_on: - - mstdb - - mstredis + - db + - redis + ports: + - "127.0.0.1:3011:4000" - mstsidekiq: - image: pluralcafe/mastodon:stable + sidekiq: + image: pluralcafe/mastodon:edge restart: always - env_file: .docker/mastodon/.env.production - environment: - - DB_POOL=10 - command: bundle exec sidekiq -q default -q mailers -q pull -q push + env_file: ./mastodon/.env.production + command: bundle exec sidekiq -q default -q mailers -q push -q pull -q scheduler depends_on: - - mstdb - - mstes - - mstredis + - ipv6nat + - db + - redis networks: - - external_network - - mstdb_network - - mstes_network - - mstredis_network + - external + - db_network + - redis_network volumes: - - ./public/system:/mastodon/public/system + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + - ./html/system:/mastodon/public/system - mstbarkeep: + # after the Twitter migration of november 2022, separating ingress only makes sense + sidekiq-ingress: + image: pluralcafe/mastodon:edge + restart: always + env_file: ./mastodon/.env.production + command: bundle exec sidekiq -q default -q ingress + depends_on: + - ipv6nat + - db + - redis + networks: + - external + - db_network + - redis_network + volumes: + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + - ./html/system:/mastodon/public/system + + barkeep: image: pluralcafe/barkeep restart: always - env_file: ./.docker/mastodon/.env.ambassador + env_file: ./mastodon/.env.ambassador command: yarn start depends_on: - - mstdb + - db + - ipv6nat + - mastodon-web networks: - - external_network - - mstdb_network + - external + - db_network + volumes: + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro networks: - external_network: + external: driver: bridge enable_ipv6: true ipam: driver: default config: - subnet: 172.18.0.0/16 - - subnet: fd00:dead:beef::/48 - mstdb_network: - internal: true - mstes_network: - internal: true - mstredis_network: - internal: true - mststreaming_network: - internal: true - mstweb_network: - internal: true - netdata_network: + - subnet: fd00:0000:0000::/48 + db_network: internal: true + redis_network: + internal: true diff --git a/deploy/scripts/backup.sh b/deploy/scripts/backup.sh index ae2c774..df49890 100755 --- a/deploy/scripts/backup.sh +++ b/deploy/scripts/backup.sh @@ -11,7 +11,7 @@ COMPOSE="$COMPOSE -f $YML_LOC" if [ "$1" == 'daily' ]; then - find $BACKUP_LOC -type f -name postgres-daily.* -mtime +7 -delete + find $BACKUP_LOC -type f -name postgres-daily.* -mtime +3 -delete $COMPOSE exec -T -u postgres db sh -c "umask 0377 && /usr/local/bin/pg_dump -Fc -h db -d postgres -U postgres" > "$BACKUP_LOC/postgres-daily.$(date -Iseconds).pgsql" $COMPOSE run -T --rm rails rake mastodon:media:remove_remote fi diff --git a/deploy/scripts/update-containers.sh b/deploy/scripts/update-containers.sh deleted file mode 100644 index 302731c..0000000 --- a/deploy/scripts/update-containers.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -[ -z "$COMPOSE" ] && COMPOSE="$(command -v docker-compose)" -[ -z "$COMPOSE" ] && COMPOSE="/usr/local/bin/docker-compose" - -cd "$HOME" || exit - -[ -z "$NGINX_WEBHOOK" ] || printf "Content-Type: text/plain\r\n\r\n" - -($COMPOSE pull 2>&1 | grep --silent "Downloaded newer") && { - $COMPOSE up -d - - docker cp "$($COMPOSE ps -q mstweb):/mastodon/public/assets" public/ - docker cp "$($COMPOSE ps -q mstweb):/mastodon/public/packs" public/ - - docker system prune --all -f - - curl -sS "https://raw.githubusercontent.com/pluralcafe/utils/master/deploy/docker-compose.yml" > docker-compose.yml -} diff --git a/deploy/setup-mastodon.sh b/deploy/setup-mastodon.sh index e394184..165bbf4 100755 --- a/deploy/setup-mastodon.sh +++ b/deploy/setup-mastodon.sh @@ -1,5 +1,17 @@ #!/bin/bash +# copy files +install docker-compose.yaml /srv/plural.cafe/docker-compose.yaml +install conf/nginx.conf /etc/nginx/sites-available/pluralcafe +install conf/common-ssl.conf /etc/nginx/snippets/common-ssl.conf +install conf/robots.txt /srv/plural.cafe/html/robots.txt +install scripts/backup.sh /srv/plural.cafe/scripts/backup.sh + +install conf/.env.ambassador /srv/plural.cafe/mastodon/.env.ambassador +install conf/.env.production /srv/plural.cafe/mastodon/.env.production +# TODO some edits here + +# TODO rewrite the below [ -z "$YML_LOC" ] && YML_LOC="$(pwd)" cd $YML_LOC -- cgit