diff options
72 files changed, 1453 insertions, 223 deletions
diff --git a/Gemfile b/Gemfile index 5dc070f4d..00033545f 100644 --- a/Gemfile +++ b/Gemfile @@ -50,6 +50,7 @@ gem 'omniauth', '~> 1.9' gem 'discard', '~> 1.2' gem 'doorkeeper', '~> 5.4' +gem 'ed25519', '~> 1.2' gem 'fast_blank', '~> 1.0' gem 'fastimage' gem 'goldfinger', '~> 2.1' diff --git a/Gemfile.lock b/Gemfile.lock index 9348c6516..fe44c85ff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -201,6 +201,7 @@ GEM dotenv (= 2.7.5) railties (>= 3.2, < 6.1) e2mmap (0.1.0) + ed25519 (1.2.4) elasticsearch (7.7.0) elasticsearch-api (= 7.7.0) elasticsearch-transport (= 7.7.0) @@ -701,6 +702,7 @@ DEPENDENCIES doorkeeper (~> 5.4) dotenv-rails (~> 2.7) e2mmap (~> 0.1.0) + ed25519 (~> 1.2) fabrication (~> 2.21) faker (~> 2.12) fast_blank (~> 1.0) diff --git a/app/controllers/activitypub/claims_controller.rb b/app/controllers/activitypub/claims_controller.rb new file mode 100644 index 000000000..08ad952df --- /dev/null +++ b/app/controllers/activitypub/claims_controller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class ActivityPub::ClaimsController < ActivityPub::BaseController + include SignatureVerification + include AccountOwnedConcern + + skip_before_action :authenticate_user! + + before_action :require_signature! + before_action :set_claim_result + + def create + render json: @claim_result, serializer: ActivityPub::OneTimeKeySerializer + end + + private + + def set_claim_result + @claim_result = ::Keys::ClaimService.new.call(@account.id, params[:id]) + end +end diff --git a/app/controllers/activitypub/collections_controller.rb b/app/controllers/activitypub/collections_controller.rb index c1e7aa550..380de54f5 100644 --- a/app/controllers/activitypub/collections_controller.rb +++ b/app/controllers/activitypub/collections_controller.rb @@ -5,8 +5,9 @@ class ActivityPub::CollectionsController < ActivityPub::BaseController include AccountOwnedConcern before_action :require_signature!, if: :authorized_fetch_mode? + before_action :set_items before_action :set_size - before_action :set_statuses + before_action :set_type before_action :set_cache_headers def show @@ -16,40 +17,53 @@ class ActivityPub::CollectionsController < ActivityPub::BaseController private - def set_statuses - @statuses = scope_for_collection - @statuses = cache_collection(@statuses, Status) + def set_items + case params[:id] + when 'featured' + @items = begin + # Because in public fetch mode we cache the response, there would be no + # benefit from performing the check below, since a blocked account or domain + # would likely be served the cache from the reverse proxy anyway + + if authorized_fetch_mode? && !signed_request_account.nil? && (@account.blocking?(signed_request_account) || (!signed_request_account.domain.nil? && @account.domain_blocking?(signed_request_account.domain))) + [] + else + cache_collection(@account.pinned_statuses, Status) + end + end + when 'devices' + @items = @account.devices + else + not_found + end end def set_size case params[:id] - when 'featured' - @size = @account.pinned_statuses.count + when 'featured', 'devices' + @size = @items.size else not_found end end - def scope_for_collection + def set_type case params[:id] when 'featured' - # Because in public fetch mode we cache the response, there would be no - # benefit from performing the check below, since a blocked account or domain - # would likely be served the cache from the reverse proxy anyway - if authorized_fetch_mode? && !signed_request_account.nil? && (@account.blocking?(signed_request_account) || (!signed_request_account.domain.nil? && @account.domain_blocking?(signed_request_account.domain))) - Status.none - else - @account.pinned_statuses - end + @type = :ordered + when 'devices' + @type = :unordered + else + not_found end end def collection_presenter ActivityPub::CollectionPresenter.new( id: account_collection_url(@account, params[:id]), - type: :ordered, + type: @type, size: @size, - items: @statuses + items: @items ) end end diff --git a/app/controllers/api/v1/crypto/deliveries_controller.rb b/app/controllers/api/v1/crypto/deliveries_controller.rb new file mode 100644 index 000000000..aa9df6e03 --- /dev/null +++ b/app/controllers/api/v1/crypto/deliveries_controller.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class Api::V1::Crypto::DeliveriesController < Api::BaseController + before_action -> { doorkeeper_authorize! :crypto } + before_action :require_user! + before_action :set_current_device + + def create + devices.each do |device_params| + DeliverToDeviceService.new.call(current_account, @current_device, device_params) + end + + render_empty + end + + private + + def set_current_device + @current_device = Device.find_by!(access_token: doorkeeper_token) + end + + def resource_params + params.require(:device) + params.permit(device: [:account_id, :device_id, :type, :body, :hmac]) + end + + def devices + Array(resource_params[:device]) + end +end diff --git a/app/controllers/api/v1/crypto/encrypted_messages_controller.rb b/app/controllers/api/v1/crypto/encrypted_messages_controller.rb new file mode 100644 index 000000000..a67b03eb4 --- /dev/null +++ b/app/controllers/api/v1/crypto/encrypted_messages_controller.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +class Api::V1::Crypto::EncryptedMessagesController < Api::BaseController + LIMIT = 80 + + before_action -> { doorkeeper_authorize! :crypto } + before_action :require_user! + before_action :set_current_device + + before_action :set_encrypted_messages, only: :index + after_action :insert_pagination_headers, only: :index + + def index + render json: @encrypted_messages, each_serializer: REST::EncryptedMessageSerializer + end + + def clear + @current_device.encrypted_messages.up_to(params[:up_to_id]).delete_all + render_empty + end + + private + + def set_current_device + @current_device = Device.find_by!(access_token: doorkeeper_token) + end + + def set_encrypted_messages + @encrypted_messages = @current_device.encrypted_messages.paginate_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id)) + end + + def insert_pagination_headers + set_pagination_headers(next_path, prev_path) + end + + def next_path + api_v1_encrypted_messages_url pagination_params(max_id: pagination_max_id) if records_continue? + end + + def prev_path + api_v1_encrypted_messages_url pagination_params(min_id: pagination_since_id) unless @encrypted_messages.empty? + end + + def pagination_max_id + @encrypted_messages.last.id + end + + def pagination_since_id + @encrypted_messages.first.id + end + + def records_continue? + @encrypted_messages.size == limit_param(LIMIT) + end + + def pagination_params(core_params) + params.slice(:limit).permit(:limit).merge(core_params) + end +end diff --git a/app/controllers/api/v1/crypto/keys/claims_controller.rb b/app/controllers/api/v1/crypto/keys/claims_controller.rb new file mode 100644 index 000000000..34b21a380 --- /dev/null +++ b/app/controllers/api/v1/crypto/keys/claims_controller.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class Api::V1::Crypto::Keys::ClaimsController < Api::BaseController + before_action -> { doorkeeper_authorize! :crypto } + before_action :require_user! + before_action :set_claim_results + + def create + render json: @claim_results, each_serializer: REST::Keys::ClaimResultSerializer + end + + private + + def set_claim_results + @claim_results = devices.map { |device_params| ::Keys::ClaimService.new.call(current_account, device_params[:account_id], device_params[:device_id]) }.compact + end + + def resource_params + params.permit(device: [:account_id, :device_id]) + end + + def devices + Array(resource_params[:device]) + end +end diff --git a/app/controllers/api/v1/crypto/keys/counts_controller.rb b/app/controllers/api/v1/crypto/keys/counts_controller.rb new file mode 100644 index 000000000..ffd7151b7 --- /dev/null +++ b/app/controllers/api/v1/crypto/keys/counts_controller.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class Api::V1::Crypto::Keys::CountsController < Api::BaseController + before_action -> { doorkeeper_authorize! :crypto } + before_action :require_user! + before_action :set_current_device + + def show + render json: { one_time_keys: @current_device.one_time_keys.count } + end + + private + + def set_current_device + @current_device = Device.find_by!(access_token: doorkeeper_token) + end +end diff --git a/app/controllers/api/v1/crypto/keys/queries_controller.rb b/app/controllers/api/v1/crypto/keys/queries_controller.rb new file mode 100644 index 000000000..0851d797d --- /dev/null +++ b/app/controllers/api/v1/crypto/keys/queries_controller.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class Api::V1::Crypto::Keys::QueriesController < Api::BaseController + before_action -> { doorkeeper_authorize! :crypto } + before_action :require_user! + before_action :set_accounts + before_action :set_query_results + + def create + render json: @query_results, each_serializer: REST::Keys::QueryResultSerializer + end + + private + + def set_accounts + @accounts = Account.where(id: account_ids).includes(:devices) + end + + def set_query_results + @query_results = @accounts.map { |account| ::Keys::QueryService.new.call(account) }.compact + end + + def account_ids + Array(params[:id]).map(&:to_i) + end +end diff --git a/app/controllers/api/v1/crypto/keys/uploads_controller.rb b/app/controllers/api/v1/crypto/keys/uploads_controller.rb new file mode 100644 index 000000000..fc4abf63b --- /dev/null +++ b/app/controllers/api/v1/crypto/keys/uploads_controller.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class Api::V1::Crypto::Keys::UploadsController < Api::BaseController + before_action -> { doorkeeper_authorize! :crypto } + before_action :require_user! + + def create + device = Device.find_or_initialize_by(access_token: doorkeeper_token) + + device.transaction do + device.account = current_account + device.update!(resource_params[:device]) + + if resource_params[:one_time_keys].present? && resource_params[:one_time_keys].is_a?(Enumerable) + resource_params[:one_time_keys].each do |one_time_key_params| + device.one_time_keys.create!(one_time_key_params) + end + end + end + + render json: device, serializer: REST::Keys::DeviceSerializer + end + + private + + def resource_params + params.permit(device: [:device_id, :name, :fingerprint_key, :identity_key], one_time_keys: [:key_id, :key, :signature]) + end +end diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb index d362b97dc..67a6cc2ec 100644 --- a/app/controllers/statuses_controller.rb +++ b/app/controllers/statuses_controller.rb @@ -42,7 +42,7 @@ class StatusesController < ApplicationController def activity expires_in 3.minutes, public: @status.distributable? && public_fetch_mode? - render_with_cache json: @status, content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter + render_with_cache json: ActivityPub::ActivityPresenter.from_status(@status), content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter end def embed diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 572b8087e..3509a6c40 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -2,6 +2,45 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def perform + case @object['type'] + when 'EncryptedMessage' + create_encrypted_message + else + create_status + end + end + + private + + def create_encrypted_message + return reject_payload! if invalid_origin?(@object['id']) || @options[:delivered_to_account_id].blank? + + target_account = Account.find(@options[:delivered_to_account_id]) + target_device = target_account.devices.find_by(device_id: @object.dig('to', 'deviceId')) + + return if target_device.nil? + + target_device.encrypted_messages.create!( + from_account: @account, + from_device_id: @object.dig('attributedTo', 'deviceId'), + type: @object['messageType'], + body: @object['cipherText'], + digest: @object.dig('digest', 'digestValue'), + message_franking: message_franking.to_token + ) + end + + def message_franking + MessageFranking.new( + hmac: @object.dig('digest', 'digestValue'), + original_franking: @object['messageFranking'], + source_account_id: @account.id, + target_account_id: @options[:delivered_to_account_id], + timestamp: Time.now.utc + ) + end + + def create_status return reject_payload! if unsupported_object_type? || invalid_origin?(@object['id']) || Tombstone.exists?(uri: @object['id']) || !related_to_local_activity? RedisLock.acquire(lock_options) do |lock| @@ -23,8 +62,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity @status end - private - def audience_to @object['to'] || @json['to'] end @@ -262,6 +299,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def poll_vote! poll = replied_to_status.preloadable_poll already_voted = true + RedisLock.acquire(poll_lock_options) do |lock| if lock.acquired? already_voted = poll.votes.where(account: @account).exists? @@ -270,20 +308,24 @@ class ActivityPub::Activity::Create < ActivityPub::Activity raise Mastodon::RaceConditionError end end + increment_voters_count! unless already_voted ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals? end def resolve_thread(status) return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri) + ThreadResolveWorker.perform_async(status.id, in_reply_to_uri) end def fetch_replies(status) collection = @object['replies'] return if collection.nil? + replies = ActivityPub::FetchRepliesService.new.call(status, collection, false) return unless replies.nil? + uri = value_or_id(collection) ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil? end @@ -291,6 +333,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def conversation_from_uri(uri) return nil if uri.nil? return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri) + begin Conversation.find_or_create_by!(uri: uri) rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique @@ -404,6 +447,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def skip_download? return @skip_download if defined?(@skip_download) + @skip_download ||= DomainBlock.reject_media?(@account.domain) end @@ -436,11 +480,13 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def forward_for_reply return unless @json['signature'].present? && reply_to_local? + ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url]) end def increment_voters_count! poll = replied_to_status.preloadable_poll + unless poll.voters_count.nil? poll.voters_count = poll.voters_count + 1 poll.save diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index 78138fb73..634ed29fa 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -22,6 +22,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' }, discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' }, voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' }, + olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' }, }.freeze def self.default_key_transform diff --git a/app/lib/inline_renderer.rb b/app/lib/inline_renderer.rb index 27e334a4d..b70814748 100644 --- a/app/lib/inline_renderer.rb +++ b/app/lib/inline_renderer.rb @@ -19,6 +19,8 @@ class InlineRenderer serializer = REST::AnnouncementSerializer when :reaction serializer = REST::ReactionSerializer + when :encrypted_message + serializer = REST::EncryptedMessageSerializer else return end diff --git a/app/models/account.rb b/app/models/account.rb index ff7386aaf..6b7ebda9e 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -49,6 +49,7 @@ # hide_collections :boolean # avatar_storage_schema_version :integer # header_storage_schema_version :integer +# devices_url :string # class Account < ApplicationRecord diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb index 499edbf4e..cca3a17fa 100644 --- a/app/models/concerns/account_associations.rb +++ b/app/models/concerns/account_associations.rb @@ -9,6 +9,7 @@ module AccountAssociations # Identity proofs has_many :identity_proofs, class_name: 'AccountIdentityProof', dependent: :destroy, inverse_of: :account + has_many :devices, dependent: :destroy, inverse_of: :account # Timelines has_many :statuses, inverse_of: :account, dependent: :destroy diff --git a/app/models/device.rb b/app/models/device.rb new file mode 100644 index 000000000..97d0d2774 --- /dev/null +++ b/app/models/device.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: devices +# +# id :bigint(8) not null, primary key +# access_token_id :bigint(8) +# account_id :bigint(8) +# device_id :string default(""), not null +# name :string default(""), not null +# fingerprint_key :text default(""), not null +# identity_key :text default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class Device < ApplicationRecord + belongs_to :access_token, class_name: 'Doorkeeper::AccessToken' + belongs_to :account + + has_many :one_time_keys, dependent: :destroy, inverse_of: :device + has_many :encrypted_messages, dependent: :destroy, inverse_of: :device + + validates :name, :fingerprint_key, :identity_key, presence: true + validates :fingerprint_key, :identity_key, ed25519_key: true + + before_save :invalidate_associations, if: -> { device_id_changed? || fingerprint_key_changed? || identity_key_changed? } + + private + + def invalidate_associations + one_time_keys.destroy_all + encrypted_messages.destroy_all + end +end diff --git a/app/models/encrypted_message.rb b/app/models/encrypted_message.rb new file mode 100644 index 000000000..5e0aba434 --- /dev/null +++ b/app/models/encrypted_message.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: encrypted_messages +# +# id :bigint(8) not null, primary key +# device_id :bigint(8) +# from_account_id :bigint(8) +# from_device_id :string default(""), not null +# type :integer default(0), not null +# body :text default(""), not null +# digest :text default(""), not null +# message_franking :text default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class EncryptedMessage < ApplicationRecord + self.inheritance_column = nil + + include Paginable + + scope :up_to, ->(id) { where(arel_table[:id].lteq(id)) } + + belongs_to :device + belongs_to :from_account, class_name: 'Account' + + around_create Mastodon::Snowflake::Callbacks + + after_commit :push_to_streaming_api + + private + + def push_to_streaming_api + Rails.logger.info(streaming_channel) + Rails.logger.info(subscribed_to_timeline?) + + return if destroyed? || !subscribed_to_timeline? + + PushEncryptedMessageWorker.perform_async(id) + end + + def subscribed_to_timeline? + Redis.current.exists("subscribed:#{streaming_channel}") + end + + def streaming_channel + "timeline:#{device.account_id}:#{device.device_id}" + end +end diff --git a/app/models/message_franking.rb b/app/models/message_franking.rb new file mode 100644 index 000000000..c72bd1cca --- /dev/null +++ b/app/models/message_franking.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class MessageFranking + attr_reader :hmac, :source_account_id, :target_account_id, + :timestamp, :original_franking + + def initialize(attributes = {}) + @hmac = attributes[:hmac] + @source_account_id = attributes[:source_account_id] + @target_account_id = attributes[:target_account_id] + @timestamp = attributes[:timestamp] + @original_franking = attributes[:original_franking] + end + + def to_token + crypt = ActiveSupport::MessageEncryptor.new(SystemKey.current_key, serializer: Oj) + crypt.encrypt_and_sign(self) + end +end diff --git a/app/models/one_time_key.rb b/app/models/one_time_key.rb new file mode 100644 index 000000000..8ada34824 --- /dev/null +++ b/app/models/one_time_key.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: one_time_keys +# +# id :bigint(8) not null, primary key +# device_id :bigint(8) +# key_id :string default(""), not null +# key :text default(""), not null +# signature :text default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class OneTimeKey < ApplicationRecord + belongs_to :device + + validates :key_id, :key, :signature, presence: true + validates :key, ed25519_key: true + validates :signature, ed25519_signature: { message: :key, verify_key: ->(one_time_key) { one_time_key.device.fingerprint_key } } +end diff --git a/app/models/system_key.rb b/app/models/system_key.rb new file mode 100644 index 000000000..f17db7c2d --- /dev/null +++ b/app/models/system_key.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: system_keys +# +# id :bigint(8) not null, primary key +# key :binary +# created_at :datetime not null +# updated_at :datetime not null +# +class SystemKey < ApplicationRecord + ROTATION_PERIOD = 1.week.freeze + + before_validation :set_key + + scope :expired, ->(now = Time.now.utc) { where(arel_table[:created_at].lt(now - ROTATION_PERIOD * 3)) } + + class << self + def current_key + previous_key = order(id: :asc).last + + if previous_key && previous_key.created_at >= ROTATION_PERIOD.ago + previous_key.key + else + create.key + end + end + end + + private + + def set_key + return if key.present? + + cipher = OpenSSL::Cipher.new('AES-256-GCM') + cipher.encrypt + + self.key = cipher.random_key + end +end diff --git a/app/presenters/activitypub/activity_presenter.rb b/app/presenters/activitypub/activity_presenter.rb new file mode 100644 index 000000000..5d174767f --- /dev/null +++ b/app/presenters/activitypub/activity_presenter.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class ActivityPub::ActivityPresenter < ActiveModelSerializers::Model + attributes :id, :type, :actor, :published, :to, :cc, :virtual_object + + class << self + def from_status(status) + new.tap do |presenter| + presenter.id = ActivityPub::TagManager.instance.activity_uri_for(status) + presenter.type = status.reblog? ? 'Announce' : 'Create' + presenter.actor = ActivityPub::TagManager.instance.uri_for(status.account) + presenter.published = status.created_at + presenter.to = ActivityPub::TagManager.instance.to(status) + presenter.cc = ActivityPub::TagManager.instance.cc(status) + + presenter.virtual_object = begin + if status.reblog? + if status.account == status.proper.account && status.proper.private_visibility? && status.local? + status.proper + else + ActivityPub::TagManager.instance.uri_for(status.proper) + end + else + status.proper + end + end + end + end + + def from_encrypted_message(encrypted_message) + new.tap do |presenter| + presenter.id = ActivityPub::TagManager.instance.generate_uri_for(nil) + presenter.type = 'Create' + presenter.actor = ActivityPub::TagManager.instance.uri_for(encrypted_message.source_account) + presenter.published = Time.now.utc + presenter.to = ActivityPub::TagManager.instance.uri_for(encrypted_message.target_account) + presenter.virtual_object = encrypted_message + end + end + end +end diff --git a/app/serializers/activitypub/activity_serializer.rb b/app/serializers/activitypub/activity_serializer.rb index d0edad786..5bdf53f03 100644 --- a/app/serializers/activitypub/activity_serializer.rb +++ b/app/serializers/activitypub/activity_serializer.rb @@ -1,52 +1,22 @@ # frozen_string_literal: true class ActivityPub::ActivitySerializer < ActivityPub::Serializer - attributes :id, :type, :actor, :published, :to, :cc - - has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, if: :serialize_object? - - attribute :proper_uri, key: :object, unless: :serialize_object? - attribute :atom_uri, if: :announce? - - def id - ActivityPub::TagManager.instance.activity_uri_for(object) + def self.serializer_for(model, options) + case model.class.name + when 'Status' + ActivityPub::NoteSerializer + when 'DeliverToDeviceService::EncryptedMessage' + ActivityPub::EncryptedMessageSerializer + else + super + end end - def type - announce? ? 'Announce' : 'Create' - end + attributes :id, :type, :actor, :published, :to, :cc - def actor - ActivityPub::TagManager.instance.uri_for(object.account) - end + has_one :virtual_object, key: :object def published - object.created_at.iso8601 - end - - def to - ActivityPub::TagManager.instance.to(object) - end - - def cc - ActivityPub::TagManager.instance.cc(object) - end - - def proper_uri - ActivityPub::TagManager.instance.uri_for(object.proper) - end - - def atom_uri - OStatus::TagManager.instance.uri_for(object) - end - - def announce? - object.reblog? - end - - def serialize_object? - return true unless announce? - # Serialize private self-boosts of local toots - object.account == object.proper.account && object.proper.private_visibility? && object.local? + object.published.iso8601 end end diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb index aa64936a7..627d4446b 100644 --- a/app/serializers/activitypub/actor_serializer.rb +++ b/app/serializers/activitypub/actor_serializer.rb @@ -7,7 +7,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer context_extensions :manually_approves_followers, :featured, :also_known_as, :moved_to, :property_value, :identity_proof, - :discoverable + :discoverable, :olm attributes :id, :type, :following, :followers, :inbox, :outbox, :featured, @@ -20,6 +20,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer has_many :virtual_tags, key: :tag has_many :virtual_attachments, key: :attachment + attribute :devices, unless: :instance_actor? attribute :moved_to, if: :moved? attribute :also_known_as, if: :also_known_as? @@ -38,7 +39,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer has_one :icon, serializer: ActivityPub::ImageSerializer, if: :avatar_exists? has_one :image, serializer: ActivityPub::ImageSerializer, if: :header_exists? - delegate :moved?, to: :object + delegate :moved?, :instance_actor?, to: :object def id object.instance_actor? ? instance_actor_url : account_url(object) @@ -68,6 +69,10 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer object.instance_actor? ? instance_actor_inbox_url : account_inbox_url(object) end + def devices + account_collection_url(object, :devices) + end + def outbox account_outbox_url(object) end diff --git a/app/serializers/activitypub/collection_serializer.rb b/app/serializers/activitypub/collection_serializer.rb index da1ba735f..00c7b786a 100644 --- a/app/serializers/activitypub/collection_serializer.rb +++ b/app/serializers/activitypub/collection_serializer.rb @@ -2,9 +2,16 @@ class ActivityPub::CollectionSerializer < ActivityPub::Serializer def self.serializer_for(model, options) - return ActivityPub::NoteSerializer if model.class.name == 'Status' - return ActivityPub::CollectionSerializer if model.class.name == 'ActivityPub::CollectionPresenter' - super + case model.class.name + when 'Status' + ActivityPub::NoteSerializer + when 'Device' + ActivityPub::DeviceSerializer + when 'ActivityPub::CollectionPresenter' + ActivityPub::CollectionSerializer + else + super + end end attribute :id, if: -> { object.id.present? } diff --git a/app/serializers/activitypub/device_serializer.rb b/app/serializers/activitypub/device_serializer.rb new file mode 100644 index 000000000..5f0fdc8af --- /dev/null +++ b/app/serializers/activitypub/device_serializer.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +class ActivityPub::DeviceSerializer < ActivityPub::Serializer + context_extensions :olm + + include RoutingHelper + + class FingerprintKeySerializer < ActivityPub::Serializer + attributes :type, :public_key_base64 + + def type + 'Ed25519Key' + end + + def public_key_base64 + object.fingerprint_key + end + end + + class IdentityKeySerializer < ActivityPub::Serializer + attributes :type, :public_key_base64 + + def type + 'Curve25519Key' + end + + def public_key_base64 + object.identity_key + end + end + + attributes :device_id, :type, :name, :claim + + has_one :fingerprint_key, serializer: FingerprintKeySerializer + has_one :identity_key, serializer: IdentityKeySerializer + + def type + 'Device' + end + + def claim + account_claim_url(object.account, id: object.device_id) + end + + def fingerprint_key + object + end + + def identity_key + object + end +end diff --git a/app/serializers/activitypub/encrypted_message_serializer.rb b/app/serializers/activitypub/encrypted_message_serializer.rb new file mode 100644 index 000000000..3c525d23e --- /dev/null +++ b/app/serializers/activitypub/encrypted_message_serializer.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +class ActivityPub::EncryptedMessageSerializer < ActivityPub::Serializer + context :security + + context_extensions :olm + + class DeviceSerializer < ActivityPub::Serializer + attributes :type, :device_id + + def type + 'Device' + end + + def device_id + object + end + end + + class DigestSerializer < ActivityPub::Serializer + attributes :type, :digest_algorithm, :digest_value + + def type + 'Digest' + end + + def digest_algorithm + 'http://www.w3.org/2000/09/xmldsig#hmac-sha256' + end + + def digest_value + object + end + end + + attributes :type, :message_type, :cipher_text, :message_franking + + has_one :attributed_to, serializer: DeviceSerializer + has_one :to, serializer: DeviceSerializer + has_one :digest, serializer: DigestSerializer + + def type + 'EncryptedMessage' + end + + def attributed_to + object.source_device.device_id + end + + def to + object.target_device_id + end + + def message_type + object.type + end + + def cipher_text + object.body + end +end diff --git a/app/serializers/activitypub/one_time_key_serializer.rb b/app/serializers/activitypub/one_time_key_serializer.rb new file mode 100644 index 000000000..5932eb5b5 --- /dev/null +++ b/app/serializers/activitypub/one_time_key_serializer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class ActivityPub::OneTimeKeySerializer < ActivityPub::Serializer + context :security + + context_extensions :olm + + class SignatureSerializer < ActivityPub::Serializer + attributes :type, :signature_value + + def type + 'Ed25519Signature' + end + + def signature_value + object.signature + end + end + + attributes :key_id, :type, :public_key_base64 + + has_one :signature, serializer: SignatureSerializer + + def type + 'Curve25519Key' + end + + def public_key_base64 + object.key + end + + def signature + object + end +end diff --git a/app/serializers/activitypub/outbox_serializer.rb b/app/serializers/activitypub/outbox_serializer.rb index 48fbad0fd..4f4f950a5 100644 --- a/app/serializers/activitypub/outbox_serializer.rb +++ b/app/serializers/activitypub/outbox_serializer.rb @@ -2,7 +2,14 @@ class ActivityPub::OutboxSerializer < ActivityPub::CollectionSerializer def self.serializer_for(model, options) - return ActivityPub::ActivitySerializer if model.is_a?(Status) - super + if model.class.name == 'ActivityPub::ActivityPresenter' + ActivityPub::ActivitySerializer + else + super + end + end + + def items + object.items.map { |status| ActivityPub::ActivityPresenter.from_status(status) } end end diff --git a/app/serializers/activitypub/undo_announce_serializer.rb b/app/serializers/activitypub/undo_announce_serializer.rb index 6758af679..a925efc18 100644 --- a/app/serializers/activitypub/undo_announce_serializer.rb +++ b/app/serializers/activitypub/undo_announce_serializer.rb @@ -3,7 +3,7 @@ class ActivityPub::UndoAnnounceSerializer < ActivityPub::Serializer attributes :id, :type, :actor, :to - has_one :object, serializer: ActivityPub::ActivitySerializer + has_one :virtual_object, key: :object, serializer: ActivityPub::ActivitySerializer def id [ActivityPub::TagManager.instance.uri_for(object.account), '#announces/', object.id, '/undo'].join @@ -20,4 +20,8 @@ class ActivityPub::UndoAnnounceSerializer < ActivityPub::Serializer def to [ActivityPub::TagManager::COLLECTIONS[:public]] end + + def virtual_object + ActivityPub::ActivityPresenter.from_status(object) + end end diff --git a/app/serializers/rest/encrypted_message_serializer.rb b/app/serializers/rest/encrypted_message_serializer.rb new file mode 100644 index 000000000..61ebc74fa --- /dev/null +++ b/app/serializers/rest/encrypted_message_serializer.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class REST::EncryptedMessageSerializer < ActiveModel::Serializer + attributes :id, :account_id, :device_id, + :type, :body, :digest, :message_franking + + def id + object.id.to_s + end + + def account_id + object.from_account_id.to_s + end + + def device_id + object.from_device_id + end +end diff --git a/app/serializers/rest/keys/claim_result_serializer.rb b/app/serializers/rest/keys/claim_result_serializer.rb new file mode 100644 index 000000000..145044f55 --- /dev/null +++ b/app/serializers/rest/keys/claim_result_serializer.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class REST::Keys::ClaimResultSerializer < ActiveModel::Serializer + attributes :account_id, :device_id, :key_id, :key, :signature + + def account_id + object.account.id.to_s + end +end diff --git a/app/serializers/rest/keys/device_serializer.rb b/app/serializers/rest/keys/device_serializer.rb new file mode 100644 index 000000000..f9b821b79 --- /dev/null +++ b/app/serializers/rest/keys/device_serializer.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +class REST::Keys::DeviceSerializer < ActiveModel::Serializer + attributes :device_id, :name, :identity_key, + :fingerprint_key +end diff --git a/app/serializers/rest/keys/query_result_serializer.rb b/app/serializers/rest/keys/query_result_serializer.rb new file mode 100644 index 000000000..8f8bdde28 --- /dev/null +++ b/app/serializers/rest/keys/query_result_serializer.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class REST::Keys::QueryResultSerializer < ActiveModel::Serializer + attributes :account_id + + has_many :devices, serializer: REST::Keys::DeviceSerializer + + def account_id + object.account.id.to_s + end +end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 7b4c53d50..f4276cece 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -76,6 +76,7 @@ class ActivityPub::ProcessAccountService < BaseService @account.shared_inbox_url = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || '' @account.followers_url = @json['followers'] || '' @account.featured_collection_url = @json['featured'] || '' + @account.devices_url = @json['devices'] || '' @account.url = url || @uri @account.uri = @uri @account.display_name = @json['name'] || '' diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb index 896699324..6a1575616 100644 --- a/app/services/backup_service.rb +++ b/app/services/backup_service.rb @@ -22,7 +22,7 @@ class BackupService < BaseService account.statuses.with_includes.reorder(nil).find_in_batches do |statuses| statuses.each do |status| - item = serialize_payload(status, ActivityPub::ActivitySerializer, signer: @account) + item = serialize_payload(ActivityPub::ActivityPresenter.from_status(status), ActivityPub::ActivitySerializer, signer: @account) item.delete(:'@context') unless item[:type] == 'Announce' || item[:object][:attachment].blank? diff --git a/app/services/deliver_to_device_service.rb b/app/services/deliver_to_device_service.rb new file mode 100644 index 000000000..71711945c --- /dev/null +++ b/app/services/deliver_to_device_service.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +class DeliverToDeviceService < BaseService + include Payloadable + + class EncryptedMessage < ActiveModelSerializers::Model + attributes :source_account, :target_account, :source_device, + :target_device_id, :type, :body, :digest, + :message_franking + end + + def call(source_account, source_device, options = {}) + @source_account = source_account + @source_device = source_device + @target_account = Account.find(options[:account_id]) + @target_device_id = options[:device_id] + @body = options[:body] + @type = options[:type] + @hmac = options[:hmac] + + set_message_franking! + + if @target_account.local? + deliver_to_local! + else + deliver_to_remote! + end + end + + private + + def set_message_franking! + @message_franking = message_franking.to_token + end + + def deliver_to_local! + target_device = @target_account.devices.find_by!(device_id: @target_device_id) + + target_device.encrypted_messages.create!( + from_account: @source_account, + from_device_id: @source_device.device_id, + type: @type, + body: @body, + digest: @hmac, + message_franking: @message_franking + ) + end + + def deliver_to_remote! + ActivityPub::DeliveryWorker.perform_async( + Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_encrypted_message(encrypted_message), ActivityPub::ActivitySerializer)), + @source_account.id, + @target_account.inbox_url + ) + end + + def message_franking + MessageFranking.new( + source_account_id: @source_account.id, + target_account_id: @target_account.id, + hmac: @hmac, + timestamp: Time.now.utc + ) + end + + def encrypted_message + EncryptedMessage.new( + source_account: @source_account, + target_account: @target_account, + source_device: @source_device, + target_device_id: @target_device_id, + type: @type, + body: @body, + digest: @hmac, + message_franking: @message_franking + ) + end +end diff --git a/app/services/keys/claim_service.rb b/app/services/keys/claim_service.rb new file mode 100644 index 000000000..672119130 --- /dev/null +++ b/app/services/keys/claim_service.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +class Keys::ClaimService < BaseService + HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze + + class Result < ActiveModelSerializers::Model + attributes :account, :device_id, :key_id, + :key, :signature + + def initialize(account, device_id, key_attributes = {}) + @account = account + @device_id = device_id + @key_id = key_attributes[:key_id] + @key = key_attributes[:key] + @signature = key_attributes[:signature] + end + end + + def call(source_account, target_account_id, device_id) + @source_account = source_account + @target_account = Account.find(target_account_id) + @device_id = device_id + + if @target_account.local? + claim_local_key! + else + claim_remote_key! + end + rescue ActiveRecord::RecordNotFound + nil + end + + private + + def claim_local_key! + device = @target_account.devices.find_by(device_id: @device_id) + key = nil + + ApplicationRecord.transaction do + key = device.one_time_keys.order(Arel.sql('random()')).first! + key.destroy! + end + + @result = Result.new(@target_account, @device_id, key) + end + + def claim_remote_key! + query_result = QueryService.new.call(@target_account) + device = query_result.find(@device_id) + + return unless device.present? && device.valid_claim_url? + + json = fetch_resource_with_post(device.claim_url) + + return unless json.present? && json['publicKeyBase64'].present? + + @result = Result.new(@target_account, @device_id, key_id: json['id'], key: json['publicKeyBase64'], signature: json.dig('signature', 'signatureValue')) + rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e + Rails.logger.debug "Claiming one-time key for #{@target_account.acct}:#{@device_id} failed: #{e}" + nil + end + + def fetch_resource_with_post(uri) + build_post_request(uri).perform do |response| + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + + body_to_json(response.body_with_limit) if response.code == 200 + end + end + + def build_post_request(uri) + Request.new(:post, uri).tap do |request| + request.on_behalf_of(@source_account, :uri) + request.add_headers(HEADERS) + end + end +end diff --git a/app/services/keys/query_service.rb b/app/services/keys/query_service.rb new file mode 100644 index 000000000..286fbd834 --- /dev/null +++ b/app/services/keys/query_service.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +class Keys::QueryService < BaseService + include JsonLdHelper + + class Result < ActiveModelSerializers::Model + attributes :account, :devices + + def initialize(account, devices) + @account = account + @devices = devices || [] + end + + def find(device_id) + @devices.find { |device| device.device_id == device_id } + end + end + + class Device < ActiveModelSerializers::Model + attributes :device_id, :name, :identity_key, :fingerprint_key + + def initialize(attributes = {}) + @device_id = attributes[:device_id] + @name = attributes[:name] + @identity_key = attributes[:identity_key] + @fingerprint_key = attributes[:fingerprint_key] + @claim_url = attributes[:claim_url] + end + + def valid_claim_url? + return false if @claim_url.blank? + + begin + parsed_url = Addressable::URI.parse(@claim_url).normalize + rescue Addressable::URI::InvalidURIError + return false + end + + %w(http https).include?(parsed_url.scheme) && parsed_url.host.present? + end + end + + def call(account) + @account = account + + if @account.local? + query_local_devices! + else + query_remote_devices! + end + + Result.new(@account, @devices) + end + + private + + def query_local_devices! + @devices = @account.devices.map { |device| Device.new(device) } + end + + def query_remote_devices! + return if @account.devices_url.blank? + + json = fetch_resource(@account.devices_url) + + return if json['items'].blank? + + @devices = json['items'].map do |device| + Device.new(device_id: device['id'], name: device['name'], identity_key: device.dig('identityKey', 'publicKeyBase64'), fingerprint_key: device.dig('fingerprintKey', 'publicKeyBase64'), claim_url: device['claim']) + end + rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e + Rails.logger.debug "Querying devices for #{@account.acct} failed: #{e}" + nil + end +end diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb index b2d868165..3822b7dc5 100644 --- a/app/services/process_mentions_service.rb +++ b/app/services/process_mentions_service.rb @@ -65,7 +65,7 @@ class ProcessMentionsService < BaseService def activitypub_json return @activitypub_json if defined?(@activitypub_json) - @activitypub_json = Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account)) + @activitypub_json = Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @status.account)) end def resolve_account_service diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb index 4b5ae9492..6866d2fac 100644 --- a/app/services/reblog_service.rb +++ b/app/services/reblog_service.rb @@ -60,6 +60,6 @@ class ReblogService < BaseService end def build_json(reblog) - Oj.dump(serialize_payload(reblog, ActivityPub::ActivitySerializer, signer: reblog.account)) + Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account)) end end diff --git a/app/validators/ed25519_key_validator.rb b/app/validators/ed25519_key_validator.rb new file mode 100644 index 000000000..00a448d5a --- /dev/null +++ b/app/validators/ed25519_key_validator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class Ed25519KeyValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + return if value.blank? + + key = Base64.decode64(value) + + record.errors[attribute] << I18n.t('crypto.errors.invalid_key') unless verified?(key) + end + + private + + def verified?(key) + Ed25519.validate_key_bytes(key) + rescue ArgumentError + false + end +end diff --git a/app/validators/ed25519_signature_validator.rb b/app/validators/ed25519_signature_validator.rb new file mode 100644 index 000000000..77a21b837 --- /dev/null +++ b/app/validators/ed25519_signature_validator.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class Ed25519SignatureValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + return if value.blank? + + verify_key = Ed25519::VerifyKey.new(Base64.decode64(option_to_value(record, :verify_key))) + signature = Base64.decode64(value) + message = option_to_value(record, :message) + + record.errors[attribute] << I18n.t('crypto.errors.invalid_signature') unless verified?(verify_key, signature, message) + end + + private + + def verified?(verify_key, signature, message) + verify_key.verify(signature, message) + rescue Ed25519::VerifyError, ArgumentError + false + end + + def option_to_value(record, key) + if options[key].is_a?(Proc) + options[key].call(record) + else + record.public_send(options[key]) + end + end +end diff --git a/app/workers/activitypub/distribution_worker.rb b/app/workers/activitypub/distribution_worker.rb index 11b6a6111..e4997ba0e 100644 --- a/app/workers/activitypub/distribution_worker.rb +++ b/app/workers/activitypub/distribution_worker.rb @@ -43,7 +43,7 @@ class ActivityPub::DistributionWorker end def payload - @payload ||= Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @account)) + @payload ||= Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @account)) end def relay! diff --git a/app/workers/activitypub/reply_distribution_worker.rb b/app/workers/activitypub/reply_distribution_worker.rb index 1ff8a657e..d4d0148ac 100644 --- a/app/workers/activitypub/reply_distribution_worker.rb +++ b/app/workers/activitypub/reply_distribution_worker.rb @@ -29,6 +29,6 @@ class ActivityPub::ReplyDistributionWorker end def payload - @payload ||= Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account)) + @payload ||= Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @status.account)) end end diff --git a/app/workers/push_conversation_worker.rb b/app/workers/push_conversation_worker.rb index 16f538215..aa858f715 100644 --- a/app/workers/push_conversation_worker.rb +++ b/app/workers/push_conversation_worker.rb @@ -2,13 +2,14 @@ class PushConversationWorker include Sidekiq::Worker + include Redisable def perform(conversation_account_id) conversation = AccountConversation.find(conversation_account_id) message = InlineRenderer.render(conversation, conversation.account, :conversation) timeline_id = "timeline:direct:#{conversation.account_id}" - Redis.current.publish(timeline_id, Oj.dump(event: :conversation, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) + redis.publish(timeline_id, Oj.dump(event: :conversation, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) rescue ActiveRecord::RecordNotFound true end diff --git a/app/workers/push_encrypted_message_worker.rb b/app/workers/push_encrypted_message_worker.rb new file mode 100644 index 000000000..031230172 --- /dev/null +++ b/app/workers/push_encrypted_message_worker.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PushEncryptedMessageWorker + include Sidekiq::Worker + include Redisable + + def perform(encrypted_message_id) + encrypted_message = EncryptedMessage.find(encrypted_message_id) + message = InlineRenderer.render(encrypted_message, nil, :encrypted_message) + timeline_id = "timeline:#{encrypted_message.device.account_id}:#{encrypted_message.device.device_id}" + + redis.publish(timeline_id, Oj.dump(event: :encrypted_message, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb index 94788a85b..bb9dd49ca 100644 --- a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb +++ b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb @@ -8,5 +8,6 @@ class Scheduler::DoorkeeperCleanupScheduler def perform Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all + SystemKey.expired.delete_all end end diff --git a/config/brakeman.ignore b/config/brakeman.ignore index 7e3828f7e..baa993c78 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -1,33 +1,13 @@ { "ignored_warnings": [ { - "warning_type": "Mass Assignment", - "warning_code": 105, - "fingerprint": "0117d2be5947ea4e4fbed9c15f23c6615b12c6892973411820c83d079808819d", - "check_name": "PermitAttributes", - "message": "Potentially dangerous key allowed for mass assignment", - "file": "app/controllers/api/v1/search_controller.rb", - "line": 30, - "link": "https://brakemanscanner.org/docs/warning_types/mass_assignment/", - "code": "params.permit(:type, :offset, :min_id, :max_id, :account_id)", - "render_path": null, - "location": { - "type": "method", - "class": "Api::V1::SearchController", - "method": "search_params" - }, - "user_input": ":account_id", - "confidence": "High", - "note": "" - }, - { "warning_type": "SQL Injection", "warning_code": 0, "fingerprint": "04dbbc249b989db2e0119bbb0f59c9818e12889d2b97c529cdc0b1526002ba4b", "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/report.rb", - "line": 90, + "line": 112, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "Admin::ActionLog.from(\"(#{[Admin::ActionLog.where(:target_type => \"Report\", :target_id => id, :created_at => ((created_at..updated_at))).unscope(:order), Admin::ActionLog.where(:target_type => \"Account\", :target_id => target_account_id, :created_at => ((created_at..updated_at))).unscope(:order), Admin::ActionLog.where(:target_type => \"Status\", :target_id => status_ids, :created_at => ((created_at..updated_at))).unscope(:order)].map do\n \"(#{query.to_sql})\"\n end.join(\" UNION ALL \")}) AS admin_action_logs\")", "render_path": null, @@ -47,7 +27,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/status.rb", - "line": 87, + "line": 100, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "result.joins(\"INNER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}\")", "render_path": null, @@ -61,39 +41,62 @@ "note": "" }, { - "warning_type": "Mass Assignment", - "warning_code": 105, - "fingerprint": "28d81cc22580ef76e912b077b245f353499aa27b3826476667224c00227af2a9", - "check_name": "PermitAttributes", - "message": "Potentially dangerous key allowed for mass assignment", - "file": "app/controllers/admin/reports_controller.rb", - "line": 56, - "link": "https://brakemanscanner.org/docs/warning_types/mass_assignment/", - "code": "params.permit(:account_id, :resolved, :target_account_id)", - "render_path": null, + "warning_type": "Dynamic Render Path", + "warning_code": 15, + "fingerprint": "20a660939f2bbf8c665e69f2844031c0564524689a9570a0091ed94846212020", + "check_name": "Render", + "message": "Render path contains parameter value", + "file": "app/views/admin/action_logs/index.html.haml", + "line": 26, + "link": "https://brakemanscanner.org/docs/warning_types/dynamic_render_path/", + "code": "render(action => Admin::ActionLogFilter.new(filter_params).results.page(params[:page]), {})", + "render_path": [ + { + "type": "controller", + "class": "Admin::ActionLogsController", + "method": "index", + "line": 8, + "file": "app/controllers/admin/action_logs_controller.rb", + "rendered": { + "name": "admin/action_logs/index", + "file": "app/views/admin/action_logs/index.html.haml" + } + } + ], "location": { - "type": "method", - "class": "Admin::ReportsController", - "method": "filter_params" + "type": "template", + "template": "admin/action_logs/index" }, - "user_input": ":account_id", - "confidence": "High", + "user_input": "params[:page]", + "confidence": "Weak", "note": "" }, { "warning_type": "Dynamic Render Path", "warning_code": 15, - "fingerprint": "4b6a895e2805578d03ceedbe1d469cc75a0c759eba093722523edb4b8683c873", + "fingerprint": "371fe16dc4c9d6ab08a20437d65be4825776107a67c38f6d4780a9c703cd44a5", "check_name": "Render", "message": "Render path contains parameter value", - "file": "app/views/admin/action_logs/index.html.haml", - "line": 4, + "file": "app/views/admin/email_domain_blocks/index.html.haml", + "line": 17, "link": "https://brakemanscanner.org/docs/warning_types/dynamic_render_path/", - "code": "render(action => Admin::ActionLog.page(params[:page]), {})", - "render_path": [{"type":"controller","class":"Admin::ActionLogsController","method":"index","line":7,"file":"app/controllers/admin/action_logs_controller.rb","rendered":{"name":"admin/action_logs/index","file":"/home/eugr/Projects/mastodon/app/views/admin/action_logs/index.html.haml"}}], + "code": "render(action => EmailDomainBlock.where(:parent_id => nil).includes(:children).order(:id => :desc).page(params[:page]), {})", + "render_path": [ + { + "type": "controller", + "class": "Admin::EmailDomainBlocksController", + "method": "index", + "line": 10, + "file": "app/controllers/admin/email_domain_blocks_controller.rb", + "rendered": { + "name": "admin/email_domain_blocks/index", + "file": "app/views/admin/email_domain_blocks/index.html.haml" + } + } + ], "location": { "type": "template", - "template": "admin/action_logs/index" + "template": "admin/email_domain_blocks/index" }, "user_input": "params[:page]", "confidence": "Weak", @@ -106,7 +109,7 @@ "check_name": "Redirect", "message": "Possible unprotected redirect", "file": "app/controllers/remote_interaction_controller.rb", - "line": 21, + "line": 24, "link": "https://brakemanscanner.org/docs/warning_types/redirect/", "code": "redirect_to(RemoteFollow.new(resource_params).interact_address_for(Status.find(params[:id])))", "render_path": null, @@ -120,32 +123,13 @@ "note": "" }, { - "warning_type": "Dynamic Render Path", - "warning_code": 15, - "fingerprint": "67afc0d5f7775fa5bd91d1912e1b5505aeedef61876347546fa20f92fd6915e6", - "check_name": "Render", - "message": "Render path contains parameter value", - "file": "app/views/stream_entries/embed.html.haml", - "line": 3, - "link": "https://brakemanscanner.org/docs/warning_types/dynamic_render_path/", - "code": "render(action => \"stream_entries/#{Account.find_local!(params[:account_username]).statuses.find(params[:id]).stream_entry.activity_type.downcase}\", { Account.find_local!(params[:account_username]).statuses.find(params[:id]).stream_entry.activity_type.downcase.to_sym => Account.find_local!(params[:account_username]).statuses.find(params[:id]).stream_entry.activity, :centered => true, :autoplay => ActiveModel::Type::Boolean.new.cast(params[:autoplay]) })", - "render_path": [{"type":"controller","class":"StatusesController","method":"embed","line":63,"file":"app/controllers/statuses_controller.rb","rendered":{"name":"stream_entries/embed","file":"/home/eugr/Projects/mastodon/app/views/stream_entries/embed.html.haml"}}], - "location": { - "type": "template", - "template": "stream_entries/embed" - }, - "user_input": "params[:id]", - "confidence": "Weak", - "note": "" - }, - { "warning_type": "SQL Injection", "warning_code": 0, "fingerprint": "6f075c1484908e3ec9bed21ab7cf3c7866be8da3881485d1c82e13093aefcbd7", "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/status.rb", - "line": 92, + "line": 105, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "result.joins(\"LEFT OUTER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}\")", "render_path": null, @@ -159,22 +143,43 @@ "note": "" }, { - "warning_type": "Dynamic Render Path", - "warning_code": 15, - "fingerprint": "8d843713d99e8403f7992f3e72251b633817cf9076ffcbbad5613859d2bbc127", - "check_name": "Render", - "message": "Render path contains parameter value", - "file": "app/views/admin/custom_emojis/index.html.haml", - "line": 45, - "link": "https://brakemanscanner.org/docs/warning_types/dynamic_render_path/", - "code": "render(action => filtered_custom_emojis.eager_load(:local_counterpart).page(params[:page]), {})", - "render_path": [{"type":"controller","class":"Admin::CustomEmojisController","method":"index","line":11,"file":"app/controllers/admin/custom_emojis_controller.rb","rendered":{"name":"admin/custom_emojis/index","file":"/home/eugr/Projects/mastodon/app/views/admin/custom_emojis/index.html.haml"}}], + "warning_type": "Mass Assignment", + "warning_code": 105, + "fingerprint": "7631e93d0099506e7c3e5c91ba8d88523b00a41a0834ae30031a5a4e8bb3020a", + "check_name": "PermitAttributes", + "message": "Potentially dangerous key allowed for mass assignment", + "file": "app/controllers/api/v2/search_controller.rb", + "line": 28, + "link": "https://brakemanscanner.org/docs/warning_types/mass_assignment/", + "code": "params.permit(:type, :offset, :min_id, :max_id, :account_id)", + "render_path": null, "location": { - "type": "template", - "template": "admin/custom_emojis/index" + "type": "method", + "class": "Api::V2::SearchController", + "method": "search_params" }, - "user_input": "params[:page]", - "confidence": "Weak", + "user_input": ":account_id", + "confidence": "High", + "note": "" + }, + { + "warning_type": "Mass Assignment", + "warning_code": 105, + "fingerprint": "8f63dec68951d9bcf7eddb15af9392b2e1333003089c41fb76688dfd3579f394", + "check_name": "PermitAttributes", + "message": "Potentially dangerous key allowed for mass assignment", + "file": "app/controllers/api/v1/crypto/deliveries_controller.rb", + "line": 23, + "link": "https://brakemanscanner.org/docs/warning_types/mass_assignment/", + "code": "params.require(:device).permit(:account_id, :device_id, :type, :body, :hmac)", + "render_path": null, + "location": { + "type": "method", + "class": "Api::V1::Crypto::DeliveriesController", + "method": "resource_params" + }, + "user_input": ":account_id", + "confidence": "High", "note": "" }, { @@ -204,10 +209,22 @@ "check_name": "Render", "message": "Render path contains parameter value", "file": "app/views/admin/accounts/index.html.haml", - "line": 47, + "line": 54, "link": "https://brakemanscanner.org/docs/warning_types/dynamic_render_path/", "code": "render(action => filtered_accounts.page(params[:page]), {})", - "render_path": [{"type":"controller","class":"Admin::AccountsController","method":"index","line":12,"file":"app/controllers/admin/accounts_controller.rb","rendered":{"name":"admin/accounts/index","file":"/home/eugr/Projects/mastodon/app/views/admin/accounts/index.html.haml"}}], + "render_path": [ + { + "type": "controller", + "class": "Admin::AccountsController", + "method": "index", + "line": 12, + "file": "app/controllers/admin/accounts_controller.rb", + "rendered": { + "name": "admin/accounts/index", + "file": "app/views/admin/accounts/index.html.haml" + } + } + ], "location": { "type": "template", "template": "admin/accounts/index" @@ -219,40 +236,40 @@ { "warning_type": "Redirect", "warning_code": 18, - "fingerprint": "ba699ddcc6552c422c4ecd50d2cd217f616a2446659e185a50b05a0f2dad8d33", + "fingerprint": "ba568ac09683f98740f663f3d850c31785900215992e8c090497d359a2563d50", "check_name": "Redirect", "message": "Possible unprotected redirect", - "file": "app/controllers/media_controller.rb", - "line": 14, + "file": "app/controllers/remote_follow_controller.rb", + "line": 21, "link": "https://brakemanscanner.org/docs/warning_types/redirect/", - "code": "redirect_to(MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original))", + "code": "redirect_to(RemoteFollow.new(resource_params).subscribe_address_for(@account))", "render_path": null, "location": { "type": "method", - "class": "MediaController", - "method": "show" + "class": "RemoteFollowController", + "method": "create" }, - "user_input": "MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original)", + "user_input": "RemoteFollow.new(resource_params).subscribe_address_for(@account)", "confidence": "High", "note": "" }, { "warning_type": "Redirect", "warning_code": 18, - "fingerprint": "bb7e94e60af41decb811bb32171f1b27e9bf3f4d01e9e511127362e22510eb11", + "fingerprint": "ba699ddcc6552c422c4ecd50d2cd217f616a2446659e185a50b05a0f2dad8d33", "check_name": "Redirect", "message": "Possible unprotected redirect", - "file": "app/controllers/remote_follow_controller.rb", - "line": 19, + "file": "app/controllers/media_controller.rb", + "line": 20, "link": "https://brakemanscanner.org/docs/warning_types/redirect/", - "code": "redirect_to(RemoteFollow.new(resource_params).subscribe_address_for(Account.find_local!(params[:account_username])))", + "code": "redirect_to(MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original))", "render_path": null, "location": { "type": "method", - "class": "RemoteFollowController", - "method": "create" + "class": "MediaController", + "method": "show" }, - "user_input": "RemoteFollow.new(resource_params).subscribe_address_for(Account.find_local!(params[:account_username]))", + "user_input": "MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original)", "confidence": "High", "note": "" }, @@ -275,27 +292,8 @@ "user_input": ":account_id", "confidence": "High", "note": "" - }, - { - "warning_type": "Dynamic Render Path", - "warning_code": 15, - "fingerprint": "fbd0fc59adb5c6d44b60e02debb31d3af11719f534c9881e21435bbff87404d6", - "check_name": "Render", - "message": "Render path contains parameter value", - "file": "app/views/stream_entries/show.html.haml", - "line": 23, - "link": "https://brakemanscanner.org/docs/warning_types/dynamic_render_path/", - "code": "render(partial => \"stream_entries/#{Account.find_local!(params[:account_username]).statuses.find(params[:id]).stream_entry.activity_type.downcase}\", { :locals => ({ Account.find_local!(params[:account_username]).statuses.find(params[:id]).stream_entry.activity_type.downcase.to_sym => Account.find_local!(params[:account_username]).statuses.find(params[:id]).stream_entry.activity, :include_threads => true }) })", - "render_path": [{"type":"controller","class":"StatusesController","method":"show","line":34,"file":"app/controllers/statuses_controller.rb","rendered":{"name":"stream_entries/show","file":"/home/eugr/Projects/mastodon/app/views/stream_entries/show.html.haml"}}], - "location": { - "type": "template", - "template": "stream_entries/show" - }, - "user_input": "params[:id]", - "confidence": "Weak", - "note": "" } ], - "updated": "2019-02-21 02:30:29 +0100", - "brakeman_version": "4.4.0" + "updated": "2020-06-01 18:18:02 +0200", + "brakeman_version": "4.8.0" } diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb index e03380cec..63cff7c59 100644 --- a/config/initializers/doorkeeper.rb +++ b/config/initializers/doorkeeper.rb @@ -95,7 +95,8 @@ Doorkeeper.configure do :'admin:read:reports', :'admin:write', :'admin:write:accounts', - :'admin:write:reports' + :'admin:write:reports', + :crypto # Change the way client credentials are retrieved from the request object. # By default it retrieves first from the `HTTP_AUTHORIZATION` header, then diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index 0667a542c..ebb7541eb 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -19,6 +19,7 @@ ActiveSupport::Inflector.inflections(:en) do |inflect| inflect.acronym 'ActivityStreams' inflect.acronym 'JsonLd' inflect.acronym 'NodeInfo' + inflect.acronym 'Ed25519' inflect.singular 'data', 'data' end diff --git a/config/locales/en.yml b/config/locales/en.yml index 116db4498..5943dd4f6 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -721,6 +721,10 @@ en: hint_html: "<strong>Tip:</strong> We won't ask you for your password again for the next hour." invalid_password: Invalid password prompt: Confirm password to continue + crypto: + errors: + invalid_key: is not a valid Ed25519 or Curve25519 key + invalid_signature: is not a valid Ed25519 signature date: formats: default: "%b %d, %Y" diff --git a/config/routes.rb b/config/routes.rb index 920a48fe7..31333999e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -79,6 +79,7 @@ Rails.application.routes.draw do resource :outbox, only: [:show], module: :activitypub resource :inbox, only: [:create], module: :activitypub + resource :claim, only: [:create], module: :activitypub resources :collections, only: [:show], module: :activitypub end @@ -339,6 +340,23 @@ Rails.application.routes.draw do end end + namespace :crypto do + resources :deliveries, only: :create + + namespace :keys do + resource :upload, only: [:create] + resource :query, only: [:create] + resource :claim, only: [:create] + resource :count, only: [:show] + end + + resources :encrypted_messages, only: [:index] do + collection do + post :clear + end + end + end + resources :conversations, only: [:index, :destroy] do member do post :read diff --git a/db/migrate/20170129000348_create_devices.rb b/db/migrate/20170129000348_create_devices.rb deleted file mode 100644 index bf8f5fc6e..000000000 --- a/db/migrate/20170129000348_create_devices.rb +++ /dev/null @@ -1,13 +0,0 @@ -class CreateDevices < ActiveRecord::Migration[5.0] - def change - create_table :devices do |t| - t.integer :account_id, null: false - t.string :registration_id, null: false, default: '' - - t.timestamps - end - - add_index :devices, :registration_id - add_index :devices, :account_id - end -end diff --git a/db/migrate/20170205175257_remove_devices.rb b/db/migrate/20170205175257_remove_devices.rb index e96ffed4d..9ef5c440e 100644 --- a/db/migrate/20170205175257_remove_devices.rb +++ b/db/migrate/20170205175257_remove_devices.rb @@ -1,5 +1,5 @@ class RemoveDevices < ActiveRecord::Migration[5.0] def change - drop_table :devices + drop_table :devices if table_exists?(:devices) end end diff --git a/db/migrate/20200516180352_create_devices.rb b/db/migrate/20200516180352_create_devices.rb new file mode 100644 index 000000000..04a628a89 --- /dev/null +++ b/db/migrate/20200516180352_create_devices.rb @@ -0,0 +1,14 @@ +class CreateDevices < ActiveRecord::Migration[5.2] + def change + create_table :devices do |t| + t.references :access_token, foreign_key: { to_table: :oauth_access_tokens, on_delete: :cascade, index: :unique } + t.references :account, foreign_key: { on_delete: :cascade } + t.string :device_id, default: '', null: false + t.string :name, default: '', null: false + t.text :fingerprint_key, default: '', null: false + t.text :identity_key, default: '', null: false + + t.timestamps + end + end +end diff --git a/db/migrate/20200516183822_create_one_time_keys.rb b/db/migrate/20200516183822_create_one_time_keys.rb new file mode 100644 index 000000000..642b9e632 --- /dev/null +++ b/db/migrate/20200516183822_create_one_time_keys.rb @@ -0,0 +1,12 @@ +class CreateOneTimeKeys < ActiveRecord::Migration[5.2] + def change + create_table :one_time_keys do |t| + t.references :device, foreign_key: { on_delete: :cascade } + t.string :key_id, default: '', null: false, index: :unique + t.text :key, default: '', null: false + t.text :signature, default: '', null: false + + t.timestamps + end + end +end diff --git a/db/migrate/20200518083523_create_encrypted_messages.rb b/db/migrate/20200518083523_create_encrypted_messages.rb new file mode 100644 index 000000000..486726303 --- /dev/null +++ b/db/migrate/20200518083523_create_encrypted_messages.rb @@ -0,0 +1,15 @@ +class CreateEncryptedMessages < ActiveRecord::Migration[5.2] + def change + create_table :encrypted_messages do |t| + t.references :device, foreign_key: { on_delete: :cascade } + t.references :from_account, foreign_key: { to_table: :accounts, on_delete: :cascade } + t.string :from_device_id, default: '', null: false + t.integer :type, default: 0, null: false + t.text :body, default: '', null: false + t.text :digest, default: '', null: false + t.text :message_franking, default: '', null: false + + t.timestamps + end + end +end diff --git a/db/migrate/20200521180606_encrypted_message_ids_to_timestamp_ids.rb b/db/migrate/20200521180606_encrypted_message_ids_to_timestamp_ids.rb new file mode 100644 index 000000000..24d43a0bf --- /dev/null +++ b/db/migrate/20200521180606_encrypted_message_ids_to_timestamp_ids.rb @@ -0,0 +1,13 @@ +class EncryptedMessageIdsToTimestampIds < ActiveRecord::Migration[5.2] + def up + safety_assured do + execute("ALTER TABLE encrypted_messages ALTER COLUMN id SET DEFAULT timestamp_id('encrypted_messages')") + end + end + + def down + execute("LOCK encrypted_messages") + execute("SELECT setval('encrypted_messages_id_seq', (SELECT MAX(id) FROM encrypted_messages))") + execute("ALTER TABLE encrypted_messages ALTER COLUMN id SET DEFAULT nextval('encrypted_messages_id_seq')") + end +end diff --git a/db/migrate/20200529214050_add_devices_url_to_accounts.rb b/db/migrate/20200529214050_add_devices_url_to_accounts.rb new file mode 100644 index 000000000..564877e5d --- /dev/null +++ b/db/migrate/20200529214050_add_devices_url_to_accounts.rb @@ -0,0 +1,5 @@ +class AddDevicesUrlToAccounts < ActiveRecord::Migration[5.2] + def change + add_column :accounts, :devices_url, :string + end +end diff --git a/db/migrate/20200601222558_create_system_keys.rb b/db/migrate/20200601222558_create_system_keys.rb new file mode 100644 index 000000000..fd9d221aa --- /dev/null +++ b/db/migrate/20200601222558_create_system_keys.rb @@ -0,0 +1,9 @@ +class CreateSystemKeys < ActiveRecord::Migration[5.2] + def change + create_table :system_keys do |t| + t.binary :key + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 31f0c96bc..e220e13fe 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_05_10_110808) do +ActiveRecord::Schema.define(version: 2020_06_01_222558) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -174,6 +174,7 @@ ActiveRecord::Schema.define(version: 2020_05_10_110808) do t.boolean "hide_collections" t.integer "avatar_storage_schema_version" t.integer "header_storage_schema_version" + t.string "devices_url" t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id" @@ -317,6 +318,19 @@ ActiveRecord::Schema.define(version: 2020_05_10_110808) do t.index ["account_id"], name: "index_custom_filters_on_account_id" end + create_table "devices", force: :cascade do |t| + t.bigint "access_token_id" + t.bigint "account_id" + t.string "device_id", default: "", null: false + t.string "name", default: "", null: false + t.text "fingerprint_key", default: "", null: false + t.text "identity_key", default: "", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["access_token_id"], name: "index_devices_on_access_token_id" + t.index ["account_id"], name: "index_devices_on_account_id" + end + create_table "domain_allows", force: :cascade do |t| t.string "domain", default: "", null: false t.datetime "created_at", null: false @@ -344,6 +358,20 @@ ActiveRecord::Schema.define(version: 2020_05_10_110808) do t.index ["domain"], name: "index_email_domain_blocks_on_domain", unique: true end + create_table "encrypted_messages", id: :bigint, default: -> { "timestamp_id('encrypted_messages'::text)" }, force: :cascade do |t| + t.bigint "device_id" + t.bigint "from_account_id" + t.string "from_device_id", default: "", null: false + t.integer "type", default: 0, null: false + t.text "body", default: "", null: false + t.text "digest", default: "", null: false + t.text "message_franking", default: "", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["device_id"], name: "index_encrypted_messages_on_device_id" + t.index ["from_account_id"], name: "index_encrypted_messages_on_from_account_id" + end + create_table "favourites", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -551,6 +579,17 @@ ActiveRecord::Schema.define(version: 2020_05_10_110808) do t.index ["uid"], name: "index_oauth_applications_on_uid", unique: true end + create_table "one_time_keys", force: :cascade do |t| + t.bigint "device_id" + t.string "key_id", default: "", null: false + t.text "key", default: "", null: false + t.text "signature", default: "", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["device_id"], name: "index_one_time_keys_on_device_id" + t.index ["key_id"], name: "index_one_time_keys_on_key_id" + end + create_table "pghero_space_stats", force: :cascade do |t| t.text "database" t.text "schema" @@ -749,6 +788,12 @@ ActiveRecord::Schema.define(version: 2020_05_10_110808) do t.index ["tag_id", "status_id"], name: "index_statuses_tags_on_tag_id_and_status_id", unique: true end + create_table "system_keys", force: :cascade do |t| + t.binary "key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "tags", force: :cascade do |t| t.string "name", default: "", null: false t.datetime "created_at", null: false @@ -883,7 +928,11 @@ ActiveRecord::Schema.define(version: 2020_05_10_110808) do add_foreign_key "conversation_mutes", "accounts", name: "fk_225b4212bb", on_delete: :cascade add_foreign_key "conversation_mutes", "conversations", on_delete: :cascade add_foreign_key "custom_filters", "accounts", on_delete: :cascade + add_foreign_key "devices", "accounts", on_delete: :cascade + add_foreign_key "devices", "oauth_access_tokens", column: "access_token_id", on_delete: :cascade add_foreign_key "email_domain_blocks", "email_domain_blocks", column: "parent_id", on_delete: :cascade + add_foreign_key "encrypted_messages", "accounts", column: "from_account_id", on_delete: :cascade + add_foreign_key "encrypted_messages", "devices", on_delete: :cascade add_foreign_key "favourites", "accounts", name: "fk_5eb6c2b873", on_delete: :cascade add_foreign_key "favourites", "statuses", name: "fk_b0e856845e", on_delete: :cascade add_foreign_key "featured_tags", "accounts", on_delete: :cascade @@ -914,6 +963,7 @@ ActiveRecord::Schema.define(version: 2020_05_10_110808) do add_foreign_key "oauth_access_tokens", "oauth_applications", column: "application_id", name: "fk_f5fc4c1ee3", on_delete: :cascade add_foreign_key "oauth_access_tokens", "users", column: "resource_owner_id", name: "fk_e84df68546", on_delete: :cascade add_foreign_key "oauth_applications", "users", column: "owner_id", name: "fk_b0988c7c0a", on_delete: :cascade + add_foreign_key "one_time_keys", "devices", on_delete: :cascade add_foreign_key "poll_votes", "accounts", on_delete: :cascade add_foreign_key "poll_votes", "polls", on_delete: :cascade add_foreign_key "polls", "accounts", on_delete: :cascade diff --git a/spec/fabricators/device_fabricator.rb b/spec/fabricators/device_fabricator.rb new file mode 100644 index 000000000..b15d8248f --- /dev/null +++ b/spec/fabricators/device_fabricator.rb @@ -0,0 +1,8 @@ +Fabricator(:device) do + access_token + account + device_id { Faker::Number.number(digits: 5) } + name { Faker::App.name } + fingerprint_key { Base64.strict_encode64(Ed25519::SigningKey.generate.verify_key.to_bytes) } + identity_key { Base64.strict_encode64(Ed25519::SigningKey.generate.verify_key.to_bytes) } +end diff --git a/spec/fabricators/encrypted_message_fabricator.rb b/spec/fabricators/encrypted_message_fabricator.rb new file mode 100644 index 000000000..e65f66302 --- /dev/null +++ b/spec/fabricators/encrypted_message_fabricator.rb @@ -0,0 +1,8 @@ +Fabricator(:encrypted_message) do + device + from_account + from_device_id { Faker::Number.number(digits: 5) } + type 0 + body "" + message_franking "" +end diff --git a/spec/fabricators/one_time_key_fabricator.rb b/spec/fabricators/one_time_key_fabricator.rb new file mode 100644 index 000000000..8794baeb5 --- /dev/null +++ b/spec/fabricators/one_time_key_fabricator.rb @@ -0,0 +1,11 @@ +Fabricator(:one_time_key) do + device + key_id { Faker::Alphanumeric.alphanumeric(number: 10) } + key { Base64.strict_encode64(Ed25519::SigningKey.generate.verify_key.to_bytes) } + + signature do |attrs| + signing_key = Ed25519::SigningKey.generate + attrs[:device].update(fingerprint_key: Base64.strict_encode64(signing_key.verify_key.to_bytes)) + Base64.strict_encode64(signing_key.sign(attrs[:key])) + end +end diff --git a/spec/fabricators/system_key_fabricator.rb b/spec/fabricators/system_key_fabricator.rb new file mode 100644 index 000000000..f808495e0 --- /dev/null +++ b/spec/fabricators/system_key_fabricator.rb @@ -0,0 +1,3 @@ +Fabricator(:system_key) do + +end diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index 5220deabb..2ac4acc12 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -579,6 +579,62 @@ RSpec.describe ActivityPub::Activity::Create do end end + context 'with an encrypted message' do + let(:recipient) { Fabricate(:account) } + let(:target_device) { Fabricate(:device, account: recipient) } + + subject { described_class.new(json, sender, delivery: true, delivered_to_account_id: recipient.id) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'EncryptedMessage', + attributedTo: { + type: 'Device', + deviceId: '1234', + }, + to: { + type: 'Device', + deviceId: target_device.device_id, + }, + messageType: 1, + cipherText: 'Foo', + messageFranking: 'Baz678', + digest: { + digestAlgorithm: 'Bar456', + digestValue: 'Foo123', + }, + } + end + + before do + subject.perform + end + + it 'creates an encrypted message' do + encrypted_message = target_device.encrypted_messages.reload.first + + expect(encrypted_message).to_not be_nil + expect(encrypted_message.from_device_id).to eq '1234' + expect(encrypted_message.from_account).to eq sender + expect(encrypted_message.type).to eq 1 + expect(encrypted_message.body).to eq 'Foo' + expect(encrypted_message.digest).to eq 'Foo123' + end + + it 'creates a message franking' do + encrypted_message = target_device.encrypted_messages.reload.first + message_franking = encrypted_message.message_franking + + crypt = ActiveSupport::MessageEncryptor.new(SystemKey.current_key, serializer: Oj) + json = crypt.decrypt_and_verify(message_franking) + + expect(json['source_account_id']).to eq sender.id + expect(json['target_account_id']).to eq recipient.id + expect(json['original_franking']).to eq 'Baz678' + end + end + context 'when sender is followed by local users' do subject { described_class.new(json, sender, delivery: true) } diff --git a/spec/models/device_spec.rb b/spec/models/device_spec.rb new file mode 100644 index 000000000..f56fbf978 --- /dev/null +++ b/spec/models/device_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe Device, type: :model do + +end diff --git a/spec/models/encrypted_message_spec.rb b/spec/models/encrypted_message_spec.rb new file mode 100644 index 000000000..1238d57b6 --- /dev/null +++ b/spec/models/encrypted_message_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe EncryptedMessage, type: :model do + +end diff --git a/spec/models/one_time_key_spec.rb b/spec/models/one_time_key_spec.rb new file mode 100644 index 000000000..34598334c --- /dev/null +++ b/spec/models/one_time_key_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe OneTimeKey, type: :model do + +end diff --git a/spec/models/system_key_spec.rb b/spec/models/system_key_spec.rb new file mode 100644 index 000000000..a138bc131 --- /dev/null +++ b/spec/models/system_key_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe SystemKey, type: :model do + +end diff --git a/streaming/index.js b/streaming/index.js index 500d577ce..d7b1df81f 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -144,13 +144,21 @@ const startWorker = (workerId) => { callbacks.forEach(callback => callback(message)); }); - const subscriptionHeartbeat = (channel) => { - const interval = 6*60; + const subscriptionHeartbeat = channels => { + if (!Array.isArray(channels)) { + channels = [channels]; + } + + const interval = 6 * 60; + const tellSubscribed = () => { - redisClient.set(`${redisPrefix}subscribed:${channel}`, '1', 'EX', interval*3); + channels.forEach(channel => redisClient.set(`${redisPrefix}subscribed:${channel}`, '1', 'EX', interval * 3)); }; + tellSubscribed(); - const heartbeat = setInterval(tellSubscribed, interval*1000); + + const heartbeat = setInterval(tellSubscribed, interval * 1000); + return () => { clearInterval(heartbeat); }; @@ -203,7 +211,7 @@ const startWorker = (workerId) => { return; } - client.query('SELECT oauth_access_tokens.resource_owner_id, users.account_id, users.chosen_languages, oauth_access_tokens.scopes FROM oauth_access_tokens INNER JOIN users ON oauth_access_tokens.resource_owner_id = users.id WHERE oauth_access_tokens.token = $1 AND oauth_access_tokens.revoked_at IS NULL LIMIT 1', [token], (err, result) => { + client.query('SELECT oauth_access_tokens.resource_owner_id, users.account_id, users.chosen_languages, oauth_access_tokens.scopes, devices.device_id FROM oauth_access_tokens INNER JOIN users ON oauth_access_tokens.resource_owner_id = users.id LEFT OUTER JOIN devices ON oauth_access_tokens.id = devices.access_token_id WHERE oauth_access_tokens.token = $1 AND oauth_access_tokens.revoked_at IS NULL LIMIT 1', [token], (err, result) => { done(); if (err) { @@ -232,6 +240,7 @@ const startWorker = (workerId) => { req.accountId = result.rows[0].account_id; req.chosenLanguages = result.rows[0].chosen_languages; req.allowNotifications = scopes.some(scope => ['read', 'read:notifications'].includes(scope)); + req.deviceId = result.rows[0].device_id; next(); }); @@ -353,11 +362,15 @@ const startWorker = (workerId) => { }); }; - const streamFrom = (id, req, output, attachCloseHandler, needsFiltering = false, notificationOnly = false) => { - const accountId = req.accountId || req.remoteAddress; - + const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false, notificationOnly = false) => { + const accountId = req.accountId || req.remoteAddress; const streamType = notificationOnly ? ' (notification)' : ''; - log.verbose(req.requestId, `Starting stream from ${id} for ${accountId}${streamType}`); + + if (!Array.isArray(ids)) { + ids = [ids]; + } + + log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}${streamType}`); const listener = message => { const { event, payload, queued_at } = JSON.parse(message); @@ -430,8 +443,11 @@ const startWorker = (workerId) => { }); }; - subscribe(`${redisPrefix}${id}`, listener); - attachCloseHandler(`${redisPrefix}${id}`, listener); + ids.forEach(id => { + subscribe(`${redisPrefix}${id}`, listener); + }); + + attachCloseHandler(ids.map(id => `${redisPrefix}${id}`), listener); }; // Setup stream output to HTTP @@ -458,9 +474,16 @@ const startWorker = (workerId) => { }; // Setup stream end for HTTP - const streamHttpEnd = (req, closeHandler = false) => (id, listener) => { + const streamHttpEnd = (req, closeHandler = false) => (ids, listener) => { + if (!Array.isArray(ids)) { + ids = [ids]; + } + req.on('close', () => { - unsubscribe(id, listener); + ids.forEach(id => { + unsubscribe(id, listener); + }); + if (closeHandler) { closeHandler(); } @@ -516,8 +539,13 @@ const startWorker = (workerId) => { app.use(errorMiddleware); app.get('/api/v1/streaming/user', (req, res) => { - const channel = `timeline:${req.accountId}`; - streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel))); + const channels = [`timeline:${req.accountId}`]; + + if (req.deviceId) { + channels.push(`timeline:${req.accountId}:${req.deviceId}`); + } + + streamFrom(channels, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channels))); }); app.get('/api/v1/streaming/user/notification', (req, res) => { @@ -597,7 +625,12 @@ const startWorker = (workerId) => { switch(location.query.stream) { case 'user': - channel = `timeline:${req.accountId}`; + channel = [`timeline:${req.accountId}`]; + + if (req.deviceId) { + channel.push(`timeline:${req.accountId}:${req.deviceId}`); + } + streamFrom(channel, req, streamToWs(req, ws), streamWsEnd(req, ws, subscriptionHeartbeat(channel))); break; case 'user:notification': |