From 0ef9d45d0581dddf2f325033c43721f42fcfca9e Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 11 Sep 2017 23:50:37 +0200 Subject: Fix error when following locked accounts (#4896) --- .../controllers/api/v1/accounts_controller_spec.rb | 43 +++++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) (limited to 'spec') diff --git a/spec/controllers/api/v1/accounts_controller_spec.rb b/spec/controllers/api/v1/accounts_controller_spec.rb index 05df2f844..c770649ec 100644 --- a/spec/controllers/api/v1/accounts_controller_spec.rb +++ b/spec/controllers/api/v1/accounts_controller_spec.rb @@ -18,25 +18,48 @@ RSpec.describe Api::V1::AccountsController, type: :controller do end describe 'POST #follow' do - let(:other_account) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } + let(:other_account) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob', locked: locked)).account } before do post :follow, params: { id: other_account.id } end - it 'returns http success' do - expect(response).to have_http_status(:success) - end + context 'with unlocked account' do + let(:locked) { false } + + it 'returns http success' do + expect(response).to have_http_status(:success) + end + + it 'returns JSON with following=true and requested=false' do + json = body_as_json - it 'returns JSON with following=true and requested=false' do - json = body_as_json + expect(json[:following]).to be true + expect(json[:requested]).to be false + end - expect(json[:following]).to be true - expect(json[:requested]).to be false + it 'creates a following relation between user and target user' do + expect(user.account.following?(other_account)).to be true + end end - it 'creates a following relation between user and target user' do - expect(user.account.following?(other_account)).to be true + context 'with locked account' do + let(:locked) { true } + + it 'returns http success' do + expect(response).to have_http_status(:success) + end + + it 'returns JSON with following=false and requested=true' do + json = body_as_json + + expect(json[:following]).to be false + expect(json[:requested]).to be true + end + + it 'creates a follow request relation between user and target user' do + expect(user.account.requested?(other_account)).to be true + end end end -- cgit From 8f8e6776306b07d5914ef39c895de0cab44ada7f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 12 Sep 2017 05:39:38 +0200 Subject: Clean up and improve generated OpenGraph tags (#4901) - Return all images as og:image - Return videos as og:image (preview) and og:video - Return profile:username on profiles --- app/helpers/application_helper.rb | 4 ++++ app/serializers/oembed_serializer.rb | 2 +- app/views/accounts/_og.html.haml | 17 +++++++++-------- app/views/accounts/show.html.haml | 2 +- app/views/stream_entries/_og_description.html.haml | 4 ++-- app/views/stream_entries/_og_image.html.haml | 22 ++++++++++++++++++---- app/views/stream_entries/show.html.haml | 10 +++++----- spec/views/stream_entries/show.html.haml_spec.rb | 8 ++++---- 8 files changed, 44 insertions(+), 25 deletions(-) (limited to 'spec') diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 61d4442c1..6d625e7db 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -42,4 +42,8 @@ module ApplicationHelper content_tag(:i, nil, attributes.merge(class: class_names.join(' '))) end + + def opengraph(property, content) + tag(:meta, content: content, property: property) + end end diff --git a/app/serializers/oembed_serializer.rb b/app/serializers/oembed_serializer.rb index af03fd47a..0c8350e2d 100644 --- a/app/serializers/oembed_serializer.rb +++ b/app/serializers/oembed_serializer.rb @@ -45,7 +45,7 @@ class OEmbedSerializer < ActiveModel::Serializer height: height, } - content_tag(:iframe, nil, attributes) + content_tag(:script, nil, src: full_asset_url('embed.js'), async: true) + content_tag(:iframe, nil, attributes) + content_tag(:script, nil, src: full_asset_url('embed.js', skip_pipeline: true), async: true) end def width diff --git a/app/views/accounts/_og.html.haml b/app/views/accounts/_og.html.haml index 3ad39f391..1d16be590 100644 --- a/app/views/accounts/_og.html.haml +++ b/app/views/accounts/_og.html.haml @@ -1,8 +1,9 @@ -%meta{ property: 'og:url', content: url }/ -%meta{ property: 'og:site_name', content: site_title }/ -%meta{ property: 'og:title', content: [yield(:page_title).strip.presence, site_title].compact.join(' - ') }/ -%meta{ property: 'og:description', content: account.note }/ -%meta{ property: 'og:image', content: full_asset_url(account.avatar.url(:original)) }/ -%meta{ property: 'og:image:width', content: '120' }/ -%meta{ property: 'og:image:height', content: '120' }/ -%meta{ property: 'twitter:card', content: 'summary' }/ += opengraph 'og:url', url += opengraph 'og:site_name', site_title += opengraph 'og:title', [yield(:page_title).strip.presence, site_title].compact.join(' - ') += opengraph 'og:description', account.note += opengraph 'og:image', full_asset_url(account.avatar.url(:original)) += opengraph 'og:image:width', '120' += opengraph 'og:image:height', '120' += opengraph 'twitter:card', 'summary' += opengraph 'profile:username', account.local_username_and_domain diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml index e0f9f869a..6c90b2c04 100644 --- a/app/views/accounts/show.html.haml +++ b/app/views/accounts/show.html.haml @@ -9,7 +9,7 @@ %link{ rel: 'alternate', type: 'application/atom+xml', href: account_url(@account, format: 'atom') }/ %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@account) }/ - %meta{ property: 'og:type', content: 'profile' }/ + = opengraph 'og:type', 'profile' = render 'og', account: @account, url: short_account_url(@account, only_path: false) - if show_landing_strip? diff --git a/app/views/stream_entries/_og_description.html.haml b/app/views/stream_entries/_og_description.html.haml index 5762aca04..d2fa99e63 100644 --- a/app/views/stream_entries/_og_description.html.haml +++ b/app/views/stream_entries/_og_description.html.haml @@ -1,4 +1,4 @@ - if activity.is_a?(Status) && activity.spoiler_text? - %meta{ property: 'og:description', content: activity.spoiler_text }/ + = opengraph 'og:description', activity.spoiler_text - else - %meta{ property: 'og:description', content: activity.content }/ + = opengraph 'og:description', activity.content diff --git a/app/views/stream_entries/_og_image.html.haml b/app/views/stream_entries/_og_image.html.haml index f725209d8..66d9619b2 100644 --- a/app/views/stream_entries/_og_image.html.haml +++ b/app/views/stream_entries/_og_image.html.haml @@ -1,6 +1,20 @@ - if activity.is_a?(Status) && activity.non_sensitive_with_media? - %meta{ property: 'og:image', content: full_asset_url(activity.media_attachments.first.file.url(:small)) }/ + - activity.media_attachments.each do |media| + - if media.image? + = opengraph 'og:image', full_asset_url(media.file.url(:original)) + = opengraph 'og:image:type', media.file_content_type + = opengraph 'og:image:width', media.file.meta['original']['width'] + = opengraph 'og:image:height', media.file.meta['original']['height'] + - elsif media.video? + = opengraph 'og:image', full_asset_url(media.file.url(:small)) + = opengraph 'og:image:type', 'image/png' + = opengraph 'og:image:width', media.file.meta['small']['width'] + = opengraph 'og:image:height', media.file.meta['small']['height'] + = opengraph 'og:video', full_asset_url(media.file.url(:original)) + = opengraph 'og:video:type', media.file_content_type + = opengraph 'og:video:width', media.file.meta['small']['width'] + = opengraph 'og:video:height', media.file.meta['small']['height'] - else - %meta{ property: 'og:image', content: full_asset_url(account.avatar.url(:original)) }/ - %meta{ property: 'og:image:width', content: '120' }/ - %meta{ property: 'og:image:height', content: '120' }/ + = opengraph 'og:image', full_asset_url(account.avatar.url(:original)) + = opengraph 'og:image:width', '120' + = opengraph 'og:image:height','120' diff --git a/app/views/stream_entries/show.html.haml b/app/views/stream_entries/show.html.haml index 5ef72f804..1bb8a32b2 100644 --- a/app/views/stream_entries/show.html.haml +++ b/app/views/stream_entries/show.html.haml @@ -6,15 +6,15 @@ %link{ rel: 'alternate', type: 'application/json+oembed', href: api_oembed_url(url: account_stream_entry_url(@account, @stream_entry), format: 'json') }/ %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@stream_entry.activity) }/ - %meta{ property: 'og:site_name', content: site_title }/ - %meta{ property: 'og:type', content: 'article' }/ - %meta{ property: 'og:title', content: "#{@account.username} on #{site_hostname}" }/ - %meta{ property: 'og:url', content: account_stream_entry_url(@account, @stream_entry) }/ + = opengraph 'og:site_name', site_title + = opengraph 'og:type', 'article' + = opengraph 'og:title', "#{@account.username} on #{site_hostname}" + = opengraph 'og:url', account_stream_entry_url(@account, @stream_entry) = render 'stream_entries/og_description', activity: @stream_entry.activity = render 'stream_entries/og_image', activity: @stream_entry.activity, account: @account - %meta{ property: 'twitter:card', content: 'summary' }/ + = opengraph 'twitter:card', 'summary_large_image' - if show_landing_strip? = render partial: 'shared/landing_strip', locals: { account: @stream_entry.account } diff --git a/spec/views/stream_entries/show.html.haml_spec.rb b/spec/views/stream_entries/show.html.haml_spec.rb index 6cc3b117a..59ea40990 100644 --- a/spec/views/stream_entries/show.html.haml_spec.rb +++ b/spec/views/stream_entries/show.html.haml_spec.rb @@ -80,9 +80,9 @@ describe 'stream_entries/show.html.haml', without_verify_partial_doubles: true d header_tags = view.content_for(:header_tags) - expect(header_tags).to match(%r{}) - expect(header_tags).to match(%r{}) - expect(header_tags).to match(%r{}) - expect(header_tags).to match(%r{}) + expect(header_tags).to match(%r{}) + expect(header_tags).to match(%r{}) + expect(header_tags).to match(%r{}) + expect(header_tags).to match(%r{}) end end -- cgit From 550ff677daef92886ab7440ad262821f4c732e85 Mon Sep 17 00:00:00 2001 From: ThibG Date: Wed, 13 Sep 2017 14:22:16 +0200 Subject: Fix ActivityPub handling of replies with WEB_DOMAIN (#4895) (#4904) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix ActivityPub handling of replies when LOCAL_DOMAIN ≠ WEB_DOMAIN (#4895) For all intents and purposes, `local_url?` is used to check if an URL refers to the Web UI or the various API endpoints of the local instances. Those things reside on `WEB_DOMAIN` and not `LOCAL_DOMAIN`. * Change local_url? spec, as all URLs handled by Mastodon are based on WEB_DOMAIN --- app/lib/tag_manager.rb | 2 +- spec/lib/tag_manager_spec.rb | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'spec') diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb index f33a20c6f..1d0a24e42 100644 --- a/app/lib/tag_manager.rb +++ b/app/lib/tag_manager.rb @@ -87,7 +87,7 @@ class TagManager def local_url?(url) uri = Addressable::URI.parse(url).normalize domain = uri.host + (uri.port ? ":#{uri.port}" : '') - TagManager.instance.local_domain?(domain) + TagManager.instance.web_domain?(domain) end def uri_for(target) diff --git a/spec/lib/tag_manager_spec.rb b/spec/lib/tag_manager_spec.rb index 1cd6e0a6f..6c7830231 100644 --- a/spec/lib/tag_manager_spec.rb +++ b/spec/lib/tag_manager_spec.rb @@ -63,23 +63,23 @@ RSpec.describe TagManager do describe '#local_url?' do around do |example| - original_local_domain = Rails.configuration.x.local_domain + original_web_domain = Rails.configuration.x.web_domain example.run - Rails.configuration.x.local_domain = original_local_domain + Rails.configuration.x.web_domain = original_web_domain end it 'returns true if the normalized string with port is local URL' do - Rails.configuration.x.local_domain = 'domain:42' + Rails.configuration.x.web_domain = 'domain:42' expect(TagManager.instance.local_url?('https://DoMaIn:42/')).to eq true end it 'returns true if the normalized string without port is local URL' do - Rails.configuration.x.local_domain = 'domain' + Rails.configuration.x.web_domain = 'domain' expect(TagManager.instance.local_url?('https://DoMaIn/')).to eq true end it 'returns false for string with irrelevant characters' do - Rails.configuration.x.local_domain = 'domain' + Rails.configuration.x.web_domain = 'domain' expect(TagManager.instance.local_url?('https://domainn/')).to eq false end end -- cgit From 9239e4ce4d4e958e62552d4a01183d0295c020f5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 14 Sep 2017 00:04:30 +0200 Subject: Uploads for admin site settings (#4913) * Improve OpenGraph tags for about pages * Add thumbnail admin setting * Fix error * Fix up --- app/controllers/admin/settings_controller.rb | 14 ++++++-- app/javascript/images/mastodon_small.jpg | Bin 25199 -> 0 bytes app/javascript/images/preview.jpg | Bin 0 -> 292252 bytes app/models/site_upload.rb | 44 +++++++++++++++++++++++ app/presenters/instance_presenter.rb | 4 +++ app/serializers/rest/instance_serializer.rb | 8 ++++- app/views/about/_og.html.haml | 10 ++++++ app/views/about/more.html.haml | 11 +----- app/views/about/show.html.haml | 11 +----- app/views/admin/settings/edit.html.haml | 5 +++ config/locales/en.yml | 3 ++ db/migrate/20170913000752_create_site_uploads.rb | 10 ++++++ db/schema.rb | 14 +++++++- spec/fabricators/site_upload_fabricator.rb | 3 ++ spec/models/site_upload_spec.rb | 5 +++ spec/views/about/show.html.haml_spec.rb | 9 ++--- 16 files changed, 123 insertions(+), 28 deletions(-) delete mode 100644 app/javascript/images/mastodon_small.jpg create mode 100644 app/javascript/images/preview.jpg create mode 100644 app/models/site_upload.rb create mode 100644 app/views/about/_og.html.haml create mode 100644 db/migrate/20170913000752_create_site_uploads.rb create mode 100644 spec/fabricators/site_upload_fabricator.rb create mode 100644 spec/models/site_upload_spec.rb (limited to 'spec') diff --git a/app/controllers/admin/settings_controller.rb b/app/controllers/admin/settings_controller.rb index c5e6fe4e5..a2f86b8a9 100644 --- a/app/controllers/admin/settings_controller.rb +++ b/app/controllers/admin/settings_controller.rb @@ -14,6 +14,7 @@ module Admin open_deletion timeline_preview bootstrap_timeline_accounts + thumbnail ).freeze BOOLEAN_SETTINGS = %w( @@ -22,14 +23,23 @@ module Admin timeline_preview ).freeze + UPLOAD_SETTINGS = %w( + thumbnail + ).freeze + def edit @admin_settings = Form::AdminSettings.new end def update settings_params.each do |key, value| - setting = Setting.where(var: key).first_or_initialize(var: key) - setting.update(value: value_for_update(key, value)) + if UPLOAD_SETTINGS.include?(key) + upload = SiteUpload.where(var: key).first_or_initialize(var: key) + upload.update(file: value) + else + setting = Setting.where(var: key).first_or_initialize(var: key) + setting.update(value: value_for_update(key, value)) + end end flash[:notice] = I18n.t('generic.changes_saved_msg') diff --git a/app/javascript/images/mastodon_small.jpg b/app/javascript/images/mastodon_small.jpg deleted file mode 100644 index 9c88ce3f7..000000000 Binary files a/app/javascript/images/mastodon_small.jpg and /dev/null differ diff --git a/app/javascript/images/preview.jpg b/app/javascript/images/preview.jpg new file mode 100644 index 000000000..ec2856748 Binary files /dev/null and b/app/javascript/images/preview.jpg differ diff --git a/app/models/site_upload.rb b/app/models/site_upload.rb new file mode 100644 index 000000000..8ffdc8313 --- /dev/null +++ b/app/models/site_upload.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: site_uploads +# +# id :integer not null, primary key +# var :string default(""), not null +# file_file_name :string +# file_content_type :string +# file_file_size :integer +# file_updated_at :datetime +# meta :json +# created_at :datetime not null +# updated_at :datetime not null +# + +class SiteUpload < ApplicationRecord + has_attached_file :file + + validates_attachment_content_type :file, content_type: /\Aimage\/.*\z/ + validates :var, presence: true, uniqueness: true + + before_save :set_meta + after_commit :clear_cache + + def cache_key + "site_uploads/#{var}" + end + + private + + def set_meta + tempfile = file.queued_for_write[:original] + + return if tempfile.nil? + + geometry = Paperclip::Geometry.from_file(tempfile) + self.meta = { width: geometry.width.to_i, height: geometry.height.to_i } + end + + def clear_cache + Rails.cache.delete(cache_key) + end +end diff --git a/app/presenters/instance_presenter.rb b/app/presenters/instance_presenter.rb index 8104b7531..c9e3c31a1 100644 --- a/app/presenters/instance_presenter.rb +++ b/app/presenters/instance_presenter.rb @@ -35,4 +35,8 @@ class InstancePresenter def source_url Mastodon::Version.source_url end + + def thumbnail + @thumbnail ||= Rails.cache.fetch('site_uploads/thumbnail') { SiteUpload.find_by(var: 'thumbnail') } + end end diff --git a/app/serializers/rest/instance_serializer.rb b/app/serializers/rest/instance_serializer.rb index a97137909..2898011fd 100644 --- a/app/serializers/rest/instance_serializer.rb +++ b/app/serializers/rest/instance_serializer.rb @@ -1,8 +1,10 @@ # frozen_string_literal: true class REST::InstanceSerializer < ActiveModel::Serializer + include RoutingHelper + attributes :uri, :title, :description, :email, - :version, :urls, :stats + :version, :urls, :stats, :thumbnail def uri Rails.configuration.x.local_domain @@ -24,6 +26,10 @@ class REST::InstanceSerializer < ActiveModel::Serializer Mastodon::Version.to_s end + def thumbnail + full_asset_url(instance_presenter.thumbnail.file.url) if instance_presenter.thumbnail + end + def stats { user_count: instance_presenter.user_count, diff --git a/app/views/about/_og.html.haml b/app/views/about/_og.html.haml new file mode 100644 index 000000000..dbd476915 --- /dev/null +++ b/app/views/about/_og.html.haml @@ -0,0 +1,10 @@ +- thumbnail = @instance_presenter.thumbnail += opengraph 'og:site_name', t('about.hosted_on', domain: site_hostname) += opengraph 'og:url', about_url += opengraph 'og:type', 'website' += opengraph 'og:title', @instance_presenter.site_title += opengraph 'og:description', strip_tags(@instance_presenter.site_description.presence || t('about.about_mastodon_html')) += opengraph 'og:image', full_asset_url(thumbnail&.file&.url || asset_pack_path('preview.jpg', protocol: :request)) += opengraph 'og:image:width', thumbnail ? thumbnail.meta['width'] : '1200' += opengraph 'og:image:height', thumbnail ? thumbnail.meta['height'] : '630' += opengraph 'twitter:card', 'summary_large_image' diff --git a/app/views/about/more.html.haml b/app/views/about/more.html.haml index 094188472..1a4e74643 100644 --- a/app/views/about/more.html.haml +++ b/app/views/about/more.html.haml @@ -3,16 +3,7 @@ - content_for :header_tags do = javascript_pack_tag 'public', integrity: true, crossorigin: 'anonymous' - - %meta{ property: 'og:site_name', content: site_title }/ - %meta{ property: 'og:url', content: about_url }/ - %meta{ property: 'og:type', content: 'website' }/ - %meta{ property: 'og:title', content: site_hostname }/ - %meta{ property: 'og:description', content: strip_tags(@instance_presenter.site_description.presence || t('about.about_mastodon_html')) }/ - %meta{ property: 'og:image', content: asset_pack_path('mastodon_small.jpg', protocol: :request) }/ - %meta{ property: 'og:image:width', content: '400' }/ - %meta{ property: 'og:image:height', content: '400' }/ - %meta{ property: 'twitter:card', content: 'summary' }/ + = render partial: 'og' .landing-page .header-wrapper.compact diff --git a/app/views/about/show.html.haml b/app/views/about/show.html.haml index 93270fe3d..0d311b895 100644 --- a/app/views/about/show.html.haml +++ b/app/views/about/show.html.haml @@ -4,16 +4,7 @@ - content_for :header_tags do %script#initial-state{ type: 'application/json' }!= json_escape(@initial_state_json) = javascript_pack_tag 'about', integrity: true, crossorigin: 'anonymous' - - %meta{ property: 'og:site_name', content: site_title }/ - %meta{ property: 'og:url', content: about_url }/ - %meta{ property: 'og:type', content: 'website' }/ - %meta{ property: 'og:title', content: site_hostname }/ - %meta{ property: 'og:description', content: strip_tags(@instance_presenter.site_description.presence || t('about.about_mastodon_html')) }/ - %meta{ property: 'og:image', content: asset_pack_path('mastodon_small.jpg', protocol: :request) }/ - %meta{ property: 'og:image:width', content: '400' }/ - %meta{ property: 'og:image:height', content: '400' }/ - %meta{ property: 'twitter:card', content: 'summary' }/ + = render partial: 'og' .landing-page .header-wrapper diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml index 50d019ec4..468166035 100644 --- a/app/views/admin/settings/edit.html.haml +++ b/app/views/admin/settings/edit.html.haml @@ -10,6 +10,11 @@ %hr/ + .fields-group + = f.input :thumbnail, as: :file, wrapper: :with_block_label, label: t('admin.settings.thumbnail.title'), hint: t('admin.settings.thumbnail.desc_html') + + %hr/ + .fields-group = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html') diff --git a/config/locales/en.yml b/config/locales/en.yml index 0cd57a64f..8b5f52b02 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -195,6 +195,9 @@ en: desc_html: You can write your own privacy policy, terms of service or other legalese. You can use HTML tags title: Custom terms of service site_title: Instance name + thumbnail: + desc_html: Used for previews via OpenGraph and API. 1200x630px recommended + title: Instance thumbnail timeline_preview: desc_html: Display public timeline on landing page title: Timeline preview diff --git a/db/migrate/20170913000752_create_site_uploads.rb b/db/migrate/20170913000752_create_site_uploads.rb new file mode 100644 index 000000000..2246e48cd --- /dev/null +++ b/db/migrate/20170913000752_create_site_uploads.rb @@ -0,0 +1,10 @@ +class CreateSiteUploads < ActiveRecord::Migration[5.1] + def change + create_table :site_uploads do |t| + t.string :var, default: '', null: false, index: { unique: true } + t.attachment :file + t.json :meta + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index d8af0a1f8..f2ca2af69 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170905165803) do +ActiveRecord::Schema.define(version: 20170913000752) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -288,6 +288,18 @@ ActiveRecord::Schema.define(version: 20170905165803) do t.index ["thing_type", "thing_id", "var"], name: "index_settings_on_thing_type_and_thing_id_and_var", unique: true end + create_table "site_uploads", force: :cascade do |t| + t.string "var", default: "", null: false + t.string "file_file_name" + t.string "file_content_type" + t.integer "file_file_size" + t.datetime "file_updated_at" + t.json "meta" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["var"], name: "index_site_uploads_on_var", unique: true + end + create_table "status_pins", force: :cascade do |t| t.bigint "account_id", null: false t.bigint "status_id", null: false diff --git a/spec/fabricators/site_upload_fabricator.rb b/spec/fabricators/site_upload_fabricator.rb new file mode 100644 index 000000000..8f4e43ac9 --- /dev/null +++ b/spec/fabricators/site_upload_fabricator.rb @@ -0,0 +1,3 @@ +Fabricator(:site_upload) do + +end diff --git a/spec/models/site_upload_spec.rb b/spec/models/site_upload_spec.rb new file mode 100644 index 000000000..8745d54b8 --- /dev/null +++ b/spec/models/site_upload_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe SiteUpload, type: :model do + +end diff --git a/spec/views/about/show.html.haml_spec.rb b/spec/views/about/show.html.haml_spec.rb index aa151dd27..724643cbc 100644 --- a/spec/views/about/show.html.haml_spec.rb +++ b/spec/views/about/show.html.haml_spec.rb @@ -15,15 +15,16 @@ describe 'about/show.html.haml', without_verify_partial_doubles: true do version_number: '1.0', source_url: 'https://github.com/tootsuite/mastodon', open_registrations: false, + thumbnail: nil, closed_registrations_message: 'yes') assign(:instance_presenter, instance_presenter) render header_tags = view.content_for(:header_tags) - expect(header_tags).to match(%r{}) - expect(header_tags).to match(%r{}) - expect(header_tags).to match(%r{}) - expect(header_tags).to match(%r{}) + expect(header_tags).to match(%r{}) + expect(header_tags).to match(%r{}) + expect(header_tags).to match(%r{}) + expect(header_tags).to match(%r{}) end end -- cgit From 3816943e6b5e86b22c35f3c068521f7a9007deec Mon Sep 17 00:00:00 2001 From: ふぁぼ原 Date: Fri, 15 Sep 2017 01:03:20 +0900 Subject: Enable to recognize most kinds of characters as URL paths (#4941) --- app/lib/formatter.rb | 2 +- app/services/fetch_link_card_service.rb | 14 ++++++--- config/initializers/twitter_regex.rb | 42 +++++++++++++++++++++++++++ spec/lib/formatter_spec.rb | 32 ++++++++++++++++++++ spec/services/fetch_link_card_service_spec.rb | 11 +++++++ 5 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 config/initializers/twitter_regex.rb (limited to 'spec') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index cacc0364f..d9f843f44 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -131,7 +131,7 @@ class Formatter end def link_html(url) - url = Addressable::URI.parse(url).display_uri.to_s + url = Addressable::URI.parse(url).to_s prefix = url.match(/\Ahttps?:\/\/(www\.)?/).to_s text = url[prefix.length, 30] suffix = url[prefix.length + 30..-1] diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 215c69fe4..4acbfae7a 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -1,9 +1,15 @@ # frozen_string_literal: true class FetchLinkCardService < BaseService - include ActionView::Helpers::TagHelper - - URL_PATTERN = %r{https?://\S+} + URL_PATTERN = %r{ + ( # $1 URL + (https?:\/\/)? # $2 Protocol (optional) + (#{Twitter::Regex[:valid_domain]}) # $3 Domain(s) + (?::(#{Twitter::Regex[:valid_port_number]}))? # $4 Port number (optional) + (/#{Twitter::Regex[:valid_url_path]}*)? # $5 URL Path and anchor + (\?#{Twitter::Regex[:valid_url_query_chars]}*#{Twitter::Regex[:valid_url_query_ending_chars]})? # $6 Query String + ) + }iox def call(status) @status = status @@ -42,7 +48,7 @@ class FetchLinkCardService < BaseService def parse_urls if @status.local? - urls = @status.text.match(URL_PATTERN).to_a.map { |uri| Addressable::URI.parse(uri).normalize } + urls = @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[0]).normalize } else html = Nokogiri::HTML(@status.text) links = html.css('a') diff --git a/config/initializers/twitter_regex.rb b/config/initializers/twitter_regex.rb new file mode 100644 index 000000000..5a0723d24 --- /dev/null +++ b/config/initializers/twitter_regex.rb @@ -0,0 +1,42 @@ +module Twitter + class Regex + + REGEXEN[:valid_general_url_path_chars] = /[^\p{White_Space}\(\)\?]/iou + REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}\(\)\?!\*';:=\,\.\$%\[\]\p{Pd}_~&\|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou + REGEXEN[:valid_url_balanced_parens] = / + \( + (?: + #{REGEXEN[:valid_general_url_path_chars]}+ + | + # allow one nested level of balanced parentheses + (?: + #{REGEXEN[:valid_general_url_path_chars]}* + \( + #{REGEXEN[:valid_general_url_path_chars]}+ + \) + #{REGEXEN[:valid_general_url_path_chars]}* + ) + ) + \) + /iox + REGEXEN[:valid_url_path] = /(?: + (?: + #{REGEXEN[:valid_general_url_path_chars]}* + (?:#{REGEXEN[:valid_url_balanced_parens]} #{REGEXEN[:valid_general_url_path_chars]}*)* + #{REGEXEN[:valid_url_path_ending_chars]} + )|(?:#{REGEXEN[:valid_general_url_path_chars]}+\/) + )/iox + REGEXEN[:valid_url] = %r{ + ( # $1 total match + (#{REGEXEN[:valid_url_preceding_chars]}) # $2 Preceeding chracter + ( # $3 URL + (https?:\/\/)? # $4 Protocol (optional) + (#{REGEXEN[:valid_domain]}) # $5 Domain(s) + (?::(#{REGEXEN[:valid_port_number]}))? # $6 Port number (optional) + (/#{REGEXEN[:valid_url_path]}*)? # $7 URL Path and anchor + (\?#{REGEXEN[:valid_url_query_chars]}*#{REGEXEN[:valid_url_query_ending_chars]})? # $8 Query String + ) + ) + }iox + end +end diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index ab04ccbab..f9b7efac5 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -89,6 +89,38 @@ RSpec.describe Formatter do end end + context 'matches a URL with Japanese path string' do + let(:text) { 'https://ja.wikipedia.org/wiki/日本' } + + it 'has valid URL' do + is_expected.to include 'href="https://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC"' + end + end + + context 'matches a URL with Korean path string' do + let(:text) { 'https://ko.wikipedia.org/wiki/대한민국' } + + it 'has valid URL' do + is_expected.to include 'href="https://ko.wikipedia.org/wiki/%EB%8C%80%ED%95%9C%EB%AF%BC%EA%B5%AD"' + end + end + + context 'matches a URL with Simplified Chinese path string' do + let(:text) { 'https://baike.baidu.com/item/中华人民共和国' } + + it 'has valid URL' do + is_expected.to include 'href="https://baike.baidu.com/item/%E4%B8%AD%E5%8D%8E%E4%BA%BA%E6%B0%91%E5%85%B1%E5%92%8C%E5%9B%BD"' + end + end + + context 'matches a URL with Traditional Chinese path string' do + let(:text) { 'https://zh.wikipedia.org/wiki/臺灣' } + + it 'has valid URL' do + is_expected.to include 'href="https://zh.wikipedia.org/wiki/%E8%87%BA%E7%81%A3"' + end + end + context 'contains HTML (script tag)' do let(:text) { '' } diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index b0aa740ac..ba61d22c3 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -12,6 +12,8 @@ RSpec.describe FetchLinkCardService do stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt')) stub_request(:head, 'http://example.com/koi8-r').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt')) + stub_request(:head, 'http://example.com/日本語').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) + stub_request(:get, 'http://example.com/日本語').to_return(request_fixture('sjis.txt')) stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404) subject.call(status) @@ -52,6 +54,15 @@ RSpec.describe FetchLinkCardService do expect(status.preview_cards.first.title).to eq("Московя начинаетъ только въ XVI ст. привлекать внимане иностранцевъ.") end end + + context do + let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') } + + it 'works with Japanese path string' do + expect(a_request(:get, 'http://example.com/日本語')).to have_been_made.at_least_once + expect(status.preview_cards.first.title).to eq("SJISのページ") + end + end end context 'in a remote status' do -- cgit From 472df245799949631646f90d894486e4dbeaaaf9 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 15 Sep 2017 00:57:08 +0200 Subject: When web UI URL used while logged out, redirect to static page (#4954) --- app/controllers/home_controller.rb | 35 +++++++++++++++++++++++++++++++- spec/controllers/home_controller_spec.rb | 2 ++ 2 files changed, 36 insertions(+), 1 deletion(-) (limited to 'spec') diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb index 1585bc810..21dde20ce 100644 --- a/app/controllers/home_controller.rb +++ b/app/controllers/home_controller.rb @@ -11,7 +11,30 @@ class HomeController < ApplicationController private def authenticate_user! - redirect_to(single_user_mode? ? account_path(Account.first) : about_path) unless user_signed_in? + return if user_signed_in? + + matches = request.path.match(/\A\/web\/(statuses|accounts)\/([\d]+)\z/) + + if matches + case matches[1] + when 'statuses' + status = Status.find_by(id: matches[2]) + + if status && (status.public_visibility? || status.unlisted_visibility?) + redirect_to(ActivityPub::TagManager.instance.url_for(status)) + return + end + when 'accounts' + account = Account.find_by(id: matches[2]) + + if account + redirect_to(ActivityPub::TagManager.instance.url_for(account)) + return + end + end + end + + redirect_to(default_redirect_path) end def set_initial_state_json @@ -28,4 +51,14 @@ class HomeController < ApplicationController admin: Account.find_local(Setting.site_contact_username), } end + + def default_redirect_path + if request.path.start_with?('/web') + new_user_session_path + elsif single_user_mode? + short_account_path(Account.first) + else + about_path + end + end end diff --git a/spec/controllers/home_controller_spec.rb b/spec/controllers/home_controller_spec.rb index d44d720b1..1077a7288 100644 --- a/spec/controllers/home_controller_spec.rb +++ b/spec/controllers/home_controller_spec.rb @@ -6,6 +6,7 @@ RSpec.describe HomeController, type: :controller do describe 'GET #index' do context 'when not signed in' do it 'redirects to about page' do + @request.path = '/' get :index expect(response).to redirect_to(about_path) end @@ -13,6 +14,7 @@ RSpec.describe HomeController, type: :controller do context 'when signed in' do let(:user) { Fabricate(:user) } + subject do sign_in(user) get :index -- cgit From 48d77ea1ebd6096a6ad5e265d99fa20e7a965276 Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Sat, 16 Sep 2017 21:59:41 +0900 Subject: Fix filterable_languages method of SettingsHelper (#4966) --- Gemfile | 2 +- Gemfile.lock | 4 ++-- app/helpers/settings_helper.rb | 2 +- app/lib/language_detector.rb | 39 ++++++++++++++----------------- app/services/post_status_service.rb | 6 +---- spec/lib/language_detector_spec.rb | 34 +++++++++++++-------------- spec/services/post_status_service_spec.rb | 9 +++---- 7 files changed, 43 insertions(+), 53 deletions(-) (limited to 'spec') diff --git a/Gemfile b/Gemfile index 051f09f78..4f4861913 100644 --- a/Gemfile +++ b/Gemfile @@ -25,7 +25,7 @@ gem 'bootsnap' gem 'browser' gem 'charlock_holmes', '~> 0.7.5' gem 'iso-639' -gem 'cld3', '~> 3.1' +gem 'cld3', '~> 3.2.0' gem 'devise', '~> 4.2' gem 'devise-two-factor', '~> 3.0' gem 'doorkeeper', '~> 4.2' diff --git a/Gemfile.lock b/Gemfile.lock index 31893dc3f..f080f15e5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -110,7 +110,7 @@ GEM activesupport charlock_holmes (0.7.5) chunky_png (1.3.8) - cld3 (3.1.3) + cld3 (3.2.0) ffi (>= 1.1.0, < 1.10.0) climate_control (0.2.0) cocaine (0.5.8) @@ -541,7 +541,7 @@ DEPENDENCIES capistrano-yarn (~> 2.0) capybara (~> 2.14) charlock_holmes (~> 0.7.5) - cld3 (~> 3.1) + cld3 (~> 3.2.0) climate_control (~> 0.2) devise (~> 4.2) devise-two-factor (~> 3.0) diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index 369a45680..14776b354 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -41,7 +41,7 @@ module SettingsHelper end def filterable_languages - I18n.available_locales.map { |locale| locale.to_s.split('-').first.to_sym }.uniq + LanguageDetector.instance.language_names.select(&HUMAN_LOCALES.method(:key?)) end def hash_to_object(hash) diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb index 1d9932b52..a42460e10 100644 --- a/app/lib/language_detector.rb +++ b/app/lib/language_detector.rb @@ -1,26 +1,31 @@ # frozen_string_literal: true class LanguageDetector - attr_reader :text, :account + include Singleton - def initialize(text, account = nil) - @text = text - @account = account + def initialize @identifier = CLD3::NNetLanguageIdentifier.new(1, 2048) end - def to_iso_s - detected_language_code || default_locale + def detect(text, account) + detect_language_code(text) || default_locale(account) end - def prepared_text - simplified_text.strip + def language_names + @language_names = + CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym } + .uniq end private - def detected_language_code - iso6391(result.language).to_sym if detected_language_reliable? + def prepare_text(text) + simplify_text(text).strip + end + + def detect_language_code(text) + result = @identifier.find_language(prepare_text(text)) + iso6391(result.language.to_s).to_sym if result.reliable? end def iso6391(bcp47) @@ -32,15 +37,7 @@ class LanguageDetector ISO_639.find(iso639).alpha2 end - def result - @result ||= @identifier.find_language(prepared_text) - end - - def detected_language_reliable? - result.reliable? - end - - def simplified_text + def simplify_text(text) text.dup.tap do |new_text| new_text.gsub!(FetchLinkCardService::URL_PATTERN, '') new_text.gsub!(Account::MENTION_RE, '') @@ -49,7 +46,7 @@ class LanguageDetector end end - def default_locale - account&.user_locale&.to_sym || nil + def default_locale(account) + account.user_locale&.to_sym end end diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 97c55c4b2..e37cd94df 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -28,7 +28,7 @@ class PostStatusService < BaseService sensitive: options[:sensitive], spoiler_text: options[:spoiler_text] || '', visibility: options[:visibility] || account.user&.setting_default_privacy, - language: detect_language_for(text, account), + language: LanguageDetector.instance.detect(text, account), application: options[:application]) attach_media(status, media) @@ -69,10 +69,6 @@ class PostStatusService < BaseService media.update(status_id: status.id) end - def detect_language_for(text, account) - LanguageDetector.new(text, account).to_iso_s - end - def process_mentions_service @process_mentions_service ||= ProcessMentionsService.new end diff --git a/spec/lib/language_detector_spec.rb b/spec/lib/language_detector_spec.rb index ec39cb6a0..d17026511 100644 --- a/spec/lib/language_detector_spec.rb +++ b/spec/lib/language_detector_spec.rb @@ -3,10 +3,10 @@ require 'rails_helper' describe LanguageDetector do - describe 'prepared_text' do + describe 'prepare_text' do it 'returns unmodified string without special cases' do string = 'just a regular string' - result = described_class.new(string).prepared_text + result = described_class.instance.send(:prepare_text, string) expect(result).to eq string end @@ -14,33 +14,35 @@ describe LanguageDetector do it 'collapses spacing in strings' do string = 'The formatting in this is very odd' - result = described_class.new(string).prepared_text + result = described_class.instance.send(:prepare_text, string) expect(result).to eq 'The formatting in this is very odd' end it 'strips usernames from strings before detection' do string = '@username Yeah, very surreal...! also @friend' - result = described_class.new(string).prepared_text + result = described_class.instance.send(:prepare_text, string) expect(result).to eq 'Yeah, very surreal...! also' end it 'strips URLs from strings before detection' do string = 'Our website is https://example.com and also http://localhost.dev' - result = described_class.new(string).prepared_text + result = described_class.instance.send(:prepare_text, string) expect(result).to eq 'Our website is and also' end it 'strips #hashtags from strings before detection' do string = 'Hey look at all the #animals and #fish' - result = described_class.new(string).prepared_text + result = described_class.instance.send(:prepare_text, string) expect(result).to eq 'Hey look at all the and' end end - describe 'to_iso_s' do + describe 'detect' do + let(:account_without_user_locale) { Fabricate(:user, locale: nil).account } + it 'detects english language for basic strings' do strings = [ "Hello and welcome to mastodon how are you today?", @@ -48,7 +50,7 @@ describe LanguageDetector do "a lot of people just want to feel righteous all the time and that's all that matters", ] strings.each do |string| - result = described_class.new(string).to_iso_s + result = described_class.instance.detect(string, account_without_user_locale) expect(result).to eq(:en), string end @@ -56,14 +58,14 @@ describe LanguageDetector do it 'detects spanish language' do string = 'Obtener un Hola y bienvenidos a Mastodon' - result = described_class.new(string).to_iso_s + result = described_class.instance.detect(string, account_without_user_locale) expect(result).to eq :es end describe 'when language can\'t be detected' do it 'uses nil when sent an empty document' do - result = described_class.new('').to_iso_s + result = described_class.instance.detect('', account_without_user_locale) expect(result).to eq nil end @@ -73,7 +75,7 @@ describe LanguageDetector do cld_result = CLD3::NNetLanguageIdentifier.new(0, 2048).find_language(string) expect(cld_result).not_to eq :en - result = described_class.new(string).to_iso_s + result = described_class.instance.detect(string, account_without_user_locale) expect(result).to eq nil end @@ -82,14 +84,13 @@ describe LanguageDetector do describe 'with an account' do it 'uses the account locale when present' do account = double(user_locale: 'fr') - result = described_class.new('', account).to_iso_s + result = described_class.instance.detect('', account) expect(result).to eq :fr end it 'uses nil when account is present but has no locale' do - account = double(user_locale: nil) - result = described_class.new('', account).to_iso_s + result = described_class.instance.detect('', account_without_user_locale) expect(result).to eq nil end @@ -97,8 +98,7 @@ describe LanguageDetector do describe 'with an `en` default locale' do it 'uses nil for undetectable string' do - string = '' - result = described_class.new(string).to_iso_s + result = described_class.instance.detect('', account_without_user_locale) expect(result).to eq nil end @@ -114,7 +114,7 @@ describe LanguageDetector do it 'uses nil for undetectable string' do string = '' - result = described_class.new(string).to_iso_s + result = described_class.instance.detect(string, account_without_user_locale) expect(result).to eq nil end diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb index 4182c4e1f..91902ff69 100644 --- a/spec/services/post_status_service_spec.rb +++ b/spec/services/post_status_service_spec.rb @@ -65,15 +65,12 @@ RSpec.describe PostStatusService do end it 'creates a status with a language set' do - detector = double(to_iso_s: :en) - allow(LanguageDetector).to receive(:new).and_return(detector) - account = Fabricate(:account) - text = 'test status text' + text = 'This is an English text.' - subject.call(account, text) + status = subject.call(account, text) - expect(LanguageDetector).to have_received(:new).with(text, account) + expect(status.language).to eq 'en' end it 'processes mentions' do -- cgit From ec36df97c4ea3da4bc177a96050c54cf8f35ba25 Mon Sep 17 00:00:00 2001 From: unarist Date: Sun, 17 Sep 2017 04:33:52 +0900 Subject: Escape URL parts on formatting local status (#4975) --- app/lib/formatter.rb | 2 +- spec/lib/formatter_spec.rb | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'spec') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index d9f843f44..575830190 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -137,7 +137,7 @@ class Formatter suffix = url[prefix.length + 30..-1] cutoff = url[prefix.length..-1].length > 30 - "#{prefix}#{text}#{suffix}" + "#{encode(prefix)}#{encode(text)}#{encode(suffix)}" end def hashtag_html(tag) diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index f9b7efac5..b714b317a 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -121,6 +121,22 @@ RSpec.describe Formatter do end end + context 'contains unsafe URL (XSS attack, visible part)' do + let(:text) { %q{http://example.com/bb} } + + it 'has escaped HTML' do + is_expected.to include '<del>b</del>' + end + end + + context 'contains unsafe URL (XSS attack, invisible part)' do + let(:text) { %q{http://example.com/blahblahblahblah/a} } + + it 'has escaped HTML' do + is_expected.to include '<script>alert("Hello")</script>' + end + end + context 'contains HTML (script tag)' do let(:text) { '' } -- cgit From 198a9a4fa42b880cba182d19dcc8edad360a4fbe Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Sun, 17 Sep 2017 12:29:43 +0900 Subject: Remove local_only scope in Status (#4977) --- app/models/status.rb | 3 +-- spec/models/status_spec.rb | 13 ------------- 2 files changed, 1 insertion(+), 15 deletions(-) (limited to 'spec') diff --git a/app/models/status.rb b/app/models/status.rb index 514cab2e4..8e82c6aa6 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -70,7 +70,6 @@ class Status < ApplicationRecord scope :without_reblogs, -> { where('statuses.reblog_of_id IS NULL') } scope :with_public_visibility, -> { where(visibility: :public) } scope :tagged_with, ->(tag) { joins(:statuses_tags).where(statuses_tags: { tag_id: tag }) } - scope :local_only, -> { left_outer_joins(:account).where(accounts: { domain: nil }) } scope :excluding_silenced_accounts, -> { left_outer_joins(:account).where(accounts: { silenced: false }) } scope :including_silenced_accounts, -> { left_outer_joins(:account).where(accounts: { silenced: true }) } scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) } @@ -221,7 +220,7 @@ class Status < ApplicationRecord private def timeline_scope(local_only = false) - starting_scope = local_only ? Status.local_only : Status + starting_scope = local_only ? Status.local : Status starting_scope .with_public_visibility .without_reblogs diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb index 484effd5e..f7ee0a609 100644 --- a/spec/models/status_spec.rb +++ b/spec/models/status_spec.rb @@ -173,19 +173,6 @@ RSpec.describe Status, type: :model do end end - describe '.local_only' do - it 'returns only statuses from local accounts' do - local_account = Fabricate(:account, domain: nil) - remote_account = Fabricate(:account, domain: 'test.com') - local_status = Fabricate(:status, account: local_account) - remote_status = Fabricate(:status, account: remote_account) - - results = described_class.local_only - expect(results).to include(local_status) - expect(results).not_to include(remote_status) - end - end - describe '.as_home_timeline' do let(:account) { Fabricate(:account) } let(:followed) { Fabricate(:account) } -- cgit From 1eab53ee1030542a5c4c56203a61eecae9768131 Mon Sep 17 00:00:00 2001 From: unarist Date: Sun, 17 Sep 2017 18:54:23 +0900 Subject: Fix an error when actor json couldn't be fetched in ResolveRemoteAccountService (#4979) * Fix an error when actor json couldn't be fetched in ResolveRemoteAccountService * Add specs --- app/services/resolve_remote_account_service.rb | 1 + .../requests/activitypub-actor-noinbox.txt | 9 +++++ spec/fixtures/requests/activitypub-actor.txt | 9 +++++ spec/fixtures/requests/activitypub-feed.txt | 47 ++++++++++++++++++++++ spec/fixtures/requests/activitypub-webfinger.txt | 7 ++++ .../resolve_remote_account_service_spec.rb | 33 +++++++++++++++ 6 files changed, 106 insertions(+) create mode 100644 spec/fixtures/requests/activitypub-actor-noinbox.txt create mode 100644 spec/fixtures/requests/activitypub-actor.txt create mode 100644 spec/fixtures/requests/activitypub-feed.txt create mode 100644 spec/fixtures/requests/activitypub-webfinger.txt (limited to 'spec') diff --git a/app/services/resolve_remote_account_service.rb b/app/services/resolve_remote_account_service.rb index 7031c98f5..57c80fc82 100644 --- a/app/services/resolve_remote_account_service.rb +++ b/app/services/resolve_remote_account_service.rb @@ -80,6 +80,7 @@ class ResolveRemoteAccountService < BaseService def activitypub_ready? !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) && + !actor_json.nil? && actor_json['inbox'].present? end diff --git a/spec/fixtures/requests/activitypub-actor-noinbox.txt b/spec/fixtures/requests/activitypub-actor-noinbox.txt new file mode 100644 index 000000000..95b4650e0 --- /dev/null +++ b/spec/fixtures/requests/activitypub-actor-noinbox.txt @@ -0,0 +1,9 @@ +HTTP/1.1 200 OK +Date: Sun, 17 Sep 2017 06:51:23 GMT +Content-Type: application/json; charset=utf-8 +X-XSS-Protection: 1; mode=block +Link: ; rel="lrdd"; type="application/xrd+xml", ; rel="alternate"; type="application/atom+xml" +Vary: Accept-Encoding +Cache-Control: max-age=0, private, must-revalidate + +{"@context":"https://www.w3.org/ns/activitystreams","id":"https://ap.example.com/users/foo","type":"Person","following":"https://ap.example.com/users/foo/following","followers":"https://ap.example.com/users/foo/followers","inbox":null,"outbox":"https://ap.example.com/users/foo/outbox","preferredUsername":"foo","name":"","summary":"\u003cp\u003etest\u003c/p\u003e","icon":"https://quitter.no/avatar/7477-300-20160211190340.png","image":"/headers/original/missing.png","publicKey":{"id":"https://ap.example.com/users/foo#main-key","owner":"https://ap.example.com/users/foo","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu3L4vnpNLzVH31MeWI39\n4F0wKeJFsLDAsNXGeOu0QF2x+h1zLWZw/agqD2R3JPU9/kaDJGPIV2Sn5zLyUA9S\n6swCCMOtn7BBR9g9sucgXJmUFB0tACH2QSgHywMAybGfmSb3LsEMNKsGJ9VsvYoh\n8lDET6X4Pyw+ZJU0/OLo/41q9w+OrGtlsTm/PuPIeXnxa6BLqnDaxC+4IcjG/FiP\nahNCTINl/1F/TgSSDZ4Taf4U9XFEIFw8wmgploELozzIzKq+t8nhQYkgAkt64euW\npva3qL5KD1mTIZQEP+LZvh3s2WHrLi3fhbdRuwQ2c0KkJA2oSTFPDpqqbPGZ3Qvu\nHQIDAQAB\n-----END PUBLIC KEY-----\n"}} \ No newline at end of file diff --git a/spec/fixtures/requests/activitypub-actor.txt b/spec/fixtures/requests/activitypub-actor.txt new file mode 100644 index 000000000..6514241cb --- /dev/null +++ b/spec/fixtures/requests/activitypub-actor.txt @@ -0,0 +1,9 @@ +HTTP/1.1 200 OK +Cache-Control: max-age=0, private, must-revalidate +Content-Type: application/activity+json; charset=utf-8 +Link: ; rel="lrdd"; type="application/xrd+xml", ; rel="alternate"; type="application/atom+xml", ; rel="alternate"; type="application/activity+json" +Vary: Accept-Encoding +X-Content-Type-Options: nosniff +X-Xss-Protection: 1; mode=block + +{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation"}],"id":"https://ap.example.com/users/foo","type":"Person","following":"https://ap.example.com/users/foo/following","followers":"https://ap.example.com/users/foo/followers","inbox":"https://ap.example.com/users/foo/inbox","outbox":"https://ap.example.com/users/foo/outbox","preferredUsername":"foo","name":"","summary":"\u003cp\u003etest\u003c/p\u003e","url":"https://ap.example.com/@foo","manuallyApprovesFollowers":false,"publicKey":{"id":"https://ap.example.com/users/foo#main-key","owner":"https://ap.example.com/users/foo","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu3L4vnpNLzVH31MeWI39\n4F0wKeJFsLDAsNXGeOu0QF2x+h1zLWZw/agqD2R3JPU9/kaDJGPIV2Sn5zLyUA9S\n6swCCMOtn7BBR9g9sucgXJmUFB0tACH2QSgHywMAybGfmSb3LsEMNKsGJ9VsvYoh\n8lDET6X4Pyw+ZJU0/OLo/41q9w+OrGtlsTm/PuPIeXnxa6BLqnDaxC+4IcjG/FiP\nahNCTINl/1F/TgSSDZ4Taf4U9XFEIFw8wmgploELozzIzKq+t8nhQYkgAkt64euW\npva3qL5KD1mTIZQEP+LZvh3s2WHrLi3fhbdRuwQ2c0KkJA2oSTFPDpqqbPGZ3Qvu\nHQIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"https://ap.example.com/inbox"},"icon":{"type":"Image","url":"https://quitter.no/avatar/7477-300-20160211190340.png"}} \ No newline at end of file diff --git a/spec/fixtures/requests/activitypub-feed.txt b/spec/fixtures/requests/activitypub-feed.txt new file mode 100644 index 000000000..84fd414c3 --- /dev/null +++ b/spec/fixtures/requests/activitypub-feed.txt @@ -0,0 +1,47 @@ +HTTP/1.1 200 OK +Cache-Control: max-age=0, private, must-revalidate +Content-Type: application/atom+xml; charset=utf-8 +Link: ; rel="lrdd"; type="application/xrd+xml", ; rel="alternate"; type="application/atom+xml", ; rel="alternate"; type="application/activity+json" +Vary: Accept-Encoding +Date: Sun, 17 Sep 2017 06:33:53 GMT + + + + https://ap.example.com/users/foo.atom + foo + test + 2017-09-16T18:50:09Z + https://ap.example.com/system/accounts/avatars/000/000/001/original/141ee5846d159cba.png?1505587809 + + https://ap.example.com/users/foo + http://activitystrea.ms/schema/1.0/person + https://ap.example.com/users/foo + foo + foo@ap.example.com + <p>test</p> + + + foo + test + public + + + + + + + https://ap.example.com/users/foo/statuses/11076 + 2017-09-13T01:23:19Z + 2017-09-13T01:23:19Z + New status by foo + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + + <p>test</p> + + public + + + + + diff --git a/spec/fixtures/requests/activitypub-webfinger.txt b/spec/fixtures/requests/activitypub-webfinger.txt new file mode 100644 index 000000000..465066d84 --- /dev/null +++ b/spec/fixtures/requests/activitypub-webfinger.txt @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +Cache-Control: max-age=0, private, must-revalidate +Content-Type: application/jrd+json; charset=utf-8 +X-Content-Type-Options: nosniff +Date: Sun, 17 Sep 2017 06:22:50 GMT + +{"subject":"acct:foo@ap.example.com","aliases":["https://ap.example.com/@foo","https://ap.example.com/users/foo"],"links":[{"rel":"http://webfinger.net/rel/profile-page","type":"text/html","href":"https://ap.example.com/@foo"},{"rel":"http://schemas.google.com/g/2010#updates-from","type":"application/atom+xml","href":"https://ap.example.com/users/foo.atom"},{"rel":"self","type":"application/activity+json","href":"https://ap.example.com/users/foo"},{"rel":"salmon","href":"https://ap.example.com/api/salmon/1"},{"rel":"magic-public-key","href":"data:application/magic-public-key,RSA.u3L4vnpNLzVH31MeWI394F0wKeJFsLDAsNXGeOu0QF2x-h1zLWZw_agqD2R3JPU9_kaDJGPIV2Sn5zLyUA9S6swCCMOtn7BBR9g9sucgXJmUFB0tACH2QSgHywMAybGfmSb3LsEMNKsGJ9VsvYoh8lDET6X4Pyw-ZJU0_OLo_41q9w-OrGtlsTm_PuPIeXnxa6BLqnDaxC-4IcjG_FiPahNCTINl_1F_TgSSDZ4Taf4U9XFEIFw8wmgploELozzIzKq-t8nhQYkgAkt64euWpva3qL5KD1mTIZQEP-LZvh3s2WHrLi3fhbdRuwQ2c0KkJA2oSTFPDpqqbPGZ3QvuHQ==.AQAB"},{"rel":"http://ostatus.org/schema/1.0/subscribe","template":"https://ap.example.com/authorize_follow?acct={uri}"}]} \ No newline at end of file diff --git a/spec/services/resolve_remote_account_service_spec.rb b/spec/services/resolve_remote_account_service_spec.rb index d0eab2310..d0bb6a137 100644 --- a/spec/services/resolve_remote_account_service_spec.rb +++ b/spec/services/resolve_remote_account_service_spec.rb @@ -72,6 +72,39 @@ RSpec.describe ResolveRemoteAccountService do end context 'with an ActivityPub account' do + before do + stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt')) + stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt')) + stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt')) + stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404) + end + + it 'fallback to OStatus if actor json could not be fetched' do + stub_request(:get, "https://ap.example.com/users/foo").to_return(status: 404) + + account = subject.call('foo@ap.example.com') + + expect(account.ostatus?).to eq true + expect(account.remote_url).to eq 'https://ap.example.com/users/foo.atom' + end + + it 'fallback to OStatus if actor json did not have inbox_url' do + stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor-noinbox.txt')) + + account = subject.call('foo@ap.example.com') + + expect(account.ostatus?).to eq true + expect(account.remote_url).to eq 'https://ap.example.com/users/foo.atom' + end + + it 'returns new remote account' do + account = subject.call('foo@ap.example.com') + + expect(account.activitypub?).to eq true + expect(account.domain).to eq 'ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + pending end -- cgit From dd6f9a1b827226346d0fff22a79538351fb33028 Mon Sep 17 00:00:00 2001 From: unarist Date: Sun, 17 Sep 2017 22:21:57 +0900 Subject: Validate uri presence for remote status (#4985) --- app/models/status.rb | 2 +- spec/models/status_spec.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'spec') diff --git a/app/models/status.rb b/app/models/status.rb index 8e82c6aa6..2a2cdcf6e 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -55,7 +55,7 @@ class Status < ApplicationRecord has_one :notification, as: :activity, dependent: :destroy has_one :stream_entry, as: :activity, inverse_of: :status - validates :uri, uniqueness: true, unless: :local? + validates :uri, uniqueness: true, presence: true, unless: :local? validates :text, presence: true, unless: :reblog? validates_with StatusLengthValidator validates :reblog, uniqueness: { scope: :account }, if: :reblog? diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb index f7ee0a609..12efcae61 100644 --- a/spec/models/status_spec.rb +++ b/spec/models/status_spec.rb @@ -516,6 +516,14 @@ RSpec.describe Status, type: :model do end end + describe 'validation' do + it 'disallow empty uri for remote status' do + alice.update(domain: 'example.com') + status = Fabricate.build(:status, uri: '', account: alice) + expect(status).to model_have_error_on_field(:uri) + end + end + describe 'after_create' do it 'saves ActivityPub uri as uri for local status' do status = Status.create(account: alice, text: 'foo') -- cgit From 81cec35dbf1b348d23363559e3f4e6b1ec3415c5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 19 Sep 2017 02:42:40 +0200 Subject: Custom emoji (#4988) * Custom emoji - In OStatus: `` - In ActivityPub: `{ type: "Emoji", name: ":coolcat:", href: "http://..." }` - In REST API: Status object includes `emojis` array (`shortcode`, `url`) - Domain blocks with reject media stop emojis - Emoji file up to 50KB - Web UI handles custom emojis - Static pages render custom emojis as `` tags Side effects: - Undo #4500 optimization, as I needed to modify it to restore shortcode handling in emojify() - Formatter#plaintext should now make sure stripped out line-breaks and paragraphs are replaced with newlines * Fix emoji at the start not being converted --- app/javascript/mastodon/emoji.js | 60 ++++++++++------ app/javascript/mastodon/reducers/statuses.js | 9 ++- app/lib/activitypub/activity/create.rb | 13 ++++ app/lib/formatter.rb | 54 +++++++++++++- app/lib/ostatus/activity/creation.rb | 20 ++++++ app/lib/ostatus/atom_serializer.rb | 4 ++ app/models/custom_emoji.rb | 38 ++++++++++ app/models/status.rb | 4 ++ app/serializers/activitypub/note_serializer.rb | 20 +++++- app/serializers/rest/status_serializer.rb | 11 +++ .../stream_entries/_detailed_status.html.haml | 2 +- app/views/stream_entries/_simple_status.html.haml | 2 +- db/migrate/20170917153509_create_custom_emojis.rb | 13 ++++ db/schema.rb | 14 +++- spec/fabricators/custom_emoji_fabricator.rb | 5 ++ spec/fixtures/files/emojo.png | Bin 0 -> 29814 bytes spec/lib/activitypub/activity/create_spec.rb | 25 +++++++ spec/lib/formatter_spec.rb | 78 +++++++++++++++++++++ spec/lib/ostatus/atom_serializer_spec.rb | 16 ++++- spec/models/custom_emoji_spec.rb | 25 +++++++ 20 files changed, 382 insertions(+), 31 deletions(-) create mode 100644 app/models/custom_emoji.rb create mode 100644 db/migrate/20170917153509_create_custom_emojis.rb create mode 100644 spec/fabricators/custom_emoji_fabricator.rb create mode 100644 spec/fixtures/files/emojo.png create mode 100644 spec/models/custom_emoji_spec.rb (limited to 'spec') diff --git a/app/javascript/mastodon/emoji.js b/app/javascript/mastodon/emoji.js index a41dfdd1d..865b85b61 100644 --- a/app/javascript/mastodon/emoji.js +++ b/app/javascript/mastodon/emoji.js @@ -3,28 +3,48 @@ import Trie from 'substring-trie'; const trie = new Trie(Object.keys(unicodeMapping)); -const emojify = str => { - let rtn = ''; - for (;;) { - let match, i = 0; - while (i < str.length && str[i] !== '<' && !(match = trie.search(str.slice(i)))) { - i += str.codePointAt(i) < 65536 ? 1 : 2; - } - if (i === str.length) - break; - else if (str[i] === '<') { - let tagend = str.indexOf('>', i + 1) + 1; - if (!tagend) - break; - rtn += str.slice(0, tagend); - str = str.slice(tagend); - } else { - const [filename, shortCode] = unicodeMapping[match]; - rtn += str.slice(0, i) + `${match}`; - str = str.slice(i + match.length); +const emojify = (str, customEmojis = {}) => { + // This walks through the string from start to end, ignoring any tags (

,
, etc.) + // and replacing valid unicode strings + // that _aren't_ within tags with an version. + // The goal is to be the same as an emojione.regUnicode replacement, but faster. + let i = -1; + let insideTag = false; + let insideShortname = false; + let shortnameStartIndex = -1; + let match; + while (++i < str.length) { + const char = str.charAt(i); + if (insideShortname && char === ':') { + const shortname = str.substring(shortnameStartIndex, i + 1); + if (shortname in customEmojis) { + const replacement = `${shortname}`; + str = str.substring(0, shortnameStartIndex) + replacement + str.substring(i + 1); + i += (replacement.length - shortname.length - 1); // jump ahead the length we've added to the string + } else { + i--; + } + insideShortname = false; + } else if (insideTag && char === '>') { + insideTag = false; + } else if (char === '<') { + insideTag = true; + insideShortname = false; + } else if (!insideTag && char === ':') { + insideShortname = true; + shortnameStartIndex = i; + } else if (!insideTag && (match = trie.search(str.substring(i)))) { + const unicodeStr = match; + if (unicodeStr in unicodeMapping) { + const [filename, shortCode] = unicodeMapping[unicodeStr]; + const alt = unicodeStr; + const replacement = `${alt}`; + str = str.substring(0, i) + replacement + str.substring(i + unicodeStr.length); + i += (replacement.length - unicodeStr.length); // jump ahead the length we've added to the string + } } } - return rtn + str; + return str; }; export default emojify; diff --git a/app/javascript/mastodon/reducers/statuses.js b/app/javascript/mastodon/reducers/statuses.js index 7f906bef6..38b23504e 100644 --- a/app/javascript/mastodon/reducers/statuses.js +++ b/app/javascript/mastodon/reducers/statuses.js @@ -58,9 +58,14 @@ const normalizeStatus = (state, status) => { } const searchContent = [status.spoiler_text, status.content].join(' ').replace(/
/g, '\n').replace(/<\/p>

/g, '\n\n'); + const emojiMap = normalStatus.emojis.reduce((obj, emoji) => { + obj[`:${emoji.shortcode}:`] = emoji.url; + return obj; + }, {}); + normalStatus.search_index = domParser.parseFromString(searchContent, 'text/html').documentElement.textContent; - normalStatus.contentHtml = emojify(normalStatus.content); - normalStatus.spoilerHtml = emojify(escapeTextContentForBrowser(normalStatus.spoiler_text || '')); + normalStatus.contentHtml = emojify(normalStatus.content, emojiMap); + normalStatus.spoilerHtml = emojify(escapeTextContentForBrowser(normalStatus.spoiler_text || ''), emojiMap); return state.update(status.id, ImmutableMap(), map => map.mergeDeep(fromJS(normalStatus))); }; diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 894759d9a..41f2b0bad 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -61,6 +61,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity process_hashtag tag, status when 'Mention' process_mention tag, status + when 'Emoji' + process_emoji tag, status end end end @@ -79,6 +81,17 @@ class ActivityPub::Activity::Create < ActivityPub::Activity account.mentions.create(status: status) end + def process_emoji(tag, _status) + shortcode = tag['name'].delete(':') + emoji = CustomEmoji.find_by(shortcode: shortcode, domain: @account.domain) + + return if !emoji.nil? || skip_download? + + emoji = CustomEmoji.new(domain: @account.domain, shortcode: shortcode) + emoji.image_remote_url = tag['href'] + emoji.save + end + def process_attachments(status) return unless @object['attachment'].is_a?(Array) diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 575830190..29fea27de 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -9,7 +9,7 @@ class Formatter include ActionView::Helpers::TextHelper - def format(status) + def format(status, options = {}) if status.reblog? prepend_reblog = status.reblog.account.acct status = status.proper @@ -19,7 +19,11 @@ class Formatter raw_content = status.text - return reformat(raw_content) unless status.local? + unless status.local? + html = reformat(raw_content) + html = encode_custom_emojis(html, status.emojis) if options[:custom_emojify] + return html + end linkable_accounts = status.mentions.map(&:account) linkable_accounts << status.account @@ -27,6 +31,7 @@ class Formatter html = raw_content html = "RT @#{prepend_reblog} #{html}" if prepend_reblog html = encode_and_link_urls(html, linkable_accounts) + html = encode_custom_emojis(html, status.emojis) if options[:custom_emojify] html = simple_format(html, {}, sanitize: false) html = html.delete("\n") @@ -39,7 +44,9 @@ class Formatter def plaintext(status) return status.text if status.local? - strip_tags(status.text) + + text = status.text.gsub(/(
|
|<\/p>)+/) { |match| "#{match}\n" } + strip_tags(text) end def simplified_format(account) @@ -76,6 +83,47 @@ class Formatter end end + def encode_custom_emojis(html, emojis) + return html if emojis.empty? + + emoji_map = emojis.map { |e| [e.shortcode, full_asset_url(e.image.url)] }.to_h + + i = -1 + inside_tag = false + inside_shortname = false + shortname_start_index = -1 + + while i + 1 < html.size + i += 1 + + if inside_shortname && html[i] == ':' + shortcode = html[shortname_start_index + 1..i - 1] + emoji = emoji_map[shortcode] + + if emoji + replacement = "\":#{shortcode}:\"" + before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : '' + html = before_html + replacement + html[i + 1..-1] + i += replacement.size - (shortcode.size + 2) - 1 + else + i -= 1 + end + + inside_shortname = false + elsif inside_tag && html[i] == '>' + inside_tag = false + elsif html[i] == '<' + inside_tag = true + inside_shortname = false + elsif !inside_tag && html[i] == ':' + inside_shortname = true + shortname_start_index = i + end + end + + html + end + def rewrite(text, entities) chars = text.to_s.to_char_a diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb index 1a23c9efa..d3f1629c4 100644 --- a/app/lib/ostatus/activity/creation.rb +++ b/app/lib/ostatus/activity/creation.rb @@ -42,6 +42,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base save_mentions(status) save_hashtags(status) save_media(status) + save_emojis(status) end if thread? && status.thread.nil? @@ -150,6 +151,25 @@ class OStatus::Activity::Creation < OStatus::Activity::Base end end + def save_emojis(parent) + do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media? + + return if do_not_download + + @xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: TagManager::XMLNS).each do |link| + next unless link['href'] && link['name'] + + shortcode = link['name'].delete(':') + emoji = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain) + + next unless emoji.nil? + + emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain) + emoji.image_remote_url = link['href'] + emoji.save + end + end + def account_from_href(href) url = Addressable::URI.parse(href).normalize diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb index b8e22a381..a6a5cb0c4 100644 --- a/app/lib/ostatus/atom_serializer.rb +++ b/app/lib/ostatus/atom_serializer.rb @@ -368,5 +368,9 @@ class OStatus::AtomSerializer end append_element(entry, 'mastodon:scope', status.visibility) + + status.emojis.each do |emoji| + append_element(entry, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode) + end end end diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb new file mode 100644 index 000000000..f4d3b16a0 --- /dev/null +++ b/app/models/custom_emoji.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: custom_emojis +# +# id :integer not null, primary key +# shortcode :string default(""), not null +# domain :string +# image_file_name :string +# image_content_type :string +# image_file_size :integer +# image_updated_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# + +class CustomEmoji < ApplicationRecord + SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}' + + SCAN_RE = /(?<=[^[:alnum:]:]|\n|^) + :(#{SHORTCODE_RE_FRAGMENT}): + (?=[^[:alnum:]:]|$)/x + + has_attached_file :image + + validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { in: 0..50.kilobytes } + validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 } + + include Remotable + + class << self + def from_text(text, domain) + return [] if text.blank? + shortcodes = text.scan(SCAN_RE).map(&:first) + where(shortcode: shortcodes, domain: domain) + end + end +end diff --git a/app/models/status.rb b/app/models/status.rb index 2a2cdcf6e..326d128d6 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -131,6 +131,10 @@ class Status < ApplicationRecord !sensitive? && media_attachments.any? end + def emojis + CustomEmoji.from_text(text, account.domain) + end + after_create :store_uri, if: :local? before_validation :prepare_contents, if: :local? diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb index 166214eee..e5d8e3f03 100644 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@ -57,7 +57,7 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer end def virtual_tags - object.mentions + object.tags + object.mentions + object.tags + object.emojis end def atom_uri @@ -137,4 +137,22 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer "##{object.name}" end end + + class CustomEmojiSerializer < ActiveModel::Serializer + include RoutingHelper + + attributes :type, :href, :name + + def type + 'Emoji' + end + + def href + full_asset_url(object.image.url) + end + + def name + ":#{object.shortcode}:" + end + end end diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index 298a3bb40..d8efa8e60 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -17,6 +17,7 @@ class REST::StatusSerializer < ActiveModel::Serializer has_many :media_attachments, serializer: REST::MediaAttachmentSerializer has_many :mentions has_many :tags + has_many :emojis def current_user? !current_user.nil? @@ -106,4 +107,14 @@ class REST::StatusSerializer < ActiveModel::Serializer tag_url(object) end end + + class CustomEmojiSerializer < ActiveModel::Serializer + include RoutingHelper + + attributes :shortcode, :url + + def url + full_asset_url(object.image.url) + end + end end diff --git a/app/views/stream_entries/_detailed_status.html.haml b/app/views/stream_entries/_detailed_status.html.haml index dd9456260..692d5a6d5 100644 --- a/app/views/stream_entries/_detailed_status.html.haml +++ b/app/views/stream_entries/_detailed_status.html.haml @@ -17,7 +17,7 @@ %p{ style: 'margin-bottom: 0' }< %span.p-summary> #{status.spoiler_text}  %a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more') - .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status) + .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status, custom_emojify: true) - if !status.media_attachments.empty? - if status.media_attachments.first.video? diff --git a/app/views/stream_entries/_simple_status.html.haml b/app/views/stream_entries/_simple_status.html.haml index 55aa97f32..f9a530d38 100644 --- a/app/views/stream_entries/_simple_status.html.haml +++ b/app/views/stream_entries/_simple_status.html.haml @@ -18,7 +18,7 @@ %p{ style: 'margin-bottom: 0' }< %span.p-summary> #{status.spoiler_text}  %a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more') - .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status) + .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status, custom_emojify: true) - unless status.media_attachments.empty? - if status.media_attachments.first.video? diff --git a/db/migrate/20170917153509_create_custom_emojis.rb b/db/migrate/20170917153509_create_custom_emojis.rb new file mode 100644 index 000000000..4040c8312 --- /dev/null +++ b/db/migrate/20170917153509_create_custom_emojis.rb @@ -0,0 +1,13 @@ +class CreateCustomEmojis < ActiveRecord::Migration[5.1] + def change + create_table :custom_emojis do |t| + t.string :shortcode, null: false, default: '' + t.string :domain + t.attachment :image + + t.timestamps + end + + add_index :custom_emojis, [:shortcode, :domain], unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index f2ca2af69..9f42d46dd 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170913000752) do +ActiveRecord::Schema.define(version: 20170917153509) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -89,6 +89,18 @@ ActiveRecord::Schema.define(version: 20170913000752) do t.index ["uri"], name: "index_conversations_on_uri", unique: true end + create_table "custom_emojis", force: :cascade do |t| + t.string "shortcode", default: "", null: false + t.string "domain" + t.string "image_file_name" + t.string "image_content_type" + t.integer "image_file_size" + t.datetime "image_updated_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true + end + create_table "domain_blocks", id: :serial, force: :cascade do |t| t.string "domain", default: "", null: false t.datetime "created_at", null: false diff --git a/spec/fabricators/custom_emoji_fabricator.rb b/spec/fabricators/custom_emoji_fabricator.rb new file mode 100644 index 000000000..18a7d23dc --- /dev/null +++ b/spec/fabricators/custom_emoji_fabricator.rb @@ -0,0 +1,5 @@ +Fabricator(:custom_emoji) do + shortcode 'coolcat' + domain nil + image { File.open(Rails.root.join('spec', 'fixtures', 'files', 'emojo.png')) } +end diff --git a/spec/fixtures/files/emojo.png b/spec/fixtures/files/emojo.png new file mode 100644 index 000000000..cb5993499 Binary files /dev/null and b/spec/fixtures/files/emojo.png differ diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index fcb044ebc..1a9520f04 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -17,6 +17,7 @@ RSpec.describe ActivityPub::Activity::Create do before do stub_request(:get, 'http://example.com/attachment.png').to_return(request_fixture('avatar.txt')) + stub_request(:get, 'http://example.com/emoji.png').to_return(body: attachment_fixture('emojo.png')) end describe '#perform' do @@ -217,5 +218,29 @@ RSpec.describe ActivityPub::Activity::Create do expect(status.tags.map(&:name)).to include('test') end end + + context 'with emojis' do + let(:object_json) do + { + id: 'bar', + type: 'Note', + content: 'Lorem ipsum :tinking:', + tag: [ + { + type: 'Emoji', + href: 'http://example.com/emoji.png', + name: 'tinking', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.emojis.map(&:shortcode)).to include('tinking') + end + end end end diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index b714b317a..71b6b78d2 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -223,6 +223,45 @@ RSpec.describe Formatter do include_examples 'encode and link URLs' end + + context 'with custom_emojify option' do + let!(:emoji) { Fabricate(:custom_emoji) } + let(:status) { Fabricate(:status, account: local_account, text: text) } + + subject { Formatter.instance.format(status, custom_emojify: true) } + + context 'with emoji at the start' do + let(:text) { ':coolcat: Beep boop' } + + it 'converts shortcode to image tag' do + is_expected.to match(/

:coolcat::coolcat: Beep boop
' } + + it 'converts shortcode to image tag' do + is_expected.to match(/

:coolcat:Beep :coolcat: boop

' } + + it 'converts shortcode to image tag' do + is_expected.to match(/Beep :coolcat::coolcat::coolcat:

' } + + it 'does not touch the shortcodes' do + is_expected.to match(/

:coolcat::coolcat:<\/p>/) + end + end + + context 'with emoji at the end' do + let(:text) { '

Beep boop
:coolcat:

' } + + it 'converts shortcode to image tag' do + is_expected.to match(/
:coolcat:Hello :coolcat:

' } + + it 'returns records used via shortcodes in text' do + is_expected.to include(emojo) + end + end + end +end -- cgit