diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a60ae684..bddfd2d27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,7 +68,9 @@ jobs: cache-version: v1 pkg-manager: yarn - run: - command: ./bin/rails assets:precompile + command: | + export NODE_OPTIONS=--openssl-legacy-provider + ./bin/rails assets:precompile name: Precompile assets - persist_to_workspace: paths: diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 39fe1bd0b..9500c5e22 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -12,6 +12,7 @@ on: - Dockerfile permissions: contents: read + packages: write jobs: build-image: @@ -20,15 +21,28 @@ jobs: - uses: actions/checkout@v3 - uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - if: github.event_name != 'pull_request' + if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' + + - name: Log in to the Github Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' + - uses: docker/metadata-action@v4 id: meta with: - images: tootsuite/mastodon + images: | + tootsuite/mastodon + ghcr.io/mastodon/mastodon flavor: | latest=auto tags: | @@ -36,11 +50,15 @@ jobs: type=pep440,pattern={{raw}} type=pep440,pattern=v{{major}}.{{minor}} type=ref,event=pr - - uses: docker/build-push-action@v3 + + - uses: docker/build-push-action@v4 with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} + provenance: false + builder: ${{ steps.buildx.outputs.name }} + push: ${{ github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} - cache-from: type=registry,ref=tootsuite/mastodon:edge - cache-to: type=inline + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.ruby-version b/.ruby-version index b0f2dcb32..818bd47ab 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.0.4 +3.0.6 diff --git a/CHANGELOG.md b/CHANGELOG.md index b1ad9e5fd..1ac230ff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,53 @@ Changelog All notable changes to this project will be documented in this file. +## [4.0.4] - 2023-04-04 + +### Fixed + +- Fix crash in `tootctl` commands making use of parallelization when Elasticsearch is enabled ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24182), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24377)) +- Fix crash in `db:setup` when Elasticsearch is enabled ([rrgeorge](https://github.com/mastodon/mastodon/pull/24302)) +- Fix user archive takeout when using OpenStack Swift or S3 providers with no ACL support ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24200)) +- Fix invalid/expired invites being processed on sign-up ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24337)) + +### Security + +- Update Ruby to 3.0.6 due to ReDoS vulnerabilities ([saizai](https://github.com/mastodon/mastodon/pull/24333)) +- Fix unescaped user input in LDAP query ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24379)) + +# [4.0.3] - 2023-03-16 + +### Added + +- Add redirection from paths with url-encoded `@` to their decoded form ([thijskh](https://github.com/mastodon/mastodon/pull/23593)) +- Add `lang` attribute to native language names in language picker in Web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23749)) +- Add headers to outgoing mails to avoid auto-replies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23597)) + +### Fixed + +- Fix “Remove all followers from the selected domains” being more destructive than it claims ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23805)) +- Fix case-sensitive check for previously used hashtags in hashtag autocompletion ([deanveloper](https://github.com/mastodon/mastodon/pull/23526)) +- Fix sidebar behavior in settings/admin UI on mobile ([wxt2005](https://github.com/mastodon/mastodon/pull/23764)) +- Fix inefficiency when searching accounts per username in admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23801)) +- Fix server error when failing to follow back followers from `/relationships` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23787)) +- Fix server error when attempting to display the edit history of a trendable post in the admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23574)) +- Fix original account being unfollowed on migration before the follow request to the new account could be sent ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21957)) +- Fix pgBouncer resetting application name on every transaction ([Gargron](https://github.com/mastodon/mastodon/pull/23958)) +- Fix unconfirmed accounts being counted as active users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23803)) +- Fix `/api/v1/streaming` sub-paths not being redirected ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23988)) +- Fix drag'n'drop upload area text that spans multiple lines not being centered ([vintprox](https://github.com/mastodon/mastodon/pull/24029)) +- Fix sidekiq jobs not triggering Elasticsearch index updates ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24046)) +- Fix tags being unnecessarily stripped from plain-text short site description ([c960657](https://github.com/mastodon/mastodon/pull/23975)) +- Fix HTML entities not being un-escaped in extracted plain-text from remote posts ([c960657](https://github.com/mastodon/mastodon/pull/24019)) +- Fix dashboard crash on ElasticSearch server error ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23751)) +- Fix incorrect post links in strikes when the account is remote ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23611)) +- Fix misleading error code when receiving invalid WebAuthn credentials ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23568)) + +### Security + +- Change user backups to use expiring URLs for download when possible ([Gargron](https://github.com/mastodon/mastodon/pull/24136)) +- Add warning for object storage misconfiguration ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24137)) + ## [4.0.2] - 2022-11-15 ### Fixed diff --git a/Dockerfile b/Dockerfile index cf311fef2..d560f1888 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN ARCH= && \ mv node-v$NODE_VER-linux-$ARCH /opt/node # Install Ruby 3.0 -ENV RUBY_VER="3.0.4" +ENV RUBY_VER="3.0.6" RUN apt-get update && \ apt-get install -y --no-install-recommends build-essential \ bison libyaml-dev libgdbm-dev libreadline-dev libjemalloc-dev \ diff --git a/app/controllers/auth/registrations_controller.rb b/app/controllers/auth/registrations_controller.rb index 14e0d9a36..c4de20231 100644 --- a/app/controllers/auth/registrations_controller.rb +++ b/app/controllers/auth/registrations_controller.rb @@ -48,7 +48,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController super(hash) resource.locale = I18n.locale - resource.invite_code = params[:invite_code] if resource.invite_code.blank? + resource.invite_code = @invite&.code if resource.invite_code.blank? resource.registration_form_time = session[:registration_form_time] resource.sign_up_ip = request.remote_ip diff --git a/app/controllers/backups_controller.rb b/app/controllers/backups_controller.rb new file mode 100644 index 000000000..0687b62c5 --- /dev/null +++ b/app/controllers/backups_controller.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class BackupsController < ApplicationController + include RoutingHelper + + skip_before_action :require_functional! + + before_action :authenticate_user! + before_action :set_backup + + def download + case Paperclip::Attachment.default_options[:storage] + when :s3 + redirect_to @backup.dump.expiring_url(10) + when :fog + if Paperclip::Attachment.default_options.dig(:storage, :fog_credentials, :openstack_temp_url_key).present? + redirect_to @backup.dump.expiring_url(Time.now.utc + 10) + else + redirect_to full_asset_url(@backup.dump.url) + end + when :filesystem + redirect_to full_asset_url(@backup.dump.url) + end + end + + private + + def set_backup + @backup = current_user.backups.find(params[:id]) + end +end diff --git a/app/controllers/relationships_controller.rb b/app/controllers/relationships_controller.rb index 96cce55e9..de5dc5879 100644 --- a/app/controllers/relationships_controller.rb +++ b/app/controllers/relationships_controller.rb @@ -19,6 +19,8 @@ class RelationshipsController < ApplicationController @form.save rescue ActionController::ParameterMissing # Do nothing + rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound + flash[:alert] = I18n.t('relationships.follow_failure') if action_from_button == 'follow' ensure redirect_to relationships_path(filter_params) end @@ -60,8 +62,8 @@ class RelationshipsController < ApplicationController 'unfollow' elsif params[:remove_from_followers] 'remove_from_followers' - elsif params[:block_domains] - 'block_domains' + elsif params[:block_domains] || params[:remove_domains_from_followers] + 'remove_domains_from_followers' end end diff --git a/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb b/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb index a50d30f06..8435155dd 100644 --- a/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb +++ b/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb @@ -52,7 +52,7 @@ module Settings end else flash[:error] = I18n.t('webauthn_credentials.create.error') - status = :internal_server_error + status = :unprocessable_entity end else flash[:error] = t('webauthn_credentials.create.error') diff --git a/app/javascript/mastodon/features/compose/components/language_dropdown.js b/app/javascript/mastodon/features/compose/components/language_dropdown.js index bf56fd0fa..bac6c1c77 100644 --- a/app/javascript/mastodon/features/compose/components/language_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/language_dropdown.js @@ -216,7 +216,7 @@ class LanguageDropdownMenu extends React.PureComponent { return (
- {lang[2]} ({lang[1]}) + {lang[2]} ({lang[1]})
); } diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js index aabeddf15..5efb4938c 100644 --- a/app/javascript/mastodon/reducers/compose.js +++ b/app/javascript/mastodon/reducers/compose.js @@ -190,11 +190,12 @@ const ignoreSuggestion = (state, position, token, completion, path) => { }; const sortHashtagsByUse = (state, tags) => { - const personalHistory = state.get('tagHistory'); + const personalHistory = state.get('tagHistory').map(tag => tag.toLowerCase()); - return tags.sort((a, b) => { - const usedA = personalHistory.includes(a.name); - const usedB = personalHistory.includes(b.name); + const tagsWithLowercase = tags.map(t => ({ ...t, lowerName: t.name.toLowerCase() })); + const sorted = tagsWithLowercase.sort((a, b) => { + const usedA = personalHistory.includes(a.lowerName); + const usedB = personalHistory.includes(b.lowerName); if (usedA === usedB) { return 0; @@ -204,6 +205,8 @@ const sortHashtagsByUse = (state, tags) => { return 1; } }); + sorted.forEach(tag => delete tag.lowerName); + return sorted; }; const insertEmoji = (state, position, emojiData, needsSpace) => { diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss index a742bfa11..41d3e9a8a 100644 --- a/app/javascript/styles/mastodon/admin.scss +++ b/app/javascript/styles/mastodon/admin.scss @@ -400,7 +400,7 @@ $content-width: 840px; position: fixed; z-index: 10; width: 100%; - height: calc(100vh - 56px); + height: calc(100% - 56px); left: 0; bottom: 0; overflow-y: auto; diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 4e4c2d6f0..fe6fae6f4 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -4498,6 +4498,7 @@ a.status-card.compact:hover { display: flex; align-items: center; justify-content: center; + text-align: center; color: $secondary-text-color; font-size: 18px; font-weight: 500; diff --git a/app/lib/admin/system_check.rb b/app/lib/admin/system_check.rb index f512635ab..89dfcef9f 100644 --- a/app/lib/admin/system_check.rb +++ b/app/lib/admin/system_check.rb @@ -2,6 +2,7 @@ class Admin::SystemCheck ACTIVE_CHECKS = [ + Admin::SystemCheck::MediaPrivacyCheck, Admin::SystemCheck::DatabaseSchemaCheck, Admin::SystemCheck::SidekiqProcessCheck, Admin::SystemCheck::RulesCheck, diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index 8aee18267..e04ec085d 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -24,7 +24,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck def running_version @running_version ||= begin Chewy.client.info['version']['number'] - rescue Faraday::ConnectionFailed + rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error nil end end diff --git a/app/lib/admin/system_check/media_privacy_check.rb b/app/lib/admin/system_check/media_privacy_check.rb new file mode 100644 index 000000000..1df05b120 --- /dev/null +++ b/app/lib/admin/system_check/media_privacy_check.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +class Admin::SystemCheck::MediaPrivacyCheck < Admin::SystemCheck::BaseCheck + include RoutingHelper + + def skip? + !current_user.can?(:view_devops) + end + + def pass? + check_media_uploads! + @failure_message.nil? + end + + def message + Admin::SystemCheck::Message.new(@failure_message, @failure_value, @failure_action, true) + end + + private + + def check_media_uploads! + if Rails.configuration.x.use_s3 + check_media_listing_inaccessible_s3! + else + check_media_listing_inaccessible! + end + end + + def check_media_listing_inaccessible! + full_url = full_asset_url(media_attachment.file.url(:original, false)) + + # Check if we can list the uploaded file. If true, that's an error + directory_url = Addressable::URI.parse(full_url) + directory_url.query = nil + filename = directory_url.path.gsub(%r{.*/}, '') + directory_url.path = directory_url.path.gsub(%r{/[^/]+\Z}, '/') + Request.new(:get, directory_url, allow_local: true).perform do |res| + if res.truncated_body&.include?(filename) + @failure_message = use_storage? ? :upload_check_privacy_error_object_storage : :upload_check_privacy_error + @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#FS' + end + end + rescue + nil + end + + def check_media_listing_inaccessible_s3! + urls_to_check = [] + paperclip_options = Paperclip::Attachment.default_options + s3_protocol = paperclip_options[:s3_protocol] + s3_host_alias = paperclip_options[:s3_host_alias] + s3_host_name = paperclip_options[:s3_host_name] + bucket_name = paperclip_options.dig(:s3_credentials, :bucket) + + urls_to_check << "#{s3_protocol}://#{s3_host_alias}/" if s3_host_alias.present? + urls_to_check << "#{s3_protocol}://#{s3_host_name}/#{bucket_name}/" + urls_to_check.uniq.each do |full_url| + check_s3_listing!(full_url) + break if @failure_message.present? + end + rescue + nil + end + + def check_s3_listing!(full_url) + bucket_url = Addressable::URI.parse(full_url) + bucket_url.path = bucket_url.path.delete_suffix(media_attachment.file.path(:original)) + bucket_url.query = "max-keys=1&x-random=#{SecureRandom.hex(10)}" + Request.new(:get, bucket_url, allow_local: true).perform do |res| + if res.truncated_body&.include?('ListBucketResult') + @failure_message = :upload_check_privacy_error_object_storage + @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#S3' + end + end + end + + def media_attachment + @media_attachment ||= begin + attachment = Account.representative.media_attachments.first + if attachment.present? + attachment.touch # rubocop:disable Rails/SkipsModelValidations + attachment + else + create_test_attachment! + end + end + end + + def create_test_attachment! + Tempfile.create(%w(test-upload .jpg), binmode: true) do |tmp_file| + tmp_file.write( + Base64.decode64( + '/9j/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAA' \ + 'AAAD/2wCEAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBA' \ + 'QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE' \ + 'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAf/AABEIAAEAAgMBEQACEQEDEQH/x' \ + 'ABKAAEAAAAAAAAAAAAAAAAAAAALEAEAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAAAAAAA' \ + 'AAAAAEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8H//2Q==' + ) + ) + tmp_file.flush + Account.representative.media_attachments.create!(file: tmp_file) + end + end +end diff --git a/app/lib/admin/system_check/message.rb b/app/lib/admin/system_check/message.rb index bfcad3bf3..ad8d4b607 100644 --- a/app/lib/admin/system_check/message.rb +++ b/app/lib/admin/system_check/message.rb @@ -1,11 +1,12 @@ # frozen_string_literal: true class Admin::SystemCheck::Message - attr_reader :key, :value, :action + attr_reader :key, :value, :action, :critical - def initialize(key, value = nil, action = nil) - @key = key - @value = value - @action = action + def initialize(key, value = nil, action = nil, critical = false) + @key = key + @value = value + @action = action + @critical = critical end end diff --git a/app/lib/plain_text_formatter.rb b/app/lib/plain_text_formatter.rb index 08aa29696..6fa2bc5d2 100644 --- a/app/lib/plain_text_formatter.rb +++ b/app/lib/plain_text_formatter.rb @@ -18,7 +18,7 @@ class PlainTextFormatter if local? text else - strip_tags(insert_newlines).chomp + html_entities.decode(strip_tags(insert_newlines)).chomp end end @@ -27,4 +27,8 @@ class PlainTextFormatter def insert_newlines text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" } end + + def html_entities + HTMLEntities.new + end end diff --git a/app/mailers/application_mailer.rb b/app/mailers/application_mailer.rb index a37682eca..4edcb75f3 100644 --- a/app/mailers/application_mailer.rb +++ b/app/mailers/application_mailer.rb @@ -7,6 +7,8 @@ class ApplicationMailer < ActionMailer::Base helper :instance helper :formatting + after_action :set_autoreply_headers! + protected def locale_for_account(account) @@ -14,4 +16,10 @@ class ApplicationMailer < ActionMailer::Base yield end end + + def set_autoreply_headers! + headers['Precedence'] = 'list' + headers['X-Auto-Response-Suppress'] = 'All' + headers['Auto-Submitted'] = 'auto-generated' + end end diff --git a/app/models/account.rb b/app/models/account.rb index 7ca0e41ef..1dd8392aa 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -107,7 +107,7 @@ class Account < ApplicationRecord scope :bots, -> { where(actor_type: %w(Application Service)) } scope :groups, -> { where(actor_type: 'Group') } scope :alphabetic, -> { order(domain: :asc, username: :asc) } - scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) } + scope :matches_username, ->(value) { where('lower((username)::text) LIKE lower(?)', "#{value}%") } scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) } scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } scope :without_unapproved, -> { left_outer_joins(:user).remote.or(left_outer_joins(:user).merge(User.approved.confirmed)) } diff --git a/app/models/backup.rb b/app/models/backup.rb index d242fd62c..8823e7cae 100644 --- a/app/models/backup.rb +++ b/app/models/backup.rb @@ -17,6 +17,6 @@ class Backup < ApplicationRecord belongs_to :user, inverse_of: :backups - has_attached_file :dump + has_attached_file :dump, s3_permissions: ->(*) { ENV['S3_PERMISSION'] == '' ? nil : 'private' } do_not_validate_attachment_file_type :dump end diff --git a/app/models/concerns/ldap_authenticable.rb b/app/models/concerns/ldap_authenticable.rb index dc5abcd5a..775df0817 100644 --- a/app/models/concerns/ldap_authenticable.rb +++ b/app/models/concerns/ldap_authenticable.rb @@ -6,7 +6,7 @@ module LdapAuthenticable class_methods do def authenticate_with_ldap(params = {}) ldap = Net::LDAP.new(ldap_options) - filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: params[:email]) + filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: Net::LDAP::Filter.escape(params[:email])) if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: params[:password])) ldap_get_user(user_info.first) diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 5cfcf7205..d0cb727ca 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -17,8 +17,8 @@ class Form::AccountBatch unfollow! when 'remove_from_followers' remove_from_followers! - when 'block_domains' - block_domains! + when 'remove_domains_from_followers' + remove_domains_from_followers! when 'approve' approve! when 'reject' @@ -35,9 +35,15 @@ class Form::AccountBatch private def follow! + error = nil + accounts.each do |target_account| FollowService.new.call(current_account, target_account) + rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound => e + error ||= e end + + raise error if error.present? end def unfollow! @@ -50,10 +56,8 @@ class Form::AccountBatch RemoveFromFollowersService.new.call(current_account, account_ids) end - def block_domains! - AfterAccountDomainBlockWorker.push_bulk(account_domains) do |domain| - [current_account.id, domain] - end + def remove_domains_from_followers! + RemoveDomainsFromFollowersService.new.call(current_account, account_domains) end def account_domains diff --git a/app/models/user.rb b/app/models/user.rb index d5a81588f..a50120608 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -480,10 +480,13 @@ class User < ApplicationRecord def prepare_new_user! BootstrapTimelineWorker.perform_async(account_id) ActivityTracker.increment('activity:accounts:local') + ActivityTracker.record('activity:logins', id) UserMailer.welcome(self).deliver_later end def prepare_returning_user! + return unless confirmed? + ActivityTracker.record('activity:logins', id) regenerate_feed! if needs_feed_update? end diff --git a/app/services/follow_migration_service.rb b/app/services/follow_migration_service.rb new file mode 100644 index 000000000..cfe9093cb --- /dev/null +++ b/app/services/follow_migration_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class FollowMigrationService < FollowService + # Follow an account with the same settings as another account, and unfollow the old account once the request is sent + # @param [Account] source_account From which to follow + # @param [Account] target_account Account to follow + # @param [Account] old_target_account Account to unfollow once the follow request has been sent to the new one + # @option [Boolean] bypass_locked Whether to immediately follow the new account even if it is locked + def call(source_account, target_account, old_target_account, bypass_locked: false) + @old_target_account = old_target_account + + follow = source_account.active_relationships.find_by(target_account: old_target_account) + reblogs = follow&.show_reblogs? + notify = follow&.notify? + languages = follow&.languages + + super(source_account, target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true) + end + + private + + def request_follow! + follow_request = @source_account.request_follow!(@target_account, **follow_options.merge(rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit])) + + if @target_account.local? + LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request') + UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true) + elsif @target_account.activitypub? + ActivityPub::MigratedFollowDeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, @old_target_account.id) + end + + follow_request + end + + def direct_follow! + follow = super + UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true) + follow + end +end diff --git a/app/services/remove_domains_from_followers_service.rb b/app/services/remove_domains_from_followers_service.rb new file mode 100644 index 000000000..d76763409 --- /dev/null +++ b/app/services/remove_domains_from_followers_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class RemoveDomainsFromFollowersService < BaseService + include Payloadable + + def call(source_account, target_domains) + source_account.passive_relationships.where(account_id: Account.where(domain: target_domains)).find_each do |follow| + follow.destroy + + create_notification(follow) if source_account.local? && !follow.account.local? && follow.account.activitypub? + end + end + + private + + def create_notification(follow) + ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.target_account_id, follow.account.inbox_url) + end + + def build_json(follow) + Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)) + end +end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 8354f0b9f..425472abd 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -12,7 +12,7 @@ - unless @system_checks.empty? .flash-message-stack - @system_checks.each do |message| - .flash-message.warning + .flash-message{ class: message.critical ? 'alert' : 'warning' } = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil) - if message.action = link_to t("admin.system_checks.#{message.key}.action"), message.action diff --git a/app/views/admin/statuses/show.html.haml b/app/views/admin/statuses/show.html.haml index 62b49de8c..4631e97f1 100644 --- a/app/views/admin/statuses/show.html.haml +++ b/app/views/admin/statuses/show.html.haml @@ -34,7 +34,7 @@ %td - if @status.trend.allowed? %abbr{ title: t('admin.trends.tags.current_score', score: @status.trend.score) }= t('admin.trends.tags.trending_rank', rank: @status.trend.rank) - - elsif @status.trend.requires_review? + - elsif @status.requires_review? = t('admin.trends.pending_review') - else = t('admin.trends.not_allowed_to_trend') diff --git a/app/views/application/_sidebar.html.haml b/app/views/application/_sidebar.html.haml index 6d18668b0..9d0efa7e1 100644 --- a/app/views/application/_sidebar.html.haml +++ b/app/views/application/_sidebar.html.haml @@ -3,7 +3,7 @@ = image_tag @instance_presenter.thumbnail&.file&.url(:'@1x') || asset_pack_path('media/images/preview.png'), alt: @instance_presenter.title .hero-widget__text - %p= @instance_presenter.description.html_safe.presence || t('about.about_mastodon_html') + %p= @instance_presenter.description.presence || t('about.about_mastodon_html') - if Setting.trends && !(user_signed_in? && !current_user.setting_trends) - trends = Trends.tags.query.allowed.limit(3) diff --git a/app/views/disputes/strikes/show.html.haml b/app/views/disputes/strikes/show.html.haml index 4a3005f72..edf7463e5 100644 --- a/app/views/disputes/strikes/show.html.haml +++ b/app/views/disputes/strikes/show.html.haml @@ -50,15 +50,15 @@ .strike-card__statuses-list__item - if (status = status_map[status_id.to_i]) .one-liner - = link_to short_account_status_url(@strike.target_account, status_id), class: 'emojify' do - = one_line_preview(status) + .emojify= one_line_preview(status) - - status.ordered_media_attachments.each do |media_attachment| - %abbr{ title: media_attachment.description } - = fa_icon 'link' - = media_attachment.file_file_name + - status.ordered_media_attachments.each do |media_attachment| + %abbr{ title: media_attachment.description } + = fa_icon 'link' + = media_attachment.file_file_name .strike-card__statuses-list__item__meta - %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) + = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank' do + %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) - unless status.application.nil? · = status.application.name diff --git a/app/views/relationships/show.html.haml b/app/views/relationships/show.html.haml index c82e639e0..cd5152b40 100644 --- a/app/views/relationships/show.html.haml +++ b/app/views/relationships/show.html.haml @@ -48,7 +48,7 @@ = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_followers')]), name: :remove_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } unless following_relationship? - = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :block_domains, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship? + = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :remove_domains_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship? .batch-table__body - if @accounts.empty? = nothing_here 'nothing-here--under-tabs' diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml index c49613fdc..d7b59af27 100644 --- a/app/views/settings/exports/show.html.haml +++ b/app/views/settings/exports/show.html.haml @@ -64,6 +64,6 @@ %td= l backup.created_at - if backup.processed? %td= number_to_human_size backup.dump_file_size - %td= table_link_to 'download', t('exports.archive_takeout.download'), backup.dump.url + %td= table_link_to 'download', t('exports.archive_takeout.download'), download_backup_url(backup) - else %td{ colspan: 2 }= t('exports.archive_takeout.in_progress') diff --git a/app/views/shared/_og.html.haml b/app/views/shared/_og.html.haml index 2941b566e..a5d99ae33 100644 --- a/app/views/shared/_og.html.haml +++ b/app/views/shared/_og.html.haml @@ -1,5 +1,5 @@ - thumbnail = @instance_presenter.thumbnail -- description ||= strip_tags(@instance_presenter.description.presence || t('about.about_mastodon_html')) +- description ||= @instance_presenter.description.presence || strip_tags(t('about.about_mastodon_html')) %meta{ name: 'description', content: description }/ diff --git a/app/views/user_mailer/backup_ready.html.haml b/app/views/user_mailer/backup_ready.html.haml index 6009d584d..7c05473f4 100644 --- a/app/views/user_mailer/backup_ready.html.haml +++ b/app/views/user_mailer/backup_ready.html.haml @@ -55,5 +55,5 @@ %tbody %tr %td.button-primary - = link_to full_asset_url(@backup.dump.url) do + = link_to download_backup_url(@backup) do %span= t 'exports.archive_takeout.download' diff --git a/app/views/user_mailer/backup_ready.text.erb b/app/views/user_mailer/backup_ready.text.erb index eb89e7d74..8ebbaae85 100644 --- a/app/views/user_mailer/backup_ready.text.erb +++ b/app/views/user_mailer/backup_ready.text.erb @@ -4,4 +4,4 @@ <%= t 'user_mailer.backup_ready.explanation' %> -=> <%= full_asset_url(@backup.dump.url) %> +=> <%= download_backup_url(@backup) %> diff --git a/app/workers/activitypub/migrated_follow_delivery_worker.rb b/app/workers/activitypub/migrated_follow_delivery_worker.rb new file mode 100644 index 000000000..17a9e515e --- /dev/null +++ b/app/workers/activitypub/migrated_follow_delivery_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ActivityPub::MigratedFollowDeliveryWorker < ActivityPub::DeliveryWorker + def perform(json, source_account_id, inbox_url, old_target_account_id, options = {}) + super(json, source_account_id, inbox_url, options) + unfollow_old_account!(old_target_account_id) + end + + private + + def unfollow_old_account!(old_target_account_id) + old_target_account = Account.find(old_target_account_id) + UnfollowService.new.call(@source_account, old_target_account, skip_unmerge: true) + rescue StandardError + true + end +end diff --git a/app/workers/unfollow_follow_worker.rb b/app/workers/unfollow_follow_worker.rb index 7203b4888..a4d57839d 100644 --- a/app/workers/unfollow_follow_worker.rb +++ b/app/workers/unfollow_follow_worker.rb @@ -10,13 +10,7 @@ class UnfollowFollowWorker old_target_account = Account.find(old_target_account_id) new_target_account = Account.find(new_target_account_id) - follow = follower_account.active_relationships.find_by(target_account: old_target_account) - reblogs = follow&.show_reblogs? - notify = follow&.notify? - languages = follow&.languages - - FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true) - UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true) + FollowMigrationService.new.call(follower_account, new_target_account, old_target_account, bypass_locked: bypass_locked) rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError true end diff --git a/bin/tootctl b/bin/tootctl index a9ebb22c6..9c7ae8b87 100755 --- a/bin/tootctl +++ b/bin/tootctl @@ -5,7 +5,9 @@ require_relative '../config/boot' require_relative '../lib/cli' begin - Mastodon::CLI.start(ARGV) + Chewy.strategy(:mastodon) do + Mastodon::CLI.start(ARGV) + end rescue Interrupt exit(130) end diff --git a/config/application.rb b/config/application.rb index 4d6c7ebf1..720ea51ec 100644 --- a/config/application.rb +++ b/config/application.rb @@ -39,6 +39,7 @@ require_relative '../lib/mastodon/rack_middleware' require_relative '../lib/devise/two_factor_ldap_authenticatable' require_relative '../lib/devise/two_factor_pam_authenticatable' require_relative '../lib/chewy/strategy/mastodon' +require_relative '../lib/chewy/strategy/bypass_with_warning' require_relative '../lib/webpacker/manifest_extensions' require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/rails/engine_extensions' diff --git a/config/database.yml b/config/database.yml index 127a78abf..6bfde618b 100644 --- a/config/database.yml +++ b/config/database.yml @@ -4,6 +4,7 @@ default: &default timeout: 5000 encoding: unicode sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %> + application_name: '' development: <<: *default diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb index 752fc3c6d..daf4a5f32 100644 --- a/config/initializers/chewy.rb +++ b/config/initializers/chewy.rb @@ -19,7 +19,7 @@ Chewy.settings = { # cycle, which takes care of checking if Elasticsearch is enabled # or not. However, mind that for the Rails console, the :urgent # strategy is set automatically with no way to override it. -Chewy.root_strategy = :mastodon +Chewy.root_strategy = :bypass_with_warning if Rails.env.production? Chewy.request_strategy = :mastodon Chewy.use_after_commit_callbacks = false diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index a2285427c..63f0d9240 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -124,6 +124,7 @@ elsif ENV['SWIFT_ENABLED'] == 'true' openstack_domain_name: ENV.fetch('SWIFT_DOMAIN_NAME') { 'default' }, openstack_region: ENV['SWIFT_REGION'], openstack_cache_ttl: ENV.fetch('SWIFT_CACHE_TTL') { 60 }, + openstack_temp_url_key: ENV['SWIFT_TEMP_URL_KEY'], }, fog_file: { 'Cache-Control' => 'public, max-age=315576000, immutable' }, diff --git a/config/locales/en.yml b/config/locales/en.yml index c5a74ad8e..4733fbdb5 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -836,6 +836,12 @@ en: message_html: You haven't defined any server rules. sidekiq_process_check: message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration + upload_check_privacy_error: + action: Check here for more information + message_html: "Your web server is misconfigured. The privacy of your users is at risk." + upload_check_privacy_error_object_storage: + action: Check here for more information + message_html: "Your object storage is misconfigured. The privacy of your users is at risk." tags: review: Review status updated_msg: Hashtag settings updated successfully @@ -1423,6 +1429,7 @@ en: relationships: activity: Account activity dormant: Dormant + follow_failure: Could not follow some of the selected accounts. follow_selected_followers: Follow selected followers followers: Followers following: Following diff --git a/config/routes.rb b/config/routes.rb index bb8a836a7..98070946e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -109,6 +109,8 @@ Rails.application.routes.draw do resource :inbox, only: [:create], module: :activitypub + get '/:encoded_at(*path)', to: redirect("/@%{path}"), constraints: { encoded_at: /%40/ } + constraints(username: /[^@\/.]+/) do get '/@:username', to: 'accounts#show', as: :short_account get '/@:username/with_replies', to: 'accounts#show', as: :short_account_with_replies @@ -217,6 +219,7 @@ Rails.application.routes.draw do resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update] get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy, format: false + get '/backups/:id/download', to: 'backups#download', as: :download_backup, format: false resource :authorize_interaction, only: [:show, :create] resource :share, only: [:show, :create] @@ -466,7 +469,9 @@ Rails.application.routes.draw do resources :list, only: :show end - resources :streaming, only: [:index] + get '/streaming', to: 'streaming#index' + get '/streaming/(*any)', to: 'streaming#index' + resources :custom_emojis, only: [:index] resources :suggestions, only: [:index, :destroy] resources :scheduled_statuses, only: [:index, :show, :update, :destroy] diff --git a/db/seeds.rb b/db/seeds.rb index 1ca300de7..c01e83f1d 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true -Dir[Rails.root.join('db', 'seeds', '*.rb')].sort.each do |seed| - load seed +Chewy.strategy(:mastodon) do + Dir[Rails.root.join('db', 'seeds', '*.rb')].sort.each do |seed| + load seed + end end diff --git a/docker-compose.yml b/docker-compose.yml index 9e504c495..f603c2f7e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: tootsuite/mastodon:v3.5.5 + image: ghcr.io/mastodon/mastodon restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: tootsuite/mastodon:v3.5.5 + image: ghcr.io/mastodon/mastodon restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: tootsuite/mastodon:v3.5.5 + image: ghcr.io/mastodon/mastodon restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/chewy/strategy/bypass_with_warning.rb b/lib/chewy/strategy/bypass_with_warning.rb new file mode 100644 index 000000000..eb6fbaab1 --- /dev/null +++ b/lib/chewy/strategy/bypass_with_warning.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Chewy + class Strategy + class BypassWithWarning < Base + def update(...) + Rails.logger.warn 'Chewy update without a root strategy' unless @warning_issued + @warning_issued = true + end + end + end +end diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index a78a28e27..4e304c903 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -53,14 +53,16 @@ module Mastodon progress.log("Processing #{item.id}") if options[:verbose] - result = ActiveRecord::Base.connection_pool.with_connection do - yield(item) - ensure - RedisConfiguration.pool.checkin if Thread.current[:redis] - Thread.current[:redis] = nil - end + Chewy.strategy(:mastodon) do + result = ActiveRecord::Base.connection_pool.with_connection do + yield(item) + ensure + RedisConfiguration.pool.checkin if Thread.current[:redis] + Thread.current[:redis] = nil + end - aggregate.increment(result) if result.is_a?(Integer) + aggregate.increment(result) if result.is_a?(Integer) + end rescue => e progress.log pastel.red("Error processing #{item.id}: #{e}") ensure diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb index c75e8401f..9832e1a27 100644 --- a/lib/mastodon/sidekiq_middleware.rb +++ b/lib/mastodon/sidekiq_middleware.rb @@ -3,8 +3,8 @@ class Mastodon::SidekiqMiddleware BACKTRACE_LIMIT = 3 - def call(*) - yield + def call(*, &block) + Chewy.strategy(:mastodon, &block) rescue Mastodon::HostValidationError # Do not retry rescue => e diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index b461a3afa..437c0ea25 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 2 + 4 end def flags diff --git a/spec/controllers/relationships_controller_spec.rb b/spec/controllers/relationships_controller_spec.rb index 2056a2ac2..bcdcfa905 100644 --- a/spec/controllers/relationships_controller_spec.rb +++ b/spec/controllers/relationships_controller_spec.rb @@ -55,7 +55,7 @@ describe RelationshipsController do end context 'when select parameter is provided' do - subject { patch :update, params: { form_account_batch: { account_ids: [poopfeast.id] }, block_domains: '' } } + subject { patch :update, params: { form_account_batch: { account_ids: [poopfeast.id] }, remove_domains_from_followers: '' } } it 'soft-blocks followers from selected domains' do poopfeast.follow!(user.account) @@ -66,6 +66,15 @@ describe RelationshipsController do expect(poopfeast.following?(user.account)).to be false end + it 'does not unfollow users from selected domains' do + user.account.follow!(poopfeast) + + sign_in user, scope: :user + subject + + expect(user.account.following?(poopfeast)).to be true + end + include_examples 'authenticate user' include_examples 'redirects back to followers page' end diff --git a/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb index fe53b4dfc..269c4d685 100644 --- a/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb +++ b/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb @@ -248,7 +248,7 @@ describe Settings::TwoFactorAuthentication::WebauthnCredentialsController do post :create, params: { credential: new_webauthn_credential, nickname: 'USB Key' } - expect(response).to have_http_status(500) + expect(response).to have_http_status(422) expect(flash[:error]).to be_present end end @@ -268,7 +268,7 @@ describe Settings::TwoFactorAuthentication::WebauthnCredentialsController do post :create, params: { credential: new_webauthn_credential, nickname: nickname } - expect(response).to have_http_status(500) + expect(response).to have_http_status(422) expect(flash[:error]).to be_present end end diff --git a/spec/lib/plain_text_formatter_spec.rb b/spec/lib/plain_text_formatter_spec.rb index c3d0ee630..81e4ae286 100644 --- a/spec/lib/plain_text_formatter_spec.rb +++ b/spec/lib/plain_text_formatter_spec.rb @@ -4,7 +4,7 @@ RSpec.describe PlainTextFormatter do describe '#to_s' do subject { described_class.new(status.text, status.local?).to_s } - context 'given a post with local status' do + context 'when status is local' do let(:status) { Fabricate(:status, text: '

a text by a nerd who uses an HTML tag in text

', uri: nil) } it 'returns the raw text' do @@ -12,12 +12,63 @@ RSpec.describe PlainTextFormatter do end end - context 'given a post with remote status' do + context 'when status is remote' do let(:remote_account) { Fabricate(:account, domain: 'remote.test', username: 'bob', url: 'https://remote.test/') } - let(:status) { Fabricate(:status, account: remote_account, text: '

Hello

') } - it 'returns tag-stripped text' do - is_expected.to eq 'Hello' + context 'when text contains inline HTML tags' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem ipsum') } + + it 'strips the tags' do + expect(subject).to eq 'Lorem ipsum' + end + end + + context 'when text contains

tags' do + let(:status) { Fabricate(:status, account: remote_account, text: '

Lorem

ipsum

') } + + it 'inserts a newline' do + expect(subject).to eq "Lorem\nipsum" + end + end + + context 'when text contains a single
tag' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem
ipsum') } + + it 'inserts a newline' do + expect(subject).to eq "Lorem\nipsum" + end + end + + context 'when text contains consecutive
tag' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem


ipsum') } + + it 'inserts a single newline' do + expect(subject).to eq "Lorem\nipsum" + end + end + + context 'when text contains HTML entity' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem & ipsum ❤') } + + it 'unescapes the entity' do + expect(subject).to eq 'Lorem & ipsum ❤' + end + end + + context 'when text contains ipsum') } + + it 'strips the tag and its contents' do + expect(subject).to eq 'Lorem ipsum' + end + end + + context 'when text contains an HTML comment tags' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem ipsum') } + + it 'strips the comment' do + expect(subject).to eq 'Lorem ipsum' + end end end end