diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index d4ddcbaab3..d707578cae 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -81,9 +81,6 @@ Rails/WhereExists: - 'app/lib/delivery_failure_tracker.rb' - 'app/lib/feed_manager.rb' - 'app/lib/suspicious_sign_in_detector.rb' - - 'app/models/poll.rb' - - 'app/models/session_activation.rb' - - 'app/models/status.rb' - 'app/policies/status_policy.rb' - 'app/serializers/rest/announcement_serializer.rb' - 'app/workers/move_worker.rb' diff --git a/app/controllers/api/v1/annual_reports_controller.rb b/app/controllers/api/v1/annual_reports_controller.rb new file mode 100644 index 0000000000..9bc8e68ac2 --- /dev/null +++ b/app/controllers/api/v1/annual_reports_controller.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class Api::V1::AnnualReportsController < Api::BaseController + before_action -> { doorkeeper_authorize! :read, :'read:accounts' }, only: :index + before_action -> { doorkeeper_authorize! :write, :'write:accounts' }, except: :index + before_action :require_user! + before_action :set_annual_report, except: :index + + def index + with_read_replica do + @presenter = AnnualReportsPresenter.new(GeneratedAnnualReport.where(account_id: current_account.id).pending) + @relationships = StatusRelationshipsPresenter.new(@presenter.statuses, current_account.id) + end + + render json: @presenter, + serializer: REST::AnnualReportsSerializer, + relationships: @relationships + end + + def read + @annual_report.view! + render_empty + end + + private + + def set_annual_report + @annual_report = GeneratedAnnualReport.find_by!(account_id: current_account.id, year: params[:id]) + end +end diff --git a/app/lib/annual_report.rb b/app/lib/annual_report.rb new file mode 100644 index 0000000000..cf4297f2a4 --- /dev/null +++ b/app/lib/annual_report.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +class AnnualReport + include DatabaseHelper + + SOURCES = [ + AnnualReport::Archetype, + AnnualReport::TypeDistribution, + AnnualReport::TopStatuses, + AnnualReport::MostUsedApps, + AnnualReport::CommonlyInteractedWithAccounts, + AnnualReport::TimeSeries, + AnnualReport::TopHashtags, + AnnualReport::MostRebloggedAccounts, + AnnualReport::Percentiles, + ].freeze + + SCHEMA = 1 + + def initialize(account, year) + @account = account + @year = year + end + + def generate + return if GeneratedAnnualReport.exists?(account: @account, year: @year) + + GeneratedAnnualReport.create( + account: @account, + year: @year, + schema_version: SCHEMA, + data: data + ) + end + + private + + def data + with_read_replica do + SOURCES.each_with_object({}) { |klass, hsh| hsh.merge!(klass.new(@account, @year).generate) } + end + end +end diff --git a/app/lib/annual_report/archetype.rb b/app/lib/annual_report/archetype.rb new file mode 100644 index 0000000000..ea9ef366df --- /dev/null +++ b/app/lib/annual_report/archetype.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class AnnualReport::Archetype < AnnualReport::Source + # Average number of posts (including replies and reblogs) made by + # each active user in a single year (2023) + AVERAGE_PER_YEAR = 113 + + def generate + { + archetype: archetype, + } + end + + private + + def archetype + if (standalone_count + replies_count + reblogs_count) < AVERAGE_PER_YEAR + :lurker + elsif reblogs_count > (standalone_count * 2) + :booster + elsif polls_count > (standalone_count * 0.1) # standalone_count includes posts with polls + :pollster + elsif replies_count > (standalone_count * 2) + :replier + else + :oracle + end + end + + def polls_count + @polls_count ||= base_scope.where.not(poll_id: nil).count + end + + def reblogs_count + @reblogs_count ||= base_scope.where.not(reblog_of_id: nil).count + end + + def replies_count + @replies_count ||= base_scope.where.not(in_reply_to_id: nil).where.not(in_reply_to_account_id: @account.id).count + end + + def standalone_count + @standalone_count ||= base_scope.without_replies.without_reblogs.count + end + + def base_scope + @account.statuses.where(id: year_as_snowflake_range) + end +end diff --git a/app/lib/annual_report/commonly_interacted_with_accounts.rb b/app/lib/annual_report/commonly_interacted_with_accounts.rb new file mode 100644 index 0000000000..af5e854c22 --- /dev/null +++ b/app/lib/annual_report/commonly_interacted_with_accounts.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AnnualReport::CommonlyInteractedWithAccounts < AnnualReport::Source + SET_SIZE = 40 + + def generate + { + commonly_interacted_with_accounts: commonly_interacted_with_accounts.map do |(account_id, count)| + { + account_id: account_id, + count: count, + } + end, + } + end + + private + + def commonly_interacted_with_accounts + @account.statuses.reorder(nil).where(id: year_as_snowflake_range).where.not(in_reply_to_account_id: @account.id).group(:in_reply_to_account_id).having('count(*) > 1').order(total: :desc).limit(SET_SIZE).pluck(Arel.sql('in_reply_to_account_id, count(*) AS total')) + end +end diff --git a/app/lib/annual_report/most_reblogged_accounts.rb b/app/lib/annual_report/most_reblogged_accounts.rb new file mode 100644 index 0000000000..e3e8a7c90b --- /dev/null +++ b/app/lib/annual_report/most_reblogged_accounts.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AnnualReport::MostRebloggedAccounts < AnnualReport::Source + SET_SIZE = 10 + + def generate + { + most_reblogged_accounts: most_reblogged_accounts.map do |(account_id, count)| + { + account_id: account_id, + count: count, + } + end, + } + end + + private + + def most_reblogged_accounts + @account.statuses.reorder(nil).where(id: year_as_snowflake_range).where.not(reblog_of_id: nil).joins(reblog: :account).group('accounts.id').having('count(*) > 1').order(total: :desc).limit(SET_SIZE).pluck(Arel.sql('accounts.id, count(*) as total')) + end +end diff --git a/app/lib/annual_report/most_used_apps.rb b/app/lib/annual_report/most_used_apps.rb new file mode 100644 index 0000000000..85ff1ff86e --- /dev/null +++ b/app/lib/annual_report/most_used_apps.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AnnualReport::MostUsedApps < AnnualReport::Source + SET_SIZE = 10 + + def generate + { + most_used_apps: most_used_apps.map do |(name, count)| + { + name: name, + count: count, + } + end, + } + end + + private + + def most_used_apps + @account.statuses.reorder(nil).where(id: year_as_snowflake_range).joins(:application).group('oauth_applications.name').order(total: :desc).limit(SET_SIZE).pluck(Arel.sql('oauth_applications.name, count(*) as total')) + end +end diff --git a/app/lib/annual_report/percentiles.rb b/app/lib/annual_report/percentiles.rb new file mode 100644 index 0000000000..9fe4698ee5 --- /dev/null +++ b/app/lib/annual_report/percentiles.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +class AnnualReport::Percentiles < AnnualReport::Source + def generate + { + percentiles: { + followers: (total_with_fewer_followers / (total_with_any_followers + 1.0)) * 100, + statuses: (total_with_fewer_statuses / (total_with_any_statuses + 1.0)) * 100, + }, + } + end + + private + + def followers_gained + @followers_gained ||= @account.passive_relationships.where("date_part('year', follows.created_at) = ?", @year).count + end + + def statuses_created + @statuses_created ||= @account.statuses.where(id: year_as_snowflake_range).count + end + + def total_with_fewer_followers + @total_with_fewer_followers ||= Follow.find_by_sql([<<~SQL.squish, { year: @year, comparison: followers_gained }]).first.total + WITH tmp0 AS ( + SELECT follows.target_account_id + FROM follows + INNER JOIN accounts ON accounts.id = follows.target_account_id + WHERE date_part('year', follows.created_at) = :year + AND accounts.domain IS NULL + GROUP BY follows.target_account_id + HAVING COUNT(*) < :comparison + ) + SELECT count(*) AS total + FROM tmp0 + SQL + end + + def total_with_fewer_statuses + @total_with_fewer_statuses ||= Status.find_by_sql([<<~SQL.squish, { comparison: statuses_created, min_id: year_as_snowflake_range.first, max_id: year_as_snowflake_range.last }]).first.total + WITH tmp0 AS ( + SELECT statuses.account_id + FROM statuses + INNER JOIN accounts ON accounts.id = statuses.account_id + WHERE statuses.id BETWEEN :min_id AND :max_id + AND accounts.domain IS NULL + GROUP BY statuses.account_id + HAVING count(*) < :comparison + ) + SELECT count(*) AS total + FROM tmp0 + SQL + end + + def total_with_any_followers + @total_with_any_followers ||= Follow.where("date_part('year', follows.created_at) = ?", @year).joins(:target_account).merge(Account.local).count('distinct follows.target_account_id') + end + + def total_with_any_statuses + @total_with_any_statuses ||= Status.where(id: year_as_snowflake_range).joins(:account).merge(Account.local).count('distinct statuses.account_id') + end +end diff --git a/app/lib/annual_report/source.rb b/app/lib/annual_report/source.rb new file mode 100644 index 0000000000..1ccb622676 --- /dev/null +++ b/app/lib/annual_report/source.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AnnualReport::Source + attr_reader :account, :year + + def initialize(account, year) + @account = account + @year = year + end + + protected + + def year_as_snowflake_range + (Mastodon::Snowflake.id_at(DateTime.new(year, 1, 1))..Mastodon::Snowflake.id_at(DateTime.new(year, 12, 31))) + end +end diff --git a/app/lib/annual_report/time_series.rb b/app/lib/annual_report/time_series.rb new file mode 100644 index 0000000000..a144bac0d1 --- /dev/null +++ b/app/lib/annual_report/time_series.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class AnnualReport::TimeSeries < AnnualReport::Source + def generate + { + time_series: (1..12).map do |month| + { + month: month, + statuses: statuses_per_month[month] || 0, + following: following_per_month[month] || 0, + followers: followers_per_month[month] || 0, + } + end, + } + end + + private + + def statuses_per_month + @statuses_per_month ||= @account.statuses.reorder(nil).where(id: year_as_snowflake_range).group(:period).pluck(Arel.sql("date_part('month', created_at)::int AS period, count(*)")).to_h + end + + def following_per_month + @following_per_month ||= @account.active_relationships.where("date_part('year', created_at) = ?", @year).group(:period).pluck(Arel.sql("date_part('month', created_at)::int AS period, count(*)")).to_h + end + + def followers_per_month + @followers_per_month ||= @account.passive_relationships.where("date_part('year', created_at) = ?", @year).group(:period).pluck(Arel.sql("date_part('month', created_at)::int AS period, count(*)")).to_h + end +end diff --git a/app/lib/annual_report/top_hashtags.rb b/app/lib/annual_report/top_hashtags.rb new file mode 100644 index 0000000000..488dacb1b4 --- /dev/null +++ b/app/lib/annual_report/top_hashtags.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AnnualReport::TopHashtags < AnnualReport::Source + SET_SIZE = 40 + + def generate + { + top_hashtags: top_hashtags.map do |(name, count)| + { + name: name, + count: count, + } + end, + } + end + + private + + def top_hashtags + Tag.joins(:statuses).where(statuses: { id: @account.statuses.where(id: year_as_snowflake_range).reorder(nil).select(:id) }).group(:id).having('count(*) > 1').order(total: :desc).limit(SET_SIZE).pluck(Arel.sql('COALESCE(tags.display_name, tags.name), count(*) AS total')) + end +end diff --git a/app/lib/annual_report/top_statuses.rb b/app/lib/annual_report/top_statuses.rb new file mode 100644 index 0000000000..112e5591ce --- /dev/null +++ b/app/lib/annual_report/top_statuses.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AnnualReport::TopStatuses < AnnualReport::Source + def generate + top_reblogs = base_scope.order(reblogs_count: :desc).first&.id + top_favourites = base_scope.where.not(id: top_reblogs).order(favourites_count: :desc).first&.id + top_replies = base_scope.where.not(id: [top_reblogs, top_favourites]).order(replies_count: :desc).first&.id + + { + top_statuses: { + by_reblogs: top_reblogs, + by_favourites: top_favourites, + by_replies: top_replies, + }, + } + end + + def base_scope + @account.statuses.with_public_visibility.joins(:status_stat).where(id: year_as_snowflake_range).reorder(nil) + end +end diff --git a/app/lib/annual_report/type_distribution.rb b/app/lib/annual_report/type_distribution.rb new file mode 100644 index 0000000000..fc12a6f1f4 --- /dev/null +++ b/app/lib/annual_report/type_distribution.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AnnualReport::TypeDistribution < AnnualReport::Source + def generate + { + type_distribution: { + total: base_scope.count, + reblogs: base_scope.where.not(reblog_of_id: nil).count, + replies: base_scope.where.not(in_reply_to_id: nil).where.not(in_reply_to_account_id: @account.id).count, + standalone: base_scope.without_replies.without_reblogs.count, + }, + } + end + + private + + def base_scope + @account.statuses.where(id: year_as_snowflake_range) + end +end diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb index ab7ea4092f..e558195290 100644 --- a/app/lib/vacuum/media_attachments_vacuum.rb +++ b/app/lib/vacuum/media_attachments_vacuum.rb @@ -27,11 +27,17 @@ class Vacuum::MediaAttachmentsVacuum end def media_attachments_past_retention_period - MediaAttachment.remote.cached.where(MediaAttachment.arel_table[:created_at].lt(@retention_period.ago)).where(MediaAttachment.arel_table[:updated_at].lt(@retention_period.ago)) + MediaAttachment + .remote + .cached + .created_before(@retention_period.ago) + .updated_before(@retention_period.ago) end def orphaned_media_attachments - MediaAttachment.unattached.where(MediaAttachment.arel_table[:created_at].lt(TTL.ago)) + MediaAttachment + .unattached + .created_before(TTL.ago) end def retention_period? diff --git a/app/models/account_summary.rb b/app/models/account_summary.rb index 0d8835b83c..2a21d09a8b 100644 --- a/app/models/account_summary.rb +++ b/app/models/account_summary.rb @@ -12,9 +12,11 @@ class AccountSummary < ApplicationRecord self.primary_key = :account_id + has_many :follow_recommendation_suppressions, primary_key: :account_id, foreign_key: :account_id, inverse_of: false + scope :safe, -> { where(sensitive: false) } scope :localized, ->(locale) { where(language: locale) } - scope :filtered, -> { joins(arel_table.join(FollowRecommendationSuppression.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:account_id].eq(FollowRecommendationSuppression.arel_table[:account_id])).join_sources).where(FollowRecommendationSuppression.arel_table[:id].eq(nil)) } + scope :filtered, -> { where.missing(:follow_recommendation_suppressions) } def self.refresh Scenic.database.refresh_materialized_view(table_name, concurrently: false, cascade: false) diff --git a/app/models/generated_annual_report.rb b/app/models/generated_annual_report.rb new file mode 100644 index 0000000000..43c97d7108 --- /dev/null +++ b/app/models/generated_annual_report.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: generated_annual_reports +# +# id :bigint(8) not null, primary key +# account_id :bigint(8) not null +# year :integer not null +# data :jsonb not null +# schema_version :integer not null +# viewed_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# + +class GeneratedAnnualReport < ApplicationRecord + belongs_to :account + + scope :pending, -> { where(viewed_at: nil) } + + def viewed? + viewed_at.present? + end + + def view! + update!(viewed_at: Time.now.utc) + end + + def account_ids + data['most_reblogged_accounts'].pluck('account_id') + data['commonly_interacted_with_accounts'].pluck('account_id') + end + + def status_ids + data['top_statuses'].values + end +end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 5af4ec46ed..90eda3dc8d 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -204,12 +204,14 @@ class MediaAttachment < ApplicationRecord validates :file, presence: true, if: :local? validates :thumbnail, absence: true, if: -> { local? && !audio_or_video? } - scope :attached, -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) } - scope :cached, -> { remote.where.not(file_file_name: nil) } - scope :local, -> { where(remote_url: '') } - scope :ordered, -> { order(id: :asc) } - scope :remote, -> { where.not(remote_url: '') } + scope :attached, -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) } + scope :cached, -> { remote.where.not(file_file_name: nil) } + scope :created_before, ->(value) { where(arel_table[:created_at].lt(value)) } + scope :local, -> { where(remote_url: '') } + scope :ordered, -> { order(id: :asc) } + scope :remote, -> { where.not(remote_url: '') } scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) } + scope :updated_before, ->(value) { where(arel_table[:updated_at].lt(value)) } attr_accessor :skip_download diff --git a/app/models/poll.rb b/app/models/poll.rb index 37149c3d86..cc4184f80a 100644 --- a/app/models/poll.rb +++ b/app/models/poll.rb @@ -57,7 +57,7 @@ class Poll < ApplicationRecord end def voted?(account) - account.id == account_id || votes.where(account: account).exists? + account.id == account_id || votes.exists?(account: account) end def own_votes(account) diff --git a/app/models/session_activation.rb b/app/models/session_activation.rb index 7f5f0d9a9a..c67180d3ba 100644 --- a/app/models/session_activation.rb +++ b/app/models/session_activation.rb @@ -41,7 +41,7 @@ class SessionActivation < ApplicationRecord class << self def active?(id) - id && where(session_id: id).exists? + id && exists?(session_id: id) end def activate(**options) diff --git a/app/models/status.rb b/app/models/status.rb index 7fc3f8c5df..18c9176b3a 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -270,7 +270,7 @@ class Status < ApplicationRecord end def reported? - @reported ||= Report.where(target_account: account).unresolved.where('? = ANY(status_ids)', id).exists? + @reported ||= Report.where(target_account: account).unresolved.exists?(['? = ANY(status_ids)', id]) end def emojis diff --git a/app/presenters/annual_reports_presenter.rb b/app/presenters/annual_reports_presenter.rb new file mode 100644 index 0000000000..001e1d37b0 --- /dev/null +++ b/app/presenters/annual_reports_presenter.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class AnnualReportsPresenter + alias read_attribute_for_serialization send + + attr_reader :annual_reports + + def initialize(annual_reports) + @annual_reports = annual_reports + end + + def accounts + @accounts ||= Account.where(id: @annual_reports.flat_map(&:account_ids)).includes(:account_stat, :moved_to_account, user: :role) + end + + def statuses + @statuses ||= Status.where(id: @annual_reports.flat_map(&:status_ids)).with_includes + end + + def self.model_name + @model_name ||= ActiveModel::Name.new(self) + end +end diff --git a/app/serializers/rest/annual_report_serializer.rb b/app/serializers/rest/annual_report_serializer.rb new file mode 100644 index 0000000000..1fb5ddb5c1 --- /dev/null +++ b/app/serializers/rest/annual_report_serializer.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class REST::AnnualReportSerializer < ActiveModel::Serializer + attributes :year, :data, :schema_version +end diff --git a/app/serializers/rest/annual_reports_serializer.rb b/app/serializers/rest/annual_reports_serializer.rb new file mode 100644 index 0000000000..ea9572be1b --- /dev/null +++ b/app/serializers/rest/annual_reports_serializer.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class REST::AnnualReportsSerializer < ActiveModel::Serializer + has_many :annual_reports, serializer: REST::AnnualReportSerializer + has_many :accounts, serializer: REST::AccountSerializer + has_many :statuses, serializer: REST::StatusSerializer +end diff --git a/app/workers/generate_annual_report_worker.rb b/app/workers/generate_annual_report_worker.rb new file mode 100644 index 0000000000..7094c1ab9c --- /dev/null +++ b/app/workers/generate_annual_report_worker.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class GenerateAnnualReportWorker + include Sidekiq::Worker + + def perform(account_id, year) + AnnualReport.new(Account.find(account_id), year).generate + rescue ActiveRecord::RecordNotFound, ActiveRecord::RecordNotUnique + true + end +end diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index 5c985e25a0..f52d0141d4 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -24,6 +24,8 @@ class Scheduler::IndexingScheduler end end + private + def indexes [AccountsIndex, TagsIndex, PublicStatusesIndex, StatusesIndex] end diff --git a/config/locales/ar.yml b/config/locales/ar.yml index 3c8c643fe7..e6d653c674 100644 --- a/config/locales/ar.yml +++ b/config/locales/ar.yml @@ -1934,6 +1934,7 @@ ar: go_to_sso_account_settings: انتقل إلى إعدادات حساب مزود الهوية الخاص بك invalid_otp_token: رمز المصادقة بخطوتين غير صالح otp_lost_help_html: إن فقدتَهُما ، يمكنك الاتصال بـ %{email} + rate_limited: عدد محاولات التحقق كثير جدًا، يرجى المحاولة مرة أخرى لاحقًا. seamless_external_login: لقد قمت بتسجيل الدخول عبر خدمة خارجية، إنّ إعدادات الكلمة السرية و البريد الإلكتروني غير متوفرة. signed_in_as: 'تم تسجيل دخولك بصفة:' verification: diff --git a/config/locales/bg.yml b/config/locales/bg.yml index c3eaa7e4c2..b9a3135448 100644 --- a/config/locales/bg.yml +++ b/config/locales/bg.yml @@ -1793,6 +1793,7 @@ bg: failed_2fa: details: 'Ето подробности на опита за влизане:' explanation: Някой се опита да влезе в акаунта ви, но предостави невалиден втори фактор за удостоверяване. + further_actions_html: Ако не бяхте вие, то препор