From cfa7930a111a0d7d6aaff9f861201e0a7cdf098d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Misty=20De=20M=C3=A9o?= Date: Mon, 17 Jul 2023 10:34:25 -0700 Subject: [PATCH] Add support for Azure blob storage (#1297) This adds support for Azure blob storage using Paperclip, similar to what's supported for S3 and other backends. It makes use of the `paperclip-azure` plugin gem. As with S3, this is not loaded by default; it's only used if the proper environment variables are set. I'm using my fork of paperclip-azure because it incorporates a few new features to make it feature-equivalent to S3 for Mastodon's purposes. --- Gemfile | 1 + Gemfile.lock | 17 ++++++++++++++++ app/helpers/application_helper.rb | 4 ++-- .../initializers/content_security_policy.rb | 1 + config/initializers/paperclip.rb | 20 +++++++++++++++++++ 5 files changed, 41 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 6a72fec54..ead8dceda 100644 --- a/Gemfile +++ b/Gemfile @@ -22,6 +22,7 @@ gem 'aws-sdk-s3', '~> 1.117', require: false gem 'fog-core', '<= 2.4.0' gem 'fog-openstack', '~> 0.3', require: false gem 'kt-paperclip', '~> 7.1' +gem 'md-paperclip-azure', '~> 2.2', require: false gem 'blurhash', '~> 0.1' gem 'active_model_serializers', '~> 0.10' diff --git a/Gemfile.lock b/Gemfile.lock index 2b4b806c5..165cfaca8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -105,6 +105,14 @@ GEM aws-sigv4 (~> 1.4) aws-sigv4 (1.5.2) aws-eventstream (~> 1, >= 1.0.2) + azure-storage-blob (2.0.3) + azure-storage-common (~> 2.0) + nokogiri (~> 1, >= 1.10.8) + azure-storage-common (2.0.4) + faraday (~> 1.0) + faraday_middleware (~> 1.0, >= 1.0.0.rc1) + net-http-persistent (~> 4.0) + nokogiri (~> 1, >= 1.10.8) bcrypt (3.1.17) better_errors (2.9.1) coderay (>= 1.0.0) @@ -254,6 +262,8 @@ GEM faraday-patron (1.0.0) faraday-rack (1.0.0) faraday-retry (1.0.3) + faraday_middleware (1.2.0) + faraday (~> 1.0) fast_blank (1.0.1) fastimage (2.2.6) ffi (1.15.5) @@ -399,6 +409,10 @@ GEM mario-redis-lock (1.2.1) redis (>= 3.0.5) matrix (0.4.2) + md-paperclip-azure (2.2.0) + addressable (~> 2.5) + azure-storage-blob (~> 2.0.1) + hashie (~> 5.0) memory_profiler (1.0.1) method_source (1.0.0) mime-types (3.4.1) @@ -410,6 +424,8 @@ GEM msgpack (1.6.0) multi_json (1.15.0) multipart-post (2.1.1) + net-http-persistent (4.0.2) + connection_pool (~> 2.2) net-imap (0.3.4) date net-protocol @@ -817,6 +833,7 @@ DEPENDENCIES lograge (~> 0.12) makara (~> 0.5) mario-redis-lock (~> 1.2) + md-paperclip-azure (~> 2.2) memory_profiler mime-types (~> 3.4.1) net-ldap (~> 0.17) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 4c20f1e14..20760a70d 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -180,11 +180,11 @@ module ApplicationHelper end def storage_host - "https://#{ENV['S3_ALIAS_HOST'].presence || ENV['S3_CLOUDFRONT_HOST']}" + "https://#{ENV['S3_ALIAS_HOST'].presence || ENV['S3_CLOUDFRONT_HOST'].presence || ENV['AZURE_ALIAS_HOST']}" end def storage_host? - ENV['S3_ALIAS_HOST'].present? || ENV['S3_CLOUDFRONT_HOST'].present? + ENV['S3_ALIAS_HOST'].present? || ENV['S3_CLOUDFRONT_HOST'].present? || ENV['AZURE_ALIAS_HOST'].present? end def quote_wrap(text, line_width: 80, break_sequence: "\n") diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb index cb5629337..c8ed0da57 100644 --- a/config/initializers/content_security_policy.rb +++ b/config/initializers/content_security_policy.rb @@ -13,6 +13,7 @@ assets_host ||= host_to_url(base_host) media_host = host_to_url(ENV['S3_ALIAS_HOST']) media_host ||= host_to_url(ENV['S3_CLOUDFRONT_HOST']) +media_host ||= host_to_url(ENV['AZURE_ALIAS_HOST']) media_host ||= host_to_url(ENV['S3_HOSTNAME']) if ENV['S3_ENABLED'] == 'true' media_host ||= assets_host diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index a2285427c..2be349119 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -132,6 +132,26 @@ elsif ENV['SWIFT_ENABLED'] == 'true' fog_host: ENV['SWIFT_OBJECT_URL'], fog_public: true ) +elsif ENV['AZURE_ENABLED'] == 'true' + require 'paperclip-azure' + + Paperclip::Attachment.default_options.merge!( + storage: :azure, + azure_options: { + protocol: 'https' + }, + azure_credentials: { + storage_account_name: ENV['AZURE_STORAGE_ACCOUNT'], + storage_access_key: ENV['AZURE_STORAGE_ACCESS_KEY'], + container: ENV['AZURE_CONTAINER_NAME'] + } + ) + if ENV.has_key?('AZURE_ALIAS_HOST') + Paperclip::Attachment.default_options.merge!( + url: ':azure_alias_url', + azure_host_alias: ENV['AZURE_ALIAS_HOST'] + ) + end else Paperclip::Attachment.default_options.merge!( storage: :filesystem,