From 03f888806a6c1a8e209456f255b769cf6f5781ae Mon Sep 17 00:00:00 2001 From: Hugo Gameiro Date: Wed, 30 Oct 2024 10:01:25 +0000 Subject: [PATCH 1/8] Add fog storage support for tootctl media remove-orphans --- lib/mastodon/cli/media.rb | 53 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb index 996b7fd92c..8439ef8ae3 100644 --- a/lib/mastodon/cli/media.rb +++ b/lib/mastodon/cli/media.rb @@ -151,7 +151,58 @@ module Mastodon::CLI end end when :fog - fail_with_message 'The fog storage driver is not supported for this operation at this time' + paperclip_instance = MediaAttachment.new.file + fog_directory = Paperclip::Attachment.default_options[:fog_directory] + connection = Fog::Storage.new(Paperclip::Attachment.default_options[:fog_credentials]) + directory = connection.directories.get(fog_directory) + + last_key = options[:start_after] + + loop do + objects = begin + directory.files.all(prefix: prefix, marker: last_key, limit: 1000) + rescue => e + progress.log(pastel.red("Error fetching list of files: #{e}")) + progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key + break + end + + break if objects.empty? + + last_key = objects.last.key + record_map = preload_records_from_mixed_objects(objects) + + objects.each do |object| + path_segments = object.key.split('/') + path_segments.delete('cache') + + unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size) + progress.log(pastel.yellow("Unrecognized file found: #{object.key}")) + next + end + + model_name = path_segments.first.classify + attachment_name = path_segments[1].singularize + record_id = path_segments[2...-2].join.to_i + file_name = path_segments.last + record = record_map.dig(model_name, record_id) + attachment = record&.public_send(attachment_name) + + progress.increment + + next unless attachment.blank? || !attachment.variant?(file_name) + + begin + object.destroy unless dry_run? + reclaimed_bytes += object.content_length + removed += 1 + + progress.log("Found and removed orphan: #{object.key}") + rescue => e + progress.log(pastel.red("Error processing #{object.key}: #{e}")) + end + end + end when :azure fail_with_message 'The azure storage driver is not supported for this operation at this time' when :filesystem From a5ac53d326a1059a487f855987daedc8001d07f3 Mon Sep 17 00:00:00 2001 From: Hugo Gameiro Date: Wed, 30 Oct 2024 10:44:43 +0000 Subject: [PATCH 2/8] Clean up --- lib/mastodon/cli/media.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb index 8439ef8ae3..76a8229311 100644 --- a/lib/mastodon/cli/media.rb +++ b/lib/mastodon/cli/media.rb @@ -151,11 +151,9 @@ module Mastodon::CLI end end when :fog - paperclip_instance = MediaAttachment.new.file fog_directory = Paperclip::Attachment.default_options[:fog_directory] connection = Fog::Storage.new(Paperclip::Attachment.default_options[:fog_credentials]) directory = connection.directories.get(fog_directory) - last_key = options[:start_after] loop do From 6434d8ab4bb9ae3a88364f4cc9c4370533c1ce57 Mon Sep 17 00:00:00 2001 From: Hugo Gameiro Date: Wed, 18 Dec 2024 11:27:05 +0000 Subject: [PATCH 3/8] Fix directory.files.all not being converted to array --- lib/mastodon/cli/media.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb index 76a8229311..a614d84508 100644 --- a/lib/mastodon/cli/media.rb +++ b/lib/mastodon/cli/media.rb @@ -158,7 +158,7 @@ module Mastodon::CLI loop do objects = begin - directory.files.all(prefix: prefix, marker: last_key, limit: 1000) + directory.files.all(prefix: prefix, marker: last_key, limit: 1000).to_a rescue => e progress.log(pastel.red("Error fetching list of files: #{e}")) progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key From b9638a6b8232e65b7b18e8ec485f4382ef91e883 Mon Sep 17 00:00:00 2001 From: Hugo Gameiro Date: Wed, 18 Dec 2024 11:33:32 +0000 Subject: [PATCH 4/8] Reduce complexity after recent refactor --- lib/mastodon/cli/media.rb | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb index a614d84508..1cf412e981 100644 --- a/lib/mastodon/cli/media.rb +++ b/lib/mastodon/cli/media.rb @@ -172,33 +172,18 @@ module Mastodon::CLI objects.each do |object| path_segments = object.key.split('/') - path_segments.delete('cache') - - unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size) - progress.log(pastel.yellow("Unrecognized file found: #{object.key}")) - next - end - - model_name = path_segments.first.classify - attachment_name = path_segments[1].singularize - record_id = path_segments[2...-2].join.to_i - file_name = path_segments.last - record = record_map.dig(model_name, record_id) - attachment = record&.public_send(attachment_name) - progress.increment - - next unless attachment.blank? || !attachment.variant?(file_name) - + next unless orphaned_file?(path_segments) begin object.destroy unless dry_run? reclaimed_bytes += object.content_length removed += 1 - progress.log("Found and removed orphan: #{object.key}") rescue => e progress.log(pastel.red("Error processing #{object.key}: #{e}")) end + rescue UnrecognizedOrphanType + progress.log(pastel.yellow("Unrecognized file found: #{object.key}")) end end when :azure From ba3e30cf93020936462b7316c8e14eadd03dd45b Mon Sep 17 00:00:00 2001 From: Hugo Gameiro Date: Wed, 18 Dec 2024 11:43:14 +0000 Subject: [PATCH 5/8] Clean trailing whitespace --- lib/mastodon/cli/media.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb index 1cf412e981..bdc7d162ca 100644 --- a/lib/mastodon/cli/media.rb +++ b/lib/mastodon/cli/media.rb @@ -173,7 +173,7 @@ module Mastodon::CLI objects.each do |object| path_segments = object.key.split('/') progress.increment - next unless orphaned_file?(path_segments) + next unless orphaned_file?(path_segments) begin object.destroy unless dry_run? reclaimed_bytes += object.content_length From bb04f0b561b2f9bbf05b0b5c89a47485d093090d Mon Sep 17 00:00:00 2001 From: Hugo Gameiro Date: Wed, 18 Dec 2024 11:48:01 +0000 Subject: [PATCH 6/8] record_map is no longer needed after refactor --- lib/mastodon/cli/media.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb index bdc7d162ca..93f0f80b4b 100644 --- a/lib/mastodon/cli/media.rb +++ b/lib/mastodon/cli/media.rb @@ -168,7 +168,6 @@ module Mastodon::CLI break if objects.empty? last_key = objects.last.key - record_map = preload_records_from_mixed_objects(objects) objects.each do |object| path_segments = object.key.split('/') From bd769970fa34549cb93e12164cbd9fc485b8cca0 Mon Sep 17 00:00:00 2001 From: Hugo Gameiro Date: Wed, 18 Dec 2024 12:06:24 +0000 Subject: [PATCH 7/8] Attempt to fix lint Layout/EmptyLineAfterGuardClause From b23a68168a85af5cab9314ba2d014adaa62c0bee Mon Sep 17 00:00:00 2001 From: Hugo Gameiro Date: Wed, 18 Dec 2024 14:47:05 +0000 Subject: [PATCH 8/8] Fix Layout/EmptyLineAfterGuardClause --- lib/mastodon/cli/media.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb index 93f0f80b4b..87721f94cb 100644 --- a/lib/mastodon/cli/media.rb +++ b/lib/mastodon/cli/media.rb @@ -173,6 +173,7 @@ module Mastodon::CLI path_segments = object.key.split('/') progress.increment next unless orphaned_file?(path_segments) + begin object.destroy unless dry_run? reclaimed_bytes += object.content_length