From ec6d1f678f0a43d829f7f97821e860c3c13a694c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 30 Apr 2025 20:19:43 +0200 Subject: [PATCH] Add color and blurhash extraction for profile pictures --- app/javascript/mastodon/api_types/accounts.ts | 10 +++ .../avatar_overlay-test.jsx.snap | 16 +++- app/javascript/mastodon/components/avatar.tsx | 11 ++- .../mastodon/components/avatar_overlay.tsx | 30 +++----- .../mastodon/components/media_attachments.jsx | 6 +- app/javascript/mastodon/components/status.jsx | 6 +- .../features/alt_text_modal/index.tsx | 13 +++- .../status/components/detailed_status.tsx | 15 +++- .../features/ui/components/audio_modal.tsx | 20 +++-- app/javascript/mastodon/models/account.ts | 2 + .../styles/mastodon/components.scss | 10 +++ app/models/account.rb | 76 ++++++++++--------- app/models/concerns/account/avatar.rb | 10 ++- app/models/media_attachment.rb | 2 +- app/serializers/rest/account_serializer.rb | 2 +- ...50430151215_add_avatar_meta_to_accounts.rb | 8 ++ db/schema.rb | 4 +- lib/paperclip/blurhash_transcoder.rb | 2 +- lib/paperclip/color_extractor.rb | 5 +- 19 files changed, 163 insertions(+), 85 deletions(-) create mode 100644 db/migrate/20250430151215_add_avatar_meta_to_accounts.rb diff --git a/app/javascript/mastodon/api_types/accounts.ts b/app/javascript/mastodon/api_types/accounts.ts index 3f8b27497f..75a95f5581 100644 --- a/app/javascript/mastodon/api_types/accounts.ts +++ b/app/javascript/mastodon/api_types/accounts.ts @@ -12,6 +12,14 @@ export interface ApiAccountRoleJSON { name: string; } +export interface ApiMetaJSON { + colors?: { + background: string; + foreground: string; + accent: string; + }; +} + // See app/serializers/rest/account_serializer.rb export interface BaseApiAccountJSON { acct: string; @@ -44,6 +52,8 @@ export interface BaseApiAccountJSON { limited?: boolean; memorial?: boolean; hide_collections: boolean; + blurhash?: string; + meta?: ApiMetaJSON; } // See app/serializers/rest/muted_account_serializer.rb diff --git a/app/javascript/mastodon/components/__tests__/__snapshots__/avatar_overlay-test.jsx.snap b/app/javascript/mastodon/components/__tests__/__snapshots__/avatar_overlay-test.jsx.snap index fbd44ecc5e..98e5a25537 100644 --- a/app/javascript/mastodon/components/__tests__/__snapshots__/avatar_overlay-test.jsx.snap +++ b/app/javascript/mastodon/components/__tests__/__snapshots__/avatar_overlay-test.jsx.snap @@ -16,7 +16,9 @@ exports[`
alice
@@ -34,7 +38,9 @@ exports[`
eve@blackhat.lair
diff --git a/app/javascript/mastodon/components/avatar.tsx b/app/javascript/mastodon/components/avatar.tsx index a2dc0b782e..a93b35e22c 100644 --- a/app/javascript/mastodon/components/avatar.tsx +++ b/app/javascript/mastodon/components/avatar.tsx @@ -2,8 +2,9 @@ import { useState, useCallback } from 'react'; import classNames from 'classnames'; +import { Blurhash } from 'mastodon/components/blurhash'; import { useHovering } from 'mastodon/hooks/useHovering'; -import { autoPlayGif } from 'mastodon/initial_state'; +import { autoPlayGif, useBlurhash } from 'mastodon/initial_state'; import type { Account } from 'mastodon/models/account'; interface Props { @@ -58,6 +59,14 @@ export const Avatar: React.FC = ({ onMouseLeave={handleMouseLeave} style={style} > + {(loading || error) && account?.blurhash && ( + + )} + {src && !error && ( )} diff --git a/app/javascript/mastodon/components/avatar_overlay.tsx b/app/javascript/mastodon/components/avatar_overlay.tsx index 0bd33fea69..0202048df6 100644 --- a/app/javascript/mastodon/components/avatar_overlay.tsx +++ b/app/javascript/mastodon/components/avatar_overlay.tsx @@ -1,3 +1,4 @@ +import { Avatar } from 'mastodon/components/avatar'; import { useHovering } from 'mastodon/hooks/useHovering'; import { autoPlayGif } from 'mastodon/initial_state'; import type { Account } from 'mastodon/models/account'; @@ -19,12 +20,6 @@ export const AvatarOverlay: React.FC = ({ }) => { const { hovering, handleMouseEnter, handleMouseLeave } = useHovering(autoPlayGif); - const accountSrc = hovering - ? account?.get('avatar') - : account?.get('avatar_static'); - const friendSrc = hovering - ? friend?.get('avatar') - : friend?.get('avatar_static'); return (
= ({ onMouseLeave={handleMouseLeave} >
-
- {accountSrc && {account?.get('acct')}} -
+
+
-
- {friendSrc && {friend?.get('acct')}} -
+
); diff --git a/app/javascript/mastodon/components/media_attachments.jsx b/app/javascript/mastodon/components/media_attachments.jsx index 63fe3e67f9..77c0a1bab2 100644 --- a/app/javascript/mastodon/components/media_attachments.jsx +++ b/app/javascript/mastodon/components/media_attachments.jsx @@ -74,9 +74,9 @@ export default class MediaAttachments extends ImmutablePureComponent { width={width} height={height} poster={audio.get('preview_url') || status.getIn(['account', 'avatar_static'])} - backgroundColor={audio.getIn(['meta', 'colors', 'background'])} - foregroundColor={audio.getIn(['meta', 'colors', 'foreground'])} - accentColor={audio.getIn(['meta', 'colors', 'accent'])} + backgroundColor={audio.getIn(['meta', 'colors', 'background']) ?? status.getIn(['account', 'meta', 'colors', 'background'])} + foregroundColor={audio.getIn(['meta', 'colors', 'foreground']) ?? status.getIn(['account', 'meta', 'colors', 'foreground'])} + accentColor={audio.getIn(['meta', 'colors', 'accent']) ?? status.getIn(['account', 'meta', 'colors', 'accent'])} duration={audio.getIn(['meta', 'original', 'duration'], 0)} /> )} diff --git a/app/javascript/mastodon/components/status.jsx b/app/javascript/mastodon/components/status.jsx index 21d596a58c..509a75fc90 100644 --- a/app/javascript/mastodon/components/status.jsx +++ b/app/javascript/mastodon/components/status.jsx @@ -480,9 +480,9 @@ class Status extends ImmutablePureComponent { alt={description} lang={language} poster={attachment.get('preview_url') || status.getIn(['account', 'avatar_static'])} - backgroundColor={attachment.getIn(['meta', 'colors', 'background'])} - foregroundColor={attachment.getIn(['meta', 'colors', 'foreground'])} - accentColor={attachment.getIn(['meta', 'colors', 'accent'])} + backgroundColor={attachment.getIn(['meta', 'colors', 'background']) ?? status.getIn(['account', 'meta', 'colors', 'background'])} + foregroundColor={attachment.getIn(['meta', 'colors', 'foreground']) ?? status.getIn(['account', 'meta', 'colors', 'foreground'])} + accentColor={attachment.getIn(['meta', 'colors', 'accent']) ?? status.getIn(['account', 'meta', 'colors', 'accent'])} duration={attachment.getIn(['meta', 'original', 'duration'], 0)} deployPictureInPicture={pictureInPicture.get('available') ? this.handleDeployPictureInPicture : undefined} sensitive={status.get('sensitive')} diff --git a/app/javascript/mastodon/features/alt_text_modal/index.tsx b/app/javascript/mastodon/features/alt_text_modal/index.tsx index 08e4a8917c..99fee4b36d 100644 --- a/app/javascript/mastodon/features/alt_text_modal/index.tsx +++ b/app/javascript/mastodon/features/alt_text_modal/index.tsx @@ -218,12 +218,19 @@ const Preview: React.FC<{ } duration={media.getIn(['meta', 'original', 'duration'], 0) as number} backgroundColor={ - media.getIn(['meta', 'colors', 'background']) as string + (media.getIn(['meta', 'colors', 'background']) as + | string + | undefined) ?? account?.meta.colors?.background } foregroundColor={ - media.getIn(['meta', 'colors', 'foreground']) as string + (media.getIn(['meta', 'colors', 'foreground']) as + | string + | undefined) ?? account?.meta.colors?.foreground + } + accentColor={ + (media.getIn(['meta', 'colors', 'accent']) as string | undefined) ?? + account?.meta.colors?.accent } - accentColor={media.getIn(['meta', 'colors', 'accent']) as string} editable /> ); diff --git a/app/javascript/mastodon/features/status/components/detailed_status.tsx b/app/javascript/mastodon/features/status/components/detailed_status.tsx index 75d995b1e0..7babaa6b44 100644 --- a/app/javascript/mastodon/features/status/components/detailed_status.tsx +++ b/app/javascript/mastodon/features/status/components/detailed_status.tsx @@ -193,9 +193,18 @@ export const DetailedStatus: React.FC<{ status.getIn(['account', 'avatar_static']) } duration={attachment.getIn(['meta', 'original', 'duration'], 0)} - backgroundColor={attachment.getIn(['meta', 'colors', 'background'])} - foregroundColor={attachment.getIn(['meta', 'colors', 'foreground'])} - accentColor={attachment.getIn(['meta', 'colors', 'accent'])} + backgroundColor={ + attachment.getIn(['meta', 'colors', 'background']) ?? + status.getIn(['account', 'meta', 'colors', 'background']) + } + foregroundColor={ + attachment.getIn(['meta', 'colors', 'foreground']) ?? + status.getIn(['account', 'meta', 'colors', 'foreground']) + } + accentColor={ + attachment.getIn(['meta', 'colors', 'accent']) ?? + status.getIn(['account', 'meta', 'colors', 'accent']) + } sensitive={status.get('sensitive')} visible={showMedia} blurhash={attachment.get('blurhash')} diff --git a/app/javascript/mastodon/features/ui/components/audio_modal.tsx b/app/javascript/mastodon/features/ui/components/audio_modal.tsx index ddbd0b8853..c7475a1f65 100644 --- a/app/javascript/mastodon/features/ui/components/audio_modal.tsx +++ b/app/javascript/mastodon/features/ui/components/audio_modal.tsx @@ -18,8 +18,8 @@ const AudioModal: React.FC<{ }> = ({ media, statusId, options, onClose, onChangeBackgroundColor }) => { const status = useAppSelector((state) => state.statuses.get(statusId)); const accountId = status?.get('account') as string | undefined; - const accountStaticAvatar = useAppSelector((state) => - accountId ? state.accounts.get(accountId)?.avatar_static : undefined, + const account = useAppSelector((state) => + accountId ? state.accounts.get(accountId) : undefined, ); useEffect(() => { @@ -47,16 +47,24 @@ const AudioModal: React.FC<{ alt={description} lang={language} poster={ - (media.get('preview_url') as string | null) ?? accountStaticAvatar + (media.get('preview_url') as string | null) ?? + account?.avatar_static } duration={media.getIn(['meta', 'original', 'duration'], 0) as number} backgroundColor={ - media.getIn(['meta', 'colors', 'background']) as string + (media.getIn(['meta', 'colors', 'background']) as + | string + | undefined) ?? account?.meta.colors?.background } foregroundColor={ - media.getIn(['meta', 'colors', 'foreground']) as string + (media.getIn(['meta', 'colors', 'foreground']) as + | string + | undefined) ?? account?.meta.colors?.foreground + } + accentColor={ + (media.getIn(['meta', 'colors', 'accent']) as string | undefined) ?? + account?.meta.colors?.accent } - accentColor={media.getIn(['meta', 'colors', 'accent']) as string} startPlaying={options.autoPlay} /> diff --git a/app/javascript/mastodon/models/account.ts b/app/javascript/mastodon/models/account.ts index 4d95d24757..99e1740964 100644 --- a/app/javascript/mastodon/models/account.ts +++ b/app/javascript/mastodon/models/account.ts @@ -94,6 +94,8 @@ export const accountDefaultValues: AccountShape = { limited: false, moved: null, hide_collections: false, + blurhash: '', + meta: {}, // This comes from `ApiMutedAccountJSON`, but we should eventually // store that in a different object. mute_expires_at: null, diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index e22a9ed9c9..caa8288383 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -2100,6 +2100,16 @@ body > [data-popper-placement] { display: inline-block; // to not show broken images } + &__preview { + position: absolute; + top: 0; + inset-inline-start: 0; + width: 100%; + height: 100%; + border-radius: var(--avatar-border-radius); + object-fit: cover; + } + &--loading { background-color: var(--surface-background-color); } diff --git a/app/models/account.rb b/app/models/account.rb index 53bf2407e8..392787b3d5 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -5,52 +5,54 @@ # Table name: accounts # # id :bigint(8) not null, primary key -# username :string default(""), not null -# domain :string -# private_key :text -# public_key :text default(""), not null -# created_at :datetime not null -# updated_at :datetime not null -# note :text default(""), not null -# display_name :string default(""), not null -# uri :string default(""), not null -# url :string -# avatar_file_name :string +# actor_type :string +# also_known_as :string is an Array +# attribution_domains :string default([]), is an Array # avatar_content_type :string +# avatar_file_name :string # avatar_file_size :integer -# avatar_updated_at :datetime -# header_file_name :string -# header_content_type :string -# header_file_size :integer -# header_updated_at :datetime # avatar_remote_url :string -# locked :boolean default(FALSE), not null -# header_remote_url :string default(""), not null -# last_webfingered_at :datetime -# inbox_url :string default(""), not null -# outbox_url :string default(""), not null -# shared_inbox_url :string default(""), not null -# followers_url :string default(""), not null -# protocol :integer default("ostatus"), not null -# memorial :boolean default(FALSE), not null -# moved_to_account_id :bigint(8) +# avatar_storage_schema_version :integer +# avatar_updated_at :datetime +# blurhash :string +# discoverable :boolean +# display_name :string default(""), not null +# domain :string # featured_collection_url :string # fields :jsonb -# actor_type :string -# discoverable :boolean -# also_known_as :string is an Array +# followers_url :string default(""), not null +# header_content_type :string +# header_file_name :string +# header_file_size :integer +# header_remote_url :string default(""), not null +# header_storage_schema_version :integer +# header_updated_at :datetime +# hide_collections :boolean +# inbox_url :string default(""), not null +# indexable :boolean default(FALSE), not null +# last_webfingered_at :datetime +# locked :boolean default(FALSE), not null +# memorial :boolean default(FALSE), not null +# meta :json +# note :text default(""), not null +# outbox_url :string default(""), not null +# private_key :text +# protocol :integer default("ostatus"), not null +# public_key :text default(""), not null +# requested_review_at :datetime +# reviewed_at :datetime +# sensitized_at :datetime +# shared_inbox_url :string default(""), not null # silenced_at :datetime # suspended_at :datetime -# hide_collections :boolean -# avatar_storage_schema_version :integer -# header_storage_schema_version :integer # suspension_origin :integer -# sensitized_at :datetime # trendable :boolean -# reviewed_at :datetime -# requested_review_at :datetime -# indexable :boolean default(FALSE), not null -# attribution_domains :string default([]), is an Array +# uri :string default(""), not null +# url :string +# username :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# moved_to_account_id :bigint(8) # class Account < ApplicationRecord diff --git a/app/models/concerns/account/avatar.rb b/app/models/concerns/account/avatar.rb index a60a289d5b..7e19cb6554 100644 --- a/app/models/concerns/account/avatar.rb +++ b/app/models/concerns/account/avatar.rb @@ -11,7 +11,13 @@ module Account::Avatar class_methods do def avatar_styles(file) styles = { original: { geometry: "#{AVATAR_GEOMETRY}#", file_geometry_parser: FastGeometryParser } } - styles[:static] = { geometry: "#{AVATAR_GEOMETRY}#", format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' + + if file.content_type == 'image/gif' + styles[:static] = { geometry: "#{AVATAR_GEOMETRY}#", format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser, blurhash: { x_comp: 3, y_comp: 3 }, extract_colors: { meta_attribute_name: :meta } } + else + styles[:original].merge!(blurhash: { x_comp: 3, y_comp: 3 }, extract_colors: { meta_attribute_name: :meta }) + end + styles end @@ -20,7 +26,7 @@ module Account::Avatar included do # Avatar upload - has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail] + has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :color_extractor] validates_attachment_content_type :avatar, content_type: AVATAR_IMAGE_MIME_TYPES validates_attachment_size :avatar, less_than: AVATAR_LIMIT remotable_attachment :avatar, AVATAR_LIMIT, suppress_errors: false diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 04b73e6b2f..0fa97f30a8 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -165,7 +165,7 @@ class MediaAttachment < ApplicationRecord }.freeze THUMBNAIL_STYLES = { - original: IMAGE_STYLES[:small].freeze, + original: IMAGE_STYLES[:small].merge(extract_colors: { meta_attribute_name: :file_meta }.freeze).freeze, }.freeze DEFAULT_STYLES = [:original].freeze diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb index 354d384464..f08d529072 100644 --- a/app/serializers/rest/account_serializer.rb +++ b/app/serializers/rest/account_serializer.rb @@ -8,7 +8,7 @@ class REST::AccountSerializer < ActiveModel::Serializer attributes :id, :username, :acct, :display_name, :locked, :bot, :discoverable, :indexable, :group, :created_at, :note, :url, :uri, :avatar, :avatar_static, :header, :header_static, - :followers_count, :following_count, :statuses_count, :last_status_at, :hide_collections + :followers_count, :following_count, :statuses_count, :last_status_at, :hide_collections, :meta, :blurhash has_one :moved_to_account, key: :moved, serializer: REST::AccountSerializer, if: :moved_and_not_nested? diff --git a/db/migrate/20250430151215_add_avatar_meta_to_accounts.rb b/db/migrate/20250430151215_add_avatar_meta_to_accounts.rb new file mode 100644 index 0000000000..574ef00569 --- /dev/null +++ b/db/migrate/20250430151215_add_avatar_meta_to_accounts.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +class AddAvatarMetaToAccounts < ActiveRecord::Migration[8.0] + def change + safety_assured { add_column :accounts, :meta, :json } + add_column :accounts, :blurhash, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index db1687ba99..9912caf0b4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2025_04_28_095029) do +ActiveRecord::Schema[8.0].define(version: 2025_04_30_151215) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -198,6 +198,8 @@ ActiveRecord::Schema[8.0].define(version: 2025_04_28_095029) do t.datetime "requested_review_at", precision: nil t.boolean "indexable", default: false, null: false t.string "attribution_domains", default: [], array: true + t.json "meta" + t.string "blurhash" t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true t.index ["domain", "id"], name: "index_accounts_on_domain_and_id" diff --git a/lib/paperclip/blurhash_transcoder.rb b/lib/paperclip/blurhash_transcoder.rb index b4ff4a12a0..f3ba1c2409 100644 --- a/lib/paperclip/blurhash_transcoder.rb +++ b/lib/paperclip/blurhash_transcoder.rb @@ -3,7 +3,7 @@ module Paperclip class BlurhashTranscoder < Paperclip::Processor def make - return @file unless options[:style] == :small || options[:blurhash] + return @file unless options[:blurhash] width, height, data = blurhash_params # Guard against segfaults if data has unexpected size diff --git a/lib/paperclip/color_extractor.rb b/lib/paperclip/color_extractor.rb index fba32ba4cb..84862d63f3 100644 --- a/lib/paperclip/color_extractor.rb +++ b/lib/paperclip/color_extractor.rb @@ -10,6 +10,8 @@ module Paperclip BINS = 10 def make + return @file unless options.key?(:extract_colors) + background_palette, foreground_palette = Rails.configuration.x.use_vips ? palettes_from_libvips : palettes_from_imagemagick background_color = background_palette.first || foreground_palette.first @@ -66,7 +68,8 @@ module Paperclip }, } - attachment.instance.file.instance_write(:meta, (attachment.instance.file.instance_read(:meta) || {}).merge(meta)) + meta_attribute_name = options[:extract_colors][:meta_attribute_name] || "#{attachment.name}_meta" + attachment.instance.public_send("#{meta_attribute_name}=", (attachment.instance.public_send(meta_attribute_name) || {}).merge(meta)) @file rescue Vips::Error => e