Add color and blurhash extraction for profile pictures

app/javascript/mastodon/api_types/accounts.ts

@@ -12,6 +12,14 @@ export interface ApiAccountRoleJSON {
   name: string;
 }
 
+export interface ApiMetaJSON {
+  colors?: {
+    background: string;
+    foreground: string;
+    accent: string;
+  };
+}
+
 // See app/serializers/rest/account_serializer.rb
 export interface BaseApiAccountJSON {
   acct: string;
@@ -44,6 +52,8 @@ export interface BaseApiAccountJSON {
   limited?: boolean;
   memorial?: boolean;
   hide_collections: boolean;
+  blurhash?: string;
+  meta?: ApiMetaJSON;
 }
 
 // See app/serializers/rest/muted_account_serializer.rb

app/javascript/mastodon/components/__tests__/__snapshots__/avatar_overlay-test.jsx.snap

@@ -16,7 +16,9 @@ exports[`<AvatarOverlay renders a overlay avatar 1`] = `
     className="account__avatar-overlay-base"
   >
     <div
-      className="account__avatar"
+      className="account__avatar account__avatar--loading"
+      onMouseEnter={[Function]}
+      onMouseLeave={[Function]}
       style={
         {
           "height": "36px",
@@ -25,7 +27,9 @@ exports[`<AvatarOverlay renders a overlay avatar 1`] = `
       }
     >
       <img
-        alt="alice"
+        alt=""
+        onError={[Function]}
+        onLoad={[Function]}
         src="/static/alice.jpg"
       />
     </div>
@@ -34,7 +38,9 @@ exports[`<AvatarOverlay renders a overlay avatar 1`] = `
     className="account__avatar-overlay-overlay"
   >
     <div
-      className="account__avatar"
+      className="account__avatar account__avatar--loading"
+      onMouseEnter={[Function]}
+      onMouseLeave={[Function]}
       style={
         {
           "height": "24px",
@@ -43,7 +49,9 @@ exports[`<AvatarOverlay renders a overlay avatar 1`] = `
       }
     >
       <img
-        alt="eve@blackhat.lair"
+        alt=""
+        onError={[Function]}
+        onLoad={[Function]}
         src="/static/eve.jpg"
       />
     </div>

app/javascript/mastodon/components/avatar.tsx

@@ -2,8 +2,9 @@ import { useState, useCallback } from 'react';
 
 import classNames from 'classnames';
 
+import { Blurhash } from 'mastodon/components/blurhash';
 import { useHovering } from 'mastodon/hooks/useHovering';
-import { autoPlayGif } from 'mastodon/initial_state';
+import { autoPlayGif, useBlurhash } from 'mastodon/initial_state';
 import type { Account } from 'mastodon/models/account';
 
 interface Props {
@@ -58,6 +59,14 @@ export const Avatar: React.FC<Props> = ({
       onMouseLeave={handleMouseLeave}
       style={style}
     >
+      {(loading || error) && account?.blurhash && (
+        <Blurhash
+          hash={account.blurhash}
+          className='account__avatar__preview'
+          dummy={!useBlurhash}
+        />
+      )}
+
       {src && !error && (
         <img src={src} alt='' onLoad={handleLoad} onError={handleError} />
       )}

app/javascript/mastodon/components/avatar_overlay.tsx

@@ -1,3 +1,4 @@
+import { Avatar } from 'mastodon/components/avatar';
 import { useHovering } from 'mastodon/hooks/useHovering';
 import { autoPlayGif } from 'mastodon/initial_state';
 import type { Account } from 'mastodon/models/account';
@@ -19,12 +20,6 @@ export const AvatarOverlay: React.FC<Props> = ({
 }) => {
   const { hovering, handleMouseEnter, handleMouseLeave } =
     useHovering(autoPlayGif);
-  const accountSrc = hovering
-    ? account?.get('avatar')
-    : account?.get('avatar_static');
-  const friendSrc = hovering
-    ? friend?.get('avatar')
-    : friend?.get('avatar_static');
 
   return (
     <div
@@ -34,20 +29,19 @@ export const AvatarOverlay: React.FC<Props> = ({
       onMouseLeave={handleMouseLeave}
     >
       <div className='account__avatar-overlay-base'>
-        <div
+        <Avatar
-          className='account__avatar'
+          account={account}
-          style={{ width: `${baseSize}px`, height: `${baseSize}px` }}
+          size={baseSize}
-        >
+          animate={hovering || autoPlayGif}
-          {accountSrc && <img src={accountSrc} alt={account?.get('acct')} />}
+        />
-        </div>
       </div>
+
       <div className='account__avatar-overlay-overlay'>
-        <div
+        <Avatar
-          className='account__avatar'
+          account={friend}
-          style={{ width: `${overlaySize}px`, height: `${overlaySize}px` }}
+          size={overlaySize}
-        >
+          animate={hovering || autoPlayGif}
-          {friendSrc && <img src={friendSrc} alt={friend?.get('acct')} />}
+        />
-        </div>
       </div>
     </div>
   );

app/javascript/mastodon/components/media_attachments.jsx

@@ -74,9 +74,9 @@ export default class MediaAttachments extends ImmutablePureComponent {
               width={width}
               height={height}
               poster={audio.get('preview_url') || status.getIn(['account', 'avatar_static'])}
-              backgroundColor={audio.getIn(['meta', 'colors', 'background'])}
+              backgroundColor={audio.getIn(['meta', 'colors', 'background']) ?? status.getIn(['account', 'meta', 'colors', 'background'])}
-              foregroundColor={audio.getIn(['meta', 'colors', 'foreground'])}
+              foregroundColor={audio.getIn(['meta', 'colors', 'foreground']) ?? status.getIn(['account', 'meta', 'colors', 'foreground'])}
-              accentColor={audio.getIn(['meta', 'colors', 'accent'])}
+              accentColor={audio.getIn(['meta', 'colors', 'accent']) ?? status.getIn(['account', 'meta', 'colors', 'accent'])}
               duration={audio.getIn(['meta', 'original', 'duration'], 0)}
             />
           )}

app/javascript/mastodon/components/status.jsx

@@ -480,9 +480,9 @@ class Status extends ImmutablePureComponent {
                 alt={description}
                 lang={language}
                 poster={attachment.get('preview_url') || status.getIn(['account', 'avatar_static'])}
-                backgroundColor={attachment.getIn(['meta', 'colors', 'background'])}
+                backgroundColor={attachment.getIn(['meta', 'colors', 'background']) ?? status.getIn(['account', 'meta', 'colors', 'background'])}
-                foregroundColor={attachment.getIn(['meta', 'colors', 'foreground'])}
+                foregroundColor={attachment.getIn(['meta', 'colors', 'foreground']) ?? status.getIn(['account', 'meta', 'colors', 'foreground'])}
-                accentColor={attachment.getIn(['meta', 'colors', 'accent'])}
+                accentColor={attachment.getIn(['meta', 'colors', 'accent']) ?? status.getIn(['account', 'meta', 'colors', 'accent'])}
                 duration={attachment.getIn(['meta', 'original', 'duration'], 0)}
                 deployPictureInPicture={pictureInPicture.get('available') ? this.handleDeployPictureInPicture : undefined}
                 sensitive={status.get('sensitive')}

app/javascript/mastodon/features/alt_text_modal/index.tsx

@@ -218,12 +218,19 @@ const Preview: React.FC<{
         }
         duration={media.getIn(['meta', 'original', 'duration'], 0) as number}
         backgroundColor={
-          media.getIn(['meta', 'colors', 'background']) as string
+          (media.getIn(['meta', 'colors', 'background']) as
+            | string
+            | undefined) ?? account?.meta.colors?.background
         }
         foregroundColor={
-          media.getIn(['meta', 'colors', 'foreground']) as string
+          (media.getIn(['meta', 'colors', 'foreground']) as
+            | string
+            | undefined) ?? account?.meta.colors?.foreground
+        }
+        accentColor={
+          (media.getIn(['meta', 'colors', 'accent']) as string | undefined) ??
+          account?.meta.colors?.accent
         }
-        accentColor={media.getIn(['meta', 'colors', 'accent']) as string}
         editable
       />
     );

app/javascript/mastodon/features/status/components/detailed_status.tsx

@@ -193,9 +193,18 @@ export const DetailedStatus: React.FC<{
             status.getIn(['account', 'avatar_static'])
           }
           duration={attachment.getIn(['meta', 'original', 'duration'], 0)}
-          backgroundColor={attachment.getIn(['meta', 'colors', 'background'])}
+          backgroundColor={
-          foregroundColor={attachment.getIn(['meta', 'colors', 'foreground'])}
+            attachment.getIn(['meta', 'colors', 'background']) ??
-          accentColor={attachment.getIn(['meta', 'colors', 'accent'])}
+            status.getIn(['account', 'meta', 'colors', 'background'])
+          }
+          foregroundColor={
+            attachment.getIn(['meta', 'colors', 'foreground']) ??
+            status.getIn(['account', 'meta', 'colors', 'foreground'])
+          }
+          accentColor={
+            attachment.getIn(['meta', 'colors', 'accent']) ??
+            status.getIn(['account', 'meta', 'colors', 'accent'])
+          }
           sensitive={status.get('sensitive')}
           visible={showMedia}
           blurhash={attachment.get('blurhash')}

app/javascript/mastodon/features/ui/components/audio_modal.tsx

@@ -18,8 +18,8 @@ const AudioModal: React.FC<{
 }> = ({ media, statusId, options, onClose, onChangeBackgroundColor }) => {
   const status = useAppSelector((state) => state.statuses.get(statusId));
   const accountId = status?.get('account') as string | undefined;
-  const accountStaticAvatar = useAppSelector((state) =>
+  const account = useAppSelector((state) =>
-    accountId ? state.accounts.get(accountId)?.avatar_static : undefined,
+    accountId ? state.accounts.get(accountId) : undefined,
   );
 
   useEffect(() => {
@@ -47,16 +47,24 @@ const AudioModal: React.FC<{
           alt={description}
           lang={language}
           poster={
-            (media.get('preview_url') as string | null) ?? accountStaticAvatar
+            (media.get('preview_url') as string | null) ??
+            account?.avatar_static
           }
           duration={media.getIn(['meta', 'original', 'duration'], 0) as number}
           backgroundColor={
-            media.getIn(['meta', 'colors', 'background']) as string
+            (media.getIn(['meta', 'colors', 'background']) as
+              | string
+              | undefined) ?? account?.meta.colors?.background
           }
           foregroundColor={
-            media.getIn(['meta', 'colors', 'foreground']) as string
+            (media.getIn(['meta', 'colors', 'foreground']) as
+              | string
+              | undefined) ?? account?.meta.colors?.foreground
+          }
+          accentColor={
+            (media.getIn(['meta', 'colors', 'accent']) as string | undefined) ??
+            account?.meta.colors?.accent
           }
-          accentColor={media.getIn(['meta', 'colors', 'accent']) as string}
           startPlaying={options.autoPlay}
         />
       </div>

app/javascript/mastodon/models/account.ts

@@ -94,6 +94,8 @@ export const accountDefaultValues: AccountShape = {
   limited: false,
   moved: null,
   hide_collections: false,
+  blurhash: '',
+  meta: {},
   // This comes from `ApiMutedAccountJSON`, but we should eventually
   // store that in a different object.
   mute_expires_at: null,

app/javascript/styles/mastodon/components.scss

@@ -2100,6 +2100,16 @@ body > [data-popper-placement] {
     display: inline-block; // to not show broken images
   }
 
+  &__preview {
+    position: absolute;
+    top: 0;
+    inset-inline-start: 0;
+    width: 100%;
+    height: 100%;
+    border-radius: var(--avatar-border-radius);
+    object-fit: cover;
+  }
+
   &--loading {
     background-color: var(--surface-background-color);
   }

app/models/account.rb

@@ -5,52 +5,54 @@
 # Table name: accounts
 #
 #  id                            :bigint(8)        not null, primary key
-#  username                      :string           default(""), not null
+#  actor_type                    :string
-#  domain                        :string
+#  also_known_as                 :string           is an Array
-#  private_key                   :text
+#  attribution_domains           :string           default([]), is an Array
-#  public_key                    :text             default(""), not null
-#  created_at                    :datetime         not null
-#  updated_at                    :datetime         not null
-#  note                          :text             default(""), not null
-#  display_name                  :string           default(""), not null
-#  uri                           :string           default(""), not null
-#  url                           :string
-#  avatar_file_name              :string
 #  avatar_content_type           :string
+#  avatar_file_name              :string
 #  avatar_file_size              :integer
-#  avatar_updated_at             :datetime
-#  header_file_name              :string
-#  header_content_type           :string
-#  header_file_size              :integer
-#  header_updated_at             :datetime
 #  avatar_remote_url             :string
-#  locked                        :boolean          default(FALSE), not null
+#  avatar_storage_schema_version :integer
-#  header_remote_url             :string           default(""), not null
+#  avatar_updated_at             :datetime
-#  last_webfingered_at           :datetime
+#  blurhash                      :string
-#  inbox_url                     :string           default(""), not null
+#  discoverable                  :boolean
-#  outbox_url                    :string           default(""), not null
+#  display_name                  :string           default(""), not null
-#  shared_inbox_url              :string           default(""), not null
+#  domain                        :string
-#  followers_url                 :string           default(""), not null
-#  protocol                      :integer          default("ostatus"), not null
-#  memorial                      :boolean          default(FALSE), not null
-#  moved_to_account_id           :bigint(8)
 #  featured_collection_url       :string
 #  fields                        :jsonb
-#  actor_type                    :string
+#  followers_url                 :string           default(""), not null
-#  discoverable                  :boolean
+#  header_content_type           :string
-#  also_known_as                 :string           is an Array
+#  header_file_name              :string
+#  header_file_size              :integer
+#  header_remote_url             :string           default(""), not null
+#  header_storage_schema_version :integer
+#  header_updated_at             :datetime
+#  hide_collections              :boolean
+#  inbox_url                     :string           default(""), not null
+#  indexable                     :boolean          default(FALSE), not null
+#  last_webfingered_at           :datetime
+#  locked                        :boolean          default(FALSE), not null
+#  memorial                      :boolean          default(FALSE), not null
+#  meta                          :json
+#  note                          :text             default(""), not null
+#  outbox_url                    :string           default(""), not null
+#  private_key                   :text
+#  protocol                      :integer          default("ostatus"), not null
+#  public_key                    :text             default(""), not null
+#  requested_review_at           :datetime
+#  reviewed_at                   :datetime
+#  sensitized_at                 :datetime
+#  shared_inbox_url              :string           default(""), not null
 #  silenced_at                   :datetime
 #  suspended_at                  :datetime
-#  hide_collections              :boolean
-#  avatar_storage_schema_version :integer
-#  header_storage_schema_version :integer
 #  suspension_origin             :integer
-#  sensitized_at                 :datetime
 #  trendable                     :boolean
-#  reviewed_at                   :datetime
+#  uri                           :string           default(""), not null
-#  requested_review_at           :datetime
+#  url                           :string
-#  indexable                     :boolean          default(FALSE), not null
+#  username                      :string           default(""), not null
-#  attribution_domains           :string           default([]), is an Array
+#  created_at                    :datetime         not null
+#  updated_at                    :datetime         not null
+#  moved_to_account_id           :bigint(8)
 #
 
 class Account < ApplicationRecord

app/models/concerns/account/avatar.rb

@@ -11,7 +11,13 @@ module Account::Avatar
   class_methods do
     def avatar_styles(file)
       styles = { original: { geometry: "#{AVATAR_GEOMETRY}#", file_geometry_parser: FastGeometryParser } }
-      styles[:static] = { geometry: "#{AVATAR_GEOMETRY}#", format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif'
+
+      if file.content_type == 'image/gif'
+        styles[:static] = { geometry: "#{AVATAR_GEOMETRY}#", format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser, blurhash: { x_comp: 3, y_comp: 3 }, extract_colors: { meta_attribute_name: :meta } }
+      else
+        styles[:original].merge!(blurhash: { x_comp: 3, y_comp: 3 }, extract_colors: { meta_attribute_name: :meta })
+      end
+
       styles
     end
 
@@ -20,7 +26,7 @@ module Account::Avatar
 
   included do
     # Avatar upload
-    has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail]
+    has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :color_extractor]
     validates_attachment_content_type :avatar, content_type: AVATAR_IMAGE_MIME_TYPES
     validates_attachment_size :avatar, less_than: AVATAR_LIMIT
     remotable_attachment :avatar, AVATAR_LIMIT, suppress_errors: false

app/models/media_attachment.rb

@@ -165,7 +165,7 @@ class MediaAttachment < ApplicationRecord
   }.freeze
 
   THUMBNAIL_STYLES = {
-    original: IMAGE_STYLES[:small].freeze,
+    original: IMAGE_STYLES[:small].merge(extract_colors: { meta_attribute_name: :file_meta }.freeze).freeze,
   }.freeze
 
   DEFAULT_STYLES = [:original].freeze

app/serializers/rest/account_serializer.rb

@@ -8,7 +8,7 @@ class REST::AccountSerializer < ActiveModel::Serializer
 
   attributes :id, :username, :acct, :display_name, :locked, :bot, :discoverable, :indexable, :group, :created_at,
              :note, :url, :uri, :avatar, :avatar_static, :header, :header_static,
-             :followers_count, :following_count, :statuses_count, :last_status_at, :hide_collections
+             :followers_count, :following_count, :statuses_count, :last_status_at, :hide_collections, :meta, :blurhash
 
   has_one :moved_to_account, key: :moved, serializer: REST::AccountSerializer, if: :moved_and_not_nested?
 

db/migrate/20250430151215_add_avatar_meta_to_accounts.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddAvatarMetaToAccounts < ActiveRecord::Migration[8.0]
+  def change
+    safety_assured { add_column :accounts, :meta, :json }
+    add_column :accounts, :blurhash, :string
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2025_04_28_095029) do
+ActiveRecord::Schema[8.0].define(version: 2025_04_30_151215) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_catalog.plpgsql"
 
@@ -198,6 +198,8 @@ ActiveRecord::Schema[8.0].define(version: 2025_04_28_095029) do
     t.datetime "requested_review_at", precision: nil
     t.boolean "indexable", default: false, null: false
     t.string "attribution_domains", default: [], array: true
+    t.json "meta"
+    t.string "blurhash"
     t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
     t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
     t.index ["domain", "id"], name: "index_accounts_on_domain_and_id"

lib/paperclip/blurhash_transcoder.rb

@@ -3,7 +3,7 @@
 module Paperclip
   class BlurhashTranscoder < Paperclip::Processor
     def make
-      return @file unless options[:style] == :small || options[:blurhash]
+      return @file unless options[:blurhash]
 
       width, height, data = blurhash_params
       # Guard against segfaults if data has unexpected size

lib/paperclip/color_extractor.rb

@@ -10,6 +10,8 @@ module Paperclip
     BINS = 10
 
     def make
+      return @file unless options.key?(:extract_colors)
+
       background_palette, foreground_palette = Rails.configuration.x.use_vips ? palettes_from_libvips : palettes_from_imagemagick
 
       background_color   = background_palette.first || foreground_palette.first
@@ -66,7 +68,8 @@ module Paperclip
         },
       }
 
-      attachment.instance.file.instance_write(:meta, (attachment.instance.file.instance_read(:meta) || {}).merge(meta))
+      meta_attribute_name = options[:extract_colors][:meta_attribute_name] || "#{attachment.name}_meta"
+      attachment.instance.public_send("#{meta_attribute_name}=", (attachment.instance.public_send(meta_attribute_name) || {}).merge(meta))
 
       @file
     rescue Vips::Error => e