diff --git a/app/models/fasp/preview_card_trend.rb b/app/models/fasp/preview_card_trend.rb new file mode 100644 index 0000000000..ba16662b60 --- /dev/null +++ b/app/models/fasp/preview_card_trend.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: fasp_preview_card_trends +# +# id :bigint(8) not null, primary key +# allowed :boolean default(FALSE), not null +# language :string not null +# rank :integer not null +# created_at :datetime not null +# updated_at :datetime not null +# fasp_provider_id :bigint(8) not null +# preview_card_id :bigint(8) not null +# +class Fasp::PreviewCardTrend < ApplicationRecord + belongs_to :preview_card + belongs_to :fasp_provider, class_name: 'Fasp::Provider' +end diff --git a/app/models/fasp/provider.rb b/app/models/fasp/provider.rb index d052905993..f928013807 100644 --- a/app/models/fasp/provider.rb +++ b/app/models/fasp/provider.rb @@ -29,6 +29,10 @@ class Fasp::Provider < ApplicationRecord before_create :create_keypair after_commit :update_remote_capabilities + scope :with_capability, lambda { |capability_name| + where('fasp_providers.capabilities @> ?::jsonb', "[{\"id\": \"#{capability_name}\", \"enabled\": true}]") + } + def enabled_capabilities=(hash) capabilities.each do |capability| capability['enabled'] = hash[capability['id']] == '1' diff --git a/app/models/fasp/status_trend.rb b/app/models/fasp/status_trend.rb new file mode 100644 index 0000000000..ad151c19a4 --- /dev/null +++ b/app/models/fasp/status_trend.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: fasp_status_trends +# +# id :bigint(8) not null, primary key +# allowed :boolean default(FALSE), not null +# language :string not null +# rank :integer not null +# created_at :datetime not null +# updated_at :datetime not null +# fasp_provider_id :bigint(8) not null +# status_id :bigint(8) not null +# +class Fasp::StatusTrend < ApplicationRecord + belongs_to :status + belongs_to :fasp_provider, class_name: 'Fasp::Provider' +end diff --git a/app/models/fasp/tag_trend.rb b/app/models/fasp/tag_trend.rb new file mode 100644 index 0000000000..59206632f8 --- /dev/null +++ b/app/models/fasp/tag_trend.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: fasp_tag_trends +# +# id :bigint(8) not null, primary key +# allowed :boolean default(FALSE), not null +# language :string not null +# rank :integer not null +# created_at :datetime not null +# updated_at :datetime not null +# fasp_provider_id :bigint(8) not null +# tag_id :bigint(8) not null +# +class Fasp::TagTrend < ApplicationRecord + belongs_to :tag + belongs_to :fasp_provider, class_name: 'Fasp::Provider' +end diff --git a/app/services/fasp/refresh_preview_card_trends_service.rb b/app/services/fasp/refresh_preview_card_trends_service.rb new file mode 100644 index 0000000000..a43dd384cc --- /dev/null +++ b/app/services/fasp/refresh_preview_card_trends_service.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +class Fasp::RefreshPreviewCardTrendsService + def call(provider, language) + results = query_trends(provider, language) + + Fasp::PreviewCardTrend.transaction do + Fasp::PreviewCardTrend.where(language:).delete_all + + (results['links'] || []).each do |link| + preview_card = fetch_preview_card(link['url']) + + next unless preview_card + + Fasp::PreviewCardTrend.create!( + fasp_provider: provider, + preview_card:, + language:, + rank: link['rank'], + allowed: !preview_card.trendable?.nil? + ) + + fetch_examples(link['examples']) + end + end + end + + private + + def fetch_preview_card(url) + FetchLinkCardForURLService.new.call(url) + end + + def fetch_examples(uris) + uris.each { |u| FetchReplyWorker.perform_async(u) } + end + + def query_trends(provider, language) + params = { language:, withinLastHours: 4, maxCount: 20 } + + Fasp::Request.new(provider).get("/trends/v0/links?#{params.to_query}") + end +end diff --git a/app/services/fasp/refresh_status_trends_service.rb b/app/services/fasp/refresh_status_trends_service.rb new file mode 100644 index 0000000000..deeacaa1f8 --- /dev/null +++ b/app/services/fasp/refresh_status_trends_service.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class Fasp::RefreshStatusTrendsService + def call(provider, language) + results = query_trends(provider, language) + + Fasp::StatusTrend.transaction do + Fasp::StatusTrend.where(language:).delete_all + + (results['content'] || []).each do |result| + status = fetch_status(result['uri']) + + next if status.nil? + + Fasp::StatusTrend.create!( + fasp_provider: provider, + status:, + language:, + rank: result['rank'], + allowed: !status.trendable?.nil? + ) + end + end + end + + private + + def fetch_status(uri) + ResolveURLService.new.call(uri) + end + + def query_trends(provider, language) + params = { language:, withinLastHours: 4, maxCount: 20 } + + Fasp::Request.new(provider).get("/trends/v0/content?#{params.to_query}") + end +end diff --git a/app/services/fasp/refresh_tag_trends_service.rb b/app/services/fasp/refresh_tag_trends_service.rb new file mode 100644 index 0000000000..8fa46fbe7d --- /dev/null +++ b/app/services/fasp/refresh_tag_trends_service.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class Fasp::RefreshTagTrendsService + def call(provider, language) + results = query_trends(provider, language) + + Fasp::TagTrend.transaction do + Fasp::TagTrend.where(language:).delete_all + + (results['hashtags'] || []).each do |result| + tag = Tag.find_or_create_by_names(result['name']).first + + Fasp::TagTrend.create!( + fasp_provider: provider, + tag:, + language:, + rank: result['rank'], + allowed: tag.trendable? + ) + + fetch_examples(result['examples']) + end + end + end + + private + + def fetch_examples(uris) + uris.each { |u| FetchReplyWorker.perform_async(u) } + end + + def query_trends(provider, language) + params = { language:, withinLastHours: 4, maxCount: 20 } + + Fasp::Request.new(provider).get("/trends/v0/hashtags?#{params.to_query}") + end +end diff --git a/app/services/fetch_link_card_for_url_service.rb b/app/services/fetch_link_card_for_url_service.rb new file mode 100644 index 0000000000..4492f5152b --- /dev/null +++ b/app/services/fetch_link_card_for_url_service.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +class FetchLinkCardForURLService < BaseService + include Redisable + include Lockable + + def call(url) + @original_url = Addressable::URI.parse(url).normalize + return if bad_url?(@original_url) + + @url = @original_url.to_s + + with_redis_lock("fetch:#{@original_url}") do + @card = PreviewCard.find_by(url: @url) + process_url if @card.nil? || @card.updated_at <= 2.weeks.ago || @card.missing_image? + end + + @card + rescue *Mastodon::HTTP_CONNECTION_ERRORS, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Encoding::UndefinedConversionError, ActiveRecord::RecordInvalid => e + Rails.logger.debug { "Error fetching link #{@url}: #{e}" } + nil + end + + private + + def process_url + @card ||= PreviewCard.new(url: @url) + + attempt_oembed || attempt_opengraph + end + + def html + return @html if defined?(@html) + + headers = { + 'Accept' => 'text/html', + 'Accept-Language' => "#{I18n.default_locale}, *;q=0.5", + 'User-Agent' => "#{Mastodon::Version.user_agent} Bot", + } + + @html = Request.new(:get, @url).add_headers(headers).perform do |res| + next unless res.code == 200 && res.mime_type == 'text/html' + + # We follow redirects, and ideally we want to save the preview card for + # the destination URL and not any link shortener in-between, so here + # we set the URL to the one of the last response in the redirect chain + @url = res.request.uri.to_s + @card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url + + @html_charset = res.charset + + res.truncated_body + end + end + + def bad_url?(uri) + # Avoid local instance URLs and invalid URLs + uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) + end + + def attempt_oembed + service = FetchOEmbedService.new + url_domain = Addressable::URI.parse(@url).normalized_host + cached_endpoint = Rails.cache.read("oembed_endpoint:#{url_domain}") + + embed = service.call(@url, cached_endpoint: cached_endpoint) unless cached_endpoint.nil? + embed ||= service.call(@url, html: html) unless html.nil? + + return false if embed.nil? + + url = Addressable::URI.parse(service.endpoint_url) + + @card.type = embed[:type] + @card.title = embed[:title] || '' + @card.author_name = embed[:author_name] || '' + @card.author_url = embed[:author_url].present? ? (url + embed[:author_url]).to_s : '' + @card.provider_name = embed[:provider_name] || '' + @card.provider_url = embed[:provider_url].present? ? (url + embed[:provider_url]).to_s : '' + @card.width = 0 + @card.height = 0 + + case @card.type + when 'link' + @card.image_remote_url = (url + embed[:thumbnail_url]).to_s if embed[:thumbnail_url].present? + when 'photo' + return false if embed[:url].blank? + + @card.embed_url = (url + embed[:url]).to_s + @card.image_remote_url = (url + embed[:url]).to_s + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 + when 'video' + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 + @card.html = Sanitize.fragment(embed[:html], Sanitize::Config::MASTODON_OEMBED) + @card.image_remote_url = (url + embed[:thumbnail_url]).to_s if embed[:thumbnail_url].present? + when 'rich' + # Most providers rely on