From 59384282ed1360d0139b5a46b39d2be289846b2c Mon Sep 17 00:00:00 2001 From: Renaud Chaput Date: Mon, 27 Jan 2025 13:52:30 +0100 Subject: [PATCH] Add an optional metric exporter (#33734) --- Gemfile | 2 ++ Gemfile.lock | 3 ++ bin/prometheus_exporter | 27 ++++++++++++++ config/initializers/prometheus_exporter.rb | 24 +++++++++++++ config/initializers/sidekiq.rb | 42 ++++++++++++++++++++++ config/puma.rb | 21 +++++++++++ 6 files changed, 119 insertions(+) create mode 100755 bin/prometheus_exporter create mode 100644 config/initializers/prometheus_exporter.rb diff --git a/Gemfile b/Gemfile index 5becc118d3..89648e8cac 100644 --- a/Gemfile +++ b/Gemfile @@ -100,6 +100,8 @@ gem 'json-ld' gem 'json-ld-preloaded', '~> 3.2' gem 'rdf-normalize', '~> 0.5' +gem 'prometheus_exporter', '~> 2.2', require: false + gem 'opentelemetry-api', '~> 1.4.0' group :opentelemetry do diff --git a/Gemfile.lock b/Gemfile.lock index eaccaf2604..a90aef56a1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -580,6 +580,8 @@ GEM net-smtp premailer (~> 1.7, >= 1.7.9) prettyprint (0.2.0) + prometheus_exporter (2.2.0) + webrick propshaft (1.1.0) actionpack (>= 7.0.0) activesupport (>= 7.0.0) @@ -984,6 +986,7 @@ DEPENDENCIES pg (~> 1.5) pghero premailer-rails + prometheus_exporter (~> 2.2) propshaft public_suffix (~> 6.0) puma (~> 6.3) diff --git a/bin/prometheus_exporter b/bin/prometheus_exporter new file mode 100755 index 0000000000..7e0304de9e --- /dev/null +++ b/bin/prometheus_exporter @@ -0,0 +1,27 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# +# This file was generated by Bundler. +# +# The application 'prometheus_exporter' is installed as part of a gem, and +# this file is here to facilitate running it. +# + +ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) + +bundle_binstub = File.expand_path("bundle", __dir__) + +if File.file?(bundle_binstub) + if File.read(bundle_binstub, 300).include?("This file was generated by Bundler") + load(bundle_binstub) + else + abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. +Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") + end +end + +require "rubygems" +require "bundler/setup" + +load Gem.bin_path("prometheus_exporter", "prometheus_exporter") diff --git a/config/initializers/prometheus_exporter.rb b/config/initializers/prometheus_exporter.rb new file mode 100644 index 0000000000..197777e3b6 --- /dev/null +++ b/config/initializers/prometheus_exporter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +if ENV['MASTODON_PROMETHEUS_EXPORTER_ENABLED'] == 'true' + if ENV['MASTODON_PROMETHEUS_EXPORTER_LOCAL'] == 'true' + require 'prometheus_exporter/server' + require 'prometheus_exporter/client' + + # bind is the address, on which the webserver will listen + # port is the port that will provide the /metrics route + server = PrometheusExporter::Server::WebServer.new bind: ENV.fetch('MASTODON_PROMETHEUS_EXPORTER_HOST', 'localhost'), port: ENV.fetch('MASTODON_PROMETHEUS_EXPORTER_PORT', '9394').to_i + server.start + + # wire up a default local client + PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(collector: server.collector) + end + + if ENV['MASTODON_PROMETHEUS_EXPORTER_WEB_DETAILED_METRICS'] == 'true' + # Optional, as those metrics might generate extra overhead and be redundant with what OTEL provides + require 'prometheus_exporter/middleware' + + # Per-action/controller request stats like HTTP status and timings + Rails.application.middleware.unshift PrometheusExporter::Middleware + end +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 5b281c4339..fca0bec422 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -22,6 +22,48 @@ Sidekiq.configure_server do |config| end end + if ENV['MASTODON_PROMETHEUS_EXPORTER_ENABLED'] == 'true' + require 'prometheus_exporter' + require 'prometheus_exporter/instrumentation' + + config.on :startup do + # Ruby process metrics (memory, GC, etc) + PrometheusExporter::Instrumentation::Process.start type: 'sidekiq' + + # Sidekiq process metrics (concurrency, busy, etc) + PrometheusExporter::Instrumentation::SidekiqProcess.start + + # ActiveRecord metrics (connection pool usage) + PrometheusExporter::Instrumentation::ActiveRecord.start( + custom_labels: { type: 'sidekiq' }, + config_labels: [:database, :host] + ) + + if ENV['MASTODON_PROMETHEUS_EXPORTER_SIDEKIQ_DETAILED_METRICS'] == 'true' + # Optional, as those metrics might generate extra overhead and be redundant with what OTEL provides + + # Per-job metrics + config.server_middleware do |chain| + chain.add PrometheusExporter::Instrumentation::Sidekiq + end + config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler + + # Per-queue metrics for queues handled by this process (size, latency, etc) + # They will be reported by every process handling those queues, so do not sum them up + PrometheusExporter::Instrumentation::SidekiqQueue.start + + # Global Sidekiq metrics (size of the global queues, number of jobs, etc) + # Will be the same for every Sidekiq process + PrometheusExporter::Instrumentation::SidekiqStats.start + end + end + + at_exit do + # Wait for the latest metrics to be reported before shutting down + PrometheusExporter::Client.default.stop(wait_timeout_seconds: 10) + end + end + config.server_middleware do |chain| chain.add Mastodon::SidekiqMiddleware end diff --git a/config/puma.rb b/config/puma.rb index ac9ccac209..4fe8f802b9 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -17,6 +17,27 @@ workers ENV.fetch('WEB_CONCURRENCY') { 2 }.to_i preload_app! +if ENV['MASTODON_PROMETHEUS_EXPORTER_ENABLED'] == 'true' + require 'prometheus_exporter' + require 'prometheus_exporter/instrumentation' + + on_worker_boot do + # Ruby process metrics (memory, GC, etc) + PrometheusExporter::Instrumentation::Process.start(type: 'puma') + + # ActiveRecord metrics (connection pool usage) + PrometheusExporter::Instrumentation::ActiveRecord.start( + custom_labels: { type: 'puma' }, # optional params + config_labels: [:database, :host] # optional params + ) + end + + after_worker_boot do + # Puma metrics + PrometheusExporter::Instrumentation::Puma.start unless PrometheusExporter::Instrumentation::Puma.started? + end +end + on_worker_boot do ActiveSupport.on_load(:active_record) do ActiveRecord::Base.establish_connection