Browse Source

Provide prometheus metrics endpoint

Makes it possible to enable prometheus metrics endpoint by setting
the PROMETHEUS_ENABLED flag
pull/3634/head
Martin Andersson 2 years ago
parent
commit
4193f4bc5b
No known key found for this signature in database GPG Key ID: 5F205569778D1FE3
  1. 110
      Cargo.lock
  2. 5
      Cargo.toml
  3. 54
      src/api/metrics.rs
  4. 2
      src/api/mod.rs
  5. 3
      src/config.rs
  6. 6
      src/db/models/organization.rs
  7. 3
      src/main.rs
  8. 47
      src/util.rs

110
Cargo.lock

@ -126,7 +126,7 @@ dependencies = [
"log",
"parking",
"polling",
"rustix",
"rustix 0.37.20",
"slab",
"socket2 0.4.9",
"waker-fn",
@ -154,7 +154,7 @@ dependencies = [
"cfg-if",
"event-listener",
"futures-lite",
"rustix",
"rustix 0.37.20",
"signal-hook",
"windows-sys 0.48.0",
]
@ -1143,6 +1143,12 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hmac"
version = "0.12.1"
@ -1366,7 +1372,7 @@ checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f"
dependencies = [
"hermit-abi 0.3.1",
"io-lifetimes",
"rustix",
"rustix 0.37.20",
"windows-sys 0.48.0",
]
@ -1493,6 +1499,12 @@ version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
[[package]]
name = "linux-raw-sys"
version = "0.3.8"
@ -1872,7 +1884,7 @@ dependencies = [
"libc",
"redox_syscall",
"smallvec",
"windows-targets",
"windows-targets 0.48.0",
]
[[package]]
@ -2098,6 +2110,47 @@ dependencies = [
"yansi",
]
[[package]]
name = "procfs"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69"
dependencies = [
"bitflags 1.3.2",
"byteorder",
"hex",
"lazy_static",
"rustix 0.36.14",
]
[[package]]
name = "prometheus"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
dependencies = [
"cfg-if",
"fnv",
"lazy_static",
"libc",
"memchr",
"parking_lot",
"procfs",
"thiserror",
]
[[package]]
name = "prometheus-static-metric"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8f30cdb09c39930b8fa5e0f23cbb895ab3f766b187403a0ba0956fc1ef4f0e5"
dependencies = [
"lazy_static",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "psl-types"
version = "2.0.11"
@ -2464,6 +2517,20 @@ dependencies = [
"winapi",
]
[[package]]
name = "rustix"
version = "0.36.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62"
dependencies = [
"bitflags 1.3.2",
"errno",
"io-lifetimes",
"libc",
"linux-raw-sys 0.1.4",
"windows-sys 0.45.0",
]
[[package]]
name = "rustix"
version = "0.37.20"
@ -2474,7 +2541,7 @@ dependencies = [
"errno",
"io-lifetimes",
"libc",
"linux-raw-sys",
"linux-raw-sys 0.3.8",
"windows-sys 0.48.0",
]
@ -2862,7 +2929,7 @@ dependencies = [
"cfg-if",
"fastrand",
"redox_syscall",
"rustix",
"rustix 0.37.20",
"windows-sys 0.48.0",
]
@ -3362,6 +3429,7 @@ dependencies = [
"html5gum",
"job_scheduler_ng",
"jsonwebtoken",
"lazy_static",
"lettre",
"libsqlite3-sys",
"log",
@ -3373,6 +3441,8 @@ dependencies = [
"paste",
"percent-encoding",
"pico-args",
"prometheus",
"prometheus-static-metric",
"rand",
"regex",
"reqwest",
@ -3608,7 +3678,7 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
dependencies = [
"windows-targets",
"windows-targets 0.48.0",
]
[[package]]
@ -3626,13 +3696,37 @@ dependencies = [
"windows_x86_64_msvc 0.42.2",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
"windows-targets 0.42.2",
]
[[package]]
name = "windows-sys"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
dependencies = [
"windows-targets",
"windows-targets 0.48.0",
]
[[package]]
name = "windows-targets"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
dependencies = [
"windows_aarch64_gnullvm 0.42.2",
"windows_aarch64_msvc 0.42.2",
"windows_i686_gnu 0.42.2",
"windows_i686_msvc 0.42.2",
"windows_x86_64_gnu 0.42.2",
"windows_x86_64_gnullvm 0.42.2",
"windows_x86_64_msvc 0.42.2",
]
[[package]]

5
Cargo.toml

@ -162,6 +162,11 @@ argon2 = "0.5.0"
# Reading a password from the cli for generating the Argon2id ADMIN_TOKEN
rpassword = "7.2.0"
prometheus = { version = "0.13.3", features = [
"process",
], default-features = false }
prometheus-static-metric = "0.5.1"
lazy_static = "1.4.0"
[patch.crates-io]
rocket = { git = 'https://github.com/SergioBenitez/Rocket', rev = 'ce441b5f46fdf5cd99cb32b8b8638835e4c2a5fa' } # v0.5 branch

54
src/api/metrics.rs

@ -0,0 +1,54 @@
use rocket::Route;
use crate::{
config::CONFIG,
db::{
models::{Organization, User},
DbConn,
},
};
use lazy_static::lazy_static;
use prometheus::{register_gauge, register_gauge_vec, Encoder, Gauge, GaugeVec, TextEncoder};
use prometheus_static_metric::make_static_metric;
pub fn routes() -> Vec<Route> {
if !CONFIG.prometheus_enabled() {
return routes![];
}
routes![metrics]
}
make_static_metric! {
pub struct UserGauge: Gauge {
"enabled" => {
enabled:"true",
disabled:"false",
},
}
}
lazy_static! {
pub static ref USER_COUNTER_VEC: GaugeVec =
register_gauge_vec!("vw_users", "Total number of users in the system", &["enabled"]).unwrap();
pub static ref USER_COUNTER: UserGauge = UserGauge::from(&USER_COUNTER_VEC);
pub static ref ORGANIZATION_COUNTER: Gauge =
register_gauge!("vw_organizations", "Total number of organizations in the system").unwrap();
}
#[get("/")]
async fn metrics(mut conn: DbConn) -> String {
let users = User::get_all(&mut conn).await;
let org_count = Organization::count(&mut conn).await;
USER_COUNTER.enabled.set(users.iter().filter(|u| u.enabled).count() as f64);
USER_COUNTER.disabled.set(users.iter().filter(|u| !u.enabled).count() as f64);
ORGANIZATION_COUNTER.set(org_count as f64);
let mut buffer = Vec::new();
let encoder = TextEncoder::new();
let metric_families = prometheus::gather();
encoder.encode(&metric_families, &mut buffer).unwrap();
String::from_utf8(buffer.clone()).unwrap()
}

2
src/api/mod.rs

@ -2,6 +2,7 @@ mod admin;
pub mod core;
mod icons;
mod identity;
mod metrics;
mod notifications;
mod push;
mod web;
@ -21,6 +22,7 @@ pub use crate::api::{
core::{event_cleanup_job, events_routes as core_events_routes},
icons::routes as icons_routes,
identity::routes as identity_routes,
metrics::routes as metrics_routes,
notifications::routes as notifications_routes,
notifications::{start_notification_server, Notify, UpdateType},
push::{

3
src/config.rs

@ -596,6 +596,9 @@ make_config! {
/// Enable groups (BETA!) (Know the risks!) |> Enables groups support for organizations (Currently contains known issues!).
org_groups_enabled: bool, false, def, false;
/// Enable Prometheus metrics |> Enables Prometheus metrics on /metrics
prometheus_enabled: bool, false, def, false;
},
/// Yubikey settings

6
src/db/models/organization.rs

@ -319,6 +319,12 @@ impl Organization {
organizations::table.load::<OrganizationDb>(conn).expect("Error loading organizations").from_db()
}}
}
pub async fn count(conn: &mut DbConn) -> i64 {
db_run! {conn: {
organizations::table.count().get_result(conn).expect("Error counting organziations")
}}
}
}
impl UserOrganization {

3
src/main.rs

@ -528,6 +528,7 @@ async fn launch_rocket(pool: db::DbPool, extra_debug: bool) -> Result<(), Error>
.mount([basepath, "/identity"].concat(), api::identity_routes())
.mount([basepath, "/icons"].concat(), api::icons_routes())
.mount([basepath, "/notifications"].concat(), api::notifications_routes())
.mount([basepath, "/metrics"].concat(), api::metrics_routes())
.register([basepath, "/"].concat(), api::web_catchers())
.register([basepath, "/api"].concat(), api::core_catchers())
.register([basepath, "/admin"].concat(), api::admin_catchers())
@ -535,7 +536,7 @@ async fn launch_rocket(pool: db::DbPool, extra_debug: bool) -> Result<(), Error>
.manage(api::start_notification_server())
.attach(util::AppHeaders())
.attach(util::Cors())
.attach(util::BetterLogging(extra_debug))
.attach(util::BetterLogging::new(extra_debug))
.ignite()
.await?;

47
src/util.rs

@ -17,6 +17,7 @@ use tokio::{
};
use crate::CONFIG;
use prometheus::{register_histogram_vec, HistogramVec};
pub struct AppHeaders();
@ -231,10 +232,27 @@ impl<'r> FromParam<'r> for SafeString {
// Log all the routes from the main paths list, and the attachments endpoint
// Effectively ignores, any static file route, and the alive endpoint
const LOGGED_ROUTES: [&str; 7] = ["/api", "/admin", "/identity", "/icons", "/attachments", "/events", "/notifications"];
const LOGGED_ROUTES: [&str; 8] =
["/api", "/admin", "/identity", "/icons", "/attachments", "/events", "/notifications", "/metrics"];
const PROMETHEUS_LABELS: [&str; 3] = ["endpoint", "method", "status"];
pub struct BetterLogging {
extra_debug: bool,
request: HistogramVec,
prometheus_enabled: bool,
}
impl BetterLogging {
pub fn new(extra_debug: bool) -> Self {
Self {
extra_debug,
request: register_histogram_vec!("http_request_duration_seconds", "Request durations", &PROMETHEUS_LABELS)
.unwrap(),
prometheus_enabled: CONFIG.prometheus_enabled(),
}
}
}
// Boolean is extra debug, when true, we ignore the whitelist above and also print the mounts
pub struct BetterLogging(pub bool);
#[rocket::async_trait]
impl Fairing for BetterLogging {
fn info(&self) -> Info {
@ -245,7 +263,7 @@ impl Fairing for BetterLogging {
}
async fn on_liftoff(&self, rocket: &Rocket<Orbit>) {
if self.0 {
if self.extra_debug {
info!(target: "routes", "Routes loaded:");
let mut routes: Vec<_> = rocket.routes().collect();
routes.sort_by_key(|r| r.uri.path());
@ -269,15 +287,18 @@ impl Fairing for BetterLogging {
}
async fn on_request(&self, request: &mut Request<'_>, _data: &mut Data<'_>) {
if self.prometheus_enabled {
request.local_cache(|| Some(time::Instant::now()));
}
let method = request.method();
if !self.0 && method == Method::Options {
if !self.extra_debug && method == Method::Options {
return;
}
let uri = request.uri();
let uri_path = uri.path();
let uri_path_str = uri_path.url_decode_lossy();
let uri_subpath = uri_path_str.strip_prefix(&CONFIG.domain_path()).unwrap_or(&uri_path_str);
if self.0 || LOGGED_ROUTES.iter().any(|r| uri_subpath.starts_with(r)) {
if self.extra_debug || LOGGED_ROUTES.iter().any(|r| uri_subpath.starts_with(r)) {
match uri.query() {
Some(q) => info!(target: "request", "{} {}?{}", method, uri_path_str, &q[..q.len().min(30)]),
None => info!(target: "request", "{} {}", method, uri_path_str),
@ -286,13 +307,13 @@ impl Fairing for BetterLogging {
}
async fn on_response<'r>(&self, request: &'r Request<'_>, response: &mut Response<'r>) {
if !self.0 && request.method() == Method::Options {
if !self.extra_debug && request.method() == Method::Options {
return;
}
let uri_path = request.uri().path();
let uri_path_str = uri_path.url_decode_lossy();
let uri_subpath = uri_path_str.strip_prefix(&CONFIG.domain_path()).unwrap_or(&uri_path_str);
if self.0 || LOGGED_ROUTES.iter().any(|r| uri_subpath.starts_with(r)) {
if self.extra_debug || LOGGED_ROUTES.iter().any(|r| uri_subpath.starts_with(r)) {
let status = response.status();
if let Some(ref route) = request.route() {
info!(target: "response", "{} => {}", route, status)
@ -300,6 +321,16 @@ impl Fairing for BetterLogging {
info!(target: "response", "{}", status)
}
}
if !self.prometheus_enabled {
return;
}
if let Some(start_time) = request.local_cache(|| None::<time::Instant>) {
let duration = start_time.elapsed();
self.request
.with_label_values(&[uri_subpath, request.method().as_str(), &response.status().to_string()])
.observe(duration.as_seconds_f64());
}
}
}

Loading…
Cancel
Save