Skip to content

[adapters] Expose pipeline process swap usage as a Prometheus metric #6383

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sjhddh wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[adapters] Expose pipeline process swap usage as a Prometheus metric #6383

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 64 additions & 1 deletion crates/adapters/src/server/metrics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,14 @@ where
threads,
);
}
if let Some(swap_bytes) = process_swap_bytes() {
self.gauge(
"process_swap_bytes",
"Amount of the process's memory that is swapped out, in bytes. A nonzero value indicates that the process is swapping, which degrades performance.",
labels,
swap_bytes,
);
}
}

/// Consumes this metrics writer and returns the output.
Expand All @@ -484,6 +492,40 @@ where
}
}

/// Returns the number of bytes of this process's memory that is currently
/// swapped out, if it can be determined.
///
/// On Linux, this is read from the `VmSwap` field of `/proc/self/status`. On
/// other platforms, or if the field is unavailable, returns `None` so that the
/// metric is simply omitted rather than reported as a misleading zero.
#[cfg(target_os = "linux")]
fn process_swap_bytes() -> Option<u64> {
let status = std::fs::read_to_string("/proc/self/status").ok()?;
parse_vm_swap_bytes(&status)
}

#[cfg(not(target_os = "linux"))]
fn process_swap_bytes() -> Option<u64> {
None
}

/// Parses the `VmSwap` field out of the contents of `/proc/self/status`,
/// returning the swap usage in bytes.
///
/// The field looks like `VmSwap:\t 1234 kB`, where the value is in kibibytes.
/// Returns `None` if the field is missing or cannot be parsed.
#[cfg(any(target_os = "linux", test))]
fn parse_vm_swap_bytes(status: &str) -> Option<u64> {
for line in status.lines() {
if let Some(rest) = line.strip_prefix("VmSwap:") {
// The value is in kB (kibibytes); convert to bytes.
let kib: u64 = rest.split_whitespace().next()?.parse().ok()?;
return Some(kib * 1024);
}
}
None
}

/// Transforms a label into the form used for formatting a Prometheus label.
struct EscapedLabel<'a> {
name: &'a str,
Expand Down Expand Up @@ -818,9 +860,30 @@ impl Histogram for ExponentialHistogramSnapshot {
mod tests {
use crate::server::metrics::{
Bucket, Histogram, JsonFormatter, LabelStack, MetricsFormatter, MetricsWriter,
PrometheusFormatter,
PrometheusFormatter, parse_vm_swap_bytes,
};

#[test]
fn parse_vm_swap() {
let status = "\
Name:\tpipeline
VmPeak:\t 123456 kB
VmSwap:\t 1024 kB
Threads:\t8
";
// 1024 kB == 1 MiB == 1048576 bytes.
assert_eq!(parse_vm_swap_bytes(status), Some(1024 * 1024));

// Missing field yields None.
assert_eq!(parse_vm_swap_bytes("VmPeak:\t 123456 kB\n"), None);

// Zero is a valid, distinct value (process not swapping).
assert_eq!(parse_vm_swap_bytes("VmSwap:\t 0 kB\n"), Some(0));

// Malformed value yields None rather than panicking.
assert_eq!(parse_vm_swap_bytes("VmSwap:\tnot-a-number kB\n"), None);
}

#[test]
fn prometheus_writer() {
assert_eq!(
Expand Down