feat: add prometheus metrics and tracing instrumentation

Cargo.lock

@@ -2,6 +2,18 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "ahash"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.4"
@@ -50,6 +62,8 @@ dependencies = [
  "chrono",
  "fastrand",
  "jsonwebtoken",
+ "metrics",
+ "metrics-exporter-prometheus",
  "redis",
  "serde",
  "serde_json",
@@ -64,6 +78,28 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
 
+[[package]]
+name = "aws-lc-rs"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9"
+dependencies = [
+ "aws-lc-sys",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-lc-sys"
+version = "0.37.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549"
+dependencies = [
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+]
+
 [[package]]
 name = "axum"
 version = "0.8.8"
@@ -156,6 +192,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
 dependencies = [
  "find-msvc-tools",
+ "jobserver",
+ "libc",
  "shlex",
 ]
 
@@ -190,6 +228,15 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "cmake"
+version = "0.1.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -215,12 +262,37 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "core-foundation"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
 [[package]]
 name = "deranged"
 version = "0.5.8"
@@ -241,6 +313,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "dunce"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
+
 [[package]]
 name = "equivalent"
 version = "1.0.2"
@@ -269,12 +347,24 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "foldhash"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
 
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.2"
@@ -284,6 +374,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
 [[package]]
 name = "futures-channel"
 version = "0.3.32"
@@ -335,6 +431,18 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasip2",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.4.1"
@@ -348,13 +456,32 @@ dependencies = [
  "wasip3",
 ]
 
+[[package]]
+name = "h2"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http",
+ "indexmap",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
- "foldhash",
+ "foldhash 0.1.5",
 ]
 
 [[package]]
@@ -362,6 +489,9 @@ name = "hashbrown"
 version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+dependencies = [
+ "foldhash 0.2.0",
+]
 
 [[package]]
 name = "heck"
@@ -424,6 +554,7 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-core",
+ "h2",
  "http",
  "http-body",
  "httparse",
@@ -433,6 +564,24 @@ dependencies = [
  "pin-utils",
  "smallvec",
  "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
+dependencies = [
+ "http",
+ "hyper",
+ "hyper-util",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-pki-types",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
 ]
 
 [[package]]
@@ -442,12 +591,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
 dependencies = [
  "bytes",
+ "futures-channel",
+ "futures-util",
  "http",
  "http-body",
  "hyper",
+ "libc",
  "pin-project-lite",
+ "socket2",
  "tokio",
  "tower-service",
+ "tracing",
 ]
 
 [[package]]
@@ -594,12 +748,28 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "ipnet"
+version = "2.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
+
 [[package]]
 name = "itoa"
 version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
 
+[[package]]
+name = "jobserver"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
+dependencies = [
+ "getrandom 0.3.4",
+ "libc",
+]
+
 [[package]]
 name = "js-sys"
 version = "0.3.88"
@@ -686,6 +856,53 @@ version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
 
+[[package]]
+name = "metrics"
+version = "0.24.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d5312e9ba3771cfa961b585728215e3d972c950a3eed9252aa093d6301277e8"
+dependencies = [
+ "ahash",
+ "portable-atomic",
+]
+
+[[package]]
+name = "metrics-exporter-prometheus"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15"
+dependencies = [
+ "base64",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "indexmap",
+ "ipnet",
+ "metrics",
+ "metrics-util",
+ "quanta",
+ "thiserror",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdfb1365fea27e6dd9dc1dbc19f570198bc86914533ad639dae939635f096be4"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown 0.16.1",
+ "metrics",
+ "quanta",
+ "rand",
+ "rand_xoshiro",
+ "sketches-ddsketch",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -752,6 +969,12 @@ version = "1.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 
+[[package]]
+name = "openssl-probe"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+
 [[package]]
 name = "parking_lot"
 version = "0.12.5"
@@ -803,6 +1026,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
 [[package]]
 name = "potential_utf"
 version = "0.1.4"
@@ -818,6 +1047,15 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
 [[package]]
 name = "prettyplease"
 version = "0.2.37"
@@ -837,6 +1075,21 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "quanta"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "once_cell",
+ "raw-cpuid",
+ "wasi",
+ "web-sys",
+ "winapi",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.44"
@@ -852,6 +1105,26 @@ version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 
+[[package]]
+name = "rand"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
+dependencies = [
+ "rand_chacha",
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.9.5",
+]
+
 [[package]]
 name = "rand_core"
 version = "0.6.4"
@@ -861,6 +1134,33 @@ dependencies = [
  "getrandom 0.2.17",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
+dependencies = [
+ "getrandom 0.3.4",
+]
+
+[[package]]
+name = "rand_xoshiro"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
+dependencies = [
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "raw-cpuid"
+version = "11.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "redis"
 version = "0.32.7"
@@ -920,6 +1220,8 @@ dependencies = [
  "chrono",
  "common",
  "fastrand",
+ "metrics",
+ "metrics-exporter-prometheus",
  "redis",
  "serde",
  "serde_json",
@@ -929,6 +1231,67 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.17",
+ "libc",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rustls"
+version = "0.23.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b"
+dependencies = [
+ "aws-lc-rs",
+ "once_cell",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
+dependencies = [
+ "openssl-probe",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
+dependencies = [
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
+dependencies = [
+ "aws-lc-rs",
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.22"
@@ -941,12 +1304,44 @@ version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 
+[[package]]
+name = "schannel"
+version = "0.1.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "security-framework"
+version = "3.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
+dependencies = [
+ "bitflags",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "semver"
 version = "1.0.27"
@@ -1056,7 +1451,7 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
 dependencies = [
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -1071,6 +1466,12 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "sketches-ddsketch"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a"
+
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -1099,6 +1500,12 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
 [[package]]
 name = "syn"
 version = "2.0.117"
@@ -1225,6 +1632,16 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
+dependencies = [
+ "rustls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.7.18"
@@ -1328,6 +1745,12 @@ dependencies = [
  "tracing-log",
 ]
 
+[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
@@ -1340,6 +1763,12 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
 [[package]]
 name = "url"
 version = "2.5.8"
@@ -1376,6 +1805,21 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
 
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
 [[package]]
 name = "wasi"
 version = "0.11.1+wasi-snapshot-preview1"
@@ -1479,6 +1923,38 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "web-sys"
+version = "0.3.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d6bb20ed2d9572df8584f6dc81d68a41a625cadc6f15999d649a70ce7e3597a"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
 [[package]]
 name = "windows-core"
 version = "0.62.2"
@@ -1538,13 +2014,22 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.60.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.53.5",
 ]
 
 [[package]]
@@ -1556,6 +2041,22 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm 0.52.6",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
+]
+
 [[package]]
 name = "windows-targets"
 version = "0.53.5"
@@ -1563,58 +2064,106 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
 dependencies = [
  "windows-link",
- "windows_aarch64_gnullvm",
+ "windows_aarch64_gnullvm 0.53.1",
- "windows_aarch64_msvc",
+ "windows_aarch64_msvc 0.53.1",
- "windows_i686_gnu",
+ "windows_i686_gnu 0.53.1",
- "windows_i686_gnullvm",
+ "windows_i686_gnullvm 0.53.1",
- "windows_i686_msvc",
+ "windows_i686_msvc 0.53.1",
- "windows_x86_64_gnu",
+ "windows_x86_64_gnu 0.53.1",
- "windows_x86_64_gnullvm",
+ "windows_x86_64_gnullvm 0.53.1",
- "windows_x86_64_msvc",
+ "windows_x86_64_msvc 0.53.1",
 ]
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
 
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
 [[package]]
 name = "windows_i686_gnullvm"
 version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.53.1"
@@ -1738,6 +2287,26 @@ dependencies = [
  "synstructure",
 ]
 
+[[package]]
+name = "zerocopy"
+version = "0.8.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "zerofrom"
 version = "0.1.6"
@@ -1759,6 +2328,12 @@ dependencies = [
  "synstructure",
 ]
 
+[[package]]
+name = "zeroize"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+
 [[package]]
 name = "zerotrie"
 version = "0.2.3"

Cargo.toml

@@ -19,3 +19,5 @@ axum = "0.8"
 redis = { version = "0.32", features = ["tokio-comp", "connection-manager"] }
 jsonwebtoken = "10"
 chrono = { version = "0.4", features = ["serde", "clock"] }
+metrics = "0.24"
+metrics-exporter-prometheus = "0.17"

auth-api/Cargo.toml

@@ -15,3 +15,5 @@ tokio.workspace = true
 tracing.workspace = true
 tracing-subscriber.workspace = true
 fastrand.workspace = true
+metrics.workspace = true
+metrics-exporter-prometheus.workspace = true

auth-api/src/main.rs

@@ -10,6 +10,7 @@ use axum::{
 };
 use chrono::{Duration, Utc};
 use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, Validation, decode, encode};
+use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
 use redis::AsyncCommands;
 use serde::{Deserialize, Serialize};
 use tracing::{info, warn};
@@ -18,6 +19,7 @@ use tracing::{info, warn};
 struct AppState {
     jwt_secret: Arc<String>,
     redis: Option<redis::aio::ConnectionManager>,
+    metrics: PrometheusHandle,
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
@@ -78,6 +80,9 @@ async fn main() -> Result<()> {
 
     let bind = std::env::var("AUTH_BIND").unwrap_or_else(|_| "0.0.0.0:8080".into());
     let jwt_secret = std::env::var("JWT_SECRET").unwrap_or_else(|_| "dev-secret-change-me".into());
+    let metrics = PrometheusBuilder::new()
+        .install_recorder()
+        .context("install prometheus recorder")?;
     let redis = if let Ok(url) = std::env::var("REDIS_URL") {
         let client = redis::Client::open(url.clone()).context("open redis client")?;
         match redis::aio::ConnectionManager::new(client).await {
@@ -97,10 +102,12 @@ async fn main() -> Result<()> {
     let state = AppState {
         jwt_secret: Arc::new(jwt_secret),
         redis,
+        metrics,
     };
 
     let app = Router::new()
         .route("/healthz", get(healthz))
+        .route("/metrics", get(metrics_endpoint))
         .route("/v1/token/dev", post(issue_dev_token))
         .route("/v1/token/validate", post(validate_token))
         .route("/v1/stripe/webhook", post(stripe_webhook))
@@ -119,10 +126,16 @@ async fn healthz() -> &'static str {
     "ok"
 }
 
+async fn metrics_endpoint(State(state): State<AppState>) -> impl IntoResponse {
+    state.metrics.render()
+}
+
+#[tracing::instrument(skip(state))]
 async fn issue_dev_token(
     State(state): State<AppState>,
     Json(req): Json<DevTokenRequest>,
 ) -> Result<Json<TokenResponse>, ApiError> {
+    let started = std::time::Instant::now();
     let now = Utc::now();
     let exp = now + Duration::hours(24);
     let claims = Claims {
@@ -155,6 +168,9 @@ async fn issue_dev_token(
             .await
             .map_err(ApiError::internal)?;
     }
+    metrics::counter!("auth_jwt_issued_total").increment(1);
+    metrics::histogram!("auth_issue_token_latency_ms")
+        .record(started.elapsed().as_secs_f64() * 1000.0);
 
     Ok(Json(TokenResponse {
         token,
@@ -162,10 +178,12 @@ async fn issue_dev_token(
     }))
 }
 
+#[tracing::instrument(skip(state, req))]
 async fn validate_token(
     State(state): State<AppState>,
     Json(req): Json<ValidateRequest>,
 ) -> Result<Json<ValidateResponse>, ApiError> {
+    let started = std::time::Instant::now();
     let decoded = decode::<Claims>(
         &req.token,
         &DecodingKey::from_secret(state.jwt_secret.as_bytes()),
@@ -185,6 +203,9 @@ async fn validate_token(
                 .to_string();
                 let _: () = redis.set_ex(key, payload, 300).await.map_err(ApiError::internal)?;
             }
+            metrics::counter!("auth_token_validate_total", "result" => "valid").increment(1);
+            metrics::histogram!("auth_validate_token_latency_ms")
+                .record(started.elapsed().as_secs_f64() * 1000.0);
             Ok(Json(ValidateResponse {
                 valid: true,
                 user_id: Some(c.sub),
@@ -197,14 +218,21 @@ async fn validate_token(
             user_id: None,
             tier: None,
             max_tunnels: None,
-        })),
+        })).map(|resp| {
+            metrics::counter!("auth_token_validate_total", "result" => "invalid").increment(1);
+            metrics::histogram!("auth_validate_token_latency_ms")
+                .record(started.elapsed().as_secs_f64() * 1000.0);
+            resp
+        }),
     }
 }
 
+#[tracing::instrument(skip(state))]
 async fn stripe_webhook(
     State(state): State<AppState>,
     Json(event): Json<StripeWebhookEvent>,
 ) -> Result<impl IntoResponse, ApiError> {
+    let started = std::time::Instant::now();
     if let Some(mut redis) = state.redis.clone() {
         let key = format!("plan:user:{}", event.user_id);
         let payload = serde_json::json!({
@@ -217,6 +245,9 @@ async fn stripe_webhook(
         .to_string();
         let _: () = redis.set_ex(key, payload, 300).await.map_err(ApiError::internal)?;
     }
+    metrics::counter!("stripe_webhook_events_total", "event_type" => event.event_type.clone()).increment(1);
+    metrics::histogram!("stripe_webhook_latency_ms")
+        .record(started.elapsed().as_secs_f64() * 1000.0);
     Ok(StatusCode::NO_CONTENT)
 }
 

relay/Cargo.toml

@@ -14,4 +14,6 @@ redis.workspace = true
 serde_json.workspace = true
 chrono.workspace = true
 serde.workspace = true
+metrics.workspace = true
+metrics-exporter-prometheus.workspace = true
 common = { path = "../common" }

relay/src/main.rs

@@ -16,6 +16,7 @@ use common::{
 };
 use redis::AsyncCommands;
 use serde::Deserialize;
+use metrics_exporter_prometheus::PrometheusBuilder;
 use tokio::{
     io::{AsyncReadExt, AsyncWriteExt, copy_bidirectional},
     net::{TcpListener, TcpStream},
@@ -436,6 +437,7 @@ impl RedisRegistry {
 
 #[tokio::main]
 async fn main() -> Result<()> {
+    init_metrics()?;
     tracing_subscriber::fmt()
         .with_env_filter(
             tracing_subscriber::EnvFilter::try_from_default_env()
@@ -459,6 +461,7 @@ async fn main() -> Result<()> {
         .with_context(|| format!("bind r2r {}", cfg.r2r_bind))?;
 
     info!(instance_id = %cfg.instance_id, region = %cfg.region, control = %cfg.control_bind, player = %cfg.player_bind, r2r = %cfg.r2r_bind, r2r_advertise = %cfg.r2r_advertise_addr, "relay started");
+    metrics::gauge!("relay_drain_state").set(0.0);
 
     let shutdown = Arc::new(Notify::new());
     let state: SharedState = Arc::new(RwLock::new(RelayState::new()));
@@ -502,6 +505,7 @@ async fn main() -> Result<()> {
     }
 
     registry.set_draining().await;
+    metrics::gauge!("relay_drain_state").set(1.0);
     shutdown.notify_waiters();
     info!("draining relay");
     tokio::time::sleep(Duration::from_secs(1)).await;
@@ -516,6 +520,7 @@ async fn run_registry_heartbeat(state: SharedState, registry: RedisRegistry, shu
             _ = shutdown.notified() => break,
             _ = ticker.tick() => {
                 let count = state.read().await.session_count();
+                metrics::gauge!("relay_active_tunnels").set(count as f64);
                 registry.heartbeat_instance(count).await;
             }
         }
@@ -538,6 +543,7 @@ async fn run_control_accept_loop(
                     Ok(v) => v,
                     Err(e) => { warn!(error = %e, "control accept failed"); continue; }
                 };
+                metrics::counter!("relay_control_accepts_total").increment(1);
                 let cfg = cfg.clone();
                 let state = state.clone();
                 let registry = registry.clone();
@@ -569,6 +575,7 @@ async fn run_player_accept_loop(
                     Ok(v) => v,
                     Err(e) => { warn!(error = %e, "player accept failed"); continue; }
                 };
+                metrics::counter!("relay_player_accepts_total").increment(1);
                 let cfg = cfg.clone();
                 let state = state.clone();
                 let registry = registry.clone();
@@ -599,6 +606,7 @@ async fn run_r2r_accept_loop(
                     Ok(v) => v,
                     Err(e) => { warn!(error = %e, "r2r accept failed"); continue; }
                 };
+                metrics::counter!("relay_r2r_accepts_total").increment(1);
                 let cfg = cfg.clone();
                 let state = state.clone();
                 let guards = guards.clone();
@@ -613,6 +621,7 @@ async fn run_r2r_accept_loop(
     Ok(())
 }
 
+#[tracing::instrument(skip(stream, state, registry, guards, cfg), fields(peer = %addr))]
 async fn handle_control_conn(
     stream: TcpStream,
     addr: SocketAddr,
@@ -622,6 +631,7 @@ async fn handle_control_conn(
     guards: Arc<RelayGuards>,
 ) -> Result<()> {
     if !guards.allow_registration_ip(&addr.ip().to_string()).await {
+        metrics::counter!("relay_rate_limited_total", "scope" => "registration_ip").increment(1);
         anyhow::bail!("registration rate limited for {}", addr.ip());
     }
 
@@ -667,6 +677,7 @@ async fn handle_control_conn(
         owner_instance_id: cfg.instance_id.clone(),
     })).await?;
     info!(peer = %addr, user_id = %user_id, fqdn = %fqdn, session_id = %session_id, "client registered");
+    metrics::counter!("relay_tunnel_registrations_total").increment(1);
 
     let write_task = tokio::spawn(async move {
         while let Some(frame) = rx.recv().await {
@@ -749,6 +760,7 @@ async fn control_read_loop(
     }
 }
 
+#[tracing::instrument(skip(stream, cfg, state, registry, guards), fields(peer = %addr))]
 async fn handle_player_conn(
     mut stream: TcpStream,
     addr: SocketAddr,
@@ -758,6 +770,7 @@ async fn handle_player_conn(
     guards: Arc<RelayGuards>,
 ) -> Result<()> {
     if !guards.allow_player_ip(&addr.ip().to_string()).await {
+        metrics::counter!("relay_rate_limited_total", "scope" => "player_ip").increment(1);
         debug!(peer = %addr, "player connect rate limited");
         return Ok(());
     }
@@ -792,6 +805,7 @@ async fn handle_player_conn(
     Ok(())
 }
 
+#[tracing::instrument(skip(stream, _cfg, state, guards), fields(peer = %addr))]
 async fn handle_r2r_conn(
     mut stream: TcpStream,
     addr: SocketAddr,
@@ -826,6 +840,7 @@ async fn handle_r2r_conn(
     .with_context(|| format!("r2r attach failed from {addr}"))
 }
 
+#[tracing::instrument(skip(player_stream, route, cfg, registry), fields(peer = %player_addr, hostname = %hostname))]
 async fn proxy_player_to_owner(
     mut player_stream: TcpStream,
     player_addr: SocketAddr,
@@ -835,18 +850,24 @@ async fn proxy_player_to_owner(
     cfg: RelayConfig,
     registry: RedisRegistry,
 ) -> Result<()> {
+    let redis_lookup_started = Instant::now();
     let owner = registry
         .lookup_instance(&route.instance_id)
         .await
         .with_context(|| format!("owner instance {} not found in redis", route.instance_id))?;
+    metrics::histogram!("relay_redis_lookup_latency_ms")
+        .record(redis_lookup_started.elapsed().as_secs_f64() * 1000.0);
     let r2r_addr = owner
         .r2r_addr
         .clone()
         .with_context(|| format!("owner {} missing r2r_addr", route.instance_id))?;
 
+    let r2r_connect_started = Instant::now();
     let mut owner_stream = timeout(cfg.r2r_connect_timeout, TcpStream::connect(&r2r_addr))
         .await
         .context("r2r connect timeout")??;
+    metrics::histogram!("relay_r2r_connect_latency_ms")
+        .record(r2r_connect_started.elapsed().as_secs_f64() * 1000.0);
 
     let prelude = RelayForwardPrelude {
         version: 1,
@@ -861,6 +882,7 @@ async fn proxy_player_to_owner(
     write_frame(&mut owner_stream, &prelude).await?;
 
     let _ = copy_bidirectional(&mut player_stream, &mut owner_stream).await?;
+    metrics::counter!("relay_r2r_forwards_total").increment(1);
     info!(peer = %player_addr, hostname = %hostname, owner = %route.instance_id, "proxied player connection to owner relay");
     Ok(())
 }
@@ -945,6 +967,7 @@ async fn attach_player_socket_to_session(
     });
 
     info!(peer = %peer_addr, hostname = %hostname, session_id = %session.session_id, stream_id = %stream_id, source, "player proxied via client stream");
+    metrics::gauge!("relay_active_player_conns").increment(1.0);
     Ok(())
 }
 
@@ -971,7 +994,9 @@ async fn run_player_reader(
             }))
             .await
             .context("send stream data to client")?;
+        metrics::counter!("relay_bytes_out_total").increment(n as u64);
     }
+    metrics::gauge!("relay_active_player_conns").decrement(1.0);
     Ok(())
 }
 
@@ -981,6 +1006,7 @@ async fn run_player_writer(
 ) -> Result<()> {
     while let Some(chunk) = rx.recv().await {
         writer.write_all(&chunk).await?;
+        metrics::counter!("relay_bytes_in_total").increment(chunk.len() as u64);
     }
     let _ = writer.shutdown().await;
     Ok(())
@@ -1062,3 +1088,14 @@ fn guess_advertise_addr(bind: &str) -> String {
         "127.0.0.1:7001".to_string()
     }
 }
+
+fn init_metrics() -> Result<()> {
+    if let Ok(bind) = std::env::var("RELAY_METRICS_BIND") {
+        let addr: std::net::SocketAddr = bind.parse().context("parse RELAY_METRICS_BIND")?;
+        PrometheusBuilder::new()
+            .with_http_listener(addr)
+            .install()
+            .context("install prometheus exporter")?;
+    }
+    Ok(())
+}