mirror of
https://github.com/vectorize-io/hindsight.git
synced 2026-09-14 19:31:49 +08:00
Nicolò Boschi
2cd0561f14
* feat(metrics): a metrics port per worker, and event-loop lag as a histogram With --workers N every worker is its own process with its own metrics, but they share one port, so a scrape of /metrics reaches one worker at random. Counters jump between processes from one scrape to the next (a rate over them reads each switch as a reset), and process_cpu_seconds_total describes a random worker, so a worker whose event loop is saturated is invisible while the pod total still looks like headroom. - HINDSIGHT_API_METRICS_WORKER_BASE_PORT (default 0, off): each worker also serves its own registry on BASE + slot. The slot (0..N-1) is claimed with an exclusive flock on a per-slot lock file; the kernel drops it when the process exits, so a respawned worker takes over its predecessor's port. /metrics on the API port is unchanged. - HINDSIGHT_API_LOOP_LAG_METRIC (default false): the existing loop-lag probe records every sample in a hindsight.event_loop.lag histogram (seconds, with sub-second buckets), independent of its log reports. * feat(metrics): one /metrics covering every worker, labelled api_worker=<slot> Replaces the per-worker ports from the previous commit. Labelling alone would not fix the scrape: each scrape still reaches one worker, the others' series go missing from about half the scrapes, and Prometheus marks them stale. So with HINDSIGHT_API_METRICS_WORKER_LABEL on, each worker claims a slot (flock on a per-slot lock file; the kernel frees it when the process exits) and publishes a snapshot of its registry every 5 s to a directory the server's workers share. /metrics, whichever worker answers, returns every live worker's series with api_worker="<slot>": its own read live, the others from their latest snapshot, never summed. A snapshot older than 15 s is a gone worker and is skipped; a corrupt one is skipped without breaking the scrape. One port and one scrape target, so no change to charts or scrape configs. The event-loop lag histogram from the previous commit is unchanged.
69 lines
2.3 KiB
Python
69 lines
2.3 KiB
Python
"""The event-loop lag probe starts only when configured, and survives without a caller reference."""
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
import pytest
|
|
|
|
from hindsight_api import loop_lag
|
|
|
|
|
|
async def test_disabled_by_default() -> None:
|
|
assert loop_lag.install(0) is None
|
|
assert not loop_lag._tasks
|
|
|
|
|
|
async def test_reports_lag_percentiles(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture) -> None:
|
|
monkeypatch.setattr(loop_lag, "_MIN_REPORT_S", 0.0)
|
|
monkeypatch.setattr(loop_lag, "_TICK_S", 0.001)
|
|
caplog.set_level(logging.INFO, logger=loop_lag.__name__)
|
|
|
|
task = loop_lag.install(0.02)
|
|
assert task is not None
|
|
# Held by the module, so the loop's weak reference is not the only one keeping it alive.
|
|
assert task in loop_lag._tasks
|
|
try:
|
|
for _ in range(100):
|
|
if any("p99=" in r.getMessage() for r in caplog.records):
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert any("[loop-lag]" in r.getMessage() and "p99=" in r.getMessage() for r in caplog.records)
|
|
finally:
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
await asyncio.sleep(0) # done callbacks run on the next loop iteration
|
|
assert task not in loop_lag._tasks
|
|
|
|
|
|
async def test_records_samples_as_a_metric_without_log_reports(
|
|
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
|
) -> None:
|
|
"""With reports off and the metric on, every tick is recorded and nothing is logged."""
|
|
import hindsight_api.metrics as metrics
|
|
|
|
monkeypatch.setattr(loop_lag, "_TICK_S", 0.001)
|
|
samples: list[float] = []
|
|
|
|
class Recorder(metrics.NoOpMetricsCollector):
|
|
def record_loop_lag(self, lag_seconds: float) -> None:
|
|
samples.append(lag_seconds)
|
|
|
|
monkeypatch.setattr(metrics, "get_metrics_collector", lambda: Recorder())
|
|
caplog.set_level(logging.INFO, logger=loop_lag.__name__)
|
|
|
|
task = loop_lag.install(0, metric=True)
|
|
assert task is not None
|
|
try:
|
|
for _ in range(100):
|
|
if len(samples) >= 5:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert len(samples) >= 5
|
|
assert all(s >= 0.0 for s in samples)
|
|
assert not any("p99=" in r.getMessage() for r in caplog.records)
|
|
finally:
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|