2Python module that provides helpers for functional testing of metrics with
4@ref scripts/docs/en/userver/functional_testing.md for an introduction.
6@ingroup userver_testsuite
9from __future__
import annotations
11from collections.abc
import Mapping
12from collections.abc
import Set
20from typing
import overload
21from typing
import TypeAlias
22from typing
import TypeVar
26class MetricType(str, enum.Enum):
28 The type of individual metric.
30 `UNSPECIFIED` compares equal to all `MetricType`s.
31 To disable this behavior, use `is` for comparisons.
34 UNSPECIFIED =
'UNSPECIFIED'
37 HIST_RATE =
'HIST_RATE'
44 Represents the value of a HIST_RATE (a.k.a. Histogram) metric.
47 @snippet testsuite/tests/metrics/test_metrics.py histogram
49 Normally obtained from MetricsSnapshot
56 def count(self) -> int:
57 return sum(self.
buckets) + self.inf
59 def percentile(self, percent: float) -> float:
60 return _do_compute_percentile(self, percent)
63 def __post_init__(self):
68 assert self.
bounds[-1] != math.inf
73MetricValue: TypeAlias = float | Histogram
76_MISSING: Any = object()
79@dataclasses.dataclass(frozen=True)
82 Metric type that contains the `labels: dict[str, str]` and
85 The type is hashable and comparable:
86 @snippet testsuite/tests/metrics/test_metrics.py values set
88 @ingroup userver_testsuite
91 labels: dict[str, str]
96 _type: MetricType = MetricType.UNSPECIFIED
99 def __eq__(self, other: object) -> bool:
100 if not isinstance(other, Metric):
101 return NotImplemented
102 return self.
labels == other.labels
and self.value == other.value
and _type_eq(self._type, other._type)
104 def __hash__(self) -> int:
108 def __post_init__(self):
109 if isinstance(self.value, Histogram):
110 assert self._type
in (MetricType.HIST_RATE, MetricType.UNSPECIFIED)
112 assert self._type
is not MetricType.HIST_RATE
115 def type(self) -> MetricType:
122 def default(self, o):
123 if isinstance(o, Metric):
124 result = {
'labels': o.labels,
'value': o.value}
125 if o.type()
is not MetricType.UNSPECIFIED:
126 result[
'type'] = o.type()
128 elif isinstance(o, Histogram):
129 return dataclasses.asdict(o)
130 if isinstance(o, set):
132 return super().default(o)
137 Snapshot of captured metrics that mimics the dict interface. Metrics have
138 the 'dict[str(path), Set[Metric]]' format.
140 Example with @ref pytest_userver.client.ClientMonitor.metrics "await monitor_client.metrics(path_prefix, labels)":
141 @snippet samples/testsuite-support/tests/test_metrics.py metrics metrics
143 There are 3 ways to construct a `MetricsSnapshot`:
145 1. The constructor itself, taking a ready `dict[str(path), Set[Metric]]`.
146 Useful when dealing with just a few simple metrics, or when generating metrics programmatically,
147 or when combining or transforming snapshots. This format is also well-suited for individual metrics
148 with a large number of labels.
149 2. `from_dict` / `from_json`, taking the flat `json` userver metrics format
150 (a list of `{"labels": ..., "value": ...}` per path).
151 This is an alternative to the constructor for loading from a JSON file.
152 Note that `from_layered_dict` is often more terse and more readable.
153 3. `from_layered_dict`, taking a layered dict format that avoids repeating labels for every metric.
154 Recommended in tests that have many metrics sharing a label structure.
155 Can be written out in code or loaded from a JSON file using `load_json` fixture.
157 @ingroup userver_testsuite
162 values: Mapping[str, Set[Metric]],
164 common_prefix: str =
'',
165 common_labels: Mapping[str, str] |
None =
None,
168 @param values Metrics keyed by path, as a `dict[str(path), Set[Metric]]`
169 (the same format `MetricsSnapshot` itself exposes via `items()`).
170 @param common_prefix If provided, prepended to each path (separated by a dot).
171 @param common_labels If provided, these labels are added to every metric,
172 merged with (and overridden by) that metric's own labels.
174 self.
_values = _apply_common_prefix_labels(values, common_prefix, common_labels)
179 """Returns a list of metrics by specified path"""
183 """Returns count of metrics paths"""
187 """Returns a (path, list) iterable over the metrics"""
192 Returns True if metric with specified path is in the snapshot, False otherwise.
198 Compares the snapshot with a dict of metrics or with another
199 snapshot. A path mapped to an empty set of metrics is treated the
200 same as an absent path.
202 if isinstance(other, MetricsSnapshot):
203 other_values: Mapping[str, Set[Metric]] = other._values
204 elif isinstance(other, Mapping):
207 return NotImplemented
208 return _drop_empty_paths(self.
_values) == _drop_empty_paths(other_values)
210 def __repr__(self) -> str:
213 def __str__(self) -> str:
216 def get(self, path: str, default=
None):
218 Returns an list of metrics by path or default if there's no such path
223 """Returns a (path, list) iterable over the metrics"""
227 """Returns an iterable over paths of metrics"""
231 """Returns an iterable over lists of metrics"""
237 labels: dict[str, str] |
None =
None,
239 ) -> MetricsSnapshot:
241 Returns a new MetricsSnapshot restricted to the metrics whose path starts with `prefix` as a whole
242 '.'-separated segment, and, if `labels` is given, whose labels are a superset of `labels` (same
243 subset-match semantics as `require_labels` in `metrics_at`).
247 * A non-empty string: matches paths equal to `prefix`, or starting with `prefix` followed by a whole
248 '.'-separated segment boundary (`prefix='a.b'` matches paths `'a.b'` and `'a.b.c'`, but not `'a.bc'`).
249 The matched `prefix` (and the following '.', if any) is stripped from the start of every surviving
250 metric's path, so e.g. slicing `'a.b.c'` by `prefix='a.b'` makes it accessible as `'c'`, and slicing
251 `'a.b'` by `prefix='a.b'` makes it accessible as `''`.
252 * `''`: matches paths equal to `''`, or starting with a literal leading `'.'`; the leading `'.'` is
253 stripped. Only relevant for the rare case of a metric path that itself starts with a dot.
254 * `None`: matches every path unconditionally. Use this to filter by `labels` only.
256 Calling `sliced()` several times in a row composes: each `prefix` is matched against the
257 already-stripped paths of the previous `sliced()` call, and `labels` requirements accumulate.
259 Slicing only ever affects *filtering* (which metrics are visible, and under which path): it never
260 touches the `Metric` objects themselves. `metrics_at()`, `value_at()` and iteration over a sliced
261 snapshot all keep returning the exact same, untouched `Metric` objects (same labels, value, identity),
262 just possibly under a shorter path and/or a smaller surrounding set.
264 Intended use: carve out a small, closed slice of a snapshot (e.g. one metric path with a handful of
265 varying labels) to compare it with `==` against a compact expected snapshot, or to look up several
266 label combinations with `value_at`/`metrics_at` without repeating the common prefix and labels in every call.
268 @throws AssertionError if stripping `prefix` from a metric path would leave an empty remainder,
269 e.g. path 'a.b.' sliced by prefix='a.b', which would be indistinguishable from path 'a.b'.
271 @snippet testsuite/tests/metrics/test_sliced.py sliced snippet
273 result: dict[str, set[Metric]] = {}
278 remainder = _strip_prefix_segment(path, prefix)
279 if remainder
is None:
283 metric_set = {metric
for metric
in metric_set
if labels.items() <= metric.labels.items()}
287 result.setdefault(remainder, set())
288 result[remainder] |= metric_set
294 sliced_snapshot._sliced_prefix = prefix
296 sliced_snapshot._sliced_prefix = f
'{self._sliced_prefix}.{prefix}'
298 return sliced_snapshot
302 Returns a new MetricsSnapshot with the `prefix` accumulated from the preceding (possibly chained)
303 `sliced()` call(s) prepended back to every surviving metric's path. Does not mutate `self`.
305 Metrics that were filtered out by `sliced()` (because their path did not match `prefix`, or their
306 labels did not match `labels`) do NOT come back: `unsliced()` only restores the *path* of what
307 remains in the snapshot, it does not undo the filtering itself.
309 If this snapshot was never `sliced()` (i.e. `self` is the original snapshot, or the result of
310 operations other than `sliced()`), returns an equivalent snapshot unchanged.
315 result = {(f
'{prefix}.{path}' if path
else prefix): metric_set
for path, metric_set
in self.
_values.
items()}
322 labels: dict[str, str] |
None =
None,
323 ) -> MetricValue: ...
329 labels: dict[str, str] |
None,
332 ) -> MetricValue | T: ...
337 labels: dict[str, str] |
None =
None,
339 default: Any = _MISSING,
340 ) -> MetricValue | Any:
342 Returns a single metric value at specified path. If a dict of labels
343 is provided, does en exact match of labels (i.e. {} stands for no
344 labels; {'a': 'b', 'c': 'd'} matches only {'a': 'b', 'c': 'd'} or
345 {'c': 'd', 'a': 'b'} but neither match {'a': 'b'} nor {'a': 'b', 'c': 'd', 'e': 'f'}).
347 If `default` is provided, it is returned instead of asserting when
348 the metric is not found.
350 @throws AssertionError if not one metric by path and no `default` is given
352 @snippet samples/testsuite-support/tests/test_metrics.py metrics metrics
354 entry = self.
get(path, set())
355 assert entry
or default
is not _MISSING, f
'No metrics found by path "{path}"' + (
356 f
' after slicing "{self._sliced_prefix}"' if self.
_sliced_prefix is not None else ''
359 if labels
is not None:
361 filtered_entries = {x
for x
in entry
if x.labels == full_labels}
362 assert filtered_entries
or default
is not _MISSING, (
363 f
'No metrics found by path "{path}" and labels {full_labels}. Possible values: {entry}'
365 assert len(filtered_entries) <= 1, (
366 f
'Multiple metrics found by path "{path}" and labels {full_labels}: {filtered_entries}'
368 entry = filtered_entries
370 assert len(entry) <= 1, f
'Multiple metrics found by path "{path}": {entry}'
372 if default
is not _MISSING
and not entry:
374 return next(iter(entry)).value
379 require_labels: dict[str, str] |
None =
None,
382 Metrics path must exactly equal the given `path`.
383 A required subset of labels is specified by `require_labels`
385 require_labels={'a':'b', 'c':'d'}
386 { 'a':'b', 'c':'d'} - exact match
387 { 'a':'b', 'c':'d', 'e': 'f', 'h':'k'} - match
388 { 'a':'x', 'c':'d'} - no match, incorrect value for label 'a'
389 { 'a' : 'b'} - required label not found
391 @snippet samples/testsuite-support/tests/test_metrics.py metrics metrics
393 entry = self.
get(path, set())
394 full_require_labels = (
398 if full_require_labels
is not None:
399 return [metric
for metric
in entry
if full_require_labels.items() <= metric.labels.items()]
406 require_labels: dict[str, str] |
None =
None,
410 return bool(self.
metrics_at(path, require_labels))
414 other: Mapping[str, Set[Metric]],
416 ignore_zeros: bool =
False,
419 @deprecated Use `==` operator instead, which produces a nice diff
420 automatically via `pytest_assertrepr_compare`. To ignore zero-rate
421 metrics, use `without_zero_rates()` on the snapshots before comparing.
423 lhs = _flatten_snapshot(self, ignore_zeros=ignore_zeros)
424 rhs = _flatten_snapshot(other, ignore_zeros=ignore_zeros)
425 assert lhs == rhs, _diff_metric_snapshots(lhs, rhs, ignore_zeros)
429 Returns a new snapshot with "empty" RATE and HIST_RATE metrics
430 removed: a RATE metric is removed if its value is zero, a HIST_RATE
431 metric is removed if its histogram has zero count in every bucket
432 and in `inf`. GAUGE (and untyped) metrics are kept as-is, because a
433 zero GAUGE value can be meaningful.
437 path: {metric
for metric
in metric_set
if not _is_zero_rate_or_histogram(metric)}
444 Multiline linear print:
445 path: (label=value),(label=value) TYPE VALUE
446 path: (label=value),(label=value) TYPE VALUE
449 assert 'some.thing.sensor' in metric, metric.pretty_print()
453 def _iterate_over_mset(path, mset):
454 """print (pretty) one metrics set - for given path"""
462 data_for_every_path = [_iterate_over_mset(path, mset)
for path, mset
in self.
_values.
items()]
466 return '\n'.join(itertools.chain(*data_for_every_path))
469 def from_dict(data: Mapping[str, Any]) -> MetricsSnapshot:
471 Construct MetricsSnapshot from a JSON dict in the `json` userver metrics format.
476 labels=element[
'labels'],
477 value=_parse_metric_value(element[
'value']),
478 _type=MetricType[element.get(
'type',
'UNSPECIFIED')],
480 for element
in metrics_list
482 for path, metrics_list
in data.items()
489 Construct MetricsSnapshot from a JSON string in the `json` userver metrics format.
491 return MetricsSnapshot.from_dict(json.loads(json_str))
495 data: Mapping[str, Any],
497 common_prefix: str =
'',
498 common_labels: Mapping[str, str] |
None =
None,
499 ) -> MetricsSnapshot:
501 Construct MetricsSnapshot from a layered dict format that avoids
502 repeating a label's name for every metric that only differs by
505 Top-level keys of `data` are metric paths, used as-is. Within a
506 path's value, each dict key names a label as `'name = value'`
507 (with exactly one space on each side of `=`: everything before is
508 the label name, everything after is its value); the corresponding
509 child value is interpreted the same way recursively, so several
510 labels can be layered one inside another. A dict with `bounds` and
511 `buckets` keys is a leaf value instead of being recursed into,
512 parsed as a `Histogram`; any other non-dict value is a plain leaf
515 If `common_prefix` is provided, it is prepended to each path
516 (separated by a dot) so that paths in `data` can omit a shared prefix.
518 If `common_labels` is provided, these labels are added to every
519 metric in the snapshot, merged with any labels from the layered dict
522 Example: `{'a': {'x = foo': 1, 'x = bar': 2}}` is equivalent to
523 `MetricsSnapshot({'a': {Metric({'x': 'foo'}, 1), Metric({'x': 'bar'}, 2)}})`.
526 {path: _collect_layered_metrics(node, {})
for path, node
in data.items()},
527 common_prefix=common_prefix,
528 common_labels=common_labels,
533 Serialize to a JSON string
537 {path: random.sample(list(metrics), len(metrics))
for path, metrics
in self.
_values.
items()},
538 cls=_MetricsJSONEncoder,
542def _apply_common_prefix_labels(
543 values: Mapping[str, Set[Metric]],
545 common_labels: Mapping[str, str] |
None,
546) -> Mapping[str, Set[Metric]]:
548 prefix = f
'{common_prefix}.' if common_prefix
else ''
551 dataclasses.replace(metric, labels={**common_labels, **metric.labels})
for metric
in metric_set
553 for path, metric_set
in values.items()
556 return {f
'{common_prefix}.{path}': metric_set
for path, metric_set
in values.items()}
560def _drop_empty_paths(values: Mapping[str, Set[Metric]]) -> dict[str, Set[Metric]]:
561 return {path: metric_set
for path, metric_set
in values.items()
if metric_set}
564def _strip_prefix_segment(path: str, prefix: str) -> str |
None:
565 assert not prefix.endswith(
'.'), f
'prefix "{prefix}" must not end with "."'
568 dotted_prefix = f
'{prefix}.'
569 if path.startswith(dotted_prefix):
570 remainder = path.removeprefix(dotted_prefix)
577 f
'Metric path "{path}" becomes empty right after prefix "{prefix}", '
578 f
'which would make prefix matching ambiguous. This looks like a malformed metric path.'
584def _is_zero_rate_or_histogram(metric: Metric) -> bool:
585 if isinstance(metric.value, Histogram):
586 return metric.value.count() == 0
587 return metric.type() == MetricType.RATE
and metric.value == 0
590def _type_eq(lhs: MetricType, rhs: MetricType) -> bool:
591 return lhs == rhs
or lhs == MetricType.UNSPECIFIED
or rhs == MetricType.UNSPECIFIED
595 """Returns labels as a tuple of sorted items"""
596 return tuple(sorted(metric.labels.items()))
600 """Formats a single metric line as "path: (labels) TYPE VALUE", skipping the labels part (and the extra
601 space) entirely if there are no labels. `forced_type` overrides `metric`'s own type, if given"""
602 labels_str =
','.join(f
'({key}={label_value})' for key, label_value
in _get_labels_tuple(metric))
603 metric_type = forced_type
if forced_type
is not None else metric.type()
604 parts = [f
'{path}:', labels_str, metric_type.value, str(metric.value)]
605 return ' '.join(part
for part
in parts
if part)
609 values: Mapping[str, Set[Metric]],
612 other: Mapping[str, Set[Metric]],
615 Renders a snapshot as a set of "path: (labels) TYPE VALUE" strings
616 (the same format as `MetricsSnapshot.pretty_print`), suitable for use
617 with a generic set-based diff (e.g. testsuite's `CompareVisitor`
620 A metric with an UNSPECIFIED type has its type resolved from the
621 matching (by path and labels) metric in `other`, if any, mirroring the
622 wildcard-matching behavior of `Metric.__eq__`.
624 other_snapshot = other
if isinstance(other, MetricsSnapshot)
else MetricsSnapshot(other)
627 for path, metric_set
in values.items()
628 for metric
in metric_set
632def _resolve_type(path: str, metric: Metric, other_snapshot: MetricsSnapshot) -> MetricType:
633 if metric.type() != MetricType.UNSPECIFIED:
635 for other_metric
in other_snapshot.metrics_at(path, require_labels=metric.labels):
636 if other_metric.labels == metric.labels
and other_metric.type() != MetricType.UNSPECIFIED:
637 return other_metric.type()
641def _do_compute_percentile(hist: Histogram, percent: float) -> float:
644 value_lists = [[bound] * bucket
for (bucket, bound)
in zip(hist.buckets, hist.bounds, strict=
True)] + [
645 [math.inf] * hist.inf
647 values = [item
for sublist
in value_lists
for item
in sublist]
653 pivot = (len(values) - 1) * percent
654 floor = math.floor(pivot)
655 ceil = math.ceil(pivot)
657 return values[int(pivot)]
658 part1 = values[int(floor)] * (ceil - pivot)
659 part2 = values[int(ceil)] * (pivot - floor)
663def _is_histogram_dict(node: Any) -> bool:
664 return isinstance(node, dict)
and 'bounds' in node
and 'buckets' in node
667_LABEL_SEPARATOR =
' = '
670def _collect_layered_metrics(node: Any, labels: dict[str, str]) -> set[Metric]:
671 if isinstance(node, dict)
and not _is_histogram_dict(node):
673 for key, child
in node.items():
674 assert _LABEL_SEPARATOR
in key, f
"Expected a label key like 'name = value', got '{key}'"
675 label_name, label_value = key.split(_LABEL_SEPARATOR, 1)
676 assert not label_name.endswith(
' ')
and not label_value.startswith(
' '), (
677 f
"Expected exactly one space on each side of '=' in a label key, got '{key}'"
679 result |= _collect_layered_metrics(child, {**labels, label_name: label_value})
681 return {
Metric(dict(labels), _parse_metric_value(node))}
684def _parse_metric_value(value: Any) -> MetricValue:
685 if isinstance(value, dict):
687 bounds=value[
'bounds'],
688 buckets=value[
'buckets'],
691 elif isinstance(value, float):
693 elif isinstance(value, int):
696 raise Exception(f
'Failed to parse metric value from {value!r}')
699_FlattenedSnapshot: TypeAlias = Set[tuple[str, Metric]]
702def _flatten_snapshot(values, ignore_zeros: bool) -> _FlattenedSnapshot:
705 for path, metrics
in values.items()
706 for metric
in metrics
707 if metric.value != 0
or not ignore_zeros
711def _diff_metric_snapshots(
712 lhs: _FlattenedSnapshot,
713 rhs: _FlattenedSnapshot,
716 def extra_metrics_message(extra, base):
718 f
' path={path!r} labels={metric.labels!r} value={metric.value}'
719 for path, metric
in sorted(extra, key=
lambda pair: pair[0])
720 if (path, metric)
not in base
724 lines = [
'left.assert_equals(right, ignore_zeros=True) failed']
726 lines = [
'left.assert_equals(right) failed']
727 actual_extra = extra_metrics_message(lhs, rhs)
729 lines.append(
' extra in left:')
730 lines += actual_extra
732 actual_gt = extra_metrics_message(rhs, lhs)
734 lines.append(
' missing in left:')
737 return '\n'.join(lines)