userver: /home/antonyzhilin/arcadia/taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/plugins/dynamic_config.py Source File
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages Concepts
dynamic_config.py
1"""
2Supply dynamic configs for the service in testsuite.
3"""
4
5# pylint: disable=redefined-outer-name
6import contextlib
7import copy
8import dataclasses
9import datetime
10import json
11import pathlib
12import typing
13
14import pytest
15
16from pytest_userver.plugins import caches
17
18USERVER_CONFIG_HOOKS = [
19 'userver_config_dynconf_cache',
20 'userver_config_dynconf_fallback',
21 'userver_config_dynconf_url',
22]
23USERVER_CACHE_CONTROL_HOOKS = {
24 'dynamic-config-client-updater': '_userver_dynconfig_cache_control',
25}
26
27
28class BaseError(Exception):
29 """Base class for exceptions from this module"""
30
31
33 """Config parameter was not found and no default was provided"""
34
35
37 """
38 Calling `dynamic_config.get` before defaults are fetched from the service.
39 Try adding a dependency on `service_client` in your fixture.
40 """
41
42
44 """Dynamic config defaults action returned invalid response"""
45
46
48 """Invalid dynamic config name in `@pytest.mark.config`"""
49
50
51ConfigDict = typing.Dict[str, typing.Any]
52
53
55 pass
56
57
58class Missing:
59 pass
60
61
62@dataclasses.dataclass
64 timestamp: str
65 dirty_keys: typing.Set[str]
66 state: ConfigDict
67 prev_state: ConfigDict
68
69 @classmethod
70 def new(
71 cls,
72 *,
73 previous: typing.Optional['_ChangelogEntry'],
74 timestamp: str,
75 ):
76 if previous:
77 prev_state = previous.state
78 else:
79 prev_state = {}
80 return cls(
81 timestamp=timestamp,
82 dirty_keys=set(),
83 state=prev_state.copy(),
84 prev_state=prev_state,
85 )
86
87 @property
88 def has_changes(self) -> bool:
89 return bool(self.dirty_keys)
90
91 def update(self, values: ConfigDict):
92 for key, value in values.items():
93 if value == self.prev_state.get(key, Missing):
94 self.dirty_keys.discard(key)
95 else:
96 self.dirty_keys.add(key)
97 self.state.update(values)
98
99
100@dataclasses.dataclass(frozen=True)
102 timestamp: str
103 values: ConfigDict
104 removed: typing.List[str]
105
106 def is_empty(self) -> bool:
107 return not self.values and not self.removed
108
109
111 timestamp: datetime.datetime
112 committed_entries: typing.List[_ChangelogEntry]
113 staged_entry: _ChangelogEntry
114
115 def __init__(self):
116 self.timestamp = datetime.datetime.fromtimestamp(
117 0,
118 datetime.timezone.utc,
119 )
120 self.committed_entries = []
121 self.staged_entry = _ChangelogEntry.new(
122 timestamp=self.service_timestamp(),
123 previous=None,
124 )
125
126 def service_timestamp(self) -> str:
127 return self.timestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
128
129 def next_timestamp(self) -> str:
130 self.timestamp += datetime.timedelta(seconds=1)
131 return self.service_timestamp()
132
133 def commit(self) -> _ChangelogEntry:
134 """Commit staged changed if any and return last committed entry."""
135 entry = self.staged_entry
136 if entry.has_changes or not self.committed_entries:
137 self.staged_entry = _ChangelogEntry.new(
138 timestamp=self.next_timestamp(),
139 previous=entry,
140 )
141 self.committed_entries.append(entry)
142 return self.committed_entries[-1]
143
144 def get_updated_since(
145 self,
146 values: ConfigDict,
147 updated_since: str,
148 ids: typing.Optional[typing.List[str]] = None,
149 ) -> Updates:
150 entry = self.commit()
151 values, removed = self._get_updated_since(values, updated_since)
152 if ids:
153 values = {name: values[name] for name in ids if name in values}
154 removed = [name for name in removed if name in ids]
155 return Updates(
156 timestamp=entry.timestamp,
157 values=values,
158 removed=removed,
159 )
160
161 def _get_updated_since(
162 self,
163 values: ConfigDict,
164 updated_since: str,
165 ) -> typing.Tuple[ConfigDict, typing.List[str]]:
166 if not updated_since:
167 return values, []
168 dirty_keys = set()
169 last_known_state = {}
170 for entry in reversed(self.committed_entries):
171 if entry.timestamp > updated_since:
172 dirty_keys.update(entry.dirty_keys)
173 else:
174 if entry.timestamp == updated_since:
175 last_known_state = entry.state
176 break
177 # We don't want to send them again
178 result = {}
179 removed = []
180 for key in dirty_keys:
181 value = values.get(key, RemoveKey)
182 if last_known_state.get(key, Missing) != value:
183 if value is RemoveKey:
184 removed.append(key)
185 else:
186 result[key] = value
187 return result, removed
188
189 def add_entries(self, values: ConfigDict):
190 self.staged_entry.update(values)
191
192 @contextlib.contextmanager
193 def rollback(self, defaults: ConfigDict):
194 try:
195 yield
196 finally:
197 self._do_rollback(defaults)
198
199 def _do_rollback(self, defaults: ConfigDict):
200 if not self.committed_entries:
201 return
202
203 maybe_dirty = set()
204 for entry in self.committed_entries:
205 maybe_dirty.update(entry.dirty_keys)
206
207 last = self.committed_entries[-1]
208 last_state = last.state
209 dirty_keys = set()
210 reverted = {}
211 for key in maybe_dirty:
212 original = defaults.get(key, RemoveKey)
213 if last_state[key] != original:
214 dirty_keys.add(key)
215 reverted[key] = original
216
217 entry = _ChangelogEntry(
218 timestamp=last.timestamp,
219 state=last.state,
220 dirty_keys=dirty_keys,
221 prev_state={},
222 )
223 self.committed_entries = [entry]
225 timestamp=self.staged_entry.timestamp,
226 dirty_keys=dirty_keys.copy(),
227 state=reverted,
228 prev_state=entry.state,
229 )
230
231
233 """
234 @brief Simple dynamic config backend.
235
236 @see @ref pytest_userver.plugins.dynamic_config.dynamic_config "dynamic_config"
237 """
238
239 def __init__(
240 self,
241 *,
242 initial_values: ConfigDict,
243 defaults: typing.Optional[ConfigDict],
244 config_cache_components: typing.Iterable[str],
245 cache_invalidation_state: caches.InvalidationState,
246 changelog: _Changelog,
247 ):
248 self._values = initial_values.copy()
249 # Defaults are only there for convenience, to allow accessing them
250 # in tests using dynamic_config.get. They are not sent to the service.
251 self._defaults = defaults
252 self._cache_invalidation_state = cache_invalidation_state
253 self._config_cache_components = config_cache_components
254 self._changelog = changelog
255
256 def set_values(self, values: ConfigDict):
257 self.set_values_unsafe(copy.deepcopy(values))
258
259 def set_values_unsafe(self, values: ConfigDict):
260 self._values.update(values)
261 self._changelog.add_entries(values)
262 self._sync_with_service()
263
264 def set(self, **values):
265 self.set_values(values)
266
267 def get_values_unsafe(self) -> ConfigDict:
268 return self._values
269
270 def get(self, key: str, default: typing.Any = None) -> typing.Any:
271 if key in self._values:
272 return copy.deepcopy(self._values[key])
273 if self._defaults is not None and key in self._defaults:
274 return copy.deepcopy(self._defaults[key])
275 if default is not None:
276 return default
277 if self._defaults is None:
279 f'Defaults for config {key!r} have not yet been fetched '
280 'from the service. Options:\n'
281 '1. add a dependency on service_client in your fixture;\n'
282 '2. pass `default` parameter to `dynamic_config.get`',
283 )
284 raise DynamicConfigNotFoundError(f'Config {key!r} is not found')
285
286 def remove_values(self, keys):
287 extra_keys = set(keys).difference(self._values.keys())
288 if extra_keys:
290 f'Attempting to remove nonexistent configs: {extra_keys}',
291 )
292 for key in keys:
293 self._values.pop(key)
294 self._changelog.add_entries({key: RemoveKey for key in keys})
295 self._sync_with_service()
296
297 def remove(self, key):
298 return self.remove_values([key])
299
300 @contextlib.contextmanager
301 def modify(self, key: str) -> typing.Any:
302 value = self.get(key)
303 yield value
304 self.set_values({key: value})
305
306 @contextlib.contextmanager
307 def modify_many(
308 self,
309 *keys: typing.Tuple[str, ...],
310 ) -> typing.Tuple[typing.Any, ...]:
311 values = tuple(self.get(key) for key in keys)
312 yield values
313 self.set_values(dict(zip(keys, values)))
314
315 def _sync_with_service(self):
316 self._cache_invalidation_state.invalidate(
318 )
319
320
321@pytest.fixture
323 request,
324 search_path,
325 object_substitute,
326 cache_invalidation_state,
327 _dynamic_config_defaults_storage,
328 config_service_defaults,
329 dynamic_config_changelog,
330 _dynconf_load_json_cached,
331 dynconf_cache_names,
332) -> DynamicConfig:
333 """
334 Fixture that allows to control dynamic config values used by the service.
335
336 Example:
337
338 @snippet core/functional_tests/basic_chaos/tests-nonchaos/handlers/test_log_request_headers.py dynamic_config usage
339
340 HTTP and gRPC client requests call `update_server_state` automatically before each request.
341
342 For main dynamic config documentation:
343
344 @see @ref dynamic_config_testsuite
345
346 See also other related fixtures:
347 * @ref pytest_userver.plugins.dynamic_config.dynamic_config "config_service_defaults"
348 * @ref pytest_userver.plugins.dynamic_config.dynamic_config "dynamic_config_fallback_patch"
349 * @ref pytest_userver.plugins.dynamic_config.dynamic_config "mock_configs_service"
350
351 @ingroup userver_testsuite_fixtures
352 """
353 config = DynamicConfig(
354 initial_values=config_service_defaults,
355 defaults=_dynamic_config_defaults_storage.snapshot,
356 config_cache_components=dynconf_cache_names,
357 cache_invalidation_state=cache_invalidation_state,
358 changelog=dynamic_config_changelog,
359 )
360
361 with dynamic_config_changelog.rollback(config_service_defaults):
362 updates = {}
363 for path in reversed(list(search_path('config.json'))):
364 values = _dynconf_load_json_cached(path)
365 updates.update(values)
366 for marker in request.node.iter_markers('config'):
367 marker_json = object_substitute(marker.kwargs)
368 updates.update(marker_json)
369 config.set_values_unsafe(updates)
370 yield config
371
372
373# @cond
374
375
376def pytest_configure(config):
377 config.addinivalue_line(
378 'markers',
379 'config: per-test dynamic config values',
380 )
381 config.addinivalue_line(
382 'markers',
383 'disable_config_check: disable config mark keys check',
384 )
385
386
387# @endcond
388
389
390@pytest.fixture(scope='session')
391def dynconf_cache_names() -> typing.Iterable[str]:
392 return tuple(USERVER_CACHE_CONTROL_HOOKS.keys())
393
394
395@pytest.fixture(scope='session')
396def _dynconf_json_cache():
397 return {}
398
399
400@pytest.fixture
401def _dynconf_load_json_cached(json_loads, _dynconf_json_cache):
402 def load(path: pathlib.Path):
403 if path not in _dynconf_json_cache:
404 _dynconf_json_cache[path] = json_loads(path.read_text())
405 return _dynconf_json_cache[path]
406
407 return load
408
409
410@pytest.fixture
411def taxi_config(dynamic_config) -> DynamicConfig:
412 """
413 Deprecated, use `dynamic_config` instead.
414 """
415 return dynamic_config
416
417
418@pytest.fixture(scope='session')
420 """
421 Override this fixture to replace some dynamic config values specifically
422 for testsuite tests:
423
424 @code
425 @pytest.fixture(scope='session')
426 def dynamic_config_fallback_patch():
427 return {"MY_CONFIG_NAME": 42}
428 @endcode
429
430 @ingroup userver_testsuite_fixtures
431 """
432 return {}
433
434
435@pytest.fixture(scope='session')
437 config_fallback_path,
438 dynamic_config_fallback_patch,
439) -> ConfigDict:
440 """
441 Fixture that returns default values for dynamic config. You may override
442 it in your local conftest.py or fixture:
443
444 @code
445 @pytest.fixture(scope='session')
446 def config_service_defaults():
447 with open('defaults.json') as fp:
448 return json.load(fp)
449 @endcode
450
451 @ingroup userver_testsuite_fixtures
452 """
453 if not config_fallback_path:
454 return dynamic_config_fallback_patch
455
456 if pathlib.Path(config_fallback_path).exists():
457 with open(config_fallback_path, 'r', encoding='utf-8') as file:
458 fallback = json.load(file)
459 fallback.update(dynamic_config_fallback_patch)
460 return fallback
461
462 raise RuntimeError(
463 'Invalid path specified in config_fallback_path fixture. '
464 'Probably invalid path was passed in --config-fallback pytest option.',
465 )
466
467
468@dataclasses.dataclass(frozen=False)
470 snapshot: typing.Optional[ConfigDict]
471
472 async def update(self, client, dynamic_config) -> None:
473 if self.snapshot is None:
474 defaults = await client.get_dynamic_config_defaults()
475 if not isinstance(defaults, dict):
477 self.snapshot = defaults
478 # pylint:disable=protected-access
479 dynamic_config._defaults = defaults
480
481
482# config_service_defaults fetches the dynamic config overrides, e.g. specified
483# in the json file, then userver_config_dynconf_fallback forwards them
484# to the service so that it has the correct dynamic config defaults.
485#
486# Independently of that, it is useful to have values for all configs, even
487# unspecified in tests, on the testsuite side. For that, we ask the service
488# for the dynamic config defaults after it's launched. It's enough to update
489# defaults once per service launch.
490@pytest.fixture(scope='session')
491def _dynamic_config_defaults_storage() -> _ConfigDefaults:
492 return _ConfigDefaults(snapshot=None)
493
494
495@pytest.fixture(scope='session')
497 """
498 Returns a function that adjusts the static configuration file for
499 the testsuite.
500 Sets `dynamic-config.fs-cache-path` to a file that is reset after the tests
501 to avoid leaking dynamic config values between test sessions.
502
503 @ingroup userver_testsuite_fixtures
504 """
505
506 def patch_config(config, _config_vars) -> None:
507 components = config['components_manager']['components']
508 dynamic_config_component = components.get('dynamic-config', None) or {}
509 if dynamic_config_component.get('fs-cache-path', '') == '':
510 return
511
512 cache_path = service_tmpdir / 'configs' / 'config_cache.json'
513
514 if cache_path.is_file():
515 # To avoid leaking dynamic config values between test sessions
516 cache_path.unlink()
517
518 dynamic_config_component['fs-cache-path'] = str(cache_path)
519
520 return patch_config
521
522
523@pytest.fixture(scope='session')
524def userver_config_dynconf_fallback(config_service_defaults):
525 """
526 Returns a function that adjusts the static configuration file for
527 the testsuite.
528 Removes `dynamic-config.defaults-path`.
529 Updates `dynamic-config.defaults` with `config_service_defaults`.
530
531 @ingroup userver_testsuite_fixtures
532 """
533
534 def extract_defaults_dict(component_config, config_vars) -> dict:
535 defaults_field = component_config.get('defaults', None) or {}
536 if isinstance(defaults_field, dict):
537 return defaults_field
538 elif isinstance(defaults_field, str):
539 if defaults_field.startswith('$'):
540 return config_vars.get(defaults_field[1:], {})
541 assert False, f'Unexpected static config option `dynamic-config.defaults`: {defaults_field!r}'
542
543 def _patch_config(config_yaml, config_vars):
544 components = config_yaml['components_manager']['components']
545 if components.get('dynamic-config', None) is None:
546 components['dynamic-config'] = {}
547 dynconf_component = components['dynamic-config']
548
549 dynconf_component.pop('defaults-path', None)
550 defaults = dict(
551 extract_defaults_dict(dynconf_component, config_vars),
552 **config_service_defaults,
553 )
554 dynconf_component['defaults'] = defaults
555
556 return _patch_config
557
558
559@pytest.fixture(scope='session')
560def userver_config_dynconf_url(mockserver_info):
561 """
562 Returns a function that adjusts the static configuration file for
563 the testsuite.
564 Sets the `dynamic-config-client.config-url` to the value of mockserver
565 configs-service, so that the
566 @ref pytest_userver.plugins.dynamic_config.mock_configs_service
567 "mock_configs_service" fixture could work.
568
569 @ingroup userver_testsuite_fixtures
570 """
571
572 def _patch_config(config, _config_vars) -> None:
573 components = config['components_manager']['components']
574 client = components.get('dynamic-config-client', None)
575 if client:
576 client['config-url'] = mockserver_info.url('configs-service')
577 client['append-path-to-url'] = True
578
579 return _patch_config
580
581
582# @cond
583
584
585# TODO publish _Changelog and document how to use it in custom config service
586# mocks.
587@pytest.fixture(scope='session')
588def dynamic_config_changelog() -> _Changelog:
589 return _Changelog()
590
591
592# @endcond
593
594
595@pytest.fixture
597 mockserver,
598 dynamic_config: DynamicConfig,
599 dynamic_config_changelog: _Changelog,
600) -> None:
601 """
602 Adds a mockserver handler that forwards dynamic_config to service's
603 `dynamic-config-client` component.
604
605 @ingroup userver_testsuite_fixtures
606 """
607
608 @mockserver.json_handler('/configs-service/configs/values')
609 def _mock_configs(request):
610 updates = dynamic_config_changelog.get_updated_since(
611 dynamic_config.get_values_unsafe(),
612 request.json.get('updated_since', ''),
613 request.json.get('ids'),
614 )
615 response = {'configs': updates.values, 'updated_at': updates.timestamp}
616 if updates.removed:
617 response['removed'] = updates.removed
618 return response
619
620 @mockserver.json_handler('/configs-service/configs/status')
621 def _mock_configs_status(_request):
622 return {
623 'updated_at': dynamic_config_changelog.timestamp.strftime(
624 '%Y-%m-%dT%H:%M:%SZ',
625 ),
626 }
627
628
629@pytest.fixture
630def _userver_dynconfig_cache_control(dynamic_config_changelog: _Changelog):
631 def cache_control(updater, timestamp):
632 entry = dynamic_config_changelog.commit()
633 if entry.timestamp == timestamp:
634 updater.exclude()
635 else:
636 updater.incremental()
637 return entry.timestamp
638
639 return cache_control
640
641
642_CHECK_CONFIG_ERROR = (
643 'Your are trying to override config value using '
644 '@pytest.mark.config({}) '
645 'that does not seem to be used by your service.\n\n'
646 'In case you really need to disable this check please add the '
647 'following mark to your testcase:\n\n'
648 '@pytest.mark.disable_config_check'
649)
650
651
652# Should be invoked after _dynamic_config_defaults_storage is filled.
653@pytest.fixture
654def _check_config_marks(
655 request,
656 _dynamic_config_defaults_storage,
657) -> typing.Callable[[], None]:
658 def check():
659 config_defaults = _dynamic_config_defaults_storage.snapshot
660 assert config_defaults is not None
661
662 if request.node.get_closest_marker('disable_config_check'):
663 return
664
665 unknown_configs = [
666 key for marker in request.node.iter_markers('config') for key in marker.kwargs if key not in config_defaults
667 ]
668
669 if unknown_configs:
670 message = _CHECK_CONFIG_ERROR.format(
671 ', '.join(f'{key}=...' for key in sorted(unknown_configs)),
672 )
673 raise UnknownConfigError(message)
674
675 return check