userver: /data/code/userver/testsuite/pytest_plugins/pytest_userver/plugins/dynamic_config.py Source File
Loading...
Searching...
No Matches
dynamic_config.py
1"""
2Supply dynamic configs for the service in testsuite.
3"""
4
5# pylint: disable=redefined-outer-name
6import contextlib
7import copy
8import dataclasses
9import datetime
10import json
11import pathlib
12import typing
13
14import pytest
15
16from pytest_userver.plugins import caches
17
18USERVER_CONFIG_HOOKS = [
19 'userver_config_dynconf_cache',
20 'userver_config_dynconf_fallback',
21 'userver_config_dynconf_url',
22]
23USERVER_CACHE_CONTROL_HOOKS = {
24 'dynamic-config-client-updater': '_userver_dynconfig_cache_control',
25}
26
27_CONFIG_CACHES = tuple(USERVER_CACHE_CONTROL_HOOKS.keys())
28
29
30class BaseError(Exception):
31 """Base class for exceptions from this module"""
32
33
35 """Config parameter was not found and no default was provided"""
36
37
39 """Dynamic config defaults action returned invalid response"""
40
41
42ConfigDict = typing.Dict[str, typing.Any]
43
44
46 pass
47
48
49class Missing:
50 pass
51
52
53@dataclasses.dataclass
55 timestamp: str
56 dirty_keys: typing.Set[str]
57 state: ConfigDict
58 prev_state: ConfigDict
59
60 @classmethod
61 def new(
62 cls,
63 *,
64 previous: typing.Optional['_ChangelogEntry'],
65 timestamp: str,
66 ):
67 if previous:
68 prev_state = previous.state
69 else:
70 prev_state = {}
71 return cls(
72 timestamp=timestamp,
73 dirty_keys=set(),
74 state=prev_state.copy(),
75 prev_state=prev_state,
76 )
77
78 @property
79 def has_changes(self) -> bool:
80 return bool(self.dirty_keysdirty_keys)
81
82 def update(self, values: ConfigDict):
83 for key, value in values.items():
84 if value == self.prev_state.get(key, Missing):
85 self.dirty_keysdirty_keys.discard(key)
86 else:
87 self.dirty_keysdirty_keys.add(key)
88 self.state.update(values)
89
90
91@dataclasses.dataclass(frozen=True)
92class Updates:
93 timestamp: str
94 values: ConfigDict
95 removed: typing.List[str]
96
97 def is_empty(self) -> bool:
98 return not self.values and not self.removed
99
100
102 timestamp: datetime.datetime
103 commited_entries: typing.List[_ChangelogEntry]
104 staged_entry: _ChangelogEntry
105
106 def __init__(self):
107 self.timestamptimestamp = datetime.datetime.fromtimestamp(
108 0, datetime.timezone.utc,
109 )
111 self.staged_entrystaged_entry = _ChangelogEntry.new(
112 timestamp=self.service_timestamp(), previous=None,
113 )
114
115 def service_timestamp(self) -> str:
116 return self.timestamptimestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
117
118 def next_timestamp(self) -> str:
119 self.timestamptimestamp += datetime.timedelta(seconds=1)
120 return self.service_timestamp()
121
122 def commit(self) -> _ChangelogEntry:
123 """Commit staged changed if any and return last commited entry."""
124 entry = self.staged_entrystaged_entry
125 if entry.has_changes or not self.commited_entriescommited_entries:
126 self.staged_entrystaged_entry = _ChangelogEntry.new(
127 timestamp=self.next_timestamp(), previous=entry,
128 )
129 self.commited_entriescommited_entries.append(entry)
131
132 def get_updated_since(
133 self,
134 values: ConfigDict,
135 updated_since: str,
136 ids: typing.Optional[typing.List[str]] = None,
137 ) -> Updates:
138 entry = self.commit()
139 values, removed = self._get_updated_since(values, updated_since)
140 if ids:
141 values = {name: values[name] for name in ids if name in values}
142 removed = [name for name in removed if name in ids]
143 return Updates(
144 timestamp=entry.timestamp, values=values, removed=removed,
145 )
146
147 def _get_updated_since(
148 self, values: ConfigDict, updated_since: str,
149 ) -> typing.Tuple[ConfigDict, typing.List[str]]:
150 if not updated_since:
151 return values, []
152 dirty_keys = set()
153 last_known_state = {}
154 for entry in reversed(self.commited_entriescommited_entries):
155 if entry.timestamp > updated_since:
156 dirty_keys.update(entry.dirty_keys)
157 else:
158 if entry.timestamp == updated_since:
159 last_known_state = entry.state
160 break
161 # We don't want to send them again
162 result = {}
163 removed = []
164 for key in dirty_keys:
165 value = values.get(key, RemoveKey)
166 if last_known_state.get(key, Missing) != value:
167 if value is RemoveKey:
168 removed.append(key)
169 else:
170 result[key] = value
171 return result, removed
172
173 def add_entries(self, values: ConfigDict):
174 self.staged_entrystaged_entry.update(values)
175
176 @contextlib.contextmanager
177 def rollback(self, defaults: ConfigDict):
178 try:
179 yield
180 finally:
181 self._do_rollback(defaults)
182
183 def _do_rollback(self, defaults: ConfigDict):
185 return
186
187 maybe_dirty = set()
188 for entry in self.commited_entriescommited_entries:
189 maybe_dirty.update(entry.dirty_keys)
190
192 last_state = last.state
193 dirty_keys = set()
194 reverted = {}
195 for key in maybe_dirty:
196 original = defaults.get(key, RemoveKey)
197 if last_state[key] != original:
198 dirty_keys.add(key)
199 reverted[key] = original
200
201 entry = _ChangelogEntry(
202 timestamp=last.timestamp,
203 state=last.state,
204 dirty_keys=dirty_keys,
205 prev_state={},
206 )
209 timestamp=self.staged_entrystaged_entry.timestamp,
210 dirty_keys=dirty_keys.copy(),
211 state=reverted,
212 prev_state=entry.state,
213 )
214
215
217 """Simple dynamic config backend."""
218
219 def __init__(
220 self,
221 *,
222 initial_values: ConfigDict,
223 config_cache_components: typing.Iterable[str],
224 cache_invalidation_state: caches.InvalidationState,
225 changelog: _Changelog,
226 ):
227 self._values = initial_values.copy()
228 self._cache_invalidation_state = cache_invalidation_state
229 self._config_cache_components = config_cache_components
230 self._changelog = changelog
231
232 def set_values(self, values: ConfigDict):
233 self.set_values_unsafe(copy.deepcopy(values))
234
235 def set_values_unsafe(self, values: ConfigDict):
236 self._values.update(values)
237 self._changelog.add_entries(values)
238 self._sync_with_service()
239
240 def set(self, **values):
241 self.set_values(values)
242
243 def get_values_unsafe(self) -> ConfigDict:
244 return self._values
245
246 def get(self, key: str, default: typing.Any = None) -> typing.Any:
247 if key not in self._values:
248 if default is not None:
249 return default
250 raise DynamicConfigNotFoundError(f'Config {key!r} is not found')
251 return copy.deepcopy(self._values[key])
252
253 def remove_values(self, keys):
254 extra_keys = set(keys).difference(self._values.keys())
255 if extra_keys:
257 f'Attempting to remove nonexistent configs: {extra_keys}',
258 )
259 for key in keys:
260 self._values.pop(key)
261 self._changelog.add_entries({key: RemoveKey for key in keys})
262 self._sync_with_service()
263
264 def remove(self, key):
265 return self.remove_values([key])
266
267 @contextlib.contextmanager
268 def modify(self, key: str) -> typing.Any:
269 value = self.get(key)
270 yield value
271 self.set_values({key: value})
272
273 @contextlib.contextmanager
274 def modify_many(
275 self, *keys: typing.Tuple[str, ...],
276 ) -> typing.Tuple[typing.Any, ...]:
277 values = tuple(self.get(key) for key in keys)
278 yield values
279 self.set_values(dict(zip(keys, values)))
280
281 def _sync_with_service(self):
282 self._cache_invalidation_state.invalidate(
284 )
285
286
287@pytest.fixture
289 request,
290 search_path,
291 object_substitute,
292 cache_invalidation_state,
293 _config_service_defaults_updated,
294 dynamic_config_changelog,
295 _dynconfig_load_json_cached,
296 dynconf_cache_names,
297) -> DynamicConfig:
298 """
299 Fixture that allows to control dynamic config values used by the service.
300
301 After change to the config, be sure to call:
302 @code
303 await service_client.update_server_state()
304 @endcode
305
306 HTTP client requests call it automatically before each request.
307
308 @ingroup userver_testsuite_fixtures
309 """
310 config = DynamicConfig(
311 initial_values=_config_service_defaults_updated.snapshot,
312 config_cache_components=dynconf_cache_names,
313 cache_invalidation_state=cache_invalidation_state,
314 changelog=dynamic_config_changelog,
315 )
316 updates = {}
317 with dynamic_config_changelog.rollback(
318 _config_service_defaults_updated.snapshot,
319 ):
320 for path in reversed(list(search_path('config.json'))):
321 values = _dynconfig_load_json_cached(path)
322 updates.update(values)
323 for marker in request.node.iter_markers('config'):
324 marker_json = object_substitute(marker.kwargs)
325 updates.update(marker_json)
326 config.set_values_unsafe(updates)
327 yield config
328
329
330@pytest.fixture(scope='session')
331def dynconf_cache_names():
332 return tuple(_CONFIG_CACHES)
333
334
335@pytest.fixture(scope='session')
336def _dynconfig_json_cache():
337 return {}
338
339
340@pytest.fixture
341def _dynconfig_load_json_cached(json_loads, _dynconfig_json_cache):
342 def load(path: pathlib.Path):
343 if path not in _dynconfig_json_cache:
344 _dynconfig_json_cache[path] = json_loads(path.read_text())
345 return _dynconfig_json_cache[path]
346
347 return load
348
349
350@pytest.fixture
351def taxi_config(dynamic_config) -> DynamicConfig:
352 """
353 Deprecated, use `dynamic_config` instead.
354 """
355 return dynamic_config
356
357
358@pytest.fixture(scope='session')
360 """
361 Override this fixture to replace some dynamic config values specifically
362 for testsuite tests:
363
364 @code
365 @pytest.fixture(scope='session')
366 def dynamic_config_fallback_patch():
367 return {"MY_CONFIG_NAME": 42}
368 @endcode
369
370 @ingroup userver_testsuite_fixtures
371 """
372 return {}
373
374
375@pytest.fixture(scope='session')
377 config_fallback_path, dynamic_config_fallback_patch,
378) -> ConfigDict:
379 """
380 Fixture that returns default values for dynamic config. You may override
381 it in your local conftest.py or fixture:
382
383 @code
384 @pytest.fixture(scope='session')
385 def config_service_defaults():
386 with open('defaults.json') as fp:
387 return json.load(fp)
388 @endcode
389
390 @ingroup userver_testsuite_fixtures
391 """
392 if not config_fallback_path:
393 return dynamic_config_fallback_patch
394
395 if pathlib.Path(config_fallback_path).exists():
396 with open(config_fallback_path, 'r', encoding='utf-8') as file:
397 fallback = json.load(file)
398 fallback.update(dynamic_config_fallback_patch)
399 return fallback
400
401 raise RuntimeError(
402 'Either provide the path to dynamic config defaults file using '
403 '--config-fallback pytest option, or override '
404 f'{config_service_defaults.__name__} fixture to provide custom '
405 'dynamic config loading behavior.',
406 )
407
408
409@dataclasses.dataclass(frozen=False)
411 snapshot: ConfigDict
412
413 async def update(self, client, dynamic_config) -> None:
414 if not self.snapshotsnapshot:
415 values = await client.get_dynamic_config_defaults()
416 if not isinstance(values, dict):
418 # There may already be some config overrides from the current test.
419 values.update(dynamic_config.get_values_unsafe())
420 self.snapshotsnapshot = values
421 dynamic_config.set_values(self.snapshotsnapshot)
422
423
424# If there is no config_fallback_path, then we want to ask the service
425# for the dynamic config defaults after it's launched. It's enough to update
426# defaults once per service launch.
427@pytest.fixture(scope='package')
428def _config_service_defaults_updated(config_service_defaults):
429 return _ConfigDefaults(snapshot=config_service_defaults)
430
431
432@pytest.fixture(scope='session')
433def userver_config_dynconf_cache(service_tmpdir):
434 def patch_config(config, _config_vars) -> None:
435 components = config['components_manager']['components']
436 dynamic_config_component = components.get('dynamic-config', None) or {}
437 if dynamic_config_component.get('fs-cache-path', '') == '':
438 return
439
440 cache_path = service_tmpdir / 'configs' / 'config_cache.json'
441
442 if cache_path.is_file():
443 # To avoid leaking dynamic config values between test sessions
444 cache_path.unlink()
445
446 dynamic_config_component['fs-cache-path'] = str(cache_path)
447
448 return patch_config
449
450
451@pytest.fixture(scope='session')
452def userver_config_dynconf_fallback(config_service_defaults):
453 """
454 Returns a function that adjusts the static configuration file for
455 the testsuite.
456 Sets `dynamic-config.defaults-path` according to `config_service_defaults`.
457 Updates `dynamic-config.defaults` with `config_service_defaults`.
458
459 @ingroup userver_testsuite_fixtures
460 """
461
462 def extract_defaults_dict(component_config, config_vars) -> dict:
463 defaults_field = component_config.get('defaults', None) or {}
464 if isinstance(defaults_field, dict):
465 return defaults_field
466 elif isinstance(defaults_field, str):
467 if defaults_field.startswith('$'):
468 return config_vars.get(defaults_field[1:], {})
469 assert False, (
470 f'Unexpected static config option '
471 f'`dynamic-config.defaults`: {defaults_field!r}'
472 )
473
474 def _patch_config(config_yaml, config_vars):
475 components = config_yaml['components_manager']['components']
476 if components.get('dynamic-config', None) is None:
477 components['dynamic-config'] = {}
478 dynconf_component = components['dynamic-config']
479
480 dynconf_component.pop('defaults-path', None)
481 defaults = dict(
482 extract_defaults_dict(dynconf_component, config_vars),
483 **config_service_defaults,
484 )
485 dynconf_component['defaults'] = defaults
486
487 return _patch_config
488
489
490@pytest.fixture(scope='session')
491def userver_config_dynconf_url(mockserver_info):
492 """
493 Returns a function that adjusts the static configuration file for
494 the testsuite.
495 Sets the `dynamic-config-client.config-url` to the value of mockserver
496 configs-service, so that the
497 @ref pytest_userver.plugins.dynamic_config.mock_configs_service
498 "mock_configs_service" fixture could work.
499
500 @ingroup userver_testsuite_fixtures
501 """
502
503 def _patch_config(config, _config_vars) -> None:
504 components = config['components_manager']['components']
505 client = components.get('dynamic-config-client', None)
506 if client:
507 client['config-url'] = mockserver_info.url('configs-service')
508 client['append-path-to-url'] = True
509
510 return _patch_config
511
512
513# TODO publish _Changelog and document how to use it in custom config service
514# mocks.
515@pytest.fixture(scope='session')
516def dynamic_config_changelog() -> _Changelog:
517 return _Changelog()
518
519
520@pytest.fixture
522 mockserver,
523 dynamic_config: DynamicConfig,
524 dynamic_config_changelog: _Changelog,
525) -> None:
526 """
527 Adds a mockserver handler that forwards dynamic_config to service's
528 `dynamic-config-client` component.
529
530 @ingroup userver_testsuite_fixtures
531 """
532
533 @mockserver.json_handler('/configs-service/configs/values')
534 def _mock_configs(request):
535 updates = dynamic_config_changelog.get_updated_since(
536 dynamic_config.get_values_unsafe(),
537 request.json.get('updated_since', ''),
538 request.json.get('ids'),
539 )
540 response = {'configs': updates.values, 'updated_at': updates.timestamp}
541 if updates.removed:
542 response['removed'] = updates.removed
543 return response
544
545 @mockserver.json_handler('/configs-service/configs/status')
546 def _mock_configs_status(_request):
547 return {'updated_at': dynamic_config_changelog.timestamp}
548
549
550@pytest.fixture
551def _userver_dynconfig_cache_control(dynamic_config_changelog: _Changelog):
552 def cache_control(updater, timestamp):
553 entry = dynamic_config_changelog.commit()
554 if entry.timestamp == timestamp:
555 updater.exclude()
556 else:
557 updater.incremental()
558 return entry.timestamp
559
560 return cache_control