userver: /data/code/service_template/third_party/userver/testsuite/pytest_plugins/pytest_userver/plugins/dynamic_config.py Source File
Loading...
Searching...
No Matches
dynamic_config.py
1"""
2Supply dynamic configs for the service in testsuite.
3"""
4
5import contextlib
6import copy
7import dataclasses
8import datetime
9import json
10import pathlib
11import typing
12
13import pytest
14
15from pytest_userver.plugins import caches
16
17USERVER_CONFIG_HOOKS = [
18 'userver_config_dynconf_cache',
19 'userver_config_dynconf_fallback',
20 'userver_config_dynconf_url',
21]
22
23_CONFIG_CACHES = ['dynamic-config-client-updater']
24
25
26class BaseError(Exception):
27 """Base class for exceptions from this module"""
28
29
31 """Config parameter was not found and no default was provided"""
32
33
34ConfigDict = typing.Dict[str, typing.Any]
35
36
38 pass
39
40
41class Missing:
42 pass
43
44
45@dataclasses.dataclass
47 timestamp: str
48 dirty_keys: typing.Set[str]
49 state: ConfigDict
50
51 @classmethod
52 def new(
53 cls,
54 *,
55 previous: typing.Optional['_ChangelogEntry'],
56 timestamp: str,
57 ):
58 if previous:
59 state = previous.state.copy()
60 else:
61 state = {}
62 return cls(timestamp=timestamp, dirty_keys=set(), state=state)
63
64 @property
65 def has_changes(self) -> bool:
66 return bool(self.dirty_keysdirty_keys)
67
68 def update(self, values: ConfigDict):
69 self.state.update(values)
70 self.dirty_keysdirty_keys.update(values.keys())
71
72
73@dataclasses.dataclass(frozen=True)
74class Updates:
75 timestamp: str
76 values: ConfigDict
77 removed: typing.List[str]
78
79
81 entry: _ChangelogEntry
82 entries: typing.List[_ChangelogEntry]
83
84 def __init__(self):
85 self.timestamp = datetime.datetime.utcfromtimestamp(0)
86 self.last_entry = _ChangelogEntry.new(
87 timestamp=self.service_timestamp(), previous=None,
88 )
89 self.entriesentries = [self.last_entry]
90
91 def service_timestamp(self) -> str:
92 return self.timestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
93
94 def next_timestamp(self) -> str:
95 self.timestamp += datetime.timedelta(seconds=1)
96 return self.service_timestamp()
97
98 def tick(self):
99 self.last_entry = _ChangelogEntry.new(
100 timestamp=self.next_timestamp(), previous=self.last_entry,
101 )
102 self.entriesentries.append(self.last_entry)
103
104 def get_updated_since(
105 self,
106 values: ConfigDict,
107 updated_since: str,
108 ids: typing.Optional[typing.List[str]] = None,
109 ) -> Updates:
110 entry = self.last_entry
111 values, removed = self._get_updated_since(values, updated_since)
112 self.tick()
113 if ids:
114 values = {name: values[name] for name in ids if name in values}
115 removed = [name for name in removed if name in ids]
116 return Updates(
117 timestamp=entry.timestamp, values=values, removed=removed,
118 )
119
120 def _get_updated_since(
121 self, values: ConfigDict, updated_since: str,
122 ) -> typing.Tuple[ConfigDict, typing.List[str]]:
123 if not updated_since:
124 return values, []
125 dirty_keys = set()
126 last_known_state = {}
127 for entry in reversed(self.entriesentries):
128 if entry.timestamp > updated_since:
129 dirty_keys.update(entry.dirty_keys)
130 else:
131 last_known_state = entry.state
132 break
133 # We don't want to send them again
134 result = {}
135 removed = []
136 for key in dirty_keys:
137 value = values.get(key, RemoveKey)
138 if last_known_state.get(key, Missing) != value:
139 if value is RemoveKey:
140 removed.append(key)
141 else:
142 result[key] = value
143 return result, removed
144
145 def add_entries(self, values: ConfigDict):
146 self.last_entry.update(values)
147
148 @contextlib.contextmanager
149 def rollback(self, defaults: ConfigDict):
150 index = len(self.entriesentries) - 1
151 try:
152 yield
153 finally:
154 self._do_rollback(index, defaults)
155
156 def _do_rollback(self, first_index: int, defaults: ConfigDict):
157 dirty_keys = set()
158 for index in range(first_index, len(self.entriesentries)):
159 dirty_keys.update(self.entriesentries[index].dirty_keys)
160 if self.last_entry.has_changes:
161 self.tick()
162
163 changes = {key: defaults.get(key, RemoveKey) for key in dirty_keys}
164 self.add_entries(changes)
165 self.entriesentries = self.entriesentries[-2:]
166
167
169 """Simple dynamic config backend."""
170
171 def __init__(
172 self,
173 *,
174 initial_values: ConfigDict,
175 config_cache_components: typing.Iterable[str],
176 cache_invalidation_state: caches.InvalidationState,
177 changelog: _Changelog,
178 ):
179 self._values = initial_values.copy()
180 self._cache_invalidation_state = cache_invalidation_state
181 self._config_cache_components = config_cache_components
182 self._changelog = changelog
183
184 def set_values(self, values: ConfigDict):
185 self.set_values_unsafe(copy.deepcopy(values))
186
187 def set_values_unsafe(self, values: ConfigDict):
188 self._values.update(values)
189 self._changelog.add_entries(values)
190 self._sync_with_service()
191
192 def set(self, **values):
193 self.set_values(values)
194
195 def get_values_unsafe(self) -> ConfigDict:
196 return self._values
197
198 def get(self, key: str, default: typing.Any = None) -> typing.Any:
199 if key not in self._values:
200 if default is not None:
201 return default
202 raise DynamicConfigNotFoundError(f'Config {key!r} is not found')
203 return copy.deepcopy(self._values[key])
204
205 def remove_values(self, keys):
206 extra_keys = set(keys).difference(self._values.keys())
207 if extra_keys:
209 f'Attempting to remove nonexistent configs: {extra_keys}',
210 )
211 for key in keys:
212 self._values.pop(key)
213 self._changelog.add_entries({key: RemoveKey for key in keys})
214 self._sync_with_service()
215
216 def remove(self, key):
217 return self.remove_values([key])
218
219 @contextlib.contextmanager
220 def modify(self, key: str) -> typing.Any:
221 value = self.get(key)
222 yield value
223 self.set_values({key: value})
224
225 @contextlib.contextmanager
226 def modify_many(
227 self, *keys: typing.Tuple[str, ...],
228 ) -> typing.Tuple[typing.Any, ...]:
229 values = tuple(self.get(key) for key in keys)
230 yield values
231 self.set_values(dict(zip(keys, values)))
232
233 def _sync_with_service(self):
234 self._cache_invalidation_state.invalidate(
236 )
237
238
239def pytest_userver_caches_setup(userver_cache_config):
240 userver_cache_config.register_incremental_cache(*_CONFIG_CACHES)
241
242
243@pytest.fixture(name='dynamic_config')
245 request,
246 search_path,
247 object_substitute,
248 cache_invalidation_state,
249 _config_service_defaults_updated,
250 dynamic_config_changelog,
251 _dynconfig_load_json_cached,
252 dynconf_cache_names,
253) -> DynamicConfig:
254 """
255 Fixture that allows to control dynamic config values used by the service.
256
257 After change to the config, be sure to call:
258 @code
259 await service_client.update_server_state()
260 @endcode
261
262 HTTP client requests call it automatically before each request.
263
264 @ingroup userver_testsuite_fixtures
265 """
266 config = DynamicConfig(
267 initial_values=_config_service_defaults_updated.snapshot,
268 config_cache_components=dynconf_cache_names,
269 cache_invalidation_state=cache_invalidation_state,
270 changelog=dynamic_config_changelog,
271 )
272 updates = {}
273 with dynamic_config_changelog.rollback(
274 _config_service_defaults_updated.snapshot,
275 ):
276 for path in reversed(list(search_path('config.json'))):
277 values = _dynconfig_load_json_cached(path)
278 updates.update(values)
279 for marker in request.node.iter_markers('config'):
280 marker_json = object_substitute(marker.kwargs)
281 updates.update(marker_json)
282 config.set_values_unsafe(updates)
283 yield config
284
285
286@pytest.fixture(scope='session', name='dynconf_cache_names')
287def _dynconf_cache_names():
288 return tuple(_CONFIG_CACHES)
289
290
291@pytest.fixture(scope='session', name='_dynconfig_json_cache')
292def _dynconfig_json_cache():
293 return {}
294
295
296@pytest.fixture(name='_dynconfig_load_json_cached')
297def _dynconfig_load_json_cached(json_loads, _dynconfig_json_cache):
298 def load(path: pathlib.Path):
299 if path not in _dynconfig_json_cache:
300 _dynconfig_json_cache[path] = json_loads(path.read_text())
301 return _dynconfig_json_cache[path]
302
303 return load
304
305
306@pytest.fixture
307def taxi_config(dynamic_config) -> DynamicConfig:
308 """
309 Deprecated, use `dynamic_config` instead.
310 """
311 return dynamic_config
312
313
314@pytest.fixture(name='dynamic_config_fallback_patch', scope='session')
316 """
317 Override this fixture to replace some dynamic config values specifically
318 for testsuite tests:
319
320 @code
321 @pytest.fixture(scope='session')
322 def dynamic_config_fallback_patch():
323 return {"MY_CONFIG_NAME": 42}
324 @endcode
325
326 @ingroup userver_testsuite_fixtures
327 """
328 return {}
329
330
331@pytest.fixture(name='config_service_defaults', scope='session')
333 config_fallback_path, dynamic_config_fallback_patch,
334) -> ConfigDict:
335 """
336 Fixture that returns default values for dynamic config. You may override
337 it in your local conftest.py or fixture:
338
339 @code
340 @pytest.fixture(scope='session')
341 def config_service_defaults():
342 with open('defaults.json') as fp:
343 return json.load(fp)
344 @endcode
345
346 @ingroup userver_testsuite_fixtures
347 """
348 if not config_fallback_path:
349 return dynamic_config_fallback_patch
350
351 if pathlib.Path(config_fallback_path).exists():
352 with open(config_fallback_path, 'r', encoding='utf-8') as file:
353 fallback = json.load(file)
354 fallback.update(dynamic_config_fallback_patch)
355 return fallback
356
357 raise RuntimeError(
358 'Either provide the path to dynamic config defaults file using '
359 '--config-fallback pytest option, or override '
360 f'{_config_service_defaults.__name__} fixture to provide custom '
361 'dynamic config loading behavior.',
362 )
363
364
365@dataclasses.dataclass(frozen=False)
367 snapshot: ConfigDict
368
369 async def update(self, client, dynamic_config) -> None:
370 if not self.snapshotsnapshot:
371 values = await client.get_dynamic_config_defaults()
372 # There may already be some config overrides from the current test.
373 values.update(dynamic_config.get_values_unsafe())
374 self.snapshotsnapshot = values
375 dynamic_config.set_values(self.snapshotsnapshot)
376
377
378# If there is no config_fallback_path, then we want to ask the service
379# for the dynamic config defaults after it's launched. It's enough to update
380# defaults once per service launch.
381@pytest.fixture(scope='package')
382def _config_service_defaults_updated(config_service_defaults):
383 return _ConfigDefaults(snapshot=config_service_defaults)
384
385
386@pytest.fixture(scope='session')
387def userver_config_dynconf_cache(service_tmpdir):
388 def patch_config(config, _config_vars) -> None:
389 components = config['components_manager']['components']
390 dynamic_config_component = components.get('dynamic-config', None) or {}
391 if dynamic_config_component.get('fs-cache-path', '') == '':
392 return
393
394 cache_path = service_tmpdir / 'configs' / 'config_cache.json'
395
396 if cache_path.is_file():
397 # To avoid leaking dynamic config values between test sessions
398 cache_path.unlink()
399
400 dynamic_config_component['fs-cache-path'] = str(cache_path)
401
402 return patch_config
403
404
405@pytest.fixture(scope='session')
406def userver_config_dynconf_fallback(config_service_defaults):
407 """
408 Returns a function that adjusts the static configuration file for
409 the testsuite.
410 Sets `dynamic-config.defaults-path` according to `config_service_defaults`.
411 Updates `dynamic-config.defaults` with `config_service_defaults`.
412
413 @ingroup userver_testsuite_fixtures
414 """
415
416 def extract_defaults_dict(component_config, config_vars) -> dict:
417 defaults_field = component_config.get('defaults', None) or {}
418 if isinstance(defaults_field, dict):
419 return defaults_field
420 elif isinstance(defaults_field, str):
421 if defaults_field.startswith('$'):
422 return config_vars.get(defaults_field[1:], {})
423 assert False, (
424 f'Unexpected static config option '
425 f'`dynamic-config.defaults`: {defaults_field!r}'
426 )
427
428 def _patch_config(config_yaml, config_vars):
429 components = config_yaml['components_manager']['components']
430 if components.get('dynamic-config', None) is None:
431 components['dynamic-config'] = {}
432 dynconf_component = components['dynamic-config']
433
434 dynconf_component.pop('defaults-path', None)
435 defaults = dict(
436 extract_defaults_dict(dynconf_component, config_vars),
437 **config_service_defaults,
438 )
439 dynconf_component['defaults'] = defaults
440
441 return _patch_config
442
443
444@pytest.fixture(scope='session')
445def userver_config_dynconf_url(mockserver_info):
446 """
447 Returns a function that adjusts the static configuration file for
448 the testsuite.
449 Sets the `dynamic-config-client.config-url` to the value of mockserver
450 configs-service, so that the
451 @ref pytest_userver.plugins.dynamic_config.mock_configs_service
452 "mock_configs_service" fixture could work.
453
454 @ingroup userver_testsuite_fixtures
455 """
456
457 def _patch_config(config, _config_vars) -> None:
458 components = config['components_manager']['components']
459 client = components.get('dynamic-config-client', None)
460 if client:
461 client['config-url'] = mockserver_info.url('configs-service')
462 client['append-path-to-url'] = True
463
464 return _patch_config
465
466
467@pytest.fixture(name='dynamic_config_changelog', scope='session')
468def _dynamic_config_changelog():
469 return _Changelog()
470
471
472@pytest.fixture
474 mockserver,
475 dynamic_config: DynamicConfig,
476 dynamic_config_changelog: _Changelog,
477) -> None:
478 """
479 Adds a mockserver handler that forwards dynamic_config to service's
480 `dynamic-config-client` component.
481
482 @ingroup userver_testsuite_fixtures
483 """
484
485 @mockserver.json_handler('/configs-service/configs/values')
486 def _mock_configs(request):
487 updates = dynamic_config_changelog.get_updated_since(
488 dynamic_config.get_values_unsafe(),
489 request.json.get('updated_since', ''),
490 request.json.get('ids'),
491 )
492 response = {'configs': updates.values, 'updated_at': updates.timestamp}
493 if updates.removed:
494 response['removed'] = updates.removed
495 return response
496
497 @mockserver.json_handler('/configs-service/configs/status')
498 def _mock_configs_status(_request):
499 return {'updated_at': dynamic_config_changelog.timestamp}