userver: /data/code/userver/testsuite/pytest_plugins/pytest_userver/plugins/dynamic_config.py Source File
Loading...
Searching...
No Matches
dynamic_config.py
1"""
2Supply dynamic configs for the service in testsuite.
3"""
4
5# pylint: disable=redefined-outer-name
6import dataclasses
7import json
8import pathlib
9from typing import Callable
10from typing import Iterable
11from typing import Optional
12
13import pytest
14
15from pytest_userver import dynconf
16
17USERVER_CONFIG_HOOKS = [
18 'userver_config_dynconf_cache',
19 'userver_config_dynconf_fallback',
20 'userver_config_dynconf_url',
21]
22USERVER_CACHE_CONTROL_HOOKS = {
23 'dynamic-config-client-updater': '_userver_dynconfig_cache_control',
24}
25
26
27@pytest.fixture
29 request,
30 search_path,
31 object_substitute,
32 cache_invalidation_state,
33 _dynamic_config_defaults_storage,
34 config_service_defaults,
35 dynamic_config_changelog,
36 _dynconf_load_json_cached,
37 dynconf_cache_names,
38) -> dynconf.DynamicConfig:
39 """
40 Fixture that allows to control dynamic config values used by the service.
41
42 Example:
43
44 @snippet core/functional_tests/basic_chaos/tests-nonchaos/handlers/test_log_request_headers.py dynamic_config usage
45
46 Example with @ref kill_switches "kill switches":
47
48 @snippet core/functional_tests/dynamic_configs/tests/test_examples.py dynamic_config usage with kill switches
49
50 HTTP and gRPC client requests call `update_server_state` automatically before each request.
51
52 For main dynamic config documentation:
53
54 @see @ref dynamic_config_testsuite
55
56 See also other related fixtures:
57 * @ref pytest_userver.plugins.dynamic_config.dynamic_config "config_service_defaults"
58 * @ref pytest_userver.plugins.dynamic_config.dynamic_config "dynamic_config_fallback_patch"
59 * @ref pytest_userver.plugins.dynamic_config.dynamic_config "mock_configs_service"
60
61 @ingroup userver_testsuite_fixtures
62 """
63 config = dynconf.DynamicConfig(
64 initial_values=config_service_defaults,
65 defaults=_dynamic_config_defaults_storage.snapshot,
66 config_cache_components=dynconf_cache_names,
67 cache_invalidation_state=cache_invalidation_state,
68 changelog=dynamic_config_changelog,
69 )
70
71 with dynamic_config_changelog.rollback(config_service_defaults):
72 updates = {}
73 for path in reversed(list(search_path('config.json'))):
74 values = _dynconf_load_json_cached(path)
75 updates.update(values)
76 for marker in request.node.iter_markers('config'):
77 value_update_kwargs = {
78 key: value for key, value in marker.kwargs.items() if value is not dynconf.USE_STATIC_DEFAULT
79 }
80 value_updates_json = object_substitute(value_update_kwargs)
81 updates.update(value_updates_json)
82 config.set_values_unsafe(updates)
83
84 kill_switches_disabled = []
85 for marker in request.node.iter_markers('config'):
86 kill_switches_disabled.extend(
87 key for key, value in marker.kwargs.items() if value is dynconf.USE_STATIC_DEFAULT
88 )
89 config.switch_to_static_default(*kill_switches_disabled)
90
91 yield config
92
93
94# @cond
95
96
97def pytest_configure(config):
98 config.addinivalue_line(
99 'markers',
100 'config: per-test dynamic config values',
101 )
102 config.addinivalue_line(
103 'markers',
104 'disable_config_check: disable config mark keys check',
105 )
106
107
108# @endcond
109
110
111@pytest.fixture(scope='session')
112def dynconf_cache_names() -> Iterable[str]:
113 return tuple(USERVER_CACHE_CONTROL_HOOKS.keys())
114
115
116@pytest.fixture(scope='session')
117def _dynconf_json_cache():
118 return {}
119
120
121@pytest.fixture
122def _dynconf_load_json_cached(json_loads, _dynconf_json_cache):
123 def load(path: pathlib.Path):
124 if path not in _dynconf_json_cache:
125 _dynconf_json_cache[path] = json_loads(path.read_text())
126 return _dynconf_json_cache[path]
127
128 return load
129
130
131@pytest.fixture
132def taxi_config(dynamic_config) -> dynconf.DynamicConfig:
133 """
134 Deprecated, use `dynamic_config` instead.
135 """
136 return dynamic_config
137
138
139@pytest.fixture(scope='session')
140def dynamic_config_fallback_patch() -> dynconf.ConfigValuesDict:
141 """
142 Override this fixture to replace some dynamic config values specifically
143 for testsuite tests:
144
145 @code
146 @pytest.fixture(scope='session')
147 def dynamic_config_fallback_patch():
148 return {"MY_CONFIG_NAME": 42}
149 @endcode
150
151 @ingroup userver_testsuite_fixtures
152 """
153 return {}
154
155
156@pytest.fixture(scope='session')
158 config_fallback_path,
159 dynamic_config_fallback_patch,
160) -> dynconf.ConfigValuesDict:
161 """
162 Fixture that returns default values for dynamic config. You may override
163 it in your local conftest.py or fixture:
164
165 @code
166 @pytest.fixture(scope='session')
167 def config_service_defaults():
168 with open('defaults.json') as fp:
169 return json.load(fp)
170 @endcode
171
172 @ingroup userver_testsuite_fixtures
173 """
174 if not config_fallback_path:
175 return dynamic_config_fallback_patch
176
177 if pathlib.Path(config_fallback_path).exists():
178 with open(config_fallback_path, 'r', encoding='utf-8') as file:
179 fallback = json.load(file)
180 fallback.update(dynamic_config_fallback_patch)
181 return fallback
182
183 raise RuntimeError(
184 'Invalid path specified in config_fallback_path fixture. '
185 'Probably invalid path was passed in --config-fallback pytest option.',
186 )
187
188
189@dataclasses.dataclass(frozen=False)
191 snapshot: Optional[dynconf.ConfigValuesDict]
192
193 async def update(self, client, dynamic_config) -> None:
194 if self.snapshotsnapshot is None:
195 defaults = await client.get_dynamic_config_defaults()
196 if not isinstance(defaults, dict):
198 self.snapshotsnapshot = defaults
199 # pylint:disable=protected-access
200 dynamic_config._defaults = defaults
201
202
203# config_service_defaults fetches the dynamic config overrides, e.g. specified
204# in the json file, then userver_config_dynconf_fallback forwards them
205# to the service so that it has the correct dynamic config defaults.
206#
207# Independently of that, it is useful to have values for all configs, even
208# unspecified in tests, on the testsuite side. For that, we ask the service
209# for the dynamic config defaults after it's launched. It's enough to update
210# defaults once per service launch.
211@pytest.fixture(scope='session')
212def _dynamic_config_defaults_storage() -> _ConfigDefaults:
213 return _ConfigDefaults(snapshot=None)
214
215
216@pytest.fixture(scope='session')
218 """
219 Returns a function that adjusts the static configuration file for
220 the testsuite.
221 Sets `dynamic-config.fs-cache-path` to a file that is reset after the tests
222 to avoid leaking dynamic config values between test sessions.
223
224 @ingroup userver_testsuite_fixtures
225 """
226
227 def patch_config(config, _config_vars) -> None:
228 components = config['components_manager']['components']
229 dynamic_config_component = components.get('dynamic-config', None) or {}
230 if dynamic_config_component.get('fs-cache-path', '') == '':
231 return
232
233 cache_path = service_tmpdir / 'configs' / 'config_cache.json'
234
235 if cache_path.is_file():
236 # To avoid leaking dynamic config values between test sessions
237 cache_path.unlink()
238
239 dynamic_config_component['fs-cache-path'] = str(cache_path)
240
241 return patch_config
242
243
244@pytest.fixture(scope='session')
245def userver_config_dynconf_fallback(config_service_defaults):
246 """
247 Returns a function that adjusts the static configuration file for
248 the testsuite.
249 Removes `dynamic-config.defaults-path`.
250 Updates `dynamic-config.defaults` with `config_service_defaults`.
251
252 @ingroup userver_testsuite_fixtures
253 """
254
255 def extract_defaults_dict(component_config, config_vars) -> dict:
256 defaults_field = component_config.get('defaults', None) or {}
257 if isinstance(defaults_field, dict):
258 return defaults_field
259 elif isinstance(defaults_field, str):
260 if defaults_field.startswith('$'):
261 return config_vars.get(defaults_field[1:], {})
262 assert False, f'Unexpected static config option `dynamic-config.defaults`: {defaults_field!r}'
263
264 def _patch_config(config_yaml, config_vars):
265 components = config_yaml['components_manager']['components']
266 if components.get('dynamic-config', None) is None:
267 components['dynamic-config'] = {}
268 dynconf_component = components['dynamic-config']
269
270 dynconf_component.pop('defaults-path', None)
271 defaults = dict(
272 extract_defaults_dict(dynconf_component, config_vars),
273 **config_service_defaults,
274 )
275 dynconf_component['defaults'] = defaults
276
277 return _patch_config
278
279
280@pytest.fixture(scope='session')
281def userver_config_dynconf_url(mockserver_info):
282 """
283 Returns a function that adjusts the static configuration file for
284 the testsuite.
285 Sets the `dynamic-config-client.config-url` to the value of mockserver
286 configs-service, so that the
287 @ref pytest_userver.plugins.dynamic_config.mock_configs_service
288 "mock_configs_service" fixture could work.
289
290 @ingroup userver_testsuite_fixtures
291 """
292
293 def _patch_config(config, _config_vars) -> None:
294 components = config['components_manager']['components']
295 client = components.get('dynamic-config-client', None)
296 if client:
297 client['config-url'] = mockserver_info.url('configs-service')
298 client['append-path-to-url'] = True
299
300 return _patch_config
301
302
303# @cond
304
305
306# TODO publish dynconf._Changelog and document how to use it in custom config service
307# mocks.
308@pytest.fixture(scope='session')
309def dynamic_config_changelog() -> dynconf._Changelog:
310 return dynconf._Changelog()
311
312
313# @endcond
314
315
316@pytest.fixture
318 mockserver,
319 dynamic_config: dynconf.DynamicConfig,
320 dynamic_config_changelog: dynconf._Changelog,
321) -> None:
322 """
323 Adds a mockserver handler that forwards dynamic_config to service's
324 `dynamic-config-client` component.
325
326 @ingroup userver_testsuite_fixtures
327 """
328
329 @mockserver.json_handler('/configs-service/configs/values')
330 def _mock_configs(request):
331 updates = dynamic_config_changelog.get_updated_since(
332 dynconf._create_config_dict(
333 dynamic_config.get_values_unsafe(),
334 dynamic_config.get_kill_switches_disabled_unsafe(),
335 ),
336 request.json.get('updated_since', ''),
337 request.json.get('ids'),
338 )
339 response = {'configs': updates.values, 'updated_at': updates.timestamp}
340 if updates.removed:
341 response['removed'] = updates.removed
342 if updates.kill_switches_disabled:
343 response['kill_switches_disabled'] = updates.kill_switches_disabled
344 return response
345
346 @mockserver.json_handler('/configs-service/configs/status')
347 def _mock_configs_status(_request):
348 return {
349 'updated_at': dynamic_config_changelog.timestamp.strftime(
350 '%Y-%m-%dT%H:%M:%SZ',
351 ),
352 }
353
354
355@pytest.fixture
356def _userver_dynconfig_cache_control(dynamic_config_changelog: dynconf._Changelog):
357 def cache_control(updater, timestamp):
358 entry = dynamic_config_changelog.commit()
359 if entry.timestamp == timestamp:
360 updater.exclude()
361 else:
362 updater.incremental()
363 return entry.timestamp
364
365 return cache_control
366
367
368_CHECK_CONFIG_ERROR = (
369 'Your are trying to override config value using '
370 '@pytest.mark.config({}) '
371 'that does not seem to be used by your service.\n\n'
372 'In case you really need to disable this check please add the '
373 'following mark to your testcase:\n\n'
374 '@pytest.mark.disable_config_check'
375)
376
377
378# Should be invoked after _dynamic_config_defaults_storage is filled.
379@pytest.fixture
380def _check_config_marks(
381 request,
382 _dynamic_config_defaults_storage,
383) -> Callable[[], None]:
384 def check():
385 config_defaults = _dynamic_config_defaults_storage.snapshot
386 assert config_defaults is not None
387
388 if request.node.get_closest_marker('disable_config_check'):
389 return
390
391 unknown_configs = [
392 key for marker in request.node.iter_markers('config') for key in marker.kwargs if key not in config_defaults
393 ]
394
395 if unknown_configs:
396 message = _CHECK_CONFIG_ERROR.format(
397 ', '.join(f'{key}=...' for key in sorted(unknown_configs)),
398 )
399 raise dynconf.UnknownConfigError(message)
400
401 return check