userver: /home/antonyzhilin/arcadia/taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/plugins/config.py Source File
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages Concepts
config.py
1"""
2Work with the configuration files of the service in testsuite.
3"""
4
5# pylint: disable=redefined-outer-name
6import copy
7import dataclasses
8import itertools
9import logging
10import os
11import pathlib
12import string
13import subprocess
14import types
15from typing import Any
16from typing import Callable
17from typing import List
18from typing import Mapping
19from typing import Optional
20from typing import Union
21
22import pytest
23import yaml
24
25# flake8: noqa E266
26
40USERVER_CONFIG_HOOKS = [
41 'userver_config_substitutions',
42 'userver_config_http_server',
43 'userver_config_http_client',
44 'userver_config_logging',
45 'userver_config_logging_otlp',
46 'userver_config_testsuite',
47 'userver_config_secdist',
48 'userver_config_testsuite_middleware',
49]
50
51ServiceConfigPatch = Callable[[dict, dict], None]
52
53
54# @cond
55
56
57logger = logging.getLogger(__name__)
58
59
60class _UserverConfigPlugin:
61 def __init__(self):
62 self._config_hooks = []
63
64 @property
65 def userver_config_hooks(self):
66 return self._config_hooks
67
68 def pytest_plugin_registered(self, plugin, manager):
69 if not isinstance(plugin, types.ModuleType):
70 return
71 uhooks = getattr(plugin, 'USERVER_CONFIG_HOOKS', None)
72 if uhooks is not None:
73 self._config_hooks.extend(uhooks)
74
75
76@dataclasses.dataclass(frozen=True)
77class _UserverConfig:
78 config_yaml: dict
79 config_vars: dict
80
81
82def pytest_configure(config):
83 config.pluginmanager.register(_UserverConfigPlugin(), 'userver_config')
84
85
86def pytest_addoption(parser) -> None:
87 group = parser.getgroup('userver-config')
88 group.addoption(
89 '--service-log-level',
90 type=str.lower,
91 choices=['trace', 'debug', 'info', 'warning', 'error', 'critical'],
92 )
93 group.addoption(
94 '--service-config',
95 type=pathlib.Path,
96 help='Path to service.yaml file.',
97 )
98 group.addoption(
99 '--service-config-vars',
100 type=pathlib.Path,
101 help='Path to config_vars.yaml file.',
102 )
103 group.addoption(
104 '--service-secdist',
105 type=pathlib.Path,
106 help='Path to secure_data.json file.',
107 )
108 group.addoption(
109 '--config-fallback',
110 type=pathlib.Path,
111 help='Path to dynamic config fallback file.',
112 )
113 group.addoption(
114 '--dump-config',
115 action='store_true',
116 help='Dump config from binary before running tests',
117 )
118
119
120# @endcond
121
122
123@pytest.fixture(scope='session')
124def service_config_path(pytestconfig, service_binary) -> pathlib.Path:
125 """
126 Returns the path to service.yaml file set by command line
127 `--service-config` option.
128
129 Override this fixture to change the way path to service.yaml is provided.
130
131 @ingroup userver_testsuite_fixtures
132 """
133 if pytestconfig.option.dump_config:
134 subprocess.run([
135 service_binary,
136 '--dump-config',
137 pytestconfig.option.service_config,
138 ])
139 return pytestconfig.option.service_config
140
141
142@pytest.fixture(scope='session')
143def db_dump_schema_path(service_binary, service_tmpdir) -> pathlib.Path:
144 """
145 Runs the service binary with `--dump-db-schema` argument, dumps the 0_db_schema.sql file with database schema and
146 returns path to it.
147
148 Override this fixture to change the way to dump the database schema.
149
150 @ingroup userver_testsuite_fixtures
151
152 """
153 path = service_tmpdir.joinpath('schemas')
154 os.mkdir(path)
155 subprocess.run([
156 service_binary,
157 '--dump-db-schema',
158 path / '0_db_schema.sql',
159 ])
160 return path
161
162
163@pytest.fixture(scope='session')
164def service_config_vars_path(pytestconfig) -> Optional[pathlib.Path]:
165 """
166 Returns the path to config_vars.yaml file set by command line
167 `--service-config-vars` option.
168
169 Override this fixture to change the way path to config_vars.yaml is
170 provided.
171
172 @ingroup userver_testsuite_fixtures
173 """
174 return pytestconfig.option.service_config_vars
175
176
177@pytest.fixture(scope='session')
178def service_secdist_path(pytestconfig) -> Optional[pathlib.Path]:
179 """
180 Returns the path to secure_data.json file set by command line
181 `--service-secdist` option.
182
183 Override this fixture to change the way path to secure_data.json is
184 provided.
185
186 @ingroup userver_testsuite_fixtures
187 """
188 return pytestconfig.option.service_secdist
189
190
191@pytest.fixture(scope='session')
192def config_fallback_path(pytestconfig) -> pathlib.Path:
193 """
194 Returns the path to dynamic config fallback file set by command line
195 `--config-fallback` option.
196
197 Override this fixture to change the way path to dynamic config fallback is
198 provided.
199
200 @ingroup userver_testsuite_fixtures
201 """
202 return pytestconfig.option.config_fallback
203
204
205@pytest.fixture(scope='session')
206def service_tmpdir(service_binary, tmp_path_factory):
207 """
208 Returns the path for temporary files. The path is the same for the whole
209 session and files are not removed (at least by this fixture) between
210 tests.
211
212 @ingroup userver_testsuite_fixtures
213 """
214 return tmp_path_factory.mktemp(
215 pathlib.Path(service_binary).name,
216 numbered=False,
217 )
218
219
220@pytest.fixture(scope='session')
222 service_tmpdir,
223 service_config,
224 service_config_yaml,
225 service_config_vars,
226) -> pathlib.Path:
227 """
228 Dumps the contents of the service_config_yaml and service_config_vars into a static config for
229 testsuite and returns the path to the config file.
230
231 @ingroup userver_testsuite_fixtures
232 """
233 dst_path = service_tmpdir / 'config.yaml'
234
235 service_config_yaml = dict(service_config_yaml)
236 if not service_config_vars:
237 service_config_yaml.pop('config_vars', None)
238 else:
239 config_vars_path = service_tmpdir / 'config_vars.yaml'
240 config_vars_text = yaml.dump(service_config_vars)
241 logger.debug(
242 'userver fixture "service_config_path_temp" writes the patched static config vars to "%s":\n%s',
243 config_vars_path,
244 config_vars_text,
245 )
246 config_vars_path.write_text(config_vars_text)
247 service_config_yaml['config_vars'] = str(config_vars_path)
248
249 logger.debug(
250 'userver fixture "service_config_path_temp" writes the patched static config to "%s" equivalent to:\n%s',
251 dst_path,
252 yaml.dump(service_config),
253 )
254 dst_path.write_text(yaml.dump(service_config_yaml))
255
256 return dst_path
257
258
259@pytest.fixture(scope='session')
260def service_config_yaml(_service_config_hooked) -> dict:
261 """
262 Returns the static config values after the USERVER_CONFIG_HOOKS were
263 applied (if any). Prefer using
264 pytest_userver.plugins.config.service_config
265
266 @ingroup userver_testsuite_fixtures
267 """
268 return _service_config_hooked.config_yaml
269
270
271@pytest.fixture(scope='session')
272def service_config_vars(_service_config_hooked) -> dict:
273 """
274 Returns the static config variables (config_vars.yaml) values after the
275 USERVER_CONFIG_HOOKS were applied (if any). Prefer using
276 pytest_userver.plugins.config.service_config
277
278 @ingroup userver_testsuite_fixtures
279 """
280 return _service_config_hooked.config_vars
281
282
283def _substitute_values(config, service_config_vars: dict, service_env) -> None:
284 if isinstance(config, dict):
285 for key, value in config.items():
286 if not isinstance(value, str):
287 _substitute_values(value, service_config_vars, service_env)
288 continue
289
290 if not value.startswith('$'):
291 continue
292
293 new_value = service_config_vars.get(value[1:])
294 if new_value is not None:
295 config[key] = new_value
296 continue
297
298 env = config.get(f'{key}#env')
299 if env:
300 if service_env:
301 new_value = service_env.get(env)
302 if not new_value:
303 new_value = os.environ.get(env)
304 if new_value:
305 config[key] = new_value
306 continue
307
308 fallback = config.get(f'{key}#fallback')
309 if fallback:
310 config[key] = fallback
311 continue
312
313 config[key] = None
314
315 if isinstance(config, list):
316 for i, value in enumerate(config):
317 if not isinstance(value, str):
318 _substitute_values(value, service_config_vars, service_env)
319 continue
320
321 if not value.startswith('$'):
322 continue
323
324 new_value = service_config_vars.get(value[1:])
325 if new_value is not None:
326 config[i] = new_value
327
328
329@pytest.fixture(scope='session')
330def substitute_config_vars(service_env) -> Callable[[Any, dict], Any]:
331 """
332 A function that takes `config_yaml`, `config_vars` and applies all
333 substitutions just like the service would.
334
335 Useful when patching the service config. It's a good idea to pass
336 a component's config instead of the whole `config_yaml` to avoid
337 unnecessary work.
338
339 @warning The returned YAML is a clone, mutating it will not modify
340 the actual config while in a config hook!
341
342 @ingroup userver_testsuite_fixtures
343 """
344
345 def substitute(config_yaml, config_vars, /):
346 if config_yaml is not None and not isinstance(config_yaml, dict) and not isinstance(config_yaml, list):
347 raise TypeError(
348 f'{substitute_config_vars.__name__} can only be meaningfully '
349 'called with dict and list nodes of config_yaml, while given: '
350 f'{config_yaml!r}. Pass a containing object instead.',
351 )
352
353 config = copy.deepcopy(config_yaml)
354 _substitute_values(config, config_vars, service_env)
355 return config
356
357 return substitute
358
359
360@pytest.fixture(scope='session')
362 service_config_yaml,
363 service_config_vars,
364 substitute_config_vars,
365) -> dict:
366 """
367 Returns the static config values after the USERVER_CONFIG_HOOKS were
368 applied (if any) and with all the '$', environment and fallback variables
369 substituted.
370
371 @ingroup userver_testsuite_fixtures
372 """
373 config = substitute_config_vars(service_config_yaml, service_config_vars)
374 config.pop('config_vars', None)
375 return config
376
377
378@pytest.fixture(scope='session')
379def _original_service_config(
380 service_config_path,
381 service_config_vars_path,
382) -> _UserverConfig:
383 config_vars: dict
384 config_yaml: dict
385
386 with open(service_config_path, mode='rt') as fp:
387 config_yaml = yaml.safe_load(fp)
388
389 if service_config_vars_path:
390 with open(service_config_vars_path, mode='rt') as fp:
391 config_vars = yaml.safe_load(fp)
392 else:
393 config_vars = {}
394
395 return _UserverConfig(config_yaml=config_yaml, config_vars=config_vars)
396
397
398@pytest.fixture(scope='session')
399def _service_config_hooked(
400 daemon_scoped_mark,
401 pytestconfig,
402 request,
403 _original_service_config,
404) -> _UserverConfig:
405 config_yaml = copy.deepcopy(_original_service_config.config_yaml)
406 config_vars = copy.deepcopy(_original_service_config.config_vars)
407
408 plugin = pytestconfig.pluginmanager.get_plugin('userver_config')
409 local_hooks = (daemon_scoped_mark or {}).get('config_hooks', ())
410
411 for hook in itertools.chain(plugin.userver_config_hooks, local_hooks):
412 if not callable(hook):
413 hook_func = request.getfixturevalue(hook)
414 else:
415 hook_func = hook
416 hook_func(config_yaml, config_vars)
417
418 return _UserverConfig(config_yaml=config_yaml, config_vars=config_vars)
419
420
421@pytest.fixture(scope='session')
422def _service_config_substitution_vars(request, mockserver_info) -> Mapping[str, str]:
423 substitution_vars = {
424 'mockserver': mockserver_info.base_url.removesuffix('/'),
425 }
426 if request.config.pluginmanager.hasplugin('pytest_userver.plugins.grpc.mockserver'):
427 grpc_mockserver_endpoint = request.getfixturevalue('grpc_mockserver_endpoint')
428 substitution_vars['grpc_mockserver'] = grpc_mockserver_endpoint
429 return substitution_vars
430
431
432@pytest.fixture(scope='session')
433def userver_config_substitutions(_service_config_substitution_vars) -> ServiceConfigPatch:
434 """
435 Replaces substitution vars in all strings within `config_vars` using
436 [string.Template.substitute](https://docs.python.org/3/library/string.html#string.Template.substitute).
437
438 Substitution vars can be used as a shorthand for writing a full-fledged @ref SERVICE_CONFIG_HOOKS "config hook"
439 in many common cases.
440
441 Unlike normal `config_vars`, substitution vars can also apply to a part of a string.
442 For example, for `config_vars` entry
443
444 @code{.yaml}
445 frobnicator-url: $mockserver/frobnicator
446 @endcode
447
448 a possible patching result is as follows:
449
450 @code{.yaml}
451 frobnicator-url: http://127.0.0.1:1234/frobnicator
452 @endcode
453
454 Currently, the following substitution vars are supported:
455
456 * `mockserver` - mockserver url
457 * `grpc_mockserver` - grpc mockserver endpoint
458
459 @ingroup userver_testsuite_fixtures
460 """
461
462 def _substitute(key, value, parent: Union[list, dict]) -> None:
463 if isinstance(value, str):
464 parent[key] = string.Template(value).safe_substitute(_service_config_substitution_vars)
465 elif isinstance(value, dict):
466 for child_key, child_value in value.items():
467 _substitute(child_key, child_value, value)
468 elif isinstance(value, list):
469 for child_key, child_value in enumerate(value):
470 _substitute(child_key, child_value, value)
471
472 def patch_config(config_yaml, config_vars):
473 for key, value in config_vars.items():
474 _substitute(key, value, config_vars)
475
476 return patch_config
477
478
479@pytest.fixture(scope='session')
480def userver_config_http_server(service_port, monitor_port) -> ServiceConfigPatch:
481 """
482 Returns a function that adjusts the static configuration file for testsuite.
483 Sets the `server.listener.port` to listen on
484 @ref pytest_userver.plugins.base.service_port "service_port" fixture value;
485 sets the `server.listener-monitor.port` to listen on
486 @ref pytest_userver.plugins.base.monitor_port "monitor_port"
487 fixture value.
488
489 @ingroup userver_testsuite_fixtures
490 """
491
492 def _patch_config(config_yaml, config_vars):
493 components = config_yaml['components_manager']['components']
494 if 'server' in components:
495 server = components['server']
496 if 'listener' in server:
497 server['listener']['port'] = service_port
498
499 if 'listener-monitor' in server:
500 server['listener-monitor']['port'] = monitor_port
501
502 return _patch_config
503
504
505@pytest.fixture(scope='session')
506def allowed_url_prefixes_extra() -> List[str]:
507 """
508 By default, userver HTTP client is only allowed to talk to mockserver
509 when running in testsuite. This makes tests repeatable and encapsulated.
510
511 Override this fixture to whitelist some additional URLs.
512 It is still strongly advised to only talk to localhost in tests.
513
514 @ingroup userver_testsuite_fixtures
515 """
516 return []
517
518
519@pytest.fixture(scope='session')
521 mockserver_info,
522 mockserver_ssl_info,
523 allowed_url_prefixes_extra,
524) -> ServiceConfigPatch:
525 """
526 Returns a function that adjusts the static configuration file for testsuite.
527 Sets increased timeout and limits allowed URLs for `http-client` component.
528
529 @ingroup userver_testsuite_fixtures
530 """
531
532 def patch_config(config, config_vars):
533 components: dict = config['components_manager']['components']
534 if not {'http-client', 'testsuite-support'}.issubset(
535 components.keys(),
536 ):
537 return
538 http_client = components['http-client'] or {}
539 http_client['testsuite-enabled'] = True
540 http_client['testsuite-timeout'] = '10s'
541
542 allowed_urls = [mockserver_info.base_url]
543 if mockserver_ssl_info:
544 allowed_urls.append(mockserver_ssl_info.base_url)
545 allowed_urls += allowed_url_prefixes_extra
546 http_client['testsuite-allowed-url-prefixes'] = allowed_urls
547
548 return patch_config
549
550
551@pytest.fixture(scope='session')
553 """
554 Default log level to use in userver if no command line option was provided.
555
556 Returns 'debug'.
557
558 @ingroup userver_testsuite_fixtures
559 """
560 return 'debug'
561
562
563@pytest.fixture(scope='session')
564def userver_log_level(pytestconfig, userver_default_log_level) -> str:
565 """
566 Returns --service-log-level value if provided, otherwise returns
567 userver_default_log_level() value from fixture.
568
569 @ingroup userver_testsuite_fixtures
570 """
571 if pytestconfig.option.service_log_level:
572 return pytestconfig.option.service_log_level
573 return userver_default_log_level
574
575
576@pytest.fixture(scope='session')
577def userver_config_logging(userver_log_level, _service_logfile_path) -> ServiceConfigPatch:
578 """
579 Returns a function that adjusts the static configuration file for testsuite.
580 Sets the `logging.loggers.default` to log to `@stderr` with level set
581 from `--service-log-level` pytest configuration option.
582
583 @ingroup userver_testsuite_fixtures
584 """
585
586 if _service_logfile_path:
587 default_file_path = str(_service_logfile_path)
588 else:
589 default_file_path = '@stderr'
590
591 def _patch_config(config_yaml, config_vars):
592 components = config_yaml['components_manager']['components']
593 if 'logging' in components:
594 loggers = components['logging'].setdefault('loggers', {})
595 for logger in loggers.values():
596 logger['file_path'] = '@null'
597 loggers['default'] = {
598 'file_path': default_file_path,
599 'level': userver_log_level,
600 'overflow_behavior': 'discard',
601 }
602 config_vars['logger_level'] = userver_log_level
603
604 return _patch_config
605
606
607@pytest.fixture(scope='session')
608def userver_config_logging_otlp() -> ServiceConfigPatch:
609 """
610 Returns a function that adjusts the static configuration file for testsuite.
611 Sets the `otlp-logger.load-enabled` to `false` to disable OTLP logging and
612 leave the default file logger.
613
614 @ingroup userver_testsuite_fixtures
615 """
616
617 def _patch_config(config_yaml, config_vars):
618 components = config_yaml['components_manager']['components']
619 if 'otlp-logger' in components:
620 components['otlp-logger']['load-enabled'] = False
621
622 return _patch_config
623
624
625@pytest.fixture(scope='session')
626def userver_config_testsuite(pytestconfig, mockserver_info) -> ServiceConfigPatch:
627 """
628 Returns a function that adjusts the static configuration file for testsuite.
629
630 Sets up `testsuite-support` component, which:
631
632 - increases timeouts for userver drivers
633 - disables periodic cache updates
634 - enables testsuite tasks
635
636 Sets the `testsuite-enabled` in config_vars.yaml to `True`; sets the
637 `tests-control.testpoint-url` to mockserver URL.
638
639 @ingroup userver_testsuite_fixtures
640 """
641
642 def _set_postgresql_options(testsuite_support: dict) -> None:
643 testsuite_support['testsuite-pg-execute-timeout'] = '35s'
644 testsuite_support['testsuite-pg-statement-timeout'] = '30s'
645 testsuite_support['testsuite-pg-readonly-master-expected'] = True
646
647 def _set_redis_timeout(testsuite_support: dict) -> None:
648 testsuite_support['testsuite-redis-timeout-connect'] = '40s'
649 testsuite_support['testsuite-redis-timeout-single'] = '30s'
650 testsuite_support['testsuite-redis-timeout-all'] = '30s'
651
652 def _disable_cache_periodic_update(testsuite_support: dict) -> None:
653 testsuite_support['testsuite-periodic-update-enabled'] = False
654
655 def patch_config(config, config_vars) -> None:
656 # Don't delay tests teardown unnecessarily.
657 config['components_manager'].pop('graceful_shutdown_interval', None)
658 components: dict = config['components_manager']['components']
659 if 'testsuite-support' not in components:
660 return
661 testsuite_support = components['testsuite-support'] or {}
662 testsuite_support['testsuite-increased-timeout'] = '30s'
663 testsuite_support['testsuite-grpc-is-tls-enabled'] = False
664 _set_postgresql_options(testsuite_support)
665 _set_redis_timeout(testsuite_support)
666 service_runner = pytestconfig.option.service_runner_mode
667 if not service_runner:
668 _disable_cache_periodic_update(testsuite_support)
669 testsuite_support['testsuite-tasks-enabled'] = not service_runner
670 testsuite_support['testsuite-periodic-dumps-enabled'] = '$userver-dumps-periodic'
671 components['testsuite-support'] = testsuite_support
672
673 config_vars['testsuite-enabled'] = True
674 if 'tests-control' in components:
675 components['tests-control']['testpoint-url'] = mockserver_info.url(
676 'testpoint',
677 )
678
679 return patch_config
680
681
682@pytest.fixture(scope='session')
683def userver_config_secdist(service_secdist_path) -> ServiceConfigPatch:
684 """
685 Returns a function that adjusts the static configuration file for testsuite.
686 Sets the `default-secdist-provider.config` to the value of
687 @ref pytest_userver.plugins.config.service_secdist_path "service_secdist_path"
688 fixture.
689
690 @ingroup userver_testsuite_fixtures
691 """
692
693 def _patch_config(config_yaml, config_vars):
694 if not service_secdist_path:
695 return
696
697 components = config_yaml['components_manager']['components']
698 if 'default-secdist-provider' not in components:
699 return
700
701 if not service_secdist_path.is_file():
702 raise ValueError(
703 f'"{service_secdist_path}" is not a file. Provide a '
704 f'"--service-secdist" pytest option or override the '
705 f'"service_secdist_path" fixture.',
706 )
707 components['default-secdist-provider']['config'] = str(
708 service_secdist_path,
709 )
710
711 return _patch_config
712
713
714@pytest.fixture(scope='session')
715def userver_config_testsuite_middleware(userver_testsuite_middleware_enabled: bool) -> ServiceConfigPatch:
716 def patch_config(config_yaml, config_vars):
717 if not userver_testsuite_middleware_enabled:
718 return
719
720 components = config_yaml['components_manager']['components']
721 if 'server' not in components:
722 return
723
724 pipeline_builder = components.setdefault(
725 'default-server-middleware-pipeline-builder',
726 {},
727 )
728 middlewares = pipeline_builder.setdefault('append', [])
729 middlewares.append('testsuite-exceptions-handling-middleware')
730
731 return patch_config
732
733
734@pytest.fixture(scope='session')
736 """Whether testsuite middleware is enabled."""
737 return True