2Work with the configuration files of the service in testsuite.
16from typing
import Callable
17from typing
import List
18from typing
import Mapping
19from typing
import Optional
20from typing
import Union
40USERVER_CONFIG_HOOKS = [
41 'userver_config_substitutions',
42 'userver_config_http_server',
43 'userver_config_http_client',
44 'userver_config_logging',
45 'userver_config_logging_otlp',
46 'userver_config_testsuite',
47 'userver_config_secdist',
48 'userver_config_testsuite_middleware',
51ServiceConfigPatch = Callable[[dict, dict],
None]
57logger = logging.getLogger(__name__)
60class _UserverConfigPlugin:
62 self._config_hooks = []
65 def userver_config_hooks(self):
66 return self._config_hooks
68 def pytest_plugin_registered(self, plugin, manager):
69 if not isinstance(plugin, types.ModuleType):
71 uhooks = getattr(plugin,
'USERVER_CONFIG_HOOKS',
None)
72 if uhooks
is not None:
73 self._config_hooks.extend(uhooks)
76@dataclasses.dataclass(frozen=True)
82def pytest_configure(config):
83 config.pluginmanager.register(_UserverConfigPlugin(),
'userver_config')
86def pytest_addoption(parser) -> None:
87 group = parser.getgroup(
'userver-config')
89 '--service-log-level',
91 choices=[
'trace',
'debug',
'info',
'warning',
'error',
'critical'],
96 help=
'Path to service.yaml file.',
99 '--service-config-vars',
101 help=
'Path to config_vars.yaml file.',
106 help=
'Path to secure_data.json file.',
111 help=
'Path to dynamic config fallback file.',
116 help=
'Dump config from binary before running tests',
123@pytest.fixture(scope='session')
126 Returns the path to service.yaml file set by command line
127 `--service-config` option.
129 Override this fixture to change the way path to service.yaml is provided.
131 @ingroup userver_testsuite_fixtures
133 if pytestconfig.option.dump_config:
137 pytestconfig.option.service_config,
139 return pytestconfig.option.service_config
142@pytest.fixture(scope='session')
145 Runs the service binary with `--dump-db-schema` argument, dumps the 0_db_schema.sql file with database schema and
148 Override this fixture to change the way to dump the database schema.
150 @ingroup userver_testsuite_fixtures
153 path = service_tmpdir.joinpath(
'schemas')
158 path /
'0_db_schema.sql',
163@pytest.fixture(scope='session')
166 Returns the path to config_vars.yaml file set by command line
167 `--service-config-vars` option.
169 Override this fixture to change the way path to config_vars.yaml is
172 @ingroup userver_testsuite_fixtures
174 return pytestconfig.option.service_config_vars
177@pytest.fixture(scope='session')
180 Returns the path to secure_data.json file set by command line
181 `--service-secdist` option.
183 Override this fixture to change the way path to secure_data.json is
186 @ingroup userver_testsuite_fixtures
188 return pytestconfig.option.service_secdist
191@pytest.fixture(scope='session')
194 Returns the path to dynamic config fallback file set by command line
195 `--config-fallback` option.
197 Override this fixture to change the way path to dynamic config fallback is
200 @ingroup userver_testsuite_fixtures
202 return pytestconfig.option.config_fallback
205@pytest.fixture(scope='session')
208 Returns the path for temporary files. The path is the same for the whole
209 session and files are not removed (at least by this fixture) between
212 @ingroup userver_testsuite_fixtures
214 return tmp_path_factory.mktemp(
215 pathlib.Path(service_binary).name,
220@pytest.fixture(scope='session')
228 Dumps the contents of the service_config_yaml and service_config_vars into a static config for
229 testsuite and returns the path to the config file.
231 @ingroup userver_testsuite_fixtures
233 dst_path = service_tmpdir /
'config.yaml'
235 service_config_yaml = dict(service_config_yaml)
236 if not service_config_vars:
237 service_config_yaml.pop(
'config_vars',
None)
239 config_vars_path = service_tmpdir /
'config_vars.yaml'
240 config_vars_text = yaml.dump(service_config_vars)
242 'userver fixture "service_config_path_temp" writes the patched static config vars to "%s":\n%s',
246 config_vars_path.write_text(config_vars_text)
247 service_config_yaml[
'config_vars'] = str(config_vars_path)
250 'userver fixture "service_config_path_temp" writes the patched static config to "%s" equivalent to:\n%s',
252 yaml.dump(service_config),
254 dst_path.write_text(yaml.dump(service_config_yaml))
259@pytest.fixture(scope='session')
262 Returns the static config values after the USERVER_CONFIG_HOOKS were
263 applied (if any). Prefer using
264 pytest_userver.plugins.config.service_config
266 @ingroup userver_testsuite_fixtures
268 return _service_config_hooked.config_yaml
271@pytest.fixture(scope='session')
274 Returns the static config variables (config_vars.yaml) values after the
275 USERVER_CONFIG_HOOKS were applied (if any). Prefer using
276 pytest_userver.plugins.config.service_config
278 @ingroup userver_testsuite_fixtures
280 return _service_config_hooked.config_vars
283def _substitute_values(config, service_config_vars: dict, service_env) ->
None:
284 if isinstance(config, dict):
285 for key, value
in config.items():
286 if not isinstance(value, str):
287 _substitute_values(value, service_config_vars, service_env)
290 if not value.startswith(
'$'):
293 new_value = service_config_vars.get(value[1:])
294 if new_value
is not None:
295 config[key] = new_value
298 env = config.get(f
'{key}#env')
301 new_value = service_env.get(env)
303 new_value = os.environ.get(env)
305 config[key] = new_value
308 fallback = config.get(f
'{key}#fallback')
310 config[key] = fallback
315 if isinstance(config, list):
316 for i, value
in enumerate(config):
317 if not isinstance(value, str):
318 _substitute_values(value, service_config_vars, service_env)
321 if not value.startswith(
'$'):
324 new_value = service_config_vars.get(value[1:])
325 if new_value
is not None:
326 config[i] = new_value
329@pytest.fixture(scope='session')
332 A function that takes `config_yaml`, `config_vars` and applies all
333 substitutions just like the service would.
335 Useful when patching the service config. It's a good idea to pass
336 a component's config instead of the whole `config_yaml` to avoid
339 @warning The returned YAML is a clone, mutating it will not modify
340 the actual config while in a config hook!
342 @ingroup userver_testsuite_fixtures
345 def substitute(config_yaml, config_vars, /):
346 if config_yaml
is not None and not isinstance(config_yaml, dict)
and not isinstance(config_yaml, list):
348 f
'{substitute_config_vars.__name__} can only be meaningfully '
349 'called with dict and list nodes of config_yaml, while given: '
350 f
'{config_yaml!r}. Pass a containing object instead.',
353 config = copy.deepcopy(config_yaml)
354 _substitute_values(config, config_vars, service_env)
360@pytest.fixture(scope='session')
364 substitute_config_vars,
367 Returns the static config values after the USERVER_CONFIG_HOOKS were
368 applied (if any) and with all the '$', environment and fallback variables
371 @ingroup userver_testsuite_fixtures
374 config.pop(
'config_vars',
None)
378@pytest.fixture(scope='session')
379def _original_service_config(
381 service_config_vars_path,
386 with open(service_config_path, mode=
'rt')
as fp:
387 config_yaml = yaml.safe_load(fp)
389 if service_config_vars_path:
390 with open(service_config_vars_path, mode=
'rt')
as fp:
391 config_vars = yaml.safe_load(fp)
395 return _UserverConfig(config_yaml=config_yaml, config_vars=config_vars)
398@pytest.fixture(scope='session')
399def _service_config_hooked(
403 _original_service_config,
405 config_yaml = copy.deepcopy(_original_service_config.config_yaml)
406 config_vars = copy.deepcopy(_original_service_config.config_vars)
408 plugin = pytestconfig.pluginmanager.get_plugin(
'userver_config')
409 local_hooks = (daemon_scoped_mark
or {}).get(
'config_hooks', ())
411 for hook
in itertools.chain(plugin.userver_config_hooks, local_hooks):
412 if not callable(hook):
413 hook_func = request.getfixturevalue(hook)
416 hook_func(config_yaml, config_vars)
418 return _UserverConfig(config_yaml=config_yaml, config_vars=config_vars)
421@pytest.fixture(scope='session')
422def _service_config_substitution_vars(request, mockserver_info) -> Mapping[str, str]:
423 substitution_vars = {
424 'mockserver': mockserver_info.base_url.removesuffix(
'/'),
426 if request.config.pluginmanager.hasplugin(
'pytest_userver.plugins.grpc.mockserver'):
427 grpc_mockserver_endpoint = request.getfixturevalue(
'grpc_mockserver_endpoint')
428 substitution_vars[
'grpc_mockserver'] = grpc_mockserver_endpoint
429 return substitution_vars
432@pytest.fixture(scope='session')
435 Replaces substitution vars in all strings within `config_vars` using
436 [string.Template.substitute](https://docs.python.org/3/library/string.html#string.Template.substitute).
438 Substitution vars can be used as a shorthand for writing a full-fledged @ref SERVICE_CONFIG_HOOKS "config hook"
439 in many common cases.
441 Unlike normal `config_vars`, substitution vars can also apply to a part of a string.
442 For example, for `config_vars` entry
445 frobnicator-url: $mockserver/frobnicator
448 a possible patching result is as follows:
451 frobnicator-url: http://127.0.0.1:1234/frobnicator
454 Currently, the following substitution vars are supported:
456 * `mockserver` - mockserver url
457 * `grpc_mockserver` - grpc mockserver endpoint
459 @ingroup userver_testsuite_fixtures
462 def _substitute(key, value, parent: Union[list, dict]) ->
None:
463 if isinstance(value, str):
464 parent[key] = string.Template(value).safe_substitute(_service_config_substitution_vars)
465 elif isinstance(value, dict):
466 for child_key, child_value
in value.items():
467 _substitute(child_key, child_value, value)
468 elif isinstance(value, list):
469 for child_key, child_value
in enumerate(value):
470 _substitute(child_key, child_value, value)
472 def patch_config(config_yaml, config_vars):
473 for key, value
in config_vars.items():
474 _substitute(key, value, config_vars)
479@pytest.fixture(scope='session')
482 Returns a function that adjusts the static configuration file for testsuite.
483 Sets the `server.listener.port` to listen on
484 @ref pytest_userver.plugins.base.service_port "service_port" fixture value;
485 sets the `server.listener-monitor.port` to listen on
486 @ref pytest_userver.plugins.base.monitor_port "monitor_port"
489 @ingroup userver_testsuite_fixtures
492 def _patch_config(config_yaml, config_vars):
493 components = config_yaml[
'components_manager'][
'components']
494 if 'server' in components:
495 server = components[
'server']
496 if 'listener' in server:
497 server[
'listener'][
'port'] = service_port
499 if 'listener-monitor' in server:
500 server[
'listener-monitor'][
'port'] = monitor_port
505@pytest.fixture(scope='session')
508 By default, userver HTTP client is only allowed to talk to mockserver
509 when running in testsuite. This makes tests repeatable and encapsulated.
511 Override this fixture to whitelist some additional URLs.
512 It is still strongly advised to only talk to localhost in tests.
514 @ingroup userver_testsuite_fixtures
519@pytest.fixture(scope='session')
523 allowed_url_prefixes_extra,
524) -> ServiceConfigPatch:
526 Returns a function that adjusts the static configuration file for testsuite.
527 Sets increased timeout and limits allowed URLs for `http-client` component.
529 @ingroup userver_testsuite_fixtures
532 def patch_config(config, config_vars):
533 components: dict = config[
'components_manager'][
'components']
534 if not {
'http-client',
'testsuite-support'}.issubset(
538 http_client = components[
'http-client']
or {}
539 http_client[
'testsuite-enabled'] =
True
540 http_client[
'testsuite-timeout'] =
'10s'
542 allowed_urls = [mockserver_info.base_url]
543 if mockserver_ssl_info:
544 allowed_urls.append(mockserver_ssl_info.base_url)
545 allowed_urls += allowed_url_prefixes_extra
546 http_client[
'testsuite-allowed-url-prefixes'] = allowed_urls
551@pytest.fixture(scope='session')
554 Default log level to use in userver if no command line option was provided.
558 @ingroup userver_testsuite_fixtures
563@pytest.fixture(scope='session')
566 Returns --service-log-level value if provided, otherwise returns
567 userver_default_log_level() value from fixture.
569 @ingroup userver_testsuite_fixtures
571 if pytestconfig.option.service_log_level:
572 return pytestconfig.option.service_log_level
573 return userver_default_log_level
576@pytest.fixture(scope='session')
579 Returns a function that adjusts the static configuration file for testsuite.
580 Sets the `logging.loggers.default` to log to `@stderr` with level set
581 from `--service-log-level` pytest configuration option.
583 @ingroup userver_testsuite_fixtures
586 if _service_logfile_path:
587 default_file_path = str(_service_logfile_path)
589 default_file_path =
'@stderr'
591 def _patch_config(config_yaml, config_vars):
592 components = config_yaml[
'components_manager'][
'components']
593 if 'logging' in components:
594 loggers = components[
'logging'].setdefault(
'loggers', {})
595 for logger
in loggers.values():
596 logger[
'file_path'] =
'@null'
597 loggers[
'default'] = {
598 'file_path': default_file_path,
599 'level': userver_log_level,
600 'overflow_behavior':
'discard',
602 config_vars[
'logger_level'] = userver_log_level
607@pytest.fixture(scope='session')
610 Returns a function that adjusts the static configuration file for testsuite.
611 Sets the `otlp-logger.load-enabled` to `false` to disable OTLP logging and
612 leave the default file logger.
614 @ingroup userver_testsuite_fixtures
617 def _patch_config(config_yaml, config_vars):
618 components = config_yaml[
'components_manager'][
'components']
619 if 'otlp-logger' in components:
620 components[
'otlp-logger'][
'load-enabled'] =
False
625@pytest.fixture(scope='session')
628 Returns a function that adjusts the static configuration file for testsuite.
630 Sets up `testsuite-support` component, which:
632 - increases timeouts for userver drivers
633 - disables periodic cache updates
634 - enables testsuite tasks
636 Sets the `testsuite-enabled` in config_vars.yaml to `True`; sets the
637 `tests-control.testpoint-url` to mockserver URL.
639 @ingroup userver_testsuite_fixtures
642 def _set_postgresql_options(testsuite_support: dict) ->
None:
643 testsuite_support[
'testsuite-pg-execute-timeout'] =
'35s'
644 testsuite_support[
'testsuite-pg-statement-timeout'] =
'30s'
645 testsuite_support[
'testsuite-pg-readonly-master-expected'] =
True
647 def _set_redis_timeout(testsuite_support: dict) ->
None:
648 testsuite_support[
'testsuite-redis-timeout-connect'] =
'40s'
649 testsuite_support[
'testsuite-redis-timeout-single'] =
'30s'
650 testsuite_support[
'testsuite-redis-timeout-all'] =
'30s'
652 def _disable_cache_periodic_update(testsuite_support: dict) ->
None:
653 testsuite_support[
'testsuite-periodic-update-enabled'] =
False
655 def patch_config(config, config_vars) -> None:
657 config[
'components_manager'].pop(
'graceful_shutdown_interval',
None)
658 components: dict = config[
'components_manager'][
'components']
659 if 'testsuite-support' not in components:
661 testsuite_support = components[
'testsuite-support']
or {}
662 testsuite_support[
'testsuite-increased-timeout'] =
'30s'
663 testsuite_support[
'testsuite-grpc-is-tls-enabled'] =
False
664 _set_postgresql_options(testsuite_support)
665 _set_redis_timeout(testsuite_support)
666 service_runner = pytestconfig.option.service_runner_mode
667 if not service_runner:
668 _disable_cache_periodic_update(testsuite_support)
669 testsuite_support[
'testsuite-tasks-enabled'] =
not service_runner
670 testsuite_support[
'testsuite-periodic-dumps-enabled'] =
'$userver-dumps-periodic'
671 components[
'testsuite-support'] = testsuite_support
673 config_vars[
'testsuite-enabled'] =
True
674 if 'tests-control' in components:
675 components[
'tests-control'][
'testpoint-url'] = mockserver_info.url(
682@pytest.fixture(scope='session')
685 Returns a function that adjusts the static configuration file for testsuite.
686 Sets the `default-secdist-provider.config` to the value of
687 @ref pytest_userver.plugins.config.service_secdist_path "service_secdist_path"
690 @ingroup userver_testsuite_fixtures
693 def _patch_config(config_yaml, config_vars):
694 if not service_secdist_path:
697 components = config_yaml[
'components_manager'][
'components']
698 if 'default-secdist-provider' not in components:
701 if not service_secdist_path.is_file():
703 f
'"{service_secdist_path}" is not a file. Provide a '
704 f
'"--service-secdist" pytest option or override the '
705 f
'"service_secdist_path" fixture.',
707 components[
'default-secdist-provider'][
'config'] = str(
708 service_secdist_path,
714@pytest.fixture(scope='session')
715def userver_config_testsuite_middleware(userver_testsuite_middleware_enabled: bool) -> ServiceConfigPatch:
716 def patch_config(config_yaml, config_vars):
717 if not userver_testsuite_middleware_enabled:
720 components = config_yaml[
'components_manager'][
'components']
721 if 'server' not in components:
724 pipeline_builder = components.setdefault(
725 'default-server-middleware-pipeline-builder',
728 middlewares = pipeline_builder.setdefault(
'append', [])
729 middlewares.append(
'testsuite-exceptions-handling-middleware')
734@pytest.fixture(scope='session')
736 """Whether testsuite middleware is enabled."""