X-Git-Url: https://wannabe.guru.org/gitweb/?a=blobdiff_plain;f=config.py;h=bea701837c520ca86d584f9b48cead98ab06e838;hb=532df2c5b57c7517dfb3dddd8c1358fbadf8baf3;hp=672e1ae0328e6d7e5b99ccbc4d702adfcd516f32;hpb=a838c154135b2420d9047a101caf24a2c9f593c2;p=python_utils.git diff --git a/config.py b/config.py index 672e1ae..bea7018 100644 --- a/config.py +++ b/config.py @@ -1,7 +1,11 @@ #!/usr/bin/env python3 -"""Global configuration driven by commandline arguments (even across -different modules). Usage: +# © Copyright 2021-2022, Scott Gasch + +"""Global configuration driven by commandline arguments, environment variables +and saved configuration files. This works across several modules. + +Usage: module.py: ---------- @@ -36,7 +40,8 @@ different modules). Usage: config.parse() # Very important, this must be invoked! If you set this up and remember to invoke config.parse(), all commandline - arguments will play nicely together: + arguments will play nicely together. This is done automatically for you + if you're using the bootstrap module's initialize wrapper. % main.py -h usage: main.py [-h] @@ -67,57 +72,59 @@ import argparse import logging import os import pprint -import re import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional # This module is commonly used by others in here and should avoid # taking any unnecessary dependencies back on them. -# Note: at this point in time, logging hasn't been configured and -# anything we log will come out the root logger. - - -class LoadFromFile(argparse.Action): - """Helper to load a config file into argparse.""" - def __call__ (self, parser, namespace, values, option_string = None): - with values as f: - buf = f.read() - argv = [] - for line in buf.split(','): - line = line.strip() - line = line.strip('{') - line = line.strip('}') - m = re.match(r"^'([a-zA-Z_\-]+)'\s*:\s*(.*)$", line) - if m: - key = m.group(1) - value = m.group(2) - value = value.strip("'") - if value not in ('None', 'True', 'False'): - argv.append(f'--{key}') - argv.append(value) - parser.parse_args(argv, namespace) +# Defer logging messages until later when logging has been initialized. +SAVED_MESSAGES: List[str] = [] + +# Make a copy of the original program arguments. +PROGRAM_NAME: str = os.path.basename(sys.argv[0]) +ORIG_ARGV: List[str] = sys.argv.copy() + + +class OptionalRawFormatter(argparse.HelpFormatter): + """This formatter has the same bahavior as the normal argparse text + formatter except when the help text of an argument begins with + "RAW|". In that case, the line breaks are preserved and the text + is not wrapped. + + """ + + def _split_lines(self, text, width): + if text.startswith('RAW|'): + return text[4:].splitlines() + return argparse.HelpFormatter._split_lines(self, text, width) # A global parser that we will collect arguments into. -args = argparse.ArgumentParser( - description=f"This program uses config.py ({__file__}) for global, cross-module configuration.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, +ARGS = argparse.ArgumentParser( + description=None, + formatter_class=OptionalRawFormatter, fromfile_prefix_chars="@", + epilog=f'{PROGRAM_NAME} uses config.py ({__file__}) for global, cross-module configuration setup and parsing.', ) -config_parse_called = False -# A global configuration dictionary that will contain parsed arguments -# It is also this variable that modules use to access parsed arguments +# Keep track of if we've been called and prevent being called more +# than once. +CONFIG_PARSE_CALLED = False + + +# A global configuration dictionary that will contain parsed arguments. +# It is also this variable that modules use to access parsed arguments. +# This is the data that is most interesting to our callers; it will hold +# the configuration result. config: Dict[str, Any] = {} -# Defer logging messages until later when logging has been initialized. -saved_messages: List[str] = [] +# It would be really nice if this shit worked from interactive python def add_commandline_args(title: str, description: str = ""): """Create a new context for arguments and return a handle.""" - return args.add_argument_group(title, description) + return ARGS.add_argument_group(title, description) group = add_commandline_args( @@ -126,27 +133,39 @@ group = add_commandline_args( ) group.add_argument( '--config_loadfile', - type=open, - action=LoadFromFile, metavar='FILENAME', default=None, - help='Config file from which to read args in lieu or in addition to commandline.', + help='Config file (populated via --config_savefile) from which to read args in lieu or in addition to commandline.', ) group.add_argument( '--config_dump', default=False, action='store_true', - help='Display the global configuration on STDERR at program startup.', + help='Display the global configuration (possibly derived from multiple sources) on STDERR at program startup.', ) group.add_argument( '--config_savefile', type=str, metavar='FILENAME', default=None, - help='Populate config file compatible --config_loadfile to save config for later use.', + help='Populate config file compatible with --config_loadfile to save global config for later use.', +) +group.add_argument( + '--config_rejects_unrecognized_arguments', + default=False, + action='store_true', + help=( + 'If present, config will raise an exception if it doesn\'t recognize an argument. The ' + + 'default behavior is to ignore this so as to allow interoperability with programs that ' + + 'want to use their own argparse calls to parse their own, separate commandline args.' + ), ) +def overwrite_argparse_epilog(msg: str) -> None: + ARGS.epilog = msg + + def is_flag_already_in_argv(var: str): """Is a particular flag passed on the commandline?""" for _ in sys.argv: @@ -155,19 +174,20 @@ def is_flag_already_in_argv(var: str): return False -def parse() -> Dict[str, Any]: - import string_utils +def reorder_arg_action_groups_before_help(entry_module: Optional[str]): + reordered_action_groups = [] + for grp in ARGS._action_groups: + if entry_module is not None and entry_module in grp.title: # type: ignore + reordered_action_groups.append(grp) + elif PROGRAM_NAME in group.title: # type: ignore + reordered_action_groups.append(grp) + else: + reordered_action_groups.insert(0, grp) + return reordered_action_groups - """Main program should call this early in main()""" - global config_parse_called - if config_parse_called: - return config - config_parse_called = True - global saved_messages - # Examine the environment variables to settings that match - # known flags. - usage_message = args.format_usage() +def augment_sys_argv_from_environment_variables(): + usage_message = ARGS.format_usage() optional = False var = '' for x in usage_message.split(): @@ -190,52 +210,127 @@ def parse() -> Dict[str, Any]: if env in os.environ: if not is_flag_already_in_argv(var): value = os.environ[env] - saved_messages.append( - f'Initialized from environment: {var} = {value}' - ) - if len(chunks) == 1 and string_utils.to_bool(value): + SAVED_MESSAGES.append(f'Initialized from environment: {var} = {value}') + from string_utils import to_bool + + if len(chunks) == 1 and to_bool(value): sys.argv.append(var) elif len(chunks) > 1: sys.argv.append(var) sys.argv.append(value) var = '' env = '' + + +def augment_sys_argv_from_loadfile(): + loadfile = None + saw_other_args = False + grab_next_arg = False + for arg in sys.argv[1:]: + if 'config_loadfile' in arg: + pieces = arg.split('=') + if len(pieces) > 1: + loadfile = pieces[1] + else: + grab_next_arg = True + elif grab_next_arg: + loadfile = arg else: - next + saw_other_args = True + + if loadfile is not None: + if not os.path.exists(loadfile): + raise Exception( + f'ERROR: --config_loadfile argument must be a file, {loadfile} not found.' + ) + if saw_other_args: + msg = f'Augmenting commandline arguments with those from {loadfile}.' + else: + msg = f'Reading commandline arguments from {loadfile}.' + print(msg, file=sys.stderr) + SAVED_MESSAGES.append(msg) + + with open(loadfile, 'r') as rf: + newargs = rf.readlines() + newargs = [arg.strip('\n') for arg in newargs if 'config_savefile' not in arg] + sys.argv += newargs + + +def parse(entry_module: Optional[str]) -> Dict[str, Any]: + """Main program should call this early in main(). Note that the + bootstrap.initialize wrapper takes care of this automatically. - # Parse (possibly augmented) commandline args with argparse normally. - #config.update(vars(args.parse_args())) - known, unknown = args.parse_known_args() + """ + global CONFIG_PARSE_CALLED + if CONFIG_PARSE_CALLED: + return config + + # If we're about to do the usage message dump, put the main + # module's argument group last in the list (if possible) so that + # when the user passes -h or --help, it will be visible on the + # screen w/o scrolling. + for arg in sys.argv: + if arg in ('--help', '-h'): + if entry_module is not None: + entry_module = os.path.basename(entry_module) + ARGS._action_groups = reorder_arg_action_groups_before_help(entry_module) + + # Examine the environment for variables that match known flags. + # For a flag called --example_flag the corresponding environment + # variable would be called EXAMPLE_FLAG. If found, hackily add + # these into sys.argv to be parsed. + augment_sys_argv_from_environment_variables() + + # Look for loadfile and read/parse it if present. This also + # works by jamming these values onto sys.argv. + augment_sys_argv_from_loadfile() + + # Parse (possibly augmented, possibly completely overwritten) + # commandline args with argparse normally and populate config. + known, unknown = ARGS.parse_known_args() config.update(vars(known)) # Reconstruct the argv with unrecognized flags for the benefit of - # future argument parsers. + # future argument parsers. For example, unittest_main in python + # has some of its own flags. If we didn't recognize it, maybe + # someone else will. + if len(unknown) > 0: + if config['config_rejects_unrecognized_arguments']: + raise Exception( + f'Encountered unrecognized config argument(s) {unknown} with --config_rejects_unrecognized_arguments enabled; halting.' + ) + SAVED_MESSAGES.append(f'Config encountered unrecognized commandline arguments: {unknown}') sys.argv = sys.argv[:1] + unknown - if config['config_savefile']: - with open(config['config_savefile'], 'w') as wf: - wf.write("\n".join(sys.argv[1:])) + # Check for savefile and populate it if requested. + savefile = config['config_savefile'] + if savefile and len(savefile) > 0: + with open(savefile, 'w') as wf: + wf.write("\n".join(ORIG_ARGV[1:])) + # Also dump the config on stderr if requested. if config['config_dump']: dump_config() + + CONFIG_PARSE_CALLED = True return config def has_been_parsed() -> bool: """Has the global config been parsed yet?""" - global config_parse_called - return config_parse_called + return CONFIG_PARSE_CALLED def dump_config(): """Print the current config to stdout.""" print("Global Configuration:", file=sys.stderr) pprint.pprint(config, stream=sys.stderr) + print() def late_logging(): """Log messages saved earlier now that logging has been initialized.""" logger = logging.getLogger(__name__) - global saved_messages - for _ in saved_messages: + logger.debug('Original commandline was: %s', ORIG_ARGV) + for _ in SAVED_MESSAGES: logger.debug(_)