#!/usr/bin/env python3
-"""Global configuration driven by commandline arguments (even across
-different modules). Usage:
+# © Copyright 2021-2022, Scott Gasch
+
+"""Global configuration driven by commandline arguments, environment variables
+and saved configuration files. This works across several modules.
+
+Usage:
module.py:
----------
config.parse() # Very important, this must be invoked!
If you set this up and remember to invoke config.parse(), all commandline
- arguments will play nicely together:
+ arguments will play nicely together. This is done automatically for you
+ if you're using the bootstrap module's initialize wrapper.
% main.py -h
usage: main.py [-h]
--dry_run
Should we really do the thing?
- Arguments themselves should be accessed via config.config['arg_name']. e.g.
+ Arguments themselves should be accessed via
+ config.config['arg_name']. e.g.
if not config.config['dry_run']:
module.do_the_thing()
+
"""
import argparse
+import logging
+import os
import pprint
-import re
import sys
-from typing import Dict, Any
-
-# Note: at this point in time, logging hasn't been configured and
-# anything we log will come out the root logger.
-
-
-class LoadFromFile(argparse.Action):
- """Helper to load a config file into argparse."""
- def __call__ (self, parser, namespace, values, option_string = None):
- with values as f:
- buf = f.read()
- argv = []
- for line in buf.split(','):
- line = line.strip()
- line = line.strip('{')
- line = line.strip('}')
- m = re.match(r"^'([a-zA-Z_\-]+)'\s*:\s*(.*)$", line)
- if m:
- key = m.group(1)
- value = m.group(2)
- value = value.strip("'")
- if value not in ('None', 'True', 'False'):
- argv.append(f'--{key}')
- argv.append(value)
- parser.parse_args(argv, namespace)
+from typing import Any, Dict, List, Optional
+
+# This module is commonly used by others in here and should avoid
+# taking any unnecessary dependencies back on them.
+
+# Defer logging messages until later when logging has been initialized.
+SAVED_MESSAGES: List[str] = []
+
+# Make a copy of the original program arguments.
+PROGRAM_NAME: str = os.path.basename(sys.argv[0])
+ORIG_ARGV: List[str] = sys.argv.copy()
+
+
+class OptionalRawFormatter(argparse.HelpFormatter):
+ """This formatter has the same bahavior as the normal argparse text
+ formatter except when the help text of an argument begins with
+ "RAW|". In that case, the line breaks are preserved and the text
+ is not wrapped.
+
+ """
+
+ def _split_lines(self, text, width):
+ if text.startswith('RAW|'):
+ return text[4:].splitlines()
+ return argparse.HelpFormatter._split_lines(self, text, width)
# A global parser that we will collect arguments into.
-args = argparse.ArgumentParser(
- description=f"This program uses config.py ({__file__}) for global, cross-module configuration.",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
- fromfile_prefix_chars="@"
+ARGS = argparse.ArgumentParser(
+ description=None,
+ formatter_class=OptionalRawFormatter,
+ fromfile_prefix_chars="@",
+ epilog=f'{PROGRAM_NAME} uses config.py ({__file__}) for global, cross-module configuration setup and parsing.',
)
-config_parse_called = False
-# A global configuration dictionary that will contain parsed arguments
-# It is also this variable that modules use to access parsed arguments
+# Keep track of if we've been called and prevent being called more
+# than once.
+CONFIG_PARSE_CALLED = False
+
+
+# A global configuration dictionary that will contain parsed arguments.
+# It is also this variable that modules use to access parsed arguments.
+# This is the data that is most interesting to our callers; it will hold
+# the configuration result.
config: Dict[str, Any] = {}
+# It would be really nice if this shit worked from interactive python
+
def add_commandline_args(title: str, description: str = ""):
"""Create a new context for arguments and return a handle."""
- return args.add_argument_group(title, description)
+ return ARGS.add_argument_group(title, description)
group = add_commandline_args(
)
group.add_argument(
'--config_loadfile',
- type=open,
- action=LoadFromFile,
metavar='FILENAME',
default=None,
- help='Config file from which to read args in lieu or in addition to commandline.',
+ help='Config file (populated via --config_savefile) from which to read args in lieu or in addition to commandline.',
)
group.add_argument(
'--config_dump',
default=False,
action='store_true',
- help='Display the global configuration on STDERR at program startup.',
+ help='Display the global configuration (possibly derived from multiple sources) on STDERR at program startup.',
)
group.add_argument(
'--config_savefile',
type=str,
metavar='FILENAME',
default=None,
- help='Populate config file compatible --config_loadfile to save config for later use.',
+ help='Populate config file compatible with --config_loadfile to save global config for later use.',
+)
+group.add_argument(
+ '--config_rejects_unrecognized_arguments',
+ default=False,
+ action='store_true',
+ help=(
+ 'If present, config will raise an exception if it doesn\'t recognize an argument. The '
+ + 'default behavior is to ignore this so as to allow interoperability with programs that '
+ + 'want to use their own argparse calls to parse their own, separate commandline args.'
+ ),
)
-def parse() -> Dict[str, Any]:
- """Main program should call this early in main()"""
- global config_parse_called
- config_parse_called = True
- config.update(vars(args.parse_args()))
-
- if config['config_savefile']:
- with open(config['config_savefile'], 'w') as wf:
- wf.write("\n".join(sys.argv[1:]))
-
+def overwrite_argparse_epilog(msg: str) -> None:
+ ARGS.epilog = msg
+
+
+def is_flag_already_in_argv(var: str):
+ """Is a particular flag passed on the commandline?"""
+ for _ in sys.argv:
+ if var in _:
+ return True
+ return False
+
+
+def reorder_arg_action_groups_before_help(entry_module: Optional[str]):
+ reordered_action_groups = []
+ for grp in ARGS._action_groups:
+ if entry_module is not None and entry_module in grp.title: # type: ignore
+ reordered_action_groups.append(grp)
+ elif PROGRAM_NAME in group.title: # type: ignore
+ reordered_action_groups.append(grp)
+ else:
+ reordered_action_groups.insert(0, grp)
+ return reordered_action_groups
+
+
+def augment_sys_argv_from_environment_variables():
+ usage_message = ARGS.format_usage()
+ optional = False
+ var = ''
+ for x in usage_message.split():
+ if x[0] == '[':
+ optional = True
+ if optional:
+ var += f'{x} '
+ if x[-1] == ']':
+ optional = False
+ var = var.strip()
+ var = var.strip('[')
+ var = var.strip(']')
+ chunks = var.split()
+ if len(chunks) > 1:
+ var = var.split()[0]
+
+ # Environment vars the same as flag names without
+ # the initial -'s and in UPPERCASE.
+ env = var.strip('-').upper()
+ if env in os.environ:
+ if not is_flag_already_in_argv(var):
+ value = os.environ[env]
+ SAVED_MESSAGES.append(f'Initialized from environment: {var} = {value}')
+ from string_utils import to_bool
+
+ if len(chunks) == 1 and to_bool(value):
+ sys.argv.append(var)
+ elif len(chunks) > 1:
+ sys.argv.append(var)
+ sys.argv.append(value)
+ var = ''
+ env = ''
+
+
+def augment_sys_argv_from_loadfile():
+ loadfile = None
+ saw_other_args = False
+ grab_next_arg = False
+ for arg in sys.argv[1:]:
+ if 'config_loadfile' in arg:
+ pieces = arg.split('=')
+ if len(pieces) > 1:
+ loadfile = pieces[1]
+ else:
+ grab_next_arg = True
+ elif grab_next_arg:
+ loadfile = arg
+ else:
+ saw_other_args = True
+
+ if loadfile is not None:
+ if not os.path.exists(loadfile):
+ raise Exception(
+ f'ERROR: --config_loadfile argument must be a file, {loadfile} not found.'
+ )
+ if saw_other_args:
+ msg = f'Augmenting commandline arguments with those from {loadfile}.'
+ else:
+ msg = f'Reading commandline arguments from {loadfile}.'
+ print(msg, file=sys.stderr)
+ SAVED_MESSAGES.append(msg)
+
+ with open(loadfile, 'r') as rf:
+ newargs = rf.readlines()
+ newargs = [arg.strip('\n') for arg in newargs if 'config_savefile' not in arg]
+ sys.argv += newargs
+
+
+def parse(entry_module: Optional[str]) -> Dict[str, Any]:
+ """Main program should call this early in main(). Note that the
+ bootstrap.initialize wrapper takes care of this automatically.
+
+ """
+ global CONFIG_PARSE_CALLED
+ if CONFIG_PARSE_CALLED:
+ return config
+
+ # If we're about to do the usage message dump, put the main
+ # module's argument group last in the list (if possible) so that
+ # when the user passes -h or --help, it will be visible on the
+ # screen w/o scrolling.
+ for arg in sys.argv:
+ if arg in ('--help', '-h'):
+ if entry_module is not None:
+ entry_module = os.path.basename(entry_module)
+ ARGS._action_groups = reorder_arg_action_groups_before_help(entry_module)
+
+ # Examine the environment for variables that match known flags.
+ # For a flag called --example_flag the corresponding environment
+ # variable would be called EXAMPLE_FLAG. If found, hackily add
+ # these into sys.argv to be parsed.
+ augment_sys_argv_from_environment_variables()
+
+ # Look for loadfile and read/parse it if present. This also
+ # works by jamming these values onto sys.argv.
+ augment_sys_argv_from_loadfile()
+
+ # Parse (possibly augmented, possibly completely overwritten)
+ # commandline args with argparse normally and populate config.
+ known, unknown = ARGS.parse_known_args()
+ config.update(vars(known))
+
+ # Reconstruct the argv with unrecognized flags for the benefit of
+ # future argument parsers. For example, unittest_main in python
+ # has some of its own flags. If we didn't recognize it, maybe
+ # someone else will.
+ if len(unknown) > 0:
+ if config['config_rejects_unrecognized_arguments']:
+ raise Exception(
+ f'Encountered unrecognized config argument(s) {unknown} with --config_rejects_unrecognized_arguments enabled; halting.'
+ )
+ SAVED_MESSAGES.append(f'Config encountered unrecognized commandline arguments: {unknown}')
+ sys.argv = sys.argv[:1] + unknown
+
+ # Check for savefile and populate it if requested.
+ savefile = config['config_savefile']
+ if savefile and len(savefile) > 0:
+ with open(savefile, 'w') as wf:
+ wf.write("\n".join(ORIG_ARGV[1:]))
+
+ # Also dump the config on stderr if requested.
if config['config_dump']:
dump_config()
+ CONFIG_PARSE_CALLED = True
return config
def has_been_parsed() -> bool:
- global config_parse_called
- return config_parse_called
+ """Has the global config been parsed yet?"""
+ return CONFIG_PARSE_CALLED
def dump_config():
"""Print the current config to stdout."""
print("Global Configuration:", file=sys.stderr)
pprint.pprint(config, stream=sys.stderr)
+ print()
+
+
+def late_logging():
+ """Log messages saved earlier now that logging has been initialized."""
+ logger = logging.getLogger(__name__)
+ logger.debug('Original commandline was: %s', ORIG_ARGV)
+ for _ in SAVED_MESSAGES:
+ logger.debug(_)