#!/usr/bin/env python3
+# © Copyright 2021-2022, Scott Gasch
+
"""Global configuration driven by commandline arguments, environment variables
and saved configuration files. This works across several modules.
Usage:
- module.py:
- ----------
- import config
-
- parser = config.add_commandline_args(
- "Module",
- "Args related to module doing the thing.",
- )
- parser.add_argument(
- "--module_do_the_thing",
- type=bool,
- default=True,
- help="Should the module do the thing?"
- )
-
- main.py:
- --------
- import config
-
- def main() -> None:
+ In your file.py::
+
+ import config
+
parser = config.add_commandline_args(
- "Main",
- "A program that does the thing.",
+ "Module",
+ "Args related to module doing the thing.",
)
parser.add_argument(
- "--dry_run",
+ "--module_do_the_thing",
type=bool,
- default=False,
- help="Should we really do the thing?"
+ default=True,
+ help="Should the module do the thing?"
)
- config.parse() # Very important, this must be invoked!
+
+ In your main.py::
+
+ import config
+
+ def main() -> None:
+ parser = config.add_commandline_args(
+ "Main",
+ "A program that does the thing.",
+ )
+ parser.add_argument(
+ "--dry_run",
+ type=bool,
+ default=False,
+ help="Should we really do the thing?"
+ )
+ config.parse() # Very important, this must be invoked!
If you set this up and remember to invoke config.parse(), all commandline
arguments will play nicely together. This is done automatically for you
- if you're using the bootstrap module's initialize wrapper.
+ if you're using the :meth:`bootstrap.initialize` decorator on
+ your program's entry point. See :meth:`python_modules.bootstrap.initialize`
+ for more details.::
+
+ import bootstrap
+
+ @bootstrap.initialize
+ def main():
+ whatever
- % main.py -h
- usage: main.py [-h]
- [--module_do_the_thing MODULE_DO_THE_THING]
- [--dry_run DRY_RUN]
+ if __name__ == '__main__':
+ main()
- Module:
- Args related to module doing the thing.
+ Either way, you'll get this behavior from the commandline::
- --module_do_the_thing MODULE_DO_THE_THING
- Should the module do the thing?
+ % main.py -h
+ usage: main.py [-h]
+ [--module_do_the_thing MODULE_DO_THE_THING]
+ [--dry_run DRY_RUN]
- Main:
- A program that does the thing
+ Module:
+ Args related to module doing the thing.
- --dry_run
- Should we really do the thing?
+ --module_do_the_thing MODULE_DO_THE_THING
+ Should the module do the thing?
+
+ Main:
+ A program that does the thing
+
+ --dry_run
+ Should we really do the thing?
Arguments themselves should be accessed via
- config.config['arg_name']. e.g.
+ :code:`config.config['arg_name']`. e.g.::
- if not config.config['dry_run']:
- module.do_the_thing()
+ if not config.config['dry_run']:
+ module.do_the_thing()
"""
# taking any unnecessary dependencies back on them.
# Defer logging messages until later when logging has been initialized.
-saved_messages: List[str] = []
+SAVED_MESSAGES: List[str] = []
# Make a copy of the original program arguments.
-program_name = os.path.basename(sys.argv[0])
-original_argv = [arg for arg in sys.argv]
+PROGRAM_NAME: str = os.path.basename(sys.argv[0])
+ORIG_ARGV: List[str] = sys.argv.copy()
+
+
+class OptionalRawFormatter(argparse.HelpFormatter):
+ """This formatter has the same bahavior as the normal argparse text
+ formatter except when the help text of an argument begins with
+ "RAW|". In that case, the line breaks are preserved and the text
+ is not wrapped.
+
+ Use this, for example, when you need the helptext of an argument
+ to have its spacing preserved exactly, e.g.::
+
+ args.add_argument(
+ '--mode',
+ type=str,
+ default='PLAY',
+ choices=['CHEAT', 'AUTOPLAY', 'SELFTEST', 'PRECOMPUTE', 'PLAY'],
+ metavar='MODE',
+ help='''RAW|Our mode of operation. One of:
+
+ PLAY = play wordle with me! Pick a random solution or
+ specify a solution with --template.
+
+ CHEAT = given a --template and, optionally, --letters_in_word
+ and/or --letters_to_avoid, return the best guess word;
+
+ AUTOPLAY = given a complete word in --template, guess it step
+ by step showing work;
+
+ SELFTEST = autoplay every possible solution keeping track of
+ wins/losses and average number of guesses;
+
+ PRECOMPUTE = populate hash table with optimal guesses.
+ ''',
+ )
+
+ """
+
+ def _split_lines(self, text, width):
+ if text.startswith('RAW|'):
+ return text[4:].splitlines()
+ return argparse.HelpFormatter._split_lines(self, text, width)
# A global parser that we will collect arguments into.
-args = argparse.ArgumentParser(
+ARGS = argparse.ArgumentParser(
description=None,
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ formatter_class=OptionalRawFormatter,
fromfile_prefix_chars="@",
- epilog=f'{program_name} uses config.py ({__file__}) for global, cross-module configuration setup and parsing.'
+ epilog=f'{PROGRAM_NAME} uses config.py ({__file__}) for global, cross-module configuration setup and parsing.',
+ # I don't fully understand why but when loaded by sphinx sometimes
+ # the same module is loaded many times causing any arguments it
+ # registers via module-level code to be redefined. Work around
+ # this iff the program is 'sphinx-build'
+ conflict_handler='resolve' if PROGRAM_NAME == 'sphinx-build' else 'error',
)
# Keep track of if we've been called and prevent being called more
# than once.
-config_parse_called = False
+CONFIG_PARSE_CALLED = False
+
# A global configuration dictionary that will contain parsed arguments.
# It is also this variable that modules use to access parsed arguments.
# the configuration result.
config: Dict[str, Any] = {}
+# It would be really nice if this shit worked from interactive python
+
+
+def add_commandline_args(title: str, description: str = "") -> argparse._ArgumentGroup:
+ """Create a new context for arguments and return a handle.
-def add_commandline_args(title: str, description: str = ""):
- """Create a new context for arguments and return a handle."""
- return args.add_argument_group(title, description)
+ Args:
+ title: A title for your module's commandline arguments group.
+ description: A helpful description of your module.
+
+ Returns:
+ An argparse._ArgumentGroup to be populated by the caller.
+ """
+ return ARGS.add_argument_group(title, description)
group = add_commandline_args(
default=None,
help='Populate config file compatible with --config_loadfile to save global config for later use.',
)
+group.add_argument(
+ '--config_rejects_unrecognized_arguments',
+ default=False,
+ action='store_true',
+ help=(
+ 'If present, config will raise an exception if it doesn\'t recognize an argument. The '
+ + 'default behavior is to ignore this so as to allow interoperability with programs that '
+ + 'want to use their own argparse calls to parse their own, separate commandline args.'
+ ),
+)
+
+def overwrite_argparse_epilog(msg: str) -> None:
+ """Allows your code to override the default epilog created by
+ argparse.
-def is_flag_already_in_argv(var: str):
- """Is a particular flag passed on the commandline?"""
+ Args:
+ msg: The epilog message to substitute for the default.
+ """
+ ARGS.epilog = msg
+
+
+def is_flag_already_in_argv(var: str) -> bool:
+ """Returns true if a particular flag is passed on the commandline?
+
+ Args:
+ var: The flag to search for.
+ """
for _ in sys.argv:
if var in _:
return True
return False
-def parse(entry_module: Optional[str]) -> Dict[str, Any]:
- """Main program should call this early in main(). Note that the
- bootstrap.initialize wrapper takes care of this automatically.
+def _reorder_arg_action_groups_before_help(entry_module: Optional[str]):
+ """Internal. Used to reorder the arguments before dumping out a
+ generated help string such that the main program's arguments come
+ last.
"""
- global config_parse_called
- if config_parse_called:
- return config
+ reordered_action_groups = []
+ for grp in ARGS._action_groups:
+ if entry_module is not None and entry_module in grp.title: # type: ignore
+ reordered_action_groups.append(grp)
+ elif PROGRAM_NAME in group.title: # type: ignore
+ reordered_action_groups.append(grp)
+ else:
+ reordered_action_groups.insert(0, grp)
+ return reordered_action_groups
- global saved_messages
- # If we're about to do the usage message dump, put the main module's
- # argument group last in the list (if possible) so that when the user
- # passes -h or --help, it will be visible on the screen w/o scrolling.
- reordered_action_groups = []
- global prog
- for arg in sys.argv:
- if arg == '--help' or arg == '-h':
- for group in args._action_groups:
- if entry_module is not None and entry_module in group.title:
- reordered_action_groups.append(group)
- elif program_name in group.title:
- reordered_action_groups.append(group)
- else:
- reordered_action_groups.insert(0, group)
- args._action_groups = reordered_action_groups
+def _augment_sys_argv_from_environment_variables():
+ """Internal. Look at the system environment for variables that match
+ arg names. This is done via some munging such that:
- # Examine the environment for variables that match known flags.
- # For a flag called --example_flag the corresponding environment
- # variable would be called EXAMPLE_FLAG.
- usage_message = args.format_usage()
+ :code:`--argument_to_match`
+
+ ...is matched by:
+
+ :code:`ARGUMENT_TO_MATCH`
+
+ This allows programmers to set args via shell environment variables
+ in lieu of passing them on the cmdline.
+
+ """
+
+ usage_message = ARGS.format_usage()
optional = False
var = ''
for x in usage_message.split():
if env in os.environ:
if not is_flag_already_in_argv(var):
value = os.environ[env]
- saved_messages.append(
- f'Initialized from environment: {var} = {value}'
- )
+ SAVED_MESSAGES.append(f'Initialized from environment: {var} = {value}')
from string_utils import to_bool
+
if len(chunks) == 1 and to_bool(value):
sys.argv.append(var)
elif len(chunks) > 1:
sys.argv.append(value)
var = ''
env = ''
- else:
- next
- # Look for loadfile and read/parse it if present.
+
+def _augment_sys_argv_from_loadfile():
+ """Internal. Augment with arguments persisted in a saved file."""
+
loadfile = None
saw_other_args = False
grab_next_arg = False
saw_other_args = True
if loadfile is not None:
+ if not os.path.exists(loadfile):
+ raise Exception(
+ f'ERROR: --config_loadfile argument must be a file, {loadfile} not found.'
+ )
if saw_other_args:
msg = f'Augmenting commandline arguments with those from {loadfile}.'
- print(msg, file=sys.stderr)
- saved_messages.append(msg)
- if not os.path.exists(loadfile):
- print(f'ERROR: --config_loadfile argument must be a file, {loadfile} not found.',
- file=sys.stderr)
- sys.exit(-1)
+ else:
+ msg = f'Reading commandline arguments from {loadfile}.'
+ print(msg, file=sys.stderr)
+ SAVED_MESSAGES.append(msg)
+
with open(loadfile, 'r') as rf:
newargs = rf.readlines()
newargs = [arg.strip('\n') for arg in newargs if 'config_savefile' not in arg]
sys.argv += newargs
+
+def parse(entry_module: Optional[str]) -> Dict[str, Any]:
+ """Main program should call this early in main(). Note that the
+ :code:`bootstrap.initialize` wrapper takes care of this automatically.
+ This should only be called once per program invocation.
+
+ """
+ global CONFIG_PARSE_CALLED
+ if CONFIG_PARSE_CALLED:
+ return config
+
+ # If we're about to do the usage message dump, put the main
+ # module's argument group last in the list (if possible) so that
+ # when the user passes -h or --help, it will be visible on the
+ # screen w/o scrolling.
+ for arg in sys.argv:
+ if arg in ('--help', '-h'):
+ if entry_module is not None:
+ entry_module = os.path.basename(entry_module)
+ ARGS._action_groups = _reorder_arg_action_groups_before_help(entry_module)
+
+ # Examine the environment for variables that match known flags.
+ # For a flag called --example_flag the corresponding environment
+ # variable would be called EXAMPLE_FLAG. If found, hackily add
+ # these into sys.argv to be parsed.
+ _augment_sys_argv_from_environment_variables()
+
+ # Look for loadfile and read/parse it if present. This also
+ # works by jamming these values onto sys.argv.
+ _augment_sys_argv_from_loadfile()
+
# Parse (possibly augmented, possibly completely overwritten)
# commandline args with argparse normally and populate config.
- known, unknown = args.parse_known_args()
+ known, unknown = ARGS.parse_known_args()
config.update(vars(known))
# Reconstruct the argv with unrecognized flags for the benefit of
# future argument parsers. For example, unittest_main in python
# has some of its own flags. If we didn't recognize it, maybe
# someone else will.
+ if len(unknown) > 0:
+ if config['config_rejects_unrecognized_arguments']:
+ raise Exception(
+ f'Encountered unrecognized config argument(s) {unknown} with --config_rejects_unrecognized_arguments enabled; halting.'
+ )
+ SAVED_MESSAGES.append(f'Config encountered unrecognized commandline arguments: {unknown}')
sys.argv = sys.argv[:1] + unknown
# Check for savefile and populate it if requested.
savefile = config['config_savefile']
if savefile and len(savefile) > 0:
with open(savefile, 'w') as wf:
- wf.write(
- "\n".join(original_argv[1:])
- )
+ wf.write("\n".join(ORIG_ARGV[1:]))
# Also dump the config on stderr if requested.
if config['config_dump']:
dump_config()
- config_parse_called = True
+ CONFIG_PARSE_CALLED = True
return config
def has_been_parsed() -> bool:
- """Has the global config been parsed yet?"""
- global config_parse_called
- return config_parse_called
+ """Returns True iff the global config has already been parsed"""
+ return CONFIG_PARSE_CALLED
def dump_config():
def late_logging():
"""Log messages saved earlier now that logging has been initialized."""
logger = logging.getLogger(__name__)
- global saved_messages
- for _ in saved_messages:
+ logger.debug('Original commandline was: %s', ORIG_ARGV)
+ for _ in SAVED_MESSAGES:
logger.debug(_)
projects / python_utils.git / blobdiff