3 # © Copyright 2021-2022, Scott Gasch
5 """Global configuration driven by commandline arguments, environment variables
6 and saved configuration files. This works across several modules.
14 parser = config.add_commandline_args(
16 "Args related to module doing the thing.",
19 "--module_do_the_thing",
22 help="Should the module do the thing?"
30 parser = config.add_commandline_args(
32 "A program that does the thing.",
38 help="Should we really do the thing?"
40 config.parse() # Very important, this must be invoked!
42 If you set this up and remember to invoke config.parse(), all commandline
43 arguments will play nicely together. This is done automatically for you
44 if you're using the :meth:`bootstrap.initialize` decorator on
45 your program's entry point. See :meth:`python_modules.bootstrap.initialize`
54 if __name__ == '__main__':
57 Either way, you'll get this behavior from the commandline::
61 [--module_do_the_thing MODULE_DO_THE_THING]
65 Args related to module doing the thing.
67 --module_do_the_thing MODULE_DO_THE_THING
68 Should the module do the thing?
71 A program that does the thing
74 Should we really do the thing?
76 Arguments themselves should be accessed via
77 :code:`config.config['arg_name']`. e.g.::
79 if not config.config['dry_run']:
89 from typing import Any, Dict, List, Optional
91 # This module is commonly used by others in here and should avoid
92 # taking any unnecessary dependencies back on them.
94 # Defer logging messages until later when logging has been initialized.
95 SAVED_MESSAGES: List[str] = []
97 # Make a copy of the original program arguments.
98 PROGRAM_NAME: str = os.path.basename(sys.argv[0])
99 ORIG_ARGV: List[str] = sys.argv.copy()
102 class OptionalRawFormatter(argparse.HelpFormatter):
103 """This formatter has the same bahavior as the normal argparse text
104 formatter except when the help text of an argument begins with
105 "RAW|". In that case, the line breaks are preserved and the text
108 Use this, for example, when you need the helptext of an argument
109 to have its spacing preserved exactly, e.g.::
115 choices=['CHEAT', 'AUTOPLAY', 'SELFTEST', 'PRECOMPUTE', 'PLAY'],
117 help='''RAW|Our mode of operation. One of:
119 PLAY = play wordle with me! Pick a random solution or
120 specify a solution with --template.
122 CHEAT = given a --template and, optionally, --letters_in_word
123 and/or --letters_to_avoid, return the best guess word;
125 AUTOPLAY = given a complete word in --template, guess it step
126 by step showing work;
128 SELFTEST = autoplay every possible solution keeping track of
129 wins/losses and average number of guesses;
131 PRECOMPUTE = populate hash table with optimal guesses.
137 def _split_lines(self, text, width):
138 if text.startswith('RAW|'):
139 return text[4:].splitlines()
140 return argparse.HelpFormatter._split_lines(self, text, width)
143 # A global parser that we will collect arguments into.
144 ARGS = argparse.ArgumentParser(
146 formatter_class=OptionalRawFormatter,
147 fromfile_prefix_chars="@",
148 epilog=f'{PROGRAM_NAME} uses config.py ({__file__}) for global, cross-module configuration setup and parsing.',
149 # I don't fully understand why but when loaded by sphinx sometimes
150 # the same module is loaded many times causing any arguments it
151 # registers via module-level code to be redefined. Work around
152 # this iff the program is 'sphinx-build'
153 conflict_handler='resolve' if PROGRAM_NAME == 'sphinx-build' else 'error',
156 # Keep track of if we've been called and prevent being called more
158 CONFIG_PARSE_CALLED = False
161 # A global configuration dictionary that will contain parsed arguments.
162 # It is also this variable that modules use to access parsed arguments.
163 # This is the data that is most interesting to our callers; it will hold
164 # the configuration result.
165 config: Dict[str, Any] = {}
167 # It would be really nice if this shit worked from interactive python
170 def add_commandline_args(title: str, description: str = "") -> argparse._ArgumentGroup:
171 """Create a new context for arguments and return a handle.
174 title: A title for your module's commandline arguments group.
175 description: A helpful description of your module.
178 An argparse._ArgumentGroup to be populated by the caller.
180 return ARGS.add_argument_group(title, description)
183 group = add_commandline_args(
184 f'Global Config ({__file__})',
185 'Args that control the global config itself; how meta!',
191 help='Config file (populated via --config_savefile) from which to read args in lieu or in addition to commandline.',
197 help='Display the global configuration (possibly derived from multiple sources) on STDERR at program startup.',
204 help='Populate config file compatible with --config_loadfile to save global config for later use.',
207 '--config_rejects_unrecognized_arguments',
211 'If present, config will raise an exception if it doesn\'t recognize an argument. The '
212 + 'default behavior is to ignore this so as to allow interoperability with programs that '
213 + 'want to use their own argparse calls to parse their own, separate commandline args.'
218 def overwrite_argparse_epilog(msg: str) -> None:
219 """Allows your code to override the default epilog created by
223 msg: The epilog message to substitute for the default.
228 def is_flag_already_in_argv(var: str) -> bool:
229 """Returns true if a particular flag is passed on the commandline?
232 var: The flag to search for.
240 def _reorder_arg_action_groups_before_help(entry_module: Optional[str]):
241 """Internal. Used to reorder the arguments before dumping out a
242 generated help string such that the main program's arguments come
246 reordered_action_groups = []
247 for grp in ARGS._action_groups:
248 if entry_module is not None and entry_module in grp.title: # type: ignore
249 reordered_action_groups.append(grp)
250 elif PROGRAM_NAME in group.title: # type: ignore
251 reordered_action_groups.append(grp)
253 reordered_action_groups.insert(0, grp)
254 return reordered_action_groups
257 def print_usage() -> None:
258 """Prints the normal help usage message out."""
265 program usage help text as a string.
267 return ARGS.format_usage()
270 def _augment_sys_argv_from_environment_variables():
271 """Internal. Look at the system environment for variables that match
272 arg names. This is done via some munging such that:
274 :code:`--argument_to_match`
278 :code:`ARGUMENT_TO_MATCH`
280 This allows programmers to set args via shell environment variables
281 in lieu of passing them on the cmdline.
285 usage_message = usage()
288 for x in usage_message.split():
302 # Environment vars the same as flag names without
303 # the initial -'s and in UPPERCASE.
304 env = var.strip('-').upper()
305 if env in os.environ:
306 if not is_flag_already_in_argv(var):
307 value = os.environ[env]
308 SAVED_MESSAGES.append(f'Initialized from environment: {var} = {value}')
309 from string_utils import to_bool
311 if len(chunks) == 1 and to_bool(value):
313 elif len(chunks) > 1:
315 sys.argv.append(value)
320 def _augment_sys_argv_from_loadfile():
321 """Internal. Augment with arguments persisted in a saved file."""
324 saw_other_args = False
325 grab_next_arg = False
326 for arg in sys.argv[1:]:
327 if 'config_loadfile' in arg:
328 pieces = arg.split('=')
336 saw_other_args = True
338 if loadfile is not None:
339 if not os.path.exists(loadfile):
341 f'ERROR: --config_loadfile argument must be a file, {loadfile} not found.'
344 msg = f'Augmenting commandline arguments with those from {loadfile}.'
346 msg = f'Reading commandline arguments from {loadfile}.'
347 print(msg, file=sys.stderr)
348 SAVED_MESSAGES.append(msg)
350 with open(loadfile, 'r') as rf:
351 newargs = rf.readlines()
352 newargs = [arg.strip('\n') for arg in newargs if 'config_savefile' not in arg]
356 def parse(entry_module: Optional[str]) -> Dict[str, Any]:
357 """Main program should call this early in main(). Note that the
358 :code:`bootstrap.initialize` wrapper takes care of this automatically.
359 This should only be called once per program invocation.
362 global CONFIG_PARSE_CALLED
363 if CONFIG_PARSE_CALLED:
366 # If we're about to do the usage message dump, put the main
367 # module's argument group last in the list (if possible) so that
368 # when the user passes -h or --help, it will be visible on the
369 # screen w/o scrolling.
371 if arg in ('--help', '-h'):
372 if entry_module is not None:
373 entry_module = os.path.basename(entry_module)
374 ARGS._action_groups = _reorder_arg_action_groups_before_help(entry_module)
376 # Examine the environment for variables that match known flags.
377 # For a flag called --example_flag the corresponding environment
378 # variable would be called EXAMPLE_FLAG. If found, hackily add
379 # these into sys.argv to be parsed.
380 _augment_sys_argv_from_environment_variables()
382 # Look for loadfile and read/parse it if present. This also
383 # works by jamming these values onto sys.argv.
384 _augment_sys_argv_from_loadfile()
386 # Parse (possibly augmented, possibly completely overwritten)
387 # commandline args with argparse normally and populate config.
388 known, unknown = ARGS.parse_known_args()
389 config.update(vars(known))
391 # Reconstruct the argv with unrecognized flags for the benefit of
392 # future argument parsers. For example, unittest_main in python
393 # has some of its own flags. If we didn't recognize it, maybe
396 if config['config_rejects_unrecognized_arguments']:
398 f'Encountered unrecognized config argument(s) {unknown} with --config_rejects_unrecognized_arguments enabled; halting.'
400 SAVED_MESSAGES.append(f'Config encountered unrecognized commandline arguments: {unknown}')
401 sys.argv = sys.argv[:1] + unknown
403 # Check for savefile and populate it if requested.
404 savefile = config['config_savefile']
405 if savefile and len(savefile) > 0:
406 with open(savefile, 'w') as wf:
407 wf.write("\n".join(ORIG_ARGV[1:]))
409 # Also dump the config on stderr if requested.
410 if config['config_dump']:
413 CONFIG_PARSE_CALLED = True
417 def has_been_parsed() -> bool:
418 """Returns True iff the global config has already been parsed"""
419 return CONFIG_PARSE_CALLED
423 """Print the current config to stdout."""
424 print("Global Configuration:", file=sys.stderr)
425 pprint.pprint(config, stream=sys.stderr)
430 """Log messages saved earlier now that logging has been initialized."""
431 logger = logging.getLogger(__name__)
432 logger.debug('Original commandline was: %s', ORIG_ARGV)
433 for _ in SAVED_MESSAGES: