3 """Global configuration driven by commandline arguments, environment variables
4 and saved configuration files. This works across several modules.
12 parser = config.add_commandline_args(
14 "Args related to module doing the thing.",
17 "--module_do_the_thing",
20 help="Should the module do the thing?"
28 parser = config.add_commandline_args(
30 "A program that does the thing.",
36 help="Should we really do the thing?"
38 config.parse() # Very important, this must be invoked!
40 If you set this up and remember to invoke config.parse(), all commandline
41 arguments will play nicely together. This is done automatically for you
42 if you're using the bootstrap module's initialize wrapper.
46 [--module_do_the_thing MODULE_DO_THE_THING]
50 Args related to module doing the thing.
52 --module_do_the_thing MODULE_DO_THE_THING
53 Should the module do the thing?
56 A program that does the thing
59 Should we really do the thing?
61 Arguments themselves should be accessed via
62 config.config['arg_name']. e.g.
64 if not config.config['dry_run']:
74 from typing import Any, Dict, List, Optional
76 # This module is commonly used by others in here and should avoid
77 # taking any unnecessary dependencies back on them.
79 # Defer logging messages until later when logging has been initialized.
80 SAVED_MESSAGES: List[str] = []
82 # Make a copy of the original program arguments.
83 PROGRAM_NAME: str = os.path.basename(sys.argv[0])
84 ORIG_ARGV: List[str] = sys.argv.copy()
87 class OptionalRawFormatter(argparse.HelpFormatter):
88 """This formatter has the same bahavior as the normal argparse text
89 formatter except when the help text of an argument begins with
90 "RAW|". In that case, the line breaks are preserved and the text
95 def _split_lines(self, text, width):
96 if text.startswith('RAW|'):
97 return text[4:].splitlines()
98 return argparse.HelpFormatter._split_lines(self, text, width)
101 # A global parser that we will collect arguments into.
102 ARGS = argparse.ArgumentParser(
104 formatter_class=OptionalRawFormatter,
105 fromfile_prefix_chars="@",
106 epilog=f'{PROGRAM_NAME} uses config.py ({__file__}) for global, cross-module configuration setup and parsing.',
109 # Keep track of if we've been called and prevent being called more
111 CONFIG_PARSE_CALLED = False
114 # A global configuration dictionary that will contain parsed arguments.
115 # It is also this variable that modules use to access parsed arguments.
116 # This is the data that is most interesting to our callers; it will hold
117 # the configuration result.
118 config: Dict[str, Any] = {}
120 # It would be really nice if this shit worked from interactive python
123 def add_commandline_args(title: str, description: str = ""):
124 """Create a new context for arguments and return a handle."""
125 return ARGS.add_argument_group(title, description)
128 group = add_commandline_args(
129 f'Global Config ({__file__})',
130 'Args that control the global config itself; how meta!',
136 help='Config file (populated via --config_savefile) from which to read args in lieu or in addition to commandline.',
142 help='Display the global configuration (possibly derived from multiple sources) on STDERR at program startup.',
149 help='Populate config file compatible with --config_loadfile to save global config for later use.',
152 '--config_rejects_unrecognized_arguments',
156 'If present, config will raise an exception if it doesn\'t recognize an argument. The '
157 + 'default behavior is to ignore this so as to allow interoperability with programs that '
158 + 'want to use their own argparse calls to parse their own, separate commandline args.'
163 def overwrite_argparse_epilog(msg: str) -> None:
167 def is_flag_already_in_argv(var: str):
168 """Is a particular flag passed on the commandline?"""
175 def reorder_arg_action_groups(entry_module: Optional[str]):
176 reordered_action_groups = []
177 for grp in ARGS._action_groups:
178 if entry_module is not None and entry_module in grp.title: # type: ignore
179 reordered_action_groups.append(grp)
180 elif PROGRAM_NAME in group.title: # type: ignore
181 reordered_action_groups.append(grp)
183 reordered_action_groups.insert(0, grp)
184 return reordered_action_groups
187 def augment_sys_argv_from_environment_variables():
188 usage_message = ARGS.format_usage()
191 for x in usage_message.split():
205 # Environment vars the same as flag names without
206 # the initial -'s and in UPPERCASE.
207 env = var.strip('-').upper()
208 if env in os.environ:
209 if not is_flag_already_in_argv(var):
210 value = os.environ[env]
211 SAVED_MESSAGES.append(f'Initialized from environment: {var} = {value}')
212 from string_utils import to_bool
214 if len(chunks) == 1 and to_bool(value):
216 elif len(chunks) > 1:
218 sys.argv.append(value)
223 def augment_sys_argv_from_loadfile():
225 saw_other_args = False
226 grab_next_arg = False
227 for arg in sys.argv[1:]:
228 if 'config_loadfile' in arg:
229 pieces = arg.split('=')
237 saw_other_args = True
239 if loadfile is not None:
240 if not os.path.exists(loadfile):
242 f'ERROR: --config_loadfile argument must be a file, {loadfile} not found.'
245 msg = f'Augmenting commandline arguments with those from {loadfile}.'
247 msg = f'Reading commandline arguments from {loadfile}.'
248 print(msg, file=sys.stderr)
249 SAVED_MESSAGES.append(msg)
251 with open(loadfile, 'r') as rf:
252 newargs = rf.readlines()
253 newargs = [arg.strip('\n') for arg in newargs if 'config_savefile' not in arg]
257 def parse(entry_module: Optional[str]) -> Dict[str, Any]:
258 """Main program should call this early in main(). Note that the
259 bootstrap.initialize wrapper takes care of this automatically.
262 global CONFIG_PARSE_CALLED
263 if CONFIG_PARSE_CALLED:
266 # If we're about to do the usage message dump, put the main
267 # module's argument group last in the list (if possible) so that
268 # when the user passes -h or --help, it will be visible on the
269 # screen w/o scrolling.
271 if arg in ('--help', '-h'):
272 ARGS._action_groups = reorder_arg_action_groups(entry_module)
274 # Examine the environment for variables that match known flags.
275 # For a flag called --example_flag the corresponding environment
276 # variable would be called EXAMPLE_FLAG. If found, hackily add
277 # these into sys.argv to be parsed.
278 augment_sys_argv_from_environment_variables()
280 # Look for loadfile and read/parse it if present. This also
281 # works by jamming these values onto sys.argv.
282 augment_sys_argv_from_loadfile()
284 # Parse (possibly augmented, possibly completely overwritten)
285 # commandline args with argparse normally and populate config.
286 known, unknown = ARGS.parse_known_args()
287 config.update(vars(known))
289 # Reconstruct the argv with unrecognized flags for the benefit of
290 # future argument parsers. For example, unittest_main in python
291 # has some of its own flags. If we didn't recognize it, maybe
294 if config['config_rejects_unrecognized_arguments']:
296 f'Encountered unrecognized config argument(s) {unknown} with --config_rejects_unrecognized_arguments enabled; halting.'
298 SAVED_MESSAGES.append(f'Config encountered unrecognized commandline arguments: {unknown}')
299 sys.argv = sys.argv[:1] + unknown
301 # Check for savefile and populate it if requested.
302 savefile = config['config_savefile']
303 if savefile and len(savefile) > 0:
304 with open(savefile, 'w') as wf:
305 wf.write("\n".join(ORIG_ARGV[1:]))
307 # Also dump the config on stderr if requested.
308 if config['config_dump']:
311 CONFIG_PARSE_CALLED = True
315 def has_been_parsed() -> bool:
316 """Has the global config been parsed yet?"""
317 return CONFIG_PARSE_CALLED
321 """Print the current config to stdout."""
322 print("Global Configuration:", file=sys.stderr)
323 pprint.pprint(config, stream=sys.stderr)
328 """Log messages saved earlier now that logging has been initialized."""
329 logger = logging.getLogger(__name__)
330 for _ in SAVED_MESSAGES: