3 """Global configuration driven by commandline arguments, environment variables
4 and saved configuration files. This works across several modules.
12 parser = config.add_commandline_args(
14 "Args related to module doing the thing.",
17 "--module_do_the_thing",
20 help="Should the module do the thing?"
28 parser = config.add_commandline_args(
30 "A program that does the thing.",
36 help="Should we really do the thing?"
38 config.parse() # Very important, this must be invoked!
40 If you set this up and remember to invoke config.parse(), all commandline
41 arguments will play nicely together. This is done automatically for you
42 if you're using the bootstrap module's initialize wrapper.
46 [--module_do_the_thing MODULE_DO_THE_THING]
50 Args related to module doing the thing.
52 --module_do_the_thing MODULE_DO_THE_THING
53 Should the module do the thing?
56 A program that does the thing
59 Should we really do the thing?
61 Arguments themselves should be accessed via
62 config.config['arg_name']. e.g.
64 if not config.config['dry_run']:
74 from typing import Any, Dict, List, Optional
76 # This module is commonly used by others in here and should avoid
77 # taking any unnecessary dependencies back on them.
79 # Defer logging messages until later when logging has been initialized.
80 saved_messages: List[str] = []
82 # Make a copy of the original program arguments.
83 program_name: str = os.path.basename(sys.argv[0])
84 original_argv: List[str] = [arg for arg in sys.argv]
87 # A global parser that we will collect arguments into.
88 args = argparse.ArgumentParser(
90 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
91 fromfile_prefix_chars="@",
92 epilog=f'{program_name} uses config.py ({__file__}) for global, cross-module configuration setup and parsing.',
95 # Keep track of if we've been called and prevent being called more
97 config_parse_called = False
100 # A global configuration dictionary that will contain parsed arguments.
101 # It is also this variable that modules use to access parsed arguments.
102 # This is the data that is most interesting to our callers; it will hold
103 # the configuration result.
104 config: Dict[str, Any] = {}
105 # It would be really nice if this shit worked from interactive python
108 def add_commandline_args(title: str, description: str = ""):
109 """Create a new context for arguments and return a handle."""
110 return args.add_argument_group(title, description)
113 group = add_commandline_args(
114 f'Global Config ({__file__})',
115 'Args that control the global config itself; how meta!',
121 help='Config file (populated via --config_savefile) from which to read args in lieu or in addition to commandline.',
127 help='Display the global configuration (possibly derived from multiple sources) on STDERR at program startup.',
134 help='Populate config file compatible with --config_loadfile to save global config for later use.',
137 '--config_rejects_unrecognized_arguments',
141 'If present, config will raise an exception if it doesn\'t recognize an argument. The '
142 + 'default behavior is to ignore this so as to allow interoperability with programs that '
143 + 'want to use their own argparse calls to parse their own, separate commandline args.'
148 def is_flag_already_in_argv(var: str):
149 """Is a particular flag passed on the commandline?"""
156 def reorder_arg_action_groups(entry_module: Optional[str]):
157 global program_name, args
158 reordered_action_groups = []
159 for group in args._action_groups:
160 if entry_module is not None and entry_module in group.title: # type: ignore
161 reordered_action_groups.append(group)
162 elif program_name in group.title: # type: ignore
163 reordered_action_groups.append(group)
165 reordered_action_groups.insert(0, group)
166 return reordered_action_groups
169 def augment_sys_argv_from_environment_variables():
170 global saved_messages
171 usage_message = args.format_usage()
174 for x in usage_message.split():
188 # Environment vars the same as flag names without
189 # the initial -'s and in UPPERCASE.
190 env = var.strip('-').upper()
191 if env in os.environ:
192 if not is_flag_already_in_argv(var):
193 value = os.environ[env]
194 saved_messages.append(
195 f'Initialized from environment: {var} = {value}'
197 from string_utils import to_bool
199 if len(chunks) == 1 and to_bool(value):
201 elif len(chunks) > 1:
203 sys.argv.append(value)
208 def augment_sys_argv_from_loadfile():
209 global saved_messages
211 saw_other_args = False
212 grab_next_arg = False
213 for arg in sys.argv[1:]:
214 if 'config_loadfile' in arg:
215 pieces = arg.split('=')
223 saw_other_args = True
225 if loadfile is not None:
226 if not os.path.exists(loadfile):
228 f'ERROR: --config_loadfile argument must be a file, {loadfile} not found.'
231 msg = f'Augmenting commandline arguments with those from {loadfile}.'
233 msg = f'Reading commandline arguments from {loadfile}.'
234 print(msg, file=sys.stderr)
235 saved_messages.append(msg)
237 with open(loadfile, 'r') as rf:
238 newargs = rf.readlines()
239 newargs = [arg.strip('\n') for arg in newargs if 'config_savefile' not in arg]
243 def parse(entry_module: Optional[str]) -> Dict[str, Any]:
244 """Main program should call this early in main(). Note that the
245 bootstrap.initialize wrapper takes care of this automatically.
248 global config_parse_called
249 if config_parse_called:
251 global saved_messages
253 # If we're about to do the usage message dump, put the main
254 # module's argument group last in the list (if possible) so that
255 # when the user passes -h or --help, it will be visible on the
256 # screen w/o scrolling.
258 if arg == '--help' or arg == '-h':
259 args._action_groups = reorder_arg_action_groups(entry_module)
261 # Examine the environment for variables that match known flags.
262 # For a flag called --example_flag the corresponding environment
263 # variable would be called EXAMPLE_FLAG. If found, hackily add
264 # these into sys.argv to be parsed.
265 augment_sys_argv_from_environment_variables()
267 # Look for loadfile and read/parse it if present. This also
268 # works by jamming these values onto sys.argv.
269 augment_sys_argv_from_loadfile()
271 # Parse (possibly augmented, possibly completely overwritten)
272 # commandline args with argparse normally and populate config.
273 known, unknown = args.parse_known_args()
274 config.update(vars(known))
276 # Reconstruct the argv with unrecognized flags for the benefit of
277 # future argument parsers. For example, unittest_main in python
278 # has some of its own flags. If we didn't recognize it, maybe
281 if config['config_rejects_unrecognized_arguments']:
283 f'Encountered unrecognized config argument(s) {unknown} with --config_rejects_unrecognized_arguments enabled; halting.'
285 saved_messages.append(
286 f'Config encountered unrecognized commandline arguments: {unknown}'
288 sys.argv = sys.argv[:1] + unknown
290 # Check for savefile and populate it if requested.
291 savefile = config['config_savefile']
292 if savefile and len(savefile) > 0:
293 with open(savefile, 'w') as wf:
294 wf.write("\n".join(original_argv[1:]))
296 # Also dump the config on stderr if requested.
297 if config['config_dump']:
300 config_parse_called = True
304 def has_been_parsed() -> bool:
305 """Has the global config been parsed yet?"""
306 global config_parse_called
307 return config_parse_called
311 """Print the current config to stdout."""
312 print("Global Configuration:", file=sys.stderr)
313 pprint.pprint(config, stream=sys.stderr)
318 """Log messages saved earlier now that logging has been initialized."""
319 logger = logging.getLogger(__name__)
320 global saved_messages
321 for _ in saved_messages: