3 """Global configuration driven by commandline arguments, environment variables
4 and saved configuration files. This works across several modules.
12 parser = config.add_commandline_args(
14 "Args related to module doing the thing.",
17 "--module_do_the_thing",
20 help="Should the module do the thing?"
28 parser = config.add_commandline_args(
30 "A program that does the thing.",
36 help="Should we really do the thing?"
38 config.parse() # Very important, this must be invoked!
40 If you set this up and remember to invoke config.parse(), all commandline
41 arguments will play nicely together. This is done automatically for you
42 if you're using the bootstrap module's initialize wrapper.
46 [--module_do_the_thing MODULE_DO_THE_THING]
50 Args related to module doing the thing.
52 --module_do_the_thing MODULE_DO_THE_THING
53 Should the module do the thing?
56 A program that does the thing
59 Should we really do the thing?
61 Arguments themselves should be accessed via
62 config.config['arg_name']. e.g.
64 if not config.config['dry_run']:
74 from typing import Any, Dict, List, Optional
76 # This module is commonly used by others in here and should avoid
77 # taking any unnecessary dependencies back on them.
79 # Defer logging messages until later when logging has been initialized.
80 saved_messages: List[str] = []
82 # Make a copy of the original program arguments.
83 program_name = os.path.basename(sys.argv[0])
84 original_argv = [arg for arg in sys.argv]
87 # A global parser that we will collect arguments into.
88 args = argparse.ArgumentParser(
90 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
91 fromfile_prefix_chars="@",
92 epilog=f'------------------------------------------------------------------------------\n{program_name} uses config.py ({__file__}) for global, cross-module configuration setup and parsing.\n------------------------------------------------------------------------------'
95 # Keep track of if we've been called and prevent being called more
97 config_parse_called = False
99 # A global configuration dictionary that will contain parsed arguments.
100 # It is also this variable that modules use to access parsed arguments.
101 # This is the data that is most interesting to our callers; it will hold
102 # the configuration result.
103 config: Dict[str, Any] = {}
106 def add_commandline_args(title: str, description: str = ""):
107 """Create a new context for arguments and return a handle."""
108 return args.add_argument_group(title, description)
111 group = add_commandline_args(
112 f'Global Config ({__file__})',
113 'Args that control the global config itself; how meta!',
119 help='Config file (populated via --config_savefile) from which to read args in lieu or in addition to commandline.',
125 help='Display the global configuration on STDERR at program startup.',
132 help='Populate config file compatible with --config_loadfile to save config for later use.',
136 def is_flag_already_in_argv(var: str):
137 """Is a particular flag passed on the commandline?"""
144 def parse(entry_module: Optional[str]) -> Dict[str, Any]:
145 """Main program should call this early in main()"""
146 global config_parse_called
147 if config_parse_called:
150 global saved_messages
152 # If we're about to do the usage message dump, put the main module's
153 # argument group last in the list (if possible) so that when the user
154 # passes -h or --help, it will be visible on the screen w/o scrolling.
155 reordered_action_groups = []
158 if arg == '--help' or arg == '-h':
159 for group in args._action_groups:
160 if entry_module is not None and entry_module in group.title:
161 reordered_action_groups.append(group)
162 elif program_name in group.title:
163 reordered_action_groups.append(group)
165 reordered_action_groups.insert(0, group)
166 args._action_groups = reordered_action_groups
168 # Examine the environment variables that match known flags. For a
169 # flag called --example_flag the corresponding environment
170 # variable would be called EXAMPLE_FLAG.
171 usage_message = args.format_usage()
174 for x in usage_message.split():
188 # Environment vars the same as flag names without
189 # the initial -'s and in UPPERCASE.
190 env = var.strip('-').upper()
191 if env in os.environ:
192 if not is_flag_already_in_argv(var):
193 value = os.environ[env]
194 saved_messages.append(
195 f'Initialized from environment: {var} = {value}'
197 from string_utils import to_bool
198 if len(chunks) == 1 and to_bool(value):
200 elif len(chunks) > 1:
202 sys.argv.append(value)
208 # Look for loadfile and read/parse it if present.
210 saw_other_args = False
211 grab_next_arg = False
212 for arg in sys.argv[1:]:
213 if 'config_loadfile' in arg:
214 pieces = arg.split('=')
222 saw_other_args = True
224 if loadfile is not None:
226 msg = f'Augmenting commandline arguments with those from {loadfile}.'
227 print(msg, file=sys.stderr)
228 saved_messages.append(msg)
229 if not os.path.exists(loadfile):
230 print(f'ERROR: --config_loadfile argument must be a file, {loadfile} not found.',
233 with open(loadfile, 'r') as rf:
234 newargs = rf.readlines()
235 newargs = [arg.strip('\n') for arg in newargs if 'config_savefile' not in arg]
238 # Parse (possibly augmented, possibly completely overwritten)
239 # commandline args with argparse normally and populate config.
240 known, unknown = args.parse_known_args()
241 config.update(vars(known))
243 # Reconstruct the argv with unrecognized flags for the benefit of
244 # future argument parsers. For example, unittest_main in python
245 # has some of its own flags. If we didn't recognize it, maybe
247 sys.argv = sys.argv[:1] + unknown
249 # Check for savefile and populate it if requested.
250 savefile = config['config_savefile']
251 if savefile and len(savefile) > 0:
252 with open(savefile, 'w') as wf:
254 "\n".join(original_argv[1:])
257 # Also dump the config on stderr if requested.
258 if config['config_dump']:
261 config_parse_called = True
265 def has_been_parsed() -> bool:
266 """Has the global config been parsed yet?"""
267 global config_parse_called
268 return config_parse_called
272 """Print the current config to stdout."""
273 print("Global Configuration:", file=sys.stderr)
274 pprint.pprint(config, stream=sys.stderr)
279 """Log messages saved earlier now that logging has been initialized."""
280 logger = logging.getLogger(__name__)
281 global saved_messages
282 for _ in saved_messages: