3 """Wrapper that adds exclusive locks, timeouts, timestamp accounting,
4 max frequency, logging, etc... to running cron jobs.
11 from typing import Optional
13 from pyutils import bootstrap, config, exec_utils, stopwatch
14 from pyutils.datetimez import datetime_utils
15 from pyutils.files import file_utils, lockfile
17 logger = logging.getLogger(__name__)
19 cfg = config.add_commandline_args(
20 f'Python Cron Runner ({__file__})',
21 'Wrapper for cron commands with locking, timeouts, and accounting.',
26 metavar='LOCKFILE_PATH',
27 help='Path to the lockfile to use to ensure that two instances of a command do not execute contemporaneously.',
30 '--lockfile_audit_record',
32 metavar='LOCKFILE_AUDIT_RECORD_FILENAME',
33 help='Path to a record of when the logfile was held/released and for what reason',
40 help='Maximum time for lock acquisition + command execution. Undecorated for seconds but "3m" or "1h 15m" work too.',
45 metavar='TIMESTAMP_FILE',
47 help='The /timestamp/TIMESTAMP_FILE file tracking the work being done; files\' mtimes will be set to the last successful run of a command for accounting purposes.',
54 help='The maximum frequency with which to do this work; even if the wrapper is invoked more often than this it will not run the command. Requires --timestamp. Undecorated for seconds but "3h" or "1h 15m" work too.',
61 metavar='COMMANDLINE',
62 help='The commandline to run under a lock.',
64 config.overwrite_argparse_epilog(
68 -1000 = some internal error occurred (see exception log).
69 0 = we exited early due to not enough time passage since the last
70 invocation of --command.
71 1000 = we could not obtain the lockfile; someone else owns it.
72 else = if the --command was run successfully, cron.py will exit with
73 the same code that the subcommand exited with.
78 def run_command(timeout: Optional[int], timestamp_file: Optional[str]) -> int:
79 """Run cron command"""
80 cmd = ' '.join(config.config['command'])
81 logger.info('cron cmd = "%s"', cmd)
82 logger.debug('shell environment:')
83 for var in os.environ:
85 logger.debug('%s = %s', var, val)
86 logger.debug('____ (↓↓↓ output from the subprocess appears below here ↓↓↓) ____')
88 with stopwatch.Timer() as t:
89 ret = exec_utils.cmd_exitcode(cmd, timeout)
91 f'____ (↑↑↑ subprocess finished in {t():.2f}s, exit value was {ret} ↑↑↑) ____'
93 if timestamp_file is not None and os.path.exists(timestamp_file):
94 logger.debug('Touching %s', timestamp_file)
95 file_utils.touch_file(timestamp_file)
97 except Exception as e:
99 print('Cron subprocess failed, giving up.', file=sys.stderr)
100 logger.warning('Cron subprocess failed, giving up')
104 @bootstrap.initialize
107 if config.config['timestamp']:
108 timestamp_file = f"/timestamps/{config.config['timestamp']}"
109 if not file_utils.does_file_exist(timestamp_file):
111 '--timestamp argument\'s target file (%s) must already exist.',
116 timestamp_file = None
117 if config.config['max_frequency']:
119 'The --max_frequency argument requires the --timestamp argument.'
122 now = datetime.datetime.now()
123 if timestamp_file is not None and os.path.exists(timestamp_file):
124 max_frequency = config.config['max_frequency']
125 if max_frequency is not None:
126 max_delta = datetime_utils.parse_duration(max_frequency)
128 mtime = file_utils.get_file_mtime_as_datetime(timestamp_file)
130 if delta.total_seconds() < max_delta:
132 "It's only been %s since we last ran successfully; bailing out.",
133 datetime_utils.describe_duration_briefly(delta.total_seconds()),
137 timeout = config.config['timeout']
138 if timeout is not None:
139 timeout = datetime_utils.parse_duration(timeout)
141 logger.debug('Timeout is %ss', timeout)
142 lockfile_expiration = datetime.datetime.now().timestamp() + timeout
144 logger.debug('Timeout not specified; no lockfile expiration.')
145 lockfile_expiration = None
147 lockfile_path = config.config['lockfile']
148 if lockfile_path is not None:
149 logger.debug('Attempting to acquire lockfile %s...', lockfile_path)
151 with lockfile.LockFile(
153 do_signal_cleanup=True,
154 override_command=' '.join(config.config['command']),
155 expiration_timestamp=lockfile_expiration,
157 record = config.config['lockfile_audit_record']
158 cmd = ' '.join(config.config['command'])
161 with open(record, 'a') as wf:
162 print(f'{lockfile_path}, ACQUIRE, {start}, {cmd}', file=wf)
163 retval = run_command(timeout, timestamp_file)
165 end = datetime.datetime.now().timestamp()
166 duration = datetime_utils.describe_duration_briefly(end - start)
167 with open(record, 'a') as wf:
169 f'{lockfile_path}, RELEASE({duration}), {end}, {cmd}',
173 except lockfile.LockFileException as e:
175 msg = f'Failed to acquire {lockfile_path}, giving up.'
177 print(msg, file=sys.stderr)
180 logger.debug('No lockfile indicated; not locking anything.')
181 return run_command(timeout, timestamp_file)
184 if __name__ == '__main__':
185 # Insist that our logger.whatever('messages') make their way into
186 # syslog with a facility=LOG_CRON, please. Yes, this is hacky.
187 sys.argv.append('--logging_syslog')
188 sys.argv.append('--logging_syslog_facility=CRON')