3 """Wrapper that adds exclusive locks, timeouts, timestamp accounting,
4 max frequency, logging, etc... to running cron jobs.
11 from typing import Optional
13 from pyutils import bootstrap, config, exec_utils, stopwatch
14 from pyutils.datetimes import datetime_utils
15 from pyutils.files import file_utils, lockfile
17 logger = logging.getLogger(__name__)
19 cfg = config.add_commandline_args(
20 f"Python Cron Runner ({__file__})",
21 "Wrapper for cron commands with locking, timeouts, and accounting.",
26 metavar="LOCKFILE_PATH",
27 help="Path to the lockfile to use to ensure that two instances of a command do not execute contemporaneously.",
30 "--lockfile_audit_record",
32 metavar="LOCKFILE_AUDIT_RECORD_FILENAME",
33 help="Path to a record of when the logfile was held/released and for what reason",
40 help='Maximum time for lock acquisition + command execution. Undecorated for seconds but "3m" or "1h 15m" work too.',
45 metavar="TIMESTAMP_FILE",
47 help="The /timestamp/TIMESTAMP_FILE file tracking the work being done; files' mtimes will be set to the last successful run of a command for accounting purposes.",
54 help='The maximum frequency with which to do this work; even if the wrapper is invoked more often than this it will not run the command. Requires --timestamp. Undecorated for seconds but "3h" or "1h 15m" work too.',
61 metavar="COMMANDLINE",
62 help="The commandline to run under a lock.",
64 config.overwrite_argparse_epilog(
68 -1000 = some internal error occurred (see exception log).
69 0 = we exited early due to not enough time passage since the last
70 invocation of --command.
71 1000 = we could not obtain the lockfile; someone else owns it.
72 else = if the --command was run successfully, cron.py will exit with
73 the same code that the subcommand exited with.
78 def run_command(timeout: Optional[int], timestamp_file: Optional[str]) -> int:
79 """Run cron command"""
80 cmd = " ".join(config.config["command"])
81 logger.info('cron cmd = "%s"', cmd)
82 logger.debug("shell environment:")
83 for var in os.environ:
85 logger.debug("%s = %s", var, val)
86 logger.debug("____ (↓↓↓ output from the subprocess appears below here ↓↓↓) ____")
88 with stopwatch.Timer() as t:
89 ret = exec_utils.cmd_exitcode(cmd, timeout)
91 f"____ (↑↑↑ subprocess finished in {t():.2f}s, exit value was {ret} ↑↑↑) ____"
93 if ret == 0 and timestamp_file is not None and os.path.exists(timestamp_file):
94 logger.debug("Touching %s", timestamp_file)
95 file_utils.touch_file(timestamp_file)
98 msg = "Cron subprocess failed; giving up."
100 print("Cron subprocess failed, giving up.", file=sys.stderr)
104 @bootstrap.initialize
107 if config.config["timestamp"]:
108 timestamp_file = f"/timestamps/{config.config['timestamp']}"
109 if not file_utils.does_file_exist(timestamp_file):
111 "--timestamp argument's target file (%s) must already exist.",
116 timestamp_file = None
117 if config.config["max_frequency"]:
119 "The --max_frequency argument requires the --timestamp argument."
122 now = datetime.datetime.now()
123 if timestamp_file is not None and os.path.exists(timestamp_file):
124 max_frequency = config.config["max_frequency"]
125 if max_frequency is not None:
126 max_delta = datetime_utils.parse_duration(max_frequency)
128 mtime = file_utils.get_file_mtime_as_datetime(timestamp_file)
130 if delta.total_seconds() < max_delta:
132 "It's only been %s since we last ran successfully; bailing out.",
133 datetime_utils.describe_duration_briefly(delta.total_seconds()),
137 timeout = config.config["timeout"]
138 if timeout is not None:
139 timeout = datetime_utils.parse_duration(timeout)
141 logger.debug("Timeout is %ss", timeout)
142 lockfile_expiration = datetime.datetime.now().timestamp() + timeout
144 logger.warning("Timeout not specified; no lockfile expiration.")
145 lockfile_expiration = None
147 lockfile_path = config.config["lockfile"]
148 if lockfile_path is not None:
149 logger.debug("Attempting to acquire lockfile %s...", lockfile_path)
151 with lockfile.LockFile(
153 do_signal_cleanup=True,
154 override_command=" ".join(config.config["command"]),
155 expiration_timestamp=lockfile_expiration,
157 record = config.config["lockfile_audit_record"]
158 cmd = " ".join(config.config["command"])
161 with open(record, "a") as wf:
162 print(f"{lockfile_path}, ACQUIRE, {start}, {cmd}", file=wf)
163 retval = run_command(timeout, timestamp_file)
165 end = datetime.datetime.now().timestamp()
166 duration = datetime_utils.describe_duration_briefly(end - start)
167 with open(record, "a") as wf:
169 f"{lockfile_path}, RELEASE({duration}), {end}, {cmd}",
173 except lockfile.LockFileException:
174 msg = f"Failed to acquire {lockfile_path}, giving up."
175 logger.exception(msg)
176 print(msg, file=sys.stderr)
179 logger.debug("No lockfile indicated; not locking anything.")
180 return run_command(timeout, timestamp_file)
183 if __name__ == "__main__":
184 # Insist that our logger.whatever('messages') make their way into
185 # syslog with a facility=LOG_CRON, please. Yes, this is hacky.
186 sys.argv.append("--logging_syslog")
187 sys.argv.append("--logging_syslog_facility=CRON")