import sys
from typing import Optional
+import config
import decorator_utils
+cfg = config.add_commandline_args(
+ f'Lockfile ({__file__})',
+ 'Args related to lockfiles')
+cfg.add_argument(
+ '--lockfile_held_duration_warning_threshold_sec',
+ type=float,
+ default=10.0,
+ metavar='SECONDS',
+ help='If a lock is held for longer than this threshold we log a warning'
+)
logger = logging.getLogger(__name__)
class LockFile(object):
"""A file locking mechanism that has context-manager support so you
- can use it in a with statement.
- """
+ can use it in a with statement. e.g.
+
+ with LockFile('./foo.lock'):
+ # do a bunch of stuff... if the process dies we have a signal
+ # handler to do cleanup. Other code (in this process or another)
+ # that tries to take the same lockfile will block. There is also
+ # some logic for detecting stale locks.
+ """
def __init__(
self,
lockfile_path: str,
return True
except OSError:
pass
- logger.debug(f'Failed; I could not acquire {self.lockfile}.')
+ logger.warning(f'Could not acquire {self.lockfile}.')
return False
def acquire_with_retries(
def __enter__(self):
if self.acquire_with_retries():
+ self.locktime = datetime.datetime.now().timestamp()
return self
msg = f"Couldn't acquire {self.lockfile}; giving up."
logger.warning(msg)
raise LockFileException(msg)
def __exit__(self, type, value, traceback):
+ if self.locktime:
+ ts = datetime.datetime.now().timestamp()
+ duration = ts - self.locktime
+ if duration >= config.config['lockfile_held_duration_warning_threshold_sec']:
+ str_duration = datetime_utils.describe_duration_briefly(duration)
+ logger.warning(f'Held {self.lockfile} for {str_duration}')
self.release()
def __del__(self):
cmd = self.override_command
else:
cmd = ' '.join(sys.argv)
+ print(cmd)
contents = LockFileContents(
pid = os.getpid(),
- cmd,
+ commandline = cmd,
expiration_timestamp = self.expiration_timestamp,
)
return json.dumps(contents.__dict__)
try:
os.kill(contents.pid, 0)
except OSError:
- logger.debug('The pid seems stale; killing the lock.')
+ logger.warning(f'Lockfile {self.lockfile}\'s pid ({contents.pid}) is stale; ' +
+ 'force acquiring')
self.release()
# Has the lock expiration expired?
if contents.expiration_timestamp is not None:
now = datetime.datetime.now().timestamp()
if now > contents.expiration_datetime:
- logger.debug('The expiration time has passed; ' +
- 'killing the lock')
+ logger.warning(f'Lockfile {self.lockfile} expiration time has passed; ' +
+ 'force acquiring')
self.release()
except Exception:
pass