#!/usr/bin/env python3
+# © Copyright 2021-2022, Scott Gasch
+
+"""Two predicates that can help avoid unnecessary disk I/O by
+detecting if a particular file is identical to the contents about to
+be written or if a particular directory already contains a file that
+is identical to the one to be written. See class docs below for
+examples.
+
+"""
+
import hashlib
import logging
import os
-from typing import Any, Optional
+from typing import Any, Dict, Optional, Set
logger = logging.getLogger(__name__)
>>> os.remove(testfile)
"""
+
def __init__(self, directory: str):
super().__init__()
import file_utils
+
if not file_utils.does_directory_exist(directory):
raise ValueError(directory)
self.directory = directory
- self.md5_by_filename = {}
- self.mtime_by_filename = {}
+ self.md5_by_filename: Dict[str, str] = {}
+ self.mtime_by_filename: Dict[str, float] = {}
self._update()
def _update(self):
def _update_file(self, filename: str, mtime: Optional[float] = None):
import file_utils
+
assert file_utils.does_file_exist(filename)
if mtime is None:
mtime = file_utils.get_file_raw_mtime(filename)
+ assert mtime is not None
if self.mtime_by_filename.get(filename, 0) != mtime:
md5 = file_utils.get_file_md5(filename)
- logger.debug(f'Computed/stored {filename}\'s MD5 at ts={mtime} ({md5})')
+ logger.debug('Computed/stored %s\'s MD5 at ts=%.2f (%s)', filename, mtime, md5)
self.mtime_by_filename[filename] = mtime
self.md5_by_filename[filename] = md5
def apply(self, item: Any, filename: str) -> bool:
self._update_file(filename)
file_md5 = self.md5_by_filename.get(filename, 0)
- logger.debug(f'{filename}\'s checksum is {file_md5}')
+ logger.debug('%s\'s checksum is %s', filename, file_md5)
mem_hash = hashlib.md5()
mem_hash.update(item)
md5 = mem_hash.hexdigest()
- logger.debug(f'Item\'s checksum is {md5}')
+ logger.debug('Item\'s checksum is %s', md5)
return md5 != file_md5
>>> os.remove(testfile)
"""
+
def __init__(self, directory: str):
- self.all_md5s = set()
+ self.all_md5s: Set[str] = set()
super().__init__(directory)
def _update_file(self, filename: str, mtime: Optional[float] = None):
import file_utils
+
assert file_utils.does_file_exist(filename)
if mtime is None:
mtime = file_utils.get_file_raw_mtime(filename)
+ assert mtime is not None
if self.mtime_by_filename.get(filename, 0) != mtime:
md5 = file_utils.get_file_md5(filename)
self.mtime_by_filename[filename] = mtime
self.md5_by_filename[filename] = md5
self.all_md5s.add(md5)
- def apply(self, item: Any) -> bool:
+ def apply(self, item: Any, ignored_filename: str = None) -> bool:
+ assert ignored_filename is None
self._update()
mem_hash = hashlib.md5()
mem_hash.update(item)
if __name__ == '__main__':
import doctest
+
doctest.testmod()