3 """Utilities for working with files."""
13 from typing import Optional
15 from os.path import isfile, join, exists
16 from uuid import uuid4
19 logger = logging.getLogger(__name__)
22 def create_path_if_not_exist(path, on_error=None):
24 Attempts to create path if it does not exist. If on_error is
25 specified, it is called with an exception if one occurs, otherwise
26 exception is rethrown.
30 >>> path = os.path.join("/tmp", str(uuid.uuid4()), str(uuid.uuid4()))
31 >>> os.path.exists(path)
33 >>> create_path_if_not_exist(path)
34 >>> os.path.exists(path)
37 logger.debug(f"Creating path {path}")
38 previous_umask = os.umask(0)
43 if ex.errno != errno.EEXIST and not os.path.isdir(path):
44 if on_error is not None:
49 os.umask(previous_umask)
52 def does_file_exist(filename: str) -> bool:
53 return os.path.exists(filename) and os.path.isfile(filename)
56 def does_directory_exist(dirname: str) -> bool:
57 return os.path.exists(dirname) and os.path.isdir(dirname)
60 def does_path_exist(pathname: str) -> bool:
61 return os.path.exists(pathname)
64 def get_file_size(filename: str) -> int:
65 return os.path.getsize(filename)
68 def is_normal_file(filename: str) -> bool:
69 return os.path.isfile(filename)
72 def is_directory(filename: str) -> bool:
73 return os.path.isdir(filename)
76 def is_symlink(filename: str) -> bool:
77 return os.path.islink(filename)
80 def is_same_file(file1: str, file2: str) -> bool:
81 return os.path.samefile(file1, file2)
84 def get_file_raw_timestamps(filename: str) -> Optional[os.stat_result]:
86 return os.stat(filename)
87 except Exception as e:
92 def get_file_raw_timestamp(filename: str, extractor) -> Optional[float]:
93 tss = get_file_raw_timestamps(filename)
99 def get_file_raw_atime(filename: str) -> Optional[float]:
100 return get_file_raw_timestamp(filename, lambda x: x.st_atime)
103 def get_file_raw_mtime(filename: str) -> Optional[float]:
104 return get_file_raw_timestamp(filename, lambda x: x.st_mtime)
107 def get_file_raw_ctime(filename: str) -> Optional[float]:
108 return get_file_raw_timestamp(filename, lambda x: x.st_ctime)
111 def get_file_md5(filename: str) -> str:
112 file_hash = hashlib.md5()
113 with open(filename, "rb") as f:
116 file_hash.update(chunk)
118 return file_hash.hexdigest()
121 def set_file_raw_atime(filename: str, atime: float):
122 mtime = get_file_raw_mtime(filename)
123 os.utime(filename, (atime, mtime))
126 def set_file_raw_mtime(filename: str, mtime: float):
127 atime = get_file_raw_atime(filename)
128 os.utime(filename, (atime, mtime))
131 def set_file_raw_atime_and_mtime(filename: str, ts: float = None):
133 os.utime(filename, (ts, ts))
135 os.utime(filename, None)
138 def convert_file_timestamp_to_datetime(
139 filename: str, producer
140 ) -> Optional[datetime.datetime]:
141 ts = producer(filename)
143 return datetime.datetime.fromtimestamp(ts)
147 def get_file_atime_as_datetime(filename: str) -> Optional[datetime.datetime]:
148 return convert_file_timestamp_to_datetime(filename, get_file_raw_atime)
151 def get_file_mtime_as_datetime(filename: str) -> Optional[datetime.datetime]:
152 return convert_file_timestamp_to_datetime(filename, get_file_raw_mtime)
155 def get_file_ctime_as_datetime(filename: str) -> Optional[datetime.datetime]:
156 return convert_file_timestamp_to_datetime(filename, get_file_raw_ctime)
159 def get_file_timestamp_age_seconds(filename: str, extractor) -> Optional[int]:
161 ts = get_file_raw_timestamps(filename)
164 result = extractor(ts)
168 def get_file_atime_age_seconds(filename: str) -> Optional[int]:
169 return get_file_timestamp_age_seconds(filename, lambda x: x.st_atime)
172 def get_file_ctime_age_seconds(filename: str) -> Optional[int]:
173 return get_file_timestamp_age_seconds(filename, lambda x: x.st_ctime)
176 def get_file_mtime_age_seconds(filename: str) -> Optional[int]:
177 return get_file_timestamp_age_seconds(filename, lambda x: x.st_mtime)
180 def get_file_timestamp_timedelta(
181 filename: str, extractor
182 ) -> Optional[datetime.timedelta]:
183 age = get_file_timestamp_age_seconds(filename, extractor)
185 return datetime.timedelta(seconds=float(age))
189 def get_file_atime_timedelta(filename: str) -> Optional[datetime.timedelta]:
190 return get_file_timestamp_timedelta(filename, lambda x: x.st_atime)
193 def get_file_ctime_timedelta(filename: str) -> Optional[datetime.timedelta]:
194 return get_file_timestamp_timedelta(filename, lambda x: x.st_ctime)
197 def get_file_mtime_timedelta(filename: str) -> Optional[datetime.timedelta]:
198 return get_file_timestamp_timedelta(filename, lambda x: x.st_mtime)
201 def describe_file_timestamp(
202 filename: str, extractor, *, brief=False
204 from datetime_utils import describe_duration, describe_duration_briefly
205 age = get_file_timestamp_age_seconds(filename, extractor)
209 return describe_duration_briefly(age)
211 return describe_duration(age)
214 def describe_file_atime(filename: str, *, brief=False) -> Optional[str]:
215 return describe_file_timestamp(filename, lambda x: x.st_atime, brief=brief)
218 def describe_file_ctime(filename: str, *, brief=False) -> Optional[str]:
219 return describe_file_timestamp(filename, lambda x: x.st_ctime, brief=brief)
222 def describe_file_mtime(filename: str, *, brief=False) -> Optional[str]:
223 return describe_file_timestamp(filename, lambda x: x.st_mtime, brief=brief)
226 def touch_file(filename: str) -> bool:
227 return pathlib.Path(filename).touch()
230 def expand_globs(in_filename: str):
231 for filename in glob.glob(in_filename):
235 def get_files(directory: str):
236 for filename in os.listdir(directory):
237 full_path = join(directory, filename)
238 if isfile(full_path) and exists(full_path):
242 def get_directories(directory: str):
243 for d in os.listdir(directory):
244 full_path = join(directory, d)
245 if not isfile(full_path) and exists(full_path):
249 def get_files_recursive(directory: str):
250 for filename in get_files(directory):
252 for subdir in get_directories(directory):
253 for file_or_directory in get_files_recursive(subdir):
254 yield file_or_directory
257 class FileWriter(object):
258 def __init__(self, filename: str) -> None:
259 self.filename = filename
261 self.tempfile = f'{filename}-{uuid}.tmp'
264 def __enter__(self) -> io.TextIOWrapper:
265 assert not does_path_exist(self.tempfile)
266 self.handle = open(self.tempfile, mode="w")
269 def __exit__(self, exc_type, exc_val, exc_tb) -> bool:
270 if self.handle is not None:
272 cmd = f'/bin/mv -f {self.tempfile} {self.filename}'
275 raise Exception(f'{cmd} failed, exit value {ret>>8}')