3 """Utilities for working with files."""
11 from typing import Optional
13 from os.path import isfile, join, exists
18 logger = logging.getLogger(__name__)
21 def create_path_if_not_exist(path, on_error=None):
23 Attempts to create path if it does not exist. If on_error is
24 specified, it is called with an exception if one occurs, otherwise
25 exception is rethrown.
29 >>> path = os.path.join("/tmp", str(uuid.uuid4()), str(uuid.uuid4()))
30 >>> os.path.exists(path)
32 >>> create_path_if_not_exist(path)
33 >>> os.path.exists(path)
36 logger.debug(f"Creating path {path}")
37 previous_umask = os.umask(0)
42 if ex.errno != errno.EEXIST and not os.path.isdir(path):
43 if on_error is not None:
48 os.umask(previous_umask)
51 def does_file_exist(filename: str) -> bool:
52 return os.path.exists(filename) and os.path.isfile(filename)
55 def does_directory_exist(dirname: str) -> bool:
56 return os.path.exists(dirname) and os.path.isdir(dirname)
59 def does_path_exist(pathname: str) -> bool:
60 return os.path.exists(pathname)
63 def get_file_size(filename: str) -> int:
64 return os.path.getsize(filename)
67 def is_normal_file(filename: str) -> bool:
68 return os.path.isfile(filename)
71 def is_directory(filename: str) -> bool:
72 return os.path.isdir(filename)
75 def is_symlink(filename: str) -> bool:
76 return os.path.islink(filename)
79 def is_same_file(file1: str, file2: str) -> bool:
80 return os.path.samefile(file1, file2)
83 def get_file_raw_timestamps(filename: str) -> Optional[os.stat_result]:
85 return os.stat(filename)
86 except Exception as e:
91 def get_file_raw_timestamp(filename: str, extractor) -> Optional[float]:
92 tss = get_file_raw_timestamps(filename)
98 def get_file_raw_atime(filename: str) -> Optional[float]:
99 return get_file_raw_timestamp(filename, lambda x: x.st_atime)
102 def get_file_raw_mtime(filename: str) -> Optional[float]:
103 return get_file_raw_timestamp(filename, lambda x: x.st_mtime)
106 def get_file_raw_ctime(filename: str) -> Optional[float]:
107 return get_file_raw_timestamp(filename, lambda x: x.st_ctime)
110 def get_file_md5(filename: str) -> str:
111 file_hash = hashlib.md5()
112 with open(filename, "rb") as f:
115 file_hash.update(chunk)
117 return file_hash.hexdigest()
120 def set_file_raw_atime(filename: str, atime: float):
121 mtime = get_file_raw_mtime(filename)
122 os.utime(filename, (atime, mtime))
125 def set_file_raw_mtime(filename: str, mtime: float):
126 atime = get_file_raw_atime(filename)
127 os.utime(filename, (atime, mtime))
130 def set_file_raw_atime_and_mtime(filename: str, ts: float = None):
132 os.utime(filename, (ts, ts))
134 os.utime(filename, None)
137 def convert_file_timestamp_to_datetime(
138 filename: str, producer
139 ) -> Optional[datetime.datetime]:
140 ts = producer(filename)
142 return datetime.datetime.fromtimestamp(ts)
146 def get_file_atime_as_datetime(filename: str) -> Optional[datetime.datetime]:
147 return convert_file_timestamp_to_datetime(filename, get_file_raw_atime)
150 def get_file_mtime_as_datetime(filename: str) -> Optional[datetime.datetime]:
151 return convert_file_timestamp_to_datetime(filename, get_file_raw_mtime)
154 def get_file_ctime_as_datetime(filename: str) -> Optional[datetime.datetime]:
155 return convert_file_timestamp_to_datetime(filename, get_file_raw_ctime)
158 def get_file_timestamp_age_seconds(filename: str, extractor) -> Optional[int]:
160 ts = get_file_raw_timestamps(filename)
163 result = extractor(ts)
167 def get_file_atime_age_seconds(filename: str) -> Optional[int]:
168 return get_file_timestamp_age_seconds(filename, lambda x: x.st_atime)
171 def get_file_ctime_age_seconds(filename: str) -> Optional[int]:
172 return get_file_timestamp_age_seconds(filename, lambda x: x.st_ctime)
175 def get_file_mtime_age_seconds(filename: str) -> Optional[int]:
176 return get_file_timestamp_age_seconds(filename, lambda x: x.st_mtime)
179 def get_file_timestamp_timedelta(
180 filename: str, extractor
181 ) -> Optional[datetime.timedelta]:
182 age = get_file_timestamp_age_seconds(filename, extractor)
184 return datetime.timedelta(seconds=float(age))
188 def get_file_atime_timedelta(filename: str) -> Optional[datetime.timedelta]:
189 return get_file_timestamp_timedelta(filename, lambda x: x.st_atime)
192 def get_file_ctime_timedelta(filename: str) -> Optional[datetime.timedelta]:
193 return get_file_timestamp_timedelta(filename, lambda x: x.st_ctime)
196 def get_file_mtime_timedelta(filename: str) -> Optional[datetime.timedelta]:
197 return get_file_timestamp_timedelta(filename, lambda x: x.st_mtime)
200 def describe_file_timestamp(
201 filename: str, extractor, *, brief=False
203 age = get_file_timestamp_age_seconds(filename, extractor)
207 return datetime_utils.describe_duration_briefly(age)
209 return datetime_utils.describe_duration(age)
212 def describe_file_atime(filename: str, *, brief=False) -> Optional[str]:
213 return describe_file_timestamp(filename, lambda x: x.st_atime, brief=brief)
216 def describe_file_ctime(filename: str, *, brief=False) -> Optional[str]:
217 return describe_file_timestamp(filename, lambda x: x.st_ctime, brief=brief)
220 def describe_file_mtime(filename: str, *, brief=False) -> Optional[str]:
221 return describe_file_timestamp(filename, lambda x: x.st_mtime, brief=brief)
224 def expand_globs(in_filename: str):
225 for filename in glob.glob(in_filename):
229 def get_files(directory: str):
230 for filename in os.listdir(directory):
231 full_path = join(directory, filename)
232 if isfile(full_path) and exists(full_path):
236 def get_directories(directory: str):
237 for d in os.listdir(directory):
238 full_path = join(directory, d)
239 if not isfile(full_path) and exists(full_path):
243 def get_files_recursive(directory: str):
244 for filename in get_files(directory):
246 for subdir in get_directories(directory):
247 for file_or_directory in get_files_recursive(subdir):
248 yield file_or_directory