3 # © Copyright 2021-2023, Scott Gasch
6 This is a grab bag of file-related utilities. It has code to, for example,
7 read files transforming the text as its read, normalize pathnames, strip
8 extensions, read and manipulate atimes/mtimes/ctimes, compute a signature
9 based on a file's contents, traverse the file system recursively, etc...
23 from os.path import exists, isfile, join
24 from typing import IO, Any, Callable, List, Literal, Optional
25 from uuid import uuid4
27 logger = logging.getLogger(__name__)
30 def remove_newlines(x: str) -> str:
31 """Trivial function to be used as a line_transformer in
32 :meth:`slurp_file` for no newlines in file contents"""
33 return x.replace("\n", "")
36 def strip_whitespace(x: str) -> str:
37 """Trivial function to be used as a line_transformer in
38 :meth:`slurp_file` for no leading / trailing whitespace in
43 def remove_hash_comments(x: str) -> str:
44 """Trivial function to be used as a line_transformer in
45 :meth:`slurp_file` for no # comments in file contents"""
46 return re.sub(r"#.*$", "", x)
52 skip_blank_lines: bool = False,
53 line_transformers: Optional[List[Callable[[str], str]]] = None,
55 """Reads in a file's contents line-by-line to a memory buffer applying
56 each line transformation in turn.
59 filename: file to be read
60 skip_blank_lines: should reading skip blank lines?
61 line_transformers: little string->string transformations
64 A list of lines from the read and transformed file contents.
67 Exception: filename not found or can't be read.
72 if line_transformers is not None:
73 for x in line_transformers:
75 if not is_readable(filename):
76 raise Exception(f"{filename} can't be read.")
77 with open(filename) as rf:
79 for transformation in xforms:
80 line = transformation(line)
81 if skip_blank_lines and line == "":
87 def remove(path: str) -> None:
88 """Deletes a file. Raises if path refers to a directory or a file
92 path: the path of the file to delete
95 >>> filename = '/tmp/file_utils_test_file'
96 >>> os.system(f'touch {filename}')
98 >>> does_file_exist(filename)
101 >>> does_file_exist(filename)
107 def fix_multiple_slashes(path: str) -> str:
108 """Fixes multi-slashes in paths or path-like strings
111 path: the path in which to remove multiple slashes
113 >>> p = '/usr/local//etc/rc.d///file.txt'
114 >>> fix_multiple_slashes(p)
115 '/usr/local/etc/rc.d/file.txt'
117 >>> p = 'this is a test'
118 >>> fix_multiple_slashes(p) == p
121 return re.sub(r"/+", "/", path)
124 def delete(path: str) -> None:
125 """This is a convenience for my dumb ass who can't remember os.remove
131 def without_extension(path: str) -> str:
132 """Remove one (the last) extension from a file or path.
135 path: the path from which to remove an extension
138 the path with one extension removed.
140 See also :meth:`without_all_extensions`.
142 >>> without_extension('foobar.txt')
145 >>> without_extension('/home/scott/frapp.py')
148 >>> f = 'a.b.c.tar.gz'
150 ... f = without_extension(f)
157 >>> without_extension('foobar')
161 return os.path.splitext(path)[0]
164 def without_all_extensions(path: str) -> str:
165 """Removes all extensions from a path; handles multiple extensions
166 like foobar.tar.gz -> foobar.
169 path: the path from which to remove all extensions
172 the path with all extensions removed.
174 See also :meth:`without_extension`
176 >>> without_all_extensions('/home/scott/foobar.1.tar.gz')
181 path = without_extension(path)
185 def get_extension(path: str) -> str:
186 """Extract and return one (the last) extension from a file or path.
189 path: the path from which to extract an extension
192 The last extension from the file path.
194 See also :meth:`without_extension`, :meth:`without_all_extensions`,
195 :meth:`get_all_extensions`.
197 >>> get_extension('this_is_a_test.txt')
200 >>> get_extension('/home/scott/test.py')
203 >>> get_extension('foobar')
207 return os.path.splitext(path)[1]
210 def get_all_extensions(path: str) -> List[str]:
211 """Return the extensions of a file or path in order.
214 path: the path from which to extract all extensions.
217 a list containing each extension which may be empty.
219 See also :meth:`without_extension`, :meth:`without_all_extensions`,
220 :meth:`get_extension`.
222 >>> get_all_extensions('/home/scott/foo.tar.gz.1')
223 ['.tar', '.gz', '.1']
225 >>> get_all_extensions('/home/scott/foobar')
231 ext = get_extension(path)
232 path = without_extension(path)
240 def without_path(filespec: str) -> str:
241 """Returns the base filename without any leading path.
244 filespec: path to remove leading directories from
247 filespec without leading dir components.
249 See also :meth:`get_path`, :meth:`get_canonical_path`.
251 >>> without_path('/home/scott/foo.py')
254 >>> without_path('foo.py')
258 return os.path.split(filespec)[1]
261 def get_path(filespec: str) -> str:
262 """Returns just the path of the filespec by removing the filename and
266 filespec: path to remove filename / extension(s) from
269 filespec with just the leading directory components and no
270 filename or extension(s)
272 See also :meth:`without_path`, :meth:`get_canonical_path`.
274 >>> get_path('/home/scott/foobar.py')
277 >>> get_path('/home/scott/test.1.2.3.gz')
280 >>> get_path('~scott/frapp.txt')
284 return os.path.split(filespec)[0]
287 def get_canonical_path(filespec: str) -> str:
288 """Returns a canonicalized absolute path.
291 filespec: the path to canonicalize
294 the canonicalized path
296 See also :meth:`get_path`, :meth:`without_path`.
298 >>> get_canonical_path('/home/scott/../../home/lynn/../scott/foo.txt')
299 '/usr/home/scott/foo.txt'
302 return os.path.realpath(filespec)
305 def create_path_if_not_exist(
306 path: str, on_error: Callable[[str, OSError], None] = None
309 Attempts to create path if it does not exist already.
312 path: the path to attempt to create
313 on_error: If set, it's invoked on error conditions and passed then
314 path and OSError that it caused.
317 OSError: an exception occurred and on_error not set.
319 See also :meth:`does_file_exist`.
323 Files are created with mode 0o0777 (i.e. world read/writeable).
327 >>> path = os.path.join("/tmp", str(uuid.uuid4()), str(uuid.uuid4()))
328 >>> os.path.exists(path)
330 >>> create_path_if_not_exist(path)
331 >>> os.path.exists(path)
334 logger.debug("Creating path %s", path)
335 previous_umask = os.umask(0)
338 os.chmod(path, 0o777)
339 except OSError as ex:
340 if ex.errno != errno.EEXIST and not os.path.isdir(path):
341 if on_error is not None:
346 os.umask(previous_umask)
349 def does_file_exist(filename: str) -> bool:
350 """Returns True if a file exists and is a normal file.
353 filename: filename to check
356 True if filename exists and is a normal file.
358 See also :meth:`create_path_if_not_exist`, :meth:`is_readable`.
360 >>> does_file_exist(__file__)
362 >>> does_file_exist('/tmp/2492043r9203r9230r9230r49230r42390r4230')
365 return os.path.exists(filename) and os.path.isfile(filename)
368 def is_readable(filename: str) -> bool:
369 """Is the file readable?
372 filename: the filename to check for read access
375 True if the file exists, is a normal file, and is readable
376 by the current process. False otherwise.
378 See also :meth:`does_file_exist`, :meth:`is_writable`,
379 :meth:`is_executable`.
381 return os.access(filename, os.R_OK)
384 def is_writable(filename: str) -> bool:
385 """Is the file writable?
388 filename: the file to check for write access.
391 True if file exists, is a normal file and is writable by the
392 current process. False otherwise.
394 See also :meth:`is_readable`, :meth:`does_file_exist`.
396 return os.access(filename, os.W_OK)
399 def is_executable(filename: str) -> bool:
400 """Is the file executable?
403 filename: the file to check for execute access.
406 True if file exists, is a normal file and is executable by the
407 current process. False otherwise.
409 See also :meth:`does_file_exist`, :meth:`is_readable`,
412 return os.access(filename, os.X_OK)
415 def does_directory_exist(dirname: str) -> bool:
416 """Does the given directory exist?
419 dirname: the name of the directory to check
422 True if a path exists and is a directory, not a regular file.
424 See also :meth:`does_file_exist`.
426 >>> does_directory_exist('/tmp')
428 >>> does_directory_exist('/xyzq/21341')
431 return os.path.exists(dirname) and os.path.isdir(dirname)
434 def does_path_exist(pathname: str) -> bool:
435 """Just a more verbose wrapper around os.path.exists."""
436 return os.path.exists(pathname)
439 def get_file_size(filename: str) -> int:
440 """Returns the size of a file in bytes.
443 filename: the filename to size
446 size of filename in bytes
448 return os.path.getsize(filename)
451 def is_normal_file(filename: str) -> bool:
452 """Is that file normal (not a directory or some special file?)
455 filename: the path of the file to check
458 True if filename is a normal file.
460 See also :meth:`is_directory`, :meth:`does_file_exist`, :meth:`is_symlink`.
462 >>> is_normal_file(__file__)
465 return os.path.isfile(filename)
468 def is_directory(filename: str) -> bool:
469 """Is that path a directory (not a normal file?)
472 filename: the path of the file to check
475 True if filename is a directory
477 See also :meth:`does_directory_exist`, :meth:`is_normal_file`,
480 >>> is_directory('/tmp')
483 return os.path.isdir(filename)
486 def is_symlink(filename: str) -> bool:
487 """Is that path a symlink?
490 filename: the path of the file to check
493 True if filename is a symlink, False otherwise.
495 See also :meth:`is_directory`, :meth:`is_normal_file`.
497 >>> is_symlink('/tmp')
500 >>> is_symlink('/home')
503 return os.path.islink(filename)
506 def is_same_file(file1: str, file2: str) -> bool:
507 """Determine if two paths reference the same inode.
510 file1: the first file
511 file2: the second file
514 True if the two files are the same file.
516 See also :meth:`is_symlink`, :meth:`is_normal_file`.
518 >>> is_same_file('/tmp', '/tmp/../tmp')
521 >>> is_same_file('/tmp', '/home')
524 return os.path.samefile(file1, file2)
527 def get_file_raw_timestamps(filename: str) -> Optional[os.stat_result]:
528 """Stats the file and returns an `os.stat_result` or None on error.
531 filename: the file whose timestamps to fetch
534 the os.stat_result or None to indicate an error occurred
537 :meth:`get_file_raw_atime`,
538 :meth:`get_file_raw_ctime`,
539 :meth:`get_file_raw_mtime`,
540 :meth:`get_file_raw_timestamp`
543 return os.stat(filename)
545 logger.exception("Failed to stat path %s; returning None", filename)
549 def get_file_raw_timestamp(
550 filename: str, extractor: Callable[[os.stat_result], Optional[float]]
551 ) -> Optional[float]:
552 """Stat a file and, if successful, use extractor to fetch some
553 subset of the information in the `os.stat_result`.
556 filename: the filename to stat
557 extractor: Callable that takes a os.stat_result and produces
558 something useful(?) with it.
561 whatever the extractor produced or None on error.
564 :meth:`get_file_raw_atime`,
565 :meth:`get_file_raw_ctime`,
566 :meth:`get_file_raw_mtime`,
567 :meth:`get_file_raw_timestamps`
569 tss = get_file_raw_timestamps(filename)
571 return extractor(tss)
575 def get_file_raw_atime(filename: str) -> Optional[float]:
576 """Get a file's raw access time.
579 filename: the path to the file to stat
582 The file's raw atime (seconds since the Epoch) or
586 :meth:`get_file_atime_age_seconds`,
587 :meth:`get_file_atime_as_datetime`,
588 :meth:`get_file_atime_timedelta`,
589 :meth:`get_file_raw_ctime`,
590 :meth:`get_file_raw_mtime`,
591 :meth:`get_file_raw_timestamps`
593 return get_file_raw_timestamp(filename, lambda x: x.st_atime)
596 def get_file_raw_mtime(filename: str) -> Optional[float]:
597 """Get a file's raw modification time.
600 filename: the path to the file to stat
603 The file's raw mtime (seconds since the Epoch) or
607 :meth:`get_file_raw_atime`,
608 :meth:`get_file_raw_ctime`,
609 :meth:`get_file_mtime_age_seconds`,
610 :meth:`get_file_mtime_as_datetime`,
611 :meth:`get_file_mtime_timedelta`,
612 :meth:`get_file_raw_timestamps`
614 return get_file_raw_timestamp(filename, lambda x: x.st_mtime)
617 def get_file_raw_ctime(filename: str) -> Optional[float]:
618 """Get a file's raw creation time.
621 filename: the path to the file to stat
624 The file's raw ctime (seconds since the Epoch) or
628 :meth:`get_file_raw_atime`,
629 :meth:`get_file_ctime_age_seconds`,
630 :meth:`get_file_ctime_as_datetime`,
631 :meth:`get_file_ctime_timedelta`,
632 :meth:`get_file_raw_mtime`,
633 :meth:`get_file_raw_timestamps`
635 return get_file_raw_timestamp(filename, lambda x: x.st_ctime)
638 def get_file_md5(filename: str) -> str:
639 """Hashes filename's disk contents and returns the MD5 digest.
642 filename: the file whose contents to hash
645 the MD5 digest of the file's contents. Raises on error.
647 file_hash = hashlib.md5()
648 with open(filename, "rb") as f:
651 file_hash.update(chunk)
653 return file_hash.hexdigest()
656 def set_file_raw_atime(filename: str, atime: float) -> None:
657 """Sets a file's raw access time.
660 filename: the file whose atime should be set
661 atime: raw atime as number of seconds since the Epoch to set
664 :meth:`get_file_raw_atime`,
665 :meth:`get_file_atime_age_seconds`,
666 :meth:`get_file_atime_as_datetime`,
667 :meth:`get_file_atime_timedelta`,
668 :meth:`get_file_raw_timestamps`,
669 :meth:`set_file_raw_mtime`,
670 :meth:`set_file_raw_atime_and_mtime`,
673 mtime = get_file_raw_mtime(filename)
674 assert mtime is not None
675 os.utime(filename, (atime, mtime))
678 def set_file_raw_mtime(filename: str, mtime: float):
679 """Sets a file's raw modification time.
682 filename: the file whose mtime should be set
683 mtime: the raw mtime as number of seconds since the Epoch to set
686 :meth:`get_file_raw_mtime`,
687 :meth:`get_file_mtime_age_seconds`,
688 :meth:`get_file_mtime_as_datetime`,
689 :meth:`get_file_mtime_timedelta`,
690 :meth:`get_file_raw_timestamps`,
691 :meth:`set_file_raw_atime`,
692 :meth:`set_file_raw_atime_and_mtime`,
695 atime = get_file_raw_atime(filename)
696 assert atime is not None
697 os.utime(filename, (atime, mtime))
700 def set_file_raw_atime_and_mtime(filename: str, ts: float = None):
701 """Sets both a file's raw modification and access times.
704 filename: the file whose times to set
705 ts: the raw time to set or None to indicate time should be
706 set to the current time.
709 :meth:`get_file_raw_atime`,
710 :meth:`get_file_raw_mtime`,
711 :meth:`get_file_raw_timestamps`,
712 :meth:`set_file_raw_atime`,
713 :meth:`set_file_raw_mtime`
716 os.utime(filename, (ts, ts))
718 os.utime(filename, None)
721 def _convert_file_timestamp_to_datetime(
723 producer: Callable[[str], Optional[float]],
724 ) -> Optional[datetime.datetime]:
726 Converts a raw file timestamp into a Python datetime.
729 filename: file whose timestamps should be converted.
730 producer: source of the timestamp.
734 ts = producer(filename)
736 return datetime.datetime.fromtimestamp(ts)
740 def get_file_atime_as_datetime(filename: str) -> Optional[datetime.datetime]:
741 """Fetch a file's access time as a Python datetime.
744 filename: the file whose atime should be fetched.
747 The file's atime as a Python :class:`datetime.datetime`.
750 :meth:`get_file_raw_atime`,
751 :meth:`get_file_atime_age_seconds`,
752 :meth:`get_file_atime_timedelta`,
753 :meth:`get_file_raw_ctime`,
754 :meth:`get_file_raw_mtime`,
755 :meth:`get_file_raw_timestamps`,
756 :meth:`set_file_raw_atime`,
757 :meth:`set_file_raw_atime_and_mtime`
759 return _convert_file_timestamp_to_datetime(filename, get_file_raw_atime)
762 def get_file_mtime_as_datetime(filename: str) -> Optional[datetime.datetime]:
763 """Fetch a file's modification time as a Python datetime.
766 filename: the file whose mtime should be fetched.
769 The file's mtime as a Python :class:`datetime.datetime`.
772 :meth:`get_file_raw_mtime`,
773 :meth:`get_file_mtime_age_seconds`,
774 :meth:`get_file_mtime_timedelta`,
775 :meth:`get_file_raw_ctime`,
776 :meth:`get_file_raw_atime`,
777 :meth:`get_file_raw_timestamps`,
778 :meth:`set_file_raw_atime`,
779 :meth:`set_file_raw_atime_and_mtime`
781 return _convert_file_timestamp_to_datetime(filename, get_file_raw_mtime)
784 def get_file_ctime_as_datetime(filename: str) -> Optional[datetime.datetime]:
785 """Fetches a file's creation time as a Python datetime.
788 filename: the file whose ctime should be fetched.
791 The file's ctime as a Python :class:`datetime.datetime`.
794 :meth:`get_file_raw_ctime`,
795 :meth:`get_file_ctime_age_seconds`,
796 :meth:`get_file_ctime_timedelta`,
797 :meth:`get_file_raw_atime`,
798 :meth:`get_file_raw_mtime`,
799 :meth:`get_file_raw_timestamps`
801 return _convert_file_timestamp_to_datetime(filename, get_file_raw_ctime)
804 def _get_file_timestamp_age_seconds(filename: str, extractor) -> Optional[int]:
805 """~Internal helper"""
807 ts = get_file_raw_timestamps(filename)
810 result = extractor(ts)
814 def get_file_atime_age_seconds(filename: str) -> Optional[int]:
815 """Gets a file's access time as an age in seconds (ago).
818 filename: file whose atime should be checked.
821 The number of seconds ago that filename was last accessed.
824 :meth:`get_file_raw_atime`,
825 :meth:`get_file_atime_as_datetime`,
826 :meth:`get_file_atime_timedelta`,
827 :meth:`get_file_raw_ctime`,
828 :meth:`get_file_raw_mtime`,
829 :meth:`get_file_raw_timestamps`,
830 :meth:`set_file_raw_atime`,
831 :meth:`set_file_raw_atime_and_mtime`
833 return _get_file_timestamp_age_seconds(filename, lambda x: x.st_atime)
836 def get_file_ctime_age_seconds(filename: str) -> Optional[int]:
837 """Gets a file's creation time as an age in seconds (ago).
840 filename: file whose ctime should be checked.
843 The number of seconds ago that filename was created.
846 :meth:`get_file_raw_ctime`,
847 :meth:`get_file_ctime_age_seconds`,
848 :meth:`get_file_ctime_as_datetime`,
849 :meth:`get_file_ctime_timedelta`,
850 :meth:`get_file_raw_mtime`,
851 :meth:`get_file_raw_atime`,
852 :meth:`get_file_raw_timestamps`
854 return _get_file_timestamp_age_seconds(filename, lambda x: x.st_ctime)
857 def get_file_mtime_age_seconds(filename: str) -> Optional[int]:
858 """Gets a file's modification time as seconds (ago).
861 filename: file whose mtime should be checked.
864 The number of seconds ago that filename was last modified.
867 :meth:`get_file_raw_atime`,
868 :meth:`get_file_raw_ctime`,
869 :meth:`get_file_raw_mtime`,
870 :meth:`get_file_mtime_as_datetime`,
871 :meth:`get_file_mtime_timedelta`,
872 :meth:`get_file_raw_timestamps`,
873 :meth:`set_file_raw_atime`,
874 :meth:`set_file_raw_atime_and_mtime`
876 return _get_file_timestamp_age_seconds(filename, lambda x: x.st_mtime)
879 def _get_file_timestamp_timedelta(
880 filename: str, extractor
881 ) -> Optional[datetime.timedelta]:
882 """~Internal helper"""
883 age = _get_file_timestamp_age_seconds(filename, extractor)
885 return datetime.timedelta(seconds=float(age))
889 def get_file_atime_timedelta(filename: str) -> Optional[datetime.timedelta]:
890 """How long ago was a file accessed as a timedelta?
893 filename: the file whose atime should be checked.
896 A Python :class:`datetime.timedelta` representing how long
897 ago filename was last accessed.
900 :meth:`get_file_raw_atime`,
901 :meth:`get_file_atime_age_seconds`,
902 :meth:`get_file_atime_as_datetime`,
903 :meth:`get_file_raw_ctime`,
904 :meth:`get_file_raw_mtime`,
905 :meth:`get_file_raw_timestamps`,
906 :meth:`set_file_raw_atime`,
907 :meth:`set_file_raw_atime_and_mtime`
909 return _get_file_timestamp_timedelta(filename, lambda x: x.st_atime)
912 def get_file_ctime_timedelta(filename: str) -> Optional[datetime.timedelta]:
913 """How long ago was a file created as a timedelta?
916 filename: the file whose ctime should be checked.
919 A Python :class:`datetime.timedelta` representing how long
920 ago filename was created.
923 :meth:`get_file_raw_atime`,
924 :meth:`get_file_raw_ctime`,
925 :meth:`get_file_ctime_age_seconds`,
926 :meth:`get_file_ctime_as_datetime`,
927 :meth:`get_file_raw_mtime`,
928 :meth:`get_file_raw_timestamps`
930 return _get_file_timestamp_timedelta(filename, lambda x: x.st_ctime)
933 def get_file_mtime_timedelta(filename: str) -> Optional[datetime.timedelta]:
935 Gets a file's modification time as a Python timedelta.
938 filename: the file whose mtime should be checked.
941 A Python :class:`datetime.timedelta` representing how long
942 ago filename was last modified.
945 :meth:`get_file_raw_atime`,
946 :meth:`get_file_raw_ctime`,
947 :meth:`get_file_raw_mtime`,
948 :meth:`get_file_mtime_age_seconds`,
949 :meth:`get_file_mtime_as_datetime`,
950 :meth:`get_file_raw_timestamps`,
951 :meth:`set_file_raw_atime`,
952 :meth:`set_file_raw_atime_and_mtime`
954 return _get_file_timestamp_timedelta(filename, lambda x: x.st_mtime)
957 def describe_file_timestamp(filename: str, extractor, *, brief=False) -> Optional[str]:
958 """~Internal helper"""
959 from pyutils.datetimes.datetime_utils import (
961 describe_duration_briefly,
964 age = _get_file_timestamp_age_seconds(filename, extractor)
968 return describe_duration_briefly(age)
970 return describe_duration(age)
973 def describe_file_atime(filename: str, *, brief: bool = False) -> Optional[str]:
975 Describe how long ago a file was accessed.
978 filename: the file whose atime should be described.
979 brief: if True, describe atime briefly.
982 A string that represents how long ago filename was last
983 accessed. The description will be verbose or brief depending
984 on the brief argument.
987 :meth:`get_file_raw_atime`,
988 :meth:`get_file_atime_age_seconds`,
989 :meth:`get_file_atime_as_datetime`,
990 :meth:`get_file_atime_timedelta`,
991 :meth:`get_file_raw_ctime`,
992 :meth:`get_file_raw_mtime`,
993 :meth:`get_file_raw_timestamps`
994 :meth:`set_file_raw_atime`,
995 :meth:`set_file_raw_atime_and_mtime`
997 return describe_file_timestamp(filename, lambda x: x.st_atime, brief=brief)
1000 def describe_file_ctime(filename: str, *, brief: bool = False) -> Optional[str]:
1001 """Describes a file's creation time.
1004 filename: the file whose ctime should be described.
1005 brief: if True, describe ctime briefly.
1008 A string that represents how long ago filename was created.
1009 The description will be verbose or brief depending
1010 on the brief argument.
1013 :meth:`get_file_raw_atime`,
1014 :meth:`get_file_raw_ctime`,
1015 :meth:`get_file_ctime_age_seconds`,
1016 :meth:`get_file_ctime_as_datetime`,
1017 :meth:`get_file_ctime_timedelta`,
1018 :meth:`get_file_raw_mtime`,
1019 :meth:`get_file_raw_timestamps`
1021 return describe_file_timestamp(filename, lambda x: x.st_ctime, brief=brief)
1024 def describe_file_mtime(filename: str, *, brief: bool = False) -> Optional[str]:
1025 """Describes how long ago a file was modified.
1028 filename: the file whose mtime should be described.
1029 brief: if True, describe mtime briefly.
1032 A string that represents how long ago filename was last
1033 modified. The description will be verbose or brief depending
1034 on the brief argument.
1037 :meth:`get_file_raw_atime`,
1038 :meth:`get_file_raw_ctime`,
1039 :meth:`get_file_raw_mtime`,
1040 :meth:`get_file_mtime_age_seconds`,
1041 :meth:`get_file_mtime_as_datetime`,
1042 :meth:`get_file_mtime_timedelta`,
1043 :meth:`get_file_raw_timestamps`,
1044 :meth:`set_file_raw_atime`,
1045 :meth:`set_file_raw_atime_and_mtime`
1047 return describe_file_timestamp(filename, lambda x: x.st_mtime, brief=brief)
1050 def touch_file(filename: str, *, mode: Optional[int] = 0o666):
1051 """Like unix "touch" command's semantics: update the timestamp
1052 of a file to the current time if the file exists. Create the
1053 file if it doesn't exist.
1056 filename: the filename
1057 mode: the mode to create the file with
1061 The default creation mode is 0o666 which is world readable
1062 and writable. Override this by passing in your own mode
1063 parameter if desired.
1065 See also :meth:`set_file_raw_atime`, :meth:`set_file_raw_atime_and_mtime`,
1066 :meth:`set_file_raw_mtime`, :meth:`create_path_if_not_exist`
1068 pathlib.Path(filename, mode=mode).touch()
1071 def expand_globs(in_filename: str):
1073 Expands shell globs (* and ? wildcards) to the matching files.
1076 in_filename: the filepath to be expanded. May contain '*' and '?'
1077 globbing characters.
1080 A Generator that yields filenames that match the input pattern.
1082 See also :meth:`get_files`, :meth:`get_files_recursive`.
1084 for filename in glob.glob(in_filename):
1088 def get_files(directory: str):
1089 """Returns the files in a directory as a generator.
1092 directory: the directory to list files under.
1095 A generator that yields all files in the input directory.
1097 See also :meth:`expand_globs`, :meth:`get_files_recursive`,
1098 :meth:`get_matching_files`.
1100 for filename in os.listdir(directory):
1101 full_path = join(directory, filename)
1102 if isfile(full_path) and exists(full_path):
1106 def get_matching_files(directory: str, glob_string: str):
1108 Returns the subset of files whose name matches a glob.
1111 directory: the directory to match files within.
1112 glob_string: the globbing pattern (may include '*' and '?') to
1113 use when matching files.
1116 A generator that yields filenames in directory that match
1117 the given glob pattern.
1119 See also :meth:`get_files`, :meth:`expand_globs`.
1121 for filename in get_files(directory):
1122 if fnmatch.fnmatch(filename, glob_string):
1126 def get_directories(directory: str):
1128 Returns the subdirectories in a directory as a generator.
1131 directory: the directory to list subdirectories within.
1134 A generator that yields all subdirectories within the given
1137 See also :meth:`get_files`, :meth:`get_files_recursive`.
1139 for d in os.listdir(directory):
1140 full_path = join(directory, d)
1141 if not isfile(full_path) and exists(full_path):
1145 def get_files_recursive(directory: str):
1147 Find the files and directories under a root recursively.
1150 directory: the root directory under which to list subdirectories
1154 A generator that yields all directories and files beneath the input
1157 See also :meth:`get_files`, :meth:`get_matching_files`,
1158 :meth:`get_matching_files_recursive`
1160 for filename in get_files(directory):
1162 for subdir in get_directories(directory):
1163 for file_or_directory in get_files_recursive(subdir):
1164 yield file_or_directory
1167 def get_matching_files_recursive(directory: str, glob_string: str):
1168 """Returns the subset of files whose name matches a glob under a root recursively.
1171 directory: the root under which to search
1172 glob_string: a globbing pattern that describes the subset of
1173 files and directories to return. May contain '?' and '*'.
1176 A generator that yields all files and directories under the given root
1177 directory that match the given globbing pattern.
1179 See also :meth:`get_files_recursive`.
1182 for filename in get_files_recursive(directory):
1183 if fnmatch.fnmatch(filename, glob_string):
1187 class FileWriter(contextlib.AbstractContextManager):
1188 """A helper that writes a file to a temporary location and then
1189 moves it atomically to its ultimate destination on close.
1191 Example usage. Creates a temporary file that is populated by the
1192 print statements within the context. Until the context is exited,
1193 the true destination file does not exist so no reader of it can
1194 see partial writes due to buffering or code timing. Once the
1195 context is exited, the file is moved from its temporary location
1196 to its permanent location by a call to `/bin/mv` which should be
1199 with FileWriter('/home/bob/foobar.txt') as w:
1200 print("This is a test!", file=w)
1202 print("This is only a test...", file=w)
1205 def __init__(self, filename: str) -> None:
1208 filename: the ultimate destination file we want to populate.
1209 On exit, the file will be atomically created.
1211 self.filename = filename
1213 self.tempfile = f"{filename}-{uuid}.tmp"
1214 self.handle: Optional[IO[Any]] = None
1216 def __enter__(self) -> IO[Any]:
1217 assert not does_path_exist(self.tempfile)
1218 self.handle = open(self.tempfile, mode="w")
1222 def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
1223 if self.handle is not None:
1225 cmd = f"/bin/mv -f {self.tempfile} {self.filename}"
1226 ret = os.system(cmd)
1228 raise Exception(f"{cmd} failed, exit value {ret>>8}!")
1232 class CreateFileWithMode(contextlib.AbstractContextManager):
1233 """This helper context manager can be used instead of the typical
1234 pattern for creating a file if you want to ensure that the file
1235 created is a particular filesystem permission mode upon creation.
1237 Python's open doesn't support this; you need to set the os.umask
1238 and then create a descriptor to open via os.open, see below.
1241 >>> filename = f'/tmp/CreateFileWithModeTest.{os.getpid()}'
1242 >>> with CreateFileWithMode(filename, filesystem_mode=0o600) as wf:
1243 ... print('This is a test', file=wf)
1244 >>> result = os.stat(filename)
1246 Note: there is a high order bit set in this that is S_IFREG
1247 indicating that the file is a "normal file". Clear it with
1250 >>> print(f'{result.st_mode & 0o7777:o}')
1252 >>> with open(filename, 'r') as rf:
1253 ... contents = rf.read()
1256 >>> remove(filename)
1263 filesystem_mode: Optional[int] = 0o600,
1264 open_mode: Optional[str] = "w",
1268 filename: path of the file to create.
1269 filesystem_mode: the UNIX-style octal mode with which to create
1270 the filename. Defaults to 0o600.
1271 open_mode: the mode to use when opening the file (e.g. 'w', 'wb',
1276 If the file already exists it will be overwritten!
1279 self.filename = filename
1280 if filesystem_mode is not None:
1281 self.filesystem_mode = filesystem_mode & 0o7777
1283 self.filesystem_mode = 0o666
1284 if open_mode is not None:
1285 self.open_mode = open_mode
1287 self.open_mode = "w"
1288 self.handle: Optional[IO[Any]] = None
1289 self.old_umask = os.umask(0)
1291 def __enter__(self) -> IO[Any]:
1292 descriptor = os.open(
1294 flags=(os.O_WRONLY | os.O_CREAT | os.O_TRUNC),
1295 mode=self.filesystem_mode,
1297 self.handle = open(descriptor, self.open_mode)
1301 def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
1302 os.umask(self.old_umask)
1303 if self.handle is not None:
1308 if __name__ == "__main__":