3 # © Copyright 2021-2023, Scott Gasch
6 This is a grab bag of file-related utilities. It has code to, for example,
7 read files transforming the text as its read, normalize pathnames, strip
8 extensions, read and manipulate atimes/mtimes/ctimes, compute a signature
9 based on a file's contents, traverse the file system recursively, etc...
23 from os.path import exists, isfile, join
24 from typing import IO, Any, Callable, List, Literal, Optional
25 from uuid import uuid4
27 logger = logging.getLogger(__name__)
30 def remove_newlines(x: str) -> str:
31 """Trivial function to be used as a line_transformer in
32 :meth:`slurp_file` for no newlines in file contents"""
33 return x.replace("\n", "")
36 def strip_whitespace(x: str) -> str:
37 """Trivial function to be used as a line_transformer in
38 :meth:`slurp_file` for no leading / trailing whitespace in
43 def remove_hash_comments(x: str) -> str:
44 """Trivial function to be used as a line_transformer in
45 :meth:`slurp_file` for no # comments in file contents"""
46 return re.sub(r"#.*$", "", x)
52 skip_blank_lines: bool = False,
53 line_transformers: Optional[List[Callable[[str], str]]] = None,
55 """Reads in a file's contents line-by-line to a memory buffer applying
56 each line transformation in turn.
59 filename: file to be read
60 skip_blank_lines: should reading skip blank lines?
61 line_transformers: little string->string transformations
64 A list of lines from the read and transformed file contents.
67 Exception: filename not found or can't be read.
72 if line_transformers is not None:
73 for x in line_transformers:
75 if not is_readable(filename):
76 raise Exception(f"{filename} can't be read.")
77 with open(filename) as rf:
79 for transformation in xforms:
80 line = transformation(line)
81 if skip_blank_lines and line == "":
87 def remove(path: str) -> None:
88 """Deletes a file. Raises if path refers to a directory or a file
92 path: the path of the file to delete
95 FileNotFoundError: the path to remove does not exist
98 >>> filename = '/tmp/file_utils_test_file'
99 >>> os.system(f'touch {filename}')
101 >>> does_file_exist(filename)
104 >>> does_file_exist(filename)
107 >>> remove("/tmp/23r23r23rwdfwfwefgdfgwerhwrgewrgergerg22r")
108 Traceback (most recent call last):
110 FileNotFoundError: [Errno 2] No such file or directory: '/tmp/23r23r23rwdfwfwefgdfgwerhwrgewrgergerg22r'
115 def fix_multiple_slashes(path: str) -> str:
116 """Fixes multi-slashes in paths or path-like strings
119 path: the path in which to remove multiple slashes
121 >>> p = '/usr/local//etc/rc.d///file.txt'
122 >>> fix_multiple_slashes(p)
123 '/usr/local/etc/rc.d/file.txt'
125 >>> p = 'this is a test'
126 >>> fix_multiple_slashes(p) == p
129 return re.sub(r"/+", "/", path)
132 def delete(path: str) -> None:
133 """This is a convenience for my dumb ass who can't remember os.remove
139 def without_extension(path: str) -> str:
140 """Remove one (the last) extension from a file or path.
143 path: the path from which to remove an extension
146 the path with one extension removed.
148 See also :meth:`without_all_extensions`.
150 >>> without_extension('foobar.txt')
153 >>> without_extension('/home/scott/frapp.py')
156 >>> f = 'a.b.c.tar.gz'
158 ... f = without_extension(f)
165 >>> without_extension('foobar')
169 return os.path.splitext(path)[0]
172 def without_all_extensions(path: str) -> str:
173 """Removes all extensions from a path; handles multiple extensions
174 like foobar.tar.gz -> foobar.
177 path: the path from which to remove all extensions
180 the path with all extensions removed.
182 See also :meth:`without_extension`
184 >>> without_all_extensions('/home/scott/foobar.1.tar.gz')
189 path = without_extension(path)
193 def get_extension(path: str) -> str:
194 """Extract and return one (the last) extension from a file or path.
197 path: the path from which to extract an extension
200 The last extension from the file path.
202 See also :meth:`without_extension`, :meth:`without_all_extensions`,
203 :meth:`get_all_extensions`.
205 >>> get_extension('this_is_a_test.txt')
208 >>> get_extension('/home/scott/test.py')
211 >>> get_extension('foobar')
215 return os.path.splitext(path)[1]
218 def get_all_extensions(path: str) -> List[str]:
219 """Return the extensions of a file or path in order.
222 path: the path from which to extract all extensions.
225 a list containing each extension which may be empty.
227 See also :meth:`without_extension`, :meth:`without_all_extensions`,
228 :meth:`get_extension`.
230 >>> get_all_extensions('/home/scott/foo.tar.gz.1')
231 ['.tar', '.gz', '.1']
233 >>> get_all_extensions('/home/scott/foobar')
239 ext = get_extension(path)
240 path = without_extension(path)
248 def without_path(filespec: str) -> str:
249 """Returns the base filename without any leading path.
252 filespec: path to remove leading directories from
255 filespec without leading dir components.
257 See also :meth:`get_path`, :meth:`get_canonical_path`.
259 >>> without_path('/home/scott/foo.py')
262 >>> without_path('foo.py')
266 return os.path.split(filespec)[1]
269 def get_path(filespec: str) -> str:
270 """Returns just the path of the filespec by removing the filename and
274 filespec: path to remove filename / extension(s) from
277 filespec with just the leading directory components and no
278 filename or extension(s)
280 See also :meth:`without_path`, :meth:`get_canonical_path`.
282 >>> get_path('/home/scott/foobar.py')
285 >>> get_path('/home/scott/test.1.2.3.gz')
288 >>> get_path('~scott/frapp.txt')
292 return os.path.split(filespec)[0]
295 def get_canonical_path(filespec: str) -> str:
296 """Returns a canonicalized absolute path.
299 filespec: the path to canonicalize
302 the canonicalized path
304 See also :meth:`get_path`, :meth:`without_path`.
306 >>> get_canonical_path('/home/scott/../../home/lynn/../scott/foo.txt')
307 '/usr/home/scott/foo.txt'
310 return os.path.realpath(filespec)
313 def create_path_if_not_exist(
314 path: str, on_error: Callable[[str, OSError], None] = None
317 Attempts to create path if it does not exist already.
320 path: the path to attempt to create
321 on_error: if provided, this is invoked on error conditions and
322 passed the path and OSError that it caused
325 OSError: an exception occurred and on_error was not set.
327 See also :meth:`does_file_exist`.
330 Files are created with mode 0o0777 (i.e. world read/writeable).
334 >>> path = os.path.join("/tmp", str(uuid.uuid4()), str(uuid.uuid4()))
335 >>> os.path.exists(path)
337 >>> create_path_if_not_exist(path)
338 >>> os.path.exists(path)
341 logger.debug("Creating path %s", path)
342 previous_umask = os.umask(0)
345 os.chmod(path, 0o777)
346 except OSError as ex:
347 if ex.errno != errno.EEXIST and not os.path.isdir(path):
348 if on_error is not None:
353 os.umask(previous_umask)
356 def does_file_exist(filename: str) -> bool:
357 """Returns True if a file exists and is a normal file.
360 filename: filename to check
363 True if filename exists and is a normal file.
366 A Python core philosophy is: it's easier to ask forgiveness
367 than permission (https://docs.python.org/3/glossary.html#term-EAFP).
368 That is, code that just tries an operation and handles the set of
369 Exceptions that may arise is the preferred style. That said, this
370 function can still be useful in some situations.
372 See also :meth:`create_path_if_not_exist`, :meth:`is_readable`.
374 >>> does_file_exist(__file__)
376 >>> does_file_exist('/tmp/2492043r9203r9230r9230r49230r42390r4230')
379 return os.path.exists(filename) and os.path.isfile(filename)
382 def is_readable(filename: str) -> bool:
383 """Is the file readable?
386 filename: the filename to check for read access
389 True if the file exists, is a normal file, and is readable
390 by the current process. False otherwise.
392 See also :meth:`does_file_exist`, :meth:`is_writable`,
393 :meth:`is_executable`.
395 return os.access(filename, os.R_OK)
398 def is_writable(filename: str) -> bool:
399 """Is the file writable?
402 filename: the file to check for write access.
405 True if file exists, is a normal file and is writable by the
406 current process. False otherwise.
409 A Python core philosophy is: it's easier to ask forgiveness
410 than permission (https://docs.python.org/3/glossary.html#term-EAFP).
411 That is, code that just tries an operation and handles the set of
412 Exceptions that may arise is the preferred style. That said, this
413 function can still be useful in some situations.
415 See also :meth:`is_readable`, :meth:`does_file_exist`.
417 return os.access(filename, os.W_OK)
420 def is_executable(filename: str) -> bool:
421 """Is the file executable?
424 filename: the file to check for execute access.
427 True if file exists, is a normal file and is executable by the
428 current process. False otherwise.
431 A Python core philosophy is: it's easier to ask forgiveness
432 than permission (https://docs.python.org/3/glossary.html#term-EAFP).
433 That is, code that just tries an operation and handles the set of
434 Exceptions that may arise is the preferred style. That said, this
435 function can still be useful in some situations.
437 See also :meth:`does_file_exist`, :meth:`is_readable`,
440 return os.access(filename, os.X_OK)
443 def does_directory_exist(dirname: str) -> bool:
444 """Does the given directory exist?
447 dirname: the name of the directory to check
450 True if a path exists and is a directory, not a regular file.
452 See also :meth:`does_file_exist`.
454 >>> does_directory_exist('/tmp')
456 >>> does_directory_exist('/xyzq/21341')
459 return os.path.exists(dirname) and os.path.isdir(dirname)
462 def does_path_exist(pathname: str) -> bool:
463 """Just a more verbose wrapper around os.path.exists."""
464 return os.path.exists(pathname)
467 def get_file_size(filename: str) -> int:
468 """Returns the size of a file in bytes.
471 filename: the filename to size
474 size of filename in bytes
476 return os.path.getsize(filename)
479 def is_normal_file(filename: str) -> bool:
480 """Is that file normal (not a directory or some special file?)
483 filename: the path of the file to check
486 True if filename is a normal file.
489 A Python core philosophy is: it's easier to ask forgiveness
490 than permission (https://docs.python.org/3/glossary.html#term-EAFP).
491 That is, code that just tries an operation and handles the set of
492 Exceptions that may arise is the preferred style. That said, this
493 function can still be useful in some situations.
495 See also :meth:`is_directory`, :meth:`does_file_exist`, :meth:`is_symlink`.
497 >>> is_normal_file(__file__)
500 return os.path.isfile(filename)
503 def is_directory(filename: str) -> bool:
504 """Is that path a directory (not a normal file?)
507 filename: the path of the file to check
510 True if filename is a directory
513 A Python core philosophy is: it's easier to ask forgiveness
514 than permission (https://docs.python.org/3/glossary.html#term-EAFP).
515 That is, code that just tries an operation and handles the set of
516 Exceptions that may arise is the preferred style. That said, this
517 function can still be useful in some situations.
519 See also :meth:`does_directory_exist`, :meth:`is_normal_file`,
522 >>> is_directory('/tmp')
525 return os.path.isdir(filename)
528 def is_symlink(filename: str) -> bool:
529 """Is that path a symlink?
532 filename: the path of the file to check
535 True if filename is a symlink, False otherwise.
538 A Python core philosophy is: it's easier to ask forgiveness
539 than permission (https://docs.python.org/3/glossary.html#term-EAFP).
540 That is, code that just tries an operation and handles the set of
541 Exceptions that may arise is the preferred style. That said, this
542 function can still be useful in some situations.
544 See also :meth:`is_directory`, :meth:`is_normal_file`.
546 >>> is_symlink('/tmp')
549 >>> is_symlink('/home')
552 return os.path.islink(filename)
555 def is_same_file(file1: str, file2: str) -> bool:
556 """Determine if two paths reference the same inode.
559 file1: the first file
560 file2: the second file
563 True if the two files are the same file.
565 See also :meth:`is_symlink`, :meth:`is_normal_file`.
567 >>> is_same_file('/tmp', '/tmp/../tmp')
570 >>> is_same_file('/tmp', '/home')
573 return os.path.samefile(file1, file2)
576 def get_file_raw_timestamps(filename: str) -> Optional[os.stat_result]:
577 """Stats the file and returns an `os.stat_result` or None on error.
580 filename: the file whose timestamps to fetch
583 the os.stat_result or None to indicate an error occurred
586 :meth:`get_file_raw_atime`,
587 :meth:`get_file_raw_ctime`,
588 :meth:`get_file_raw_mtime`,
589 :meth:`get_file_raw_timestamp`
592 return os.stat(filename)
594 logger.exception("Failed to stat path %s; returning None", filename)
598 def get_file_raw_timestamp(
599 filename: str, extractor: Callable[[os.stat_result], Optional[float]]
600 ) -> Optional[float]:
601 """Stat a file and, if successful, use extractor to fetch some
602 subset of the information in the `os.stat_result`.
605 filename: the filename to stat
606 extractor: Callable that takes a os.stat_result and produces
607 something useful(?) with it.
610 whatever the extractor produced or None on error.
613 :meth:`get_file_raw_atime`,
614 :meth:`get_file_raw_ctime`,
615 :meth:`get_file_raw_mtime`,
616 :meth:`get_file_raw_timestamps`
618 tss = get_file_raw_timestamps(filename)
620 return extractor(tss)
624 def get_file_raw_atime(filename: str) -> Optional[float]:
625 """Get a file's raw access time.
628 filename: the path to the file to stat
631 The file's raw atime (seconds since the Epoch) or
635 :meth:`get_file_atime_age_seconds`,
636 :meth:`get_file_atime_as_datetime`,
637 :meth:`get_file_atime_timedelta`,
638 :meth:`get_file_raw_ctime`,
639 :meth:`get_file_raw_mtime`,
640 :meth:`get_file_raw_timestamps`
642 return get_file_raw_timestamp(filename, lambda x: x.st_atime)
645 def get_file_raw_mtime(filename: str) -> Optional[float]:
646 """Get a file's raw modification time.
649 filename: the path to the file to stat
652 The file's raw mtime (seconds since the Epoch) or
656 :meth:`get_file_raw_atime`,
657 :meth:`get_file_raw_ctime`,
658 :meth:`get_file_mtime_age_seconds`,
659 :meth:`get_file_mtime_as_datetime`,
660 :meth:`get_file_mtime_timedelta`,
661 :meth:`get_file_raw_timestamps`
663 return get_file_raw_timestamp(filename, lambda x: x.st_mtime)
666 def get_file_raw_ctime(filename: str) -> Optional[float]:
667 """Get a file's raw creation time.
670 filename: the path to the file to stat
673 The file's raw ctime (seconds since the Epoch) or
677 :meth:`get_file_raw_atime`,
678 :meth:`get_file_ctime_age_seconds`,
679 :meth:`get_file_ctime_as_datetime`,
680 :meth:`get_file_ctime_timedelta`,
681 :meth:`get_file_raw_mtime`,
682 :meth:`get_file_raw_timestamps`
684 return get_file_raw_timestamp(filename, lambda x: x.st_ctime)
687 def get_file_md5(filename: str) -> str:
688 """Hashes filename's disk contents and returns the MD5 digest.
691 filename: the file whose contents to hash
694 the MD5 digest of the file's contents. Raises on error.
696 file_hash = hashlib.md5()
697 with open(filename, "rb") as f:
700 file_hash.update(chunk)
702 return file_hash.hexdigest()
705 def set_file_raw_atime(filename: str, atime: float) -> None:
706 """Sets a file's raw access time.
709 filename: the file whose atime should be set
710 atime: raw atime as number of seconds since the Epoch to set
713 :meth:`get_file_raw_atime`,
714 :meth:`get_file_atime_age_seconds`,
715 :meth:`get_file_atime_as_datetime`,
716 :meth:`get_file_atime_timedelta`,
717 :meth:`get_file_raw_timestamps`,
718 :meth:`set_file_raw_mtime`,
719 :meth:`set_file_raw_atime_and_mtime`,
722 mtime = get_file_raw_mtime(filename)
723 assert mtime is not None
724 os.utime(filename, (atime, mtime))
727 def set_file_raw_mtime(filename: str, mtime: float):
728 """Sets a file's raw modification time.
731 filename: the file whose mtime should be set
732 mtime: the raw mtime as number of seconds since the Epoch to set
735 :meth:`get_file_raw_mtime`,
736 :meth:`get_file_mtime_age_seconds`,
737 :meth:`get_file_mtime_as_datetime`,
738 :meth:`get_file_mtime_timedelta`,
739 :meth:`get_file_raw_timestamps`,
740 :meth:`set_file_raw_atime`,
741 :meth:`set_file_raw_atime_and_mtime`,
744 atime = get_file_raw_atime(filename)
745 assert atime is not None
746 os.utime(filename, (atime, mtime))
749 def set_file_raw_atime_and_mtime(filename: str, ts: float = None):
750 """Sets both a file's raw modification and access times.
753 filename: the file whose times to set
754 ts: the raw time to set or None to indicate time should be
755 set to the current time.
758 :meth:`get_file_raw_atime`,
759 :meth:`get_file_raw_mtime`,
760 :meth:`get_file_raw_timestamps`,
761 :meth:`set_file_raw_atime`,
762 :meth:`set_file_raw_mtime`
765 os.utime(filename, (ts, ts))
767 os.utime(filename, None)
770 def _convert_file_timestamp_to_datetime(
772 producer: Callable[[str], Optional[float]],
773 ) -> Optional[datetime.datetime]:
775 Converts a raw file timestamp into a Python datetime.
778 filename: file whose timestamps should be converted.
779 producer: source of the timestamp.
783 ts = producer(filename)
785 return datetime.datetime.fromtimestamp(ts)
789 def get_file_atime_as_datetime(filename: str) -> Optional[datetime.datetime]:
790 """Fetch a file's access time as a Python datetime.
793 filename: the file whose atime should be fetched.
796 The file's atime as a Python :class:`datetime.datetime`.
799 :meth:`get_file_raw_atime`,
800 :meth:`get_file_atime_age_seconds`,
801 :meth:`get_file_atime_timedelta`,
802 :meth:`get_file_raw_ctime`,
803 :meth:`get_file_raw_mtime`,
804 :meth:`get_file_raw_timestamps`,
805 :meth:`set_file_raw_atime`,
806 :meth:`set_file_raw_atime_and_mtime`
808 return _convert_file_timestamp_to_datetime(filename, get_file_raw_atime)
811 def get_file_mtime_as_datetime(filename: str) -> Optional[datetime.datetime]:
812 """Fetch a file's modification time as a Python datetime.
815 filename: the file whose mtime should be fetched.
818 The file's mtime as a Python :class:`datetime.datetime`.
821 :meth:`get_file_raw_mtime`,
822 :meth:`get_file_mtime_age_seconds`,
823 :meth:`get_file_mtime_timedelta`,
824 :meth:`get_file_raw_ctime`,
825 :meth:`get_file_raw_atime`,
826 :meth:`get_file_raw_timestamps`,
827 :meth:`set_file_raw_atime`,
828 :meth:`set_file_raw_atime_and_mtime`
830 return _convert_file_timestamp_to_datetime(filename, get_file_raw_mtime)
833 def get_file_ctime_as_datetime(filename: str) -> Optional[datetime.datetime]:
834 """Fetches a file's creation time as a Python datetime.
837 filename: the file whose ctime should be fetched.
840 The file's ctime as a Python :class:`datetime.datetime`.
843 :meth:`get_file_raw_ctime`,
844 :meth:`get_file_ctime_age_seconds`,
845 :meth:`get_file_ctime_timedelta`,
846 :meth:`get_file_raw_atime`,
847 :meth:`get_file_raw_mtime`,
848 :meth:`get_file_raw_timestamps`
850 return _convert_file_timestamp_to_datetime(filename, get_file_raw_ctime)
853 def _get_file_timestamp_age_seconds(filename: str, extractor) -> Optional[int]:
854 """~Internal helper"""
856 ts = get_file_raw_timestamps(filename)
859 result = extractor(ts)
863 def get_file_atime_age_seconds(filename: str) -> Optional[int]:
864 """Gets a file's access time as an age in seconds (ago).
867 filename: file whose atime should be checked.
870 The number of seconds ago that filename was last accessed.
873 :meth:`get_file_raw_atime`,
874 :meth:`get_file_atime_as_datetime`,
875 :meth:`get_file_atime_timedelta`,
876 :meth:`get_file_raw_ctime`,
877 :meth:`get_file_raw_mtime`,
878 :meth:`get_file_raw_timestamps`,
879 :meth:`set_file_raw_atime`,
880 :meth:`set_file_raw_atime_and_mtime`
882 return _get_file_timestamp_age_seconds(filename, lambda x: x.st_atime)
885 def get_file_ctime_age_seconds(filename: str) -> Optional[int]:
886 """Gets a file's creation time as an age in seconds (ago).
889 filename: file whose ctime should be checked.
892 The number of seconds ago that filename was created.
895 :meth:`get_file_raw_ctime`,
896 :meth:`get_file_ctime_age_seconds`,
897 :meth:`get_file_ctime_as_datetime`,
898 :meth:`get_file_ctime_timedelta`,
899 :meth:`get_file_raw_mtime`,
900 :meth:`get_file_raw_atime`,
901 :meth:`get_file_raw_timestamps`
903 return _get_file_timestamp_age_seconds(filename, lambda x: x.st_ctime)
906 def get_file_mtime_age_seconds(filename: str) -> Optional[int]:
907 """Gets a file's modification time as seconds (ago).
910 filename: file whose mtime should be checked.
913 The number of seconds ago that filename was last modified.
916 :meth:`get_file_raw_atime`,
917 :meth:`get_file_raw_ctime`,
918 :meth:`get_file_raw_mtime`,
919 :meth:`get_file_mtime_as_datetime`,
920 :meth:`get_file_mtime_timedelta`,
921 :meth:`get_file_raw_timestamps`,
922 :meth:`set_file_raw_atime`,
923 :meth:`set_file_raw_atime_and_mtime`
925 return _get_file_timestamp_age_seconds(filename, lambda x: x.st_mtime)
928 def _get_file_timestamp_timedelta(
929 filename: str, extractor
930 ) -> Optional[datetime.timedelta]:
931 """~Internal helper"""
932 age = _get_file_timestamp_age_seconds(filename, extractor)
934 return datetime.timedelta(seconds=float(age))
938 def get_file_atime_timedelta(filename: str) -> Optional[datetime.timedelta]:
939 """How long ago was a file accessed as a timedelta?
942 filename: the file whose atime should be checked.
945 A Python :class:`datetime.timedelta` representing how long
946 ago filename was last accessed.
949 :meth:`get_file_raw_atime`,
950 :meth:`get_file_atime_age_seconds`,
951 :meth:`get_file_atime_as_datetime`,
952 :meth:`get_file_raw_ctime`,
953 :meth:`get_file_raw_mtime`,
954 :meth:`get_file_raw_timestamps`,
955 :meth:`set_file_raw_atime`,
956 :meth:`set_file_raw_atime_and_mtime`
958 return _get_file_timestamp_timedelta(filename, lambda x: x.st_atime)
961 def get_file_ctime_timedelta(filename: str) -> Optional[datetime.timedelta]:
962 """How long ago was a file created as a timedelta?
965 filename: the file whose ctime should be checked.
968 A Python :class:`datetime.timedelta` representing how long
969 ago filename was created.
972 :meth:`get_file_raw_atime`,
973 :meth:`get_file_raw_ctime`,
974 :meth:`get_file_ctime_age_seconds`,
975 :meth:`get_file_ctime_as_datetime`,
976 :meth:`get_file_raw_mtime`,
977 :meth:`get_file_raw_timestamps`
979 return _get_file_timestamp_timedelta(filename, lambda x: x.st_ctime)
982 def get_file_mtime_timedelta(filename: str) -> Optional[datetime.timedelta]:
984 Gets a file's modification time as a Python timedelta.
987 filename: the file whose mtime should be checked.
990 A Python :class:`datetime.timedelta` representing how long
991 ago filename was last modified.
994 :meth:`get_file_raw_atime`,
995 :meth:`get_file_raw_ctime`,
996 :meth:`get_file_raw_mtime`,
997 :meth:`get_file_mtime_age_seconds`,
998 :meth:`get_file_mtime_as_datetime`,
999 :meth:`get_file_raw_timestamps`,
1000 :meth:`set_file_raw_atime`,
1001 :meth:`set_file_raw_atime_and_mtime`
1003 return _get_file_timestamp_timedelta(filename, lambda x: x.st_mtime)
1006 def describe_file_timestamp(filename: str, extractor, *, brief=False) -> Optional[str]:
1007 """~Internal helper"""
1008 from pyutils.datetimes.datetime_utils import (
1010 describe_duration_briefly,
1013 age = _get_file_timestamp_age_seconds(filename, extractor)
1017 return describe_duration_briefly(age)
1019 return describe_duration(age)
1022 def describe_file_atime(filename: str, *, brief: bool = False) -> Optional[str]:
1024 Describe how long ago a file was accessed.
1027 filename: the file whose atime should be described.
1028 brief: if True, describe atime briefly.
1031 A string that represents how long ago filename was last
1032 accessed. The description will be verbose or brief depending
1033 on the brief argument.
1036 :meth:`get_file_raw_atime`,
1037 :meth:`get_file_atime_age_seconds`,
1038 :meth:`get_file_atime_as_datetime`,
1039 :meth:`get_file_atime_timedelta`,
1040 :meth:`get_file_raw_ctime`,
1041 :meth:`get_file_raw_mtime`,
1042 :meth:`get_file_raw_timestamps`
1043 :meth:`set_file_raw_atime`,
1044 :meth:`set_file_raw_atime_and_mtime`
1046 return describe_file_timestamp(filename, lambda x: x.st_atime, brief=brief)
1049 def describe_file_ctime(filename: str, *, brief: bool = False) -> Optional[str]:
1050 """Describes a file's creation time.
1053 filename: the file whose ctime should be described.
1054 brief: if True, describe ctime briefly.
1057 A string that represents how long ago filename was created.
1058 The description will be verbose or brief depending
1059 on the brief argument.
1062 :meth:`get_file_raw_atime`,
1063 :meth:`get_file_raw_ctime`,
1064 :meth:`get_file_ctime_age_seconds`,
1065 :meth:`get_file_ctime_as_datetime`,
1066 :meth:`get_file_ctime_timedelta`,
1067 :meth:`get_file_raw_mtime`,
1068 :meth:`get_file_raw_timestamps`
1070 return describe_file_timestamp(filename, lambda x: x.st_ctime, brief=brief)
1073 def describe_file_mtime(filename: str, *, brief: bool = False) -> Optional[str]:
1074 """Describes how long ago a file was modified.
1077 filename: the file whose mtime should be described.
1078 brief: if True, describe mtime briefly.
1081 A string that represents how long ago filename was last
1082 modified. The description will be verbose or brief depending
1083 on the brief argument.
1086 :meth:`get_file_raw_atime`,
1087 :meth:`get_file_raw_ctime`,
1088 :meth:`get_file_raw_mtime`,
1089 :meth:`get_file_mtime_age_seconds`,
1090 :meth:`get_file_mtime_as_datetime`,
1091 :meth:`get_file_mtime_timedelta`,
1092 :meth:`get_file_raw_timestamps`,
1093 :meth:`set_file_raw_atime`,
1094 :meth:`set_file_raw_atime_and_mtime`
1096 return describe_file_timestamp(filename, lambda x: x.st_mtime, brief=brief)
1099 def touch_file(filename: str, *, mode: Optional[int] = 0o666):
1100 """Like unix "touch" command's semantics: update the timestamp
1101 of a file to the current time if the file exists. Create the
1102 file if it doesn't exist.
1105 filename: the filename
1106 mode: the mode to create the file with
1110 The default creation mode is 0o666 which is world readable
1111 and writable. Override this by passing in your own mode
1112 parameter if desired.
1114 See also :meth:`set_file_raw_atime`, :meth:`set_file_raw_atime_and_mtime`,
1115 :meth:`set_file_raw_mtime`, :meth:`create_path_if_not_exist`
1117 pathlib.Path(filename, mode=mode).touch()
1120 def expand_globs(in_filename: str):
1122 Expands shell globs (* and ? wildcards) to the matching files.
1125 in_filename: the filepath to be expanded. May contain '*' and '?'
1126 globbing characters.
1129 A Generator that yields filenames that match the input pattern.
1131 See also :meth:`get_files`, :meth:`get_files_recursive`.
1133 for filename in glob.glob(in_filename):
1137 def get_files(directory: str):
1138 """Returns the files in a directory as a generator.
1141 directory: the directory to list files under.
1144 A generator that yields all files in the input directory.
1146 See also :meth:`expand_globs`, :meth:`get_files_recursive`,
1147 :meth:`get_matching_files`.
1149 for filename in os.listdir(directory):
1150 full_path = join(directory, filename)
1151 if isfile(full_path) and exists(full_path):
1155 def get_matching_files(directory: str, glob_string: str):
1157 Returns the subset of files whose name matches a glob.
1160 directory: the directory to match files within.
1161 glob_string: the globbing pattern (may include '*' and '?') to
1162 use when matching files.
1165 A generator that yields filenames in directory that match
1166 the given glob pattern.
1168 See also :meth:`get_files`, :meth:`expand_globs`.
1170 for filename in get_files(directory):
1171 if fnmatch.fnmatch(filename, glob_string):
1175 def get_directories(directory: str):
1177 Returns the subdirectories in a directory as a generator.
1180 directory: the directory to list subdirectories within.
1183 A generator that yields all subdirectories within the given
1186 See also :meth:`get_files`, :meth:`get_files_recursive`.
1188 for d in os.listdir(directory):
1189 full_path = join(directory, d)
1190 if not isfile(full_path) and exists(full_path):
1194 def get_files_recursive(directory: str):
1196 Find the files and directories under a root recursively.
1199 directory: the root directory under which to list subdirectories
1203 A generator that yields all directories and files beneath the input
1206 See also :meth:`get_files`, :meth:`get_matching_files`,
1207 :meth:`get_matching_files_recursive`
1209 for filename in get_files(directory):
1211 for subdir in get_directories(directory):
1212 for file_or_directory in get_files_recursive(subdir):
1213 yield file_or_directory
1216 def get_matching_files_recursive(directory: str, glob_string: str):
1217 """Returns the subset of files whose name matches a glob under a root recursively.
1220 directory: the root under which to search
1221 glob_string: a globbing pattern that describes the subset of
1222 files and directories to return. May contain '?' and '*'.
1225 A generator that yields all files and directories under the given root
1226 directory that match the given globbing pattern.
1228 See also :meth:`get_files_recursive`.
1231 for filename in get_files_recursive(directory):
1232 if fnmatch.fnmatch(filename, glob_string):
1236 class FileWriter(contextlib.AbstractContextManager):
1237 """A helper that writes a file to a temporary location and then
1238 moves it atomically to its ultimate destination on close.
1240 Example usage. Creates a temporary file that is populated by the
1241 print statements within the context. Until the context is exited,
1242 the true destination file does not exist so no reader of it can
1243 see partial writes due to buffering or code timing. Once the
1244 context is exited, the file is moved from its temporary location
1245 to its permanent location by a call to `/bin/mv` which should be
1248 with FileWriter('/home/bob/foobar.txt') as w:
1249 print("This is a test!", file=w)
1251 print("This is only a test...", file=w)
1254 def __init__(self, filename: str) -> None:
1257 filename: the ultimate destination file we want to populate.
1258 On exit, the file will be atomically created.
1260 self.filename = filename
1262 self.tempfile = f"{filename}-{uuid}.tmp"
1263 self.handle: Optional[IO[Any]] = None
1265 def __enter__(self) -> IO[Any]:
1266 assert not does_path_exist(self.tempfile)
1267 self.handle = open(self.tempfile, mode="w")
1271 def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
1272 if self.handle is not None:
1274 cmd = f"/bin/mv -f {self.tempfile} {self.filename}"
1275 ret = os.system(cmd)
1277 raise Exception(f"{cmd} failed, exit value {ret>>8}!")
1281 class CreateFileWithMode(contextlib.AbstractContextManager):
1282 """This helper context manager can be used instead of the typical
1283 pattern for creating a file if you want to ensure that the file
1284 created is a particular filesystem permission mode upon creation.
1286 Python's open doesn't support this; you need to set the os.umask
1287 and then create a descriptor to open via os.open, see below.
1290 >>> filename = f'/tmp/CreateFileWithModeTest.{os.getpid()}'
1291 >>> with CreateFileWithMode(filename, filesystem_mode=0o600) as wf:
1292 ... print('This is a test', file=wf)
1293 >>> result = os.stat(filename)
1295 Note: there is a high order bit set in this that is S_IFREG
1296 indicating that the file is a "normal file". Clear it with
1299 >>> print(f'{result.st_mode & 0o7777:o}')
1301 >>> with open(filename, 'r') as rf:
1302 ... contents = rf.read()
1305 >>> remove(filename)
1312 filesystem_mode: Optional[int] = 0o600,
1313 open_mode: Optional[str] = "w",
1317 filename: path of the file to create.
1318 filesystem_mode: the UNIX-style octal mode with which to create
1319 the filename. Defaults to 0o600.
1320 open_mode: the mode to use when opening the file (e.g. 'w', 'wb',
1325 If the file already exists it will be overwritten!
1328 self.filename = filename
1329 if filesystem_mode is not None:
1330 self.filesystem_mode = filesystem_mode & 0o7777
1332 self.filesystem_mode = 0o666
1333 if open_mode is not None:
1334 self.open_mode = open_mode
1336 self.open_mode = "w"
1337 self.handle: Optional[IO[Any]] = None
1338 self.old_umask = os.umask(0)
1340 def __enter__(self) -> IO[Any]:
1341 descriptor = os.open(
1343 flags=(os.O_WRONLY | os.O_CREAT | os.O_TRUNC),
1344 mode=self.filesystem_mode,
1346 self.handle = open(descriptor, self.open_mode)
1350 def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
1351 os.umask(self.old_umask)
1352 if self.handle is not None:
1357 if __name__ == "__main__":