3 # © Copyright 2021-2022, Scott Gasch
6 This is a grab bag of file-related utilities. It has code to, for example,
7 read files transforming the text as its read, normalize pathnames, strip
8 extensions, read and manipulate atimes/mtimes/ctimes, compute a signature
9 based on a file's contents, traverse the file system recursively, etc...
23 from os.path import exists, isfile, join
24 from typing import Callable, List, Literal, Optional, TextIO
25 from uuid import uuid4
27 logger = logging.getLogger(__name__)
30 def remove_newlines(x: str) -> str:
31 """Trivial function to be used as a line_transformer in
32 :meth:`slurp_file` for no newlines in file contents"""
33 return x.replace("\n", "")
36 def strip_whitespace(x: str) -> str:
37 """Trivial function to be used as a line_transformer in
38 :meth:`slurp_file` for no leading / trailing whitespace in
43 def remove_hash_comments(x: str) -> str:
44 """Trivial function to be used as a line_transformer in
45 :meth:`slurp_file` for no # comments in file contents"""
46 return re.sub(r"#.*$", "", x)
52 skip_blank_lines=False,
53 line_transformers: Optional[List[Callable[[str], str]]] = None,
55 """Reads in a file's contents line-by-line to a memory buffer applying
56 each line transformation in turn.
59 filename: file to be read
60 skip_blank_lines: should reading skip blank lines?
61 line_transformers: little string->string transformations
64 A list of lines from the read and transformed file contents.
69 if line_transformers is not None:
70 for x in line_transformers:
72 if not is_readable(filename):
73 raise Exception(f"{filename} can't be read.")
74 with open(filename) as rf:
76 for transformation in xforms:
77 line = transformation(line)
78 if skip_blank_lines and line == "":
84 def remove(path: str) -> None:
85 """Deletes a file. Raises if path refers to a directory or a file
89 path: the path of the file to delete
92 >>> filename = '/tmp/file_utils_test_file'
93 >>> os.system(f'touch {filename}')
95 >>> does_file_exist(filename)
98 >>> does_file_exist(filename)
104 def fix_multiple_slashes(path: str) -> str:
105 """Fixes multi-slashes in paths or path-like strings
108 path: the path in which to remove multiple slashes
110 >>> p = '/usr/local//etc/rc.d///file.txt'
111 >>> fix_multiple_slashes(p)
112 '/usr/local/etc/rc.d/file.txt'
114 >>> p = 'this is a test'
115 >>> fix_multiple_slashes(p) == p
118 return re.sub(r"/+", "/", path)
121 def delete(path: str) -> None:
122 """This is a convenience for my dumb ass who can't remember os.remove
128 def without_extension(path: str) -> str:
129 """Remove one (the last) extension from a file or path.
132 path: the path from which to remove an extension
135 the path with one extension removed.
137 See also :meth:`without_all_extensions`.
139 >>> without_extension('foobar.txt')
142 >>> without_extension('/home/scott/frapp.py')
145 >>> f = 'a.b.c.tar.gz'
147 ... f = without_extension(f)
154 >>> without_extension('foobar')
158 return os.path.splitext(path)[0]
161 def without_all_extensions(path: str) -> str:
162 """Removes all extensions from a path; handles multiple extensions
163 like foobar.tar.gz -> foobar.
166 path: the path from which to remove all extensions
169 the path with all extensions removed.
171 See also :meth:`without_extension`
173 >>> without_all_extensions('/home/scott/foobar.1.tar.gz')
178 path = without_extension(path)
182 def get_extension(path: str) -> str:
183 """Extract and return one (the last) extension from a file or path.
186 path: the path from which to extract an extension
189 The last extension from the file path.
191 See also :meth:`without_extension`, :meth:`without_all_extensions`,
192 :meth:`get_all_extensions`.
194 >>> get_extension('this_is_a_test.txt')
197 >>> get_extension('/home/scott/test.py')
200 >>> get_extension('foobar')
204 return os.path.splitext(path)[1]
207 def get_all_extensions(path: str) -> List[str]:
208 """Return the extensions of a file or path in order.
211 path: the path from which to extract all extensions.
214 a list containing each extension which may be empty.
216 See also :meth:`without_extension`, :meth:`without_all_extensions`,
217 :meth:`get_extension`.
219 >>> get_all_extensions('/home/scott/foo.tar.gz.1')
220 ['.tar', '.gz', '.1']
222 >>> get_all_extensions('/home/scott/foobar')
228 ext = get_extension(path)
229 path = without_extension(path)
237 def without_path(filespec: str) -> str:
238 """Returns the base filename without any leading path.
241 filespec: path to remove leading directories from
244 filespec without leading dir components.
246 See also :meth:`get_path`, :meth:`get_canonical_path`.
248 >>> without_path('/home/scott/foo.py')
251 >>> without_path('foo.py')
255 return os.path.split(filespec)[1]
258 def get_path(filespec: str) -> str:
259 """Returns just the path of the filespec by removing the filename and
263 filespec: path to remove filename / extension(s) from
266 filespec with just the leading directory components and no
267 filename or extension(s)
269 See also :meth:`without_path`, :meth:`get_canonical_path`.
271 >>> get_path('/home/scott/foobar.py')
274 >>> get_path('/home/scott/test.1.2.3.gz')
277 >>> get_path('~scott/frapp.txt')
281 return os.path.split(filespec)[0]
284 def get_canonical_path(filespec: str) -> str:
285 """Returns a canonicalized absolute path.
288 filespec: the path to canonicalize
291 the canonicalized path
293 See also :meth:`get_path`, :meth:`without_path`.
295 >>> get_canonical_path('/home/scott/../../home/lynn/../scott/foo.txt')
296 '/usr/home/scott/foo.txt'
299 return os.path.realpath(filespec)
302 def create_path_if_not_exist(path, on_error=None) -> None:
304 Attempts to create path if it does not exist already.
307 path: the path to attempt to create
308 on_error: If True, it's invoked on error conditions. Otherwise
309 any exceptions are raised.
311 See also :meth:`does_file_exist`.
315 Files are created with mode 0o0777 (i.e. world read/writeable).
319 >>> path = os.path.join("/tmp", str(uuid.uuid4()), str(uuid.uuid4()))
320 >>> os.path.exists(path)
322 >>> create_path_if_not_exist(path)
323 >>> os.path.exists(path)
326 logger.debug("Creating path %s", path)
327 previous_umask = os.umask(0)
330 os.chmod(path, 0o777)
331 except OSError as ex:
332 if ex.errno != errno.EEXIST and not os.path.isdir(path):
333 if on_error is not None:
338 os.umask(previous_umask)
341 def does_file_exist(filename: str) -> bool:
342 """Returns True if a file exists and is a normal file.
345 filename: filename to check
348 True if filename exists and is a normal file.
350 See also :meth:`create_path_if_not_exist`, :meth:`is_readable`.
352 >>> does_file_exist(__file__)
354 >>> does_file_exist('/tmp/2492043r9203r9230r9230r49230r42390r4230')
357 return os.path.exists(filename) and os.path.isfile(filename)
360 def is_readable(filename: str) -> bool:
361 """Is the file readable?
364 filename: the filename to check for read access
367 True if the file exists, is a normal file, and is readable
368 by the current process. False otherwise.
370 See also :meth:`does_file_exist`, :meth:`is_writable`,
371 :meth:`is_executable`.
373 return os.access(filename, os.R_OK)
376 def is_writable(filename: str) -> bool:
377 """Is the file writable?
380 filename: the file to check for write access.
383 True if file exists, is a normal file and is writable by the
384 current process. False otherwise.
386 See also :meth:`is_readable`, :meth:`does_file_exist`.
388 return os.access(filename, os.W_OK)
391 def is_executable(filename: str) -> bool:
392 """Is the file executable?
395 filename: the file to check for execute access.
398 True if file exists, is a normal file and is executable by the
399 current process. False otherwise.
401 See also :meth:`does_file_exist`, :meth:`is_readable`,
404 return os.access(filename, os.X_OK)
407 def does_directory_exist(dirname: str) -> bool:
408 """Does the given directory exist?
411 dirname: the name of the directory to check
414 True if a path exists and is a directory, not a regular file.
416 See also :meth:`does_file_exist`.
418 >>> does_directory_exist('/tmp')
420 >>> does_directory_exist('/xyzq/21341')
423 return os.path.exists(dirname) and os.path.isdir(dirname)
426 def does_path_exist(pathname: str) -> bool:
427 """Just a more verbose wrapper around os.path.exists."""
428 return os.path.exists(pathname)
431 def get_file_size(filename: str) -> int:
432 """Returns the size of a file in bytes.
435 filename: the filename to size
438 size of filename in bytes
440 return os.path.getsize(filename)
443 def is_normal_file(filename: str) -> bool:
444 """Is that file normal (not a directory or some special file?)
447 filename: the path of the file to check
450 True if filename is a normal file.
452 See also :meth:`is_directory`, :meth:`does_file_exist`, :meth:`is_symlink`.
454 >>> is_normal_file(__file__)
457 return os.path.isfile(filename)
460 def is_directory(filename: str) -> bool:
461 """Is that path a directory (not a normal file?)
464 filename: the path of the file to check
467 True if filename is a directory
469 See also :meth:`does_directory_exist`, :meth:`is_normal_file`,
472 >>> is_directory('/tmp')
475 return os.path.isdir(filename)
478 def is_symlink(filename: str) -> bool:
479 """Is that path a symlink?
482 filename: the path of the file to check
485 True if filename is a symlink, False otherwise.
487 See also :meth:`is_directory`, :meth:`is_normal_file`.
489 >>> is_symlink('/tmp')
492 >>> is_symlink('/home')
495 return os.path.islink(filename)
498 def is_same_file(file1: str, file2: str) -> bool:
499 """Determine if two paths reference the same inode.
502 file1: the first file
503 file2: the second file
506 True if the two files are the same file.
508 See also :meth:`is_symlink`, :meth:`is_normal_file`.
510 >>> is_same_file('/tmp', '/tmp/../tmp')
513 >>> is_same_file('/tmp', '/home')
516 return os.path.samefile(file1, file2)
519 def get_file_raw_timestamps(filename: str) -> Optional[os.stat_result]:
520 """Stats the file and returns an `os.stat_result` or None on error.
523 filename: the file whose timestamps to fetch
526 the os.stat_result or None to indicate an error occurred
529 :meth:`get_file_raw_atime`,
530 :meth:`get_file_raw_ctime`,
531 :meth:`get_file_raw_mtime`,
532 :meth:`get_file_raw_timestamp`
535 return os.stat(filename)
536 except Exception as e:
541 def get_file_raw_timestamp(
542 filename: str, extractor: Callable[[os.stat_result], Optional[float]]
543 ) -> Optional[float]:
544 """Stat a file and, if successful, use extractor to fetch some
545 subset of the information in the `os.stat_result`.
548 filename: the filename to stat
549 extractor: Callable that takes a os.stat_result and produces
550 something useful(?) with it.
553 whatever the extractor produced or None on error.
556 :meth:`get_file_raw_atime`,
557 :meth:`get_file_raw_ctime`,
558 :meth:`get_file_raw_mtime`,
559 :meth:`get_file_raw_timestamps`
561 tss = get_file_raw_timestamps(filename)
563 return extractor(tss)
567 def get_file_raw_atime(filename: str) -> Optional[float]:
568 """Get a file's raw access time.
571 filename: the path to the file to stat
574 The file's raw atime (seconds since the Epoch) or
578 :meth:`get_file_atime_age_seconds`,
579 :meth:`get_file_atime_as_datetime`,
580 :meth:`get_file_atime_timedelta`,
581 :meth:`get_file_raw_ctime`,
582 :meth:`get_file_raw_mtime`,
583 :meth:`get_file_raw_timestamps`
585 return get_file_raw_timestamp(filename, lambda x: x.st_atime)
588 def get_file_raw_mtime(filename: str) -> Optional[float]:
589 """Get a file's raw modification time.
592 filename: the path to the file to stat
595 The file's raw mtime (seconds since the Epoch) or
599 :meth:`get_file_raw_atime`,
600 :meth:`get_file_raw_ctime`,
601 :meth:`get_file_mtime_age_seconds`,
602 :meth:`get_file_mtime_as_datetime`,
603 :meth:`get_file_mtime_timedelta`,
604 :meth:`get_file_raw_timestamps`
606 return get_file_raw_timestamp(filename, lambda x: x.st_mtime)
609 def get_file_raw_ctime(filename: str) -> Optional[float]:
610 """Get a file's raw creation time.
613 filename: the path to the file to stat
616 The file's raw ctime (seconds since the Epoch) or
620 :meth:`get_file_raw_atime`,
621 :meth:`get_file_ctime_age_seconds`,
622 :meth:`get_file_ctime_as_datetime`,
623 :meth:`get_file_ctime_timedelta`,
624 :meth:`get_file_raw_mtime`,
625 :meth:`get_file_raw_timestamps`
627 return get_file_raw_timestamp(filename, lambda x: x.st_ctime)
630 def get_file_md5(filename: str) -> str:
631 """Hashes filename's disk contents and returns the MD5 digest.
634 filename: the file whose contents to hash
637 the MD5 digest of the file's contents. Raises on error.
639 file_hash = hashlib.md5()
640 with open(filename, "rb") as f:
643 file_hash.update(chunk)
645 return file_hash.hexdigest()
648 def set_file_raw_atime(filename: str, atime: float) -> None:
649 """Sets a file's raw access time.
652 filename: the file whose atime should be set
653 atime: raw atime as number of seconds since the Epoch to set
656 :meth:`get_file_raw_atime`,
657 :meth:`get_file_atime_age_seconds`,
658 :meth:`get_file_atime_as_datetime`,
659 :meth:`get_file_atime_timedelta`,
660 :meth:`get_file_raw_timestamps`,
661 :meth:`set_file_raw_mtime`,
662 :meth:`set_file_raw_atime_and_mtime`,
665 mtime = get_file_raw_mtime(filename)
666 assert mtime is not None
667 os.utime(filename, (atime, mtime))
670 def set_file_raw_mtime(filename: str, mtime: float):
671 """Sets a file's raw modification time.
674 filename: the file whose mtime should be set
675 mtime: the raw mtime as number of seconds since the Epoch to set
678 :meth:`get_file_raw_mtime`,
679 :meth:`get_file_mtime_age_seconds`,
680 :meth:`get_file_mtime_as_datetime`,
681 :meth:`get_file_mtime_timedelta`,
682 :meth:`get_file_raw_timestamps`,
683 :meth:`set_file_raw_atime`,
684 :meth:`set_file_raw_atime_and_mtime`,
687 atime = get_file_raw_atime(filename)
688 assert atime is not None
689 os.utime(filename, (atime, mtime))
692 def set_file_raw_atime_and_mtime(filename: str, ts: float = None):
693 """Sets both a file's raw modification and access times.
696 filename: the file whose times to set
697 ts: the raw time to set or None to indicate time should be
698 set to the current time.
701 :meth:`get_file_raw_atime`,
702 :meth:`get_file_raw_mtime`,
703 :meth:`get_file_raw_timestamps`,
704 :meth:`set_file_raw_atime`,
705 :meth:`set_file_raw_mtime`
708 os.utime(filename, (ts, ts))
710 os.utime(filename, None)
713 def _convert_file_timestamp_to_datetime(
714 filename: str, producer
715 ) -> Optional[datetime.datetime]:
717 Converts a raw file timestamp into a Python datetime.
720 filename: file whose timestamps should be converted.
721 producer: source of the timestamp.
726 ts = producer(filename)
728 return datetime.datetime.fromtimestamp(ts)
732 def get_file_atime_as_datetime(filename: str) -> Optional[datetime.datetime]:
733 """Fetch a file's access time as a Python datetime.
736 filename: the file whose atime should be fetched.
739 The file's atime as a Python :class:`datetime.datetime`.
742 :meth:`get_file_raw_atime`,
743 :meth:`get_file_atime_age_seconds`,
744 :meth:`get_file_atime_timedelta`,
745 :meth:`get_file_raw_ctime`,
746 :meth:`get_file_raw_mtime`,
747 :meth:`get_file_raw_timestamps`,
748 :meth:`set_file_raw_atime`,
749 :meth:`set_file_raw_atime_and_mtime`
751 return _convert_file_timestamp_to_datetime(filename, get_file_raw_atime)
754 def get_file_mtime_as_datetime(filename: str) -> Optional[datetime.datetime]:
755 """Fetch a file's modification time as a Python datetime.
758 filename: the file whose mtime should be fetched.
761 The file's mtime as a Python :class:`datetime.datetime`.
764 :meth:`get_file_raw_mtime`,
765 :meth:`get_file_mtime_age_seconds`,
766 :meth:`get_file_mtime_timedelta`,
767 :meth:`get_file_raw_ctime`,
768 :meth:`get_file_raw_atime`,
769 :meth:`get_file_raw_timestamps`,
770 :meth:`set_file_raw_atime`,
771 :meth:`set_file_raw_atime_and_mtime`
773 return _convert_file_timestamp_to_datetime(filename, get_file_raw_mtime)
776 def get_file_ctime_as_datetime(filename: str) -> Optional[datetime.datetime]:
777 """Fetches a file's creation time as a Python datetime.
780 filename: the file whose ctime should be fetched.
783 The file's ctime as a Python :class:`datetime.datetime`.
786 :meth:`get_file_raw_ctime`,
787 :meth:`get_file_ctime_age_seconds`,
788 :meth:`get_file_ctime_timedelta`,
789 :meth:`get_file_raw_atime`,
790 :meth:`get_file_raw_mtime`,
791 :meth:`get_file_raw_timestamps`
793 return _convert_file_timestamp_to_datetime(filename, get_file_raw_ctime)
796 def _get_file_timestamp_age_seconds(filename: str, extractor) -> Optional[int]:
797 """~Internal helper"""
799 ts = get_file_raw_timestamps(filename)
802 result = extractor(ts)
806 def get_file_atime_age_seconds(filename: str) -> Optional[int]:
807 """Gets a file's access time as an age in seconds (ago).
810 filename: file whose atime should be checked.
813 The number of seconds ago that filename was last accessed.
816 :meth:`get_file_raw_atime`,
817 :meth:`get_file_atime_as_datetime`,
818 :meth:`get_file_atime_timedelta`,
819 :meth:`get_file_raw_ctime`,
820 :meth:`get_file_raw_mtime`,
821 :meth:`get_file_raw_timestamps`,
822 :meth:`set_file_raw_atime`,
823 :meth:`set_file_raw_atime_and_mtime`
825 return _get_file_timestamp_age_seconds(filename, lambda x: x.st_atime)
828 def get_file_ctime_age_seconds(filename: str) -> Optional[int]:
829 """Gets a file's creation time as an age in seconds (ago).
832 filename: file whose ctime should be checked.
835 The number of seconds ago that filename was created.
838 :meth:`get_file_raw_ctime`,
839 :meth:`get_file_ctime_age_seconds`,
840 :meth:`get_file_ctime_as_datetime`,
841 :meth:`get_file_ctime_timedelta`,
842 :meth:`get_file_raw_mtime`,
843 :meth:`get_file_raw_atime`,
844 :meth:`get_file_raw_timestamps`
846 return _get_file_timestamp_age_seconds(filename, lambda x: x.st_ctime)
849 def get_file_mtime_age_seconds(filename: str) -> Optional[int]:
850 """Gets a file's modification time as seconds (ago).
853 filename: file whose mtime should be checked.
856 The number of seconds ago that filename was last modified.
859 :meth:`get_file_raw_atime`,
860 :meth:`get_file_raw_ctime`,
861 :meth:`get_file_raw_mtime`,
862 :meth:`get_file_mtime_as_datetime`,
863 :meth:`get_file_mtime_timedelta`,
864 :meth:`get_file_raw_timestamps`,
865 :meth:`set_file_raw_atime`,
866 :meth:`set_file_raw_atime_and_mtime`
868 return _get_file_timestamp_age_seconds(filename, lambda x: x.st_mtime)
871 def _get_file_timestamp_timedelta(
872 filename: str, extractor
873 ) -> Optional[datetime.timedelta]:
874 """~Internal helper"""
875 age = _get_file_timestamp_age_seconds(filename, extractor)
877 return datetime.timedelta(seconds=float(age))
881 def get_file_atime_timedelta(filename: str) -> Optional[datetime.timedelta]:
882 """How long ago was a file accessed as a timedelta?
885 filename: the file whose atime should be checked.
888 A Python :class:`datetime.timedelta` representing how long
889 ago filename was last accessed.
892 :meth:`get_file_raw_atime`,
893 :meth:`get_file_atime_age_seconds`,
894 :meth:`get_file_atime_as_datetime`,
895 :meth:`get_file_raw_ctime`,
896 :meth:`get_file_raw_mtime`,
897 :meth:`get_file_raw_timestamps`,
898 :meth:`set_file_raw_atime`,
899 :meth:`set_file_raw_atime_and_mtime`
901 return _get_file_timestamp_timedelta(filename, lambda x: x.st_atime)
904 def get_file_ctime_timedelta(filename: str) -> Optional[datetime.timedelta]:
905 """How long ago was a file created as a timedelta?
908 filename: the file whose ctime should be checked.
911 A Python :class:`datetime.timedelta` representing how long
912 ago filename was created.
915 :meth:`get_file_raw_atime`,
916 :meth:`get_file_raw_ctime`,
917 :meth:`get_file_ctime_age_seconds`,
918 :meth:`get_file_ctime_as_datetime`,
919 :meth:`get_file_raw_mtime`,
920 :meth:`get_file_raw_timestamps`
922 return _get_file_timestamp_timedelta(filename, lambda x: x.st_ctime)
925 def get_file_mtime_timedelta(filename: str) -> Optional[datetime.timedelta]:
927 Gets a file's modification time as a Python timedelta.
930 filename: the file whose mtime should be checked.
933 A Python :class:`datetime.timedelta` representing how long
934 ago filename was last modified.
937 :meth:`get_file_raw_atime`,
938 :meth:`get_file_raw_ctime`,
939 :meth:`get_file_raw_mtime`,
940 :meth:`get_file_mtime_age_seconds`,
941 :meth:`get_file_mtime_as_datetime`,
942 :meth:`get_file_raw_timestamps`,
943 :meth:`set_file_raw_atime`,
944 :meth:`set_file_raw_atime_and_mtime`
946 return _get_file_timestamp_timedelta(filename, lambda x: x.st_mtime)
949 def describe_file_timestamp(filename: str, extractor, *, brief=False) -> Optional[str]:
950 """~Internal helper"""
951 from pyutils.datetimes.datetime_utils import (
953 describe_duration_briefly,
956 age = _get_file_timestamp_age_seconds(filename, extractor)
960 return describe_duration_briefly(age)
962 return describe_duration(age)
965 def describe_file_atime(filename: str, *, brief=False) -> Optional[str]:
967 Describe how long ago a file was accessed.
970 filename: the file whose atime should be described.
971 brief: if True, describe atime briefly.
974 A string that represents how long ago filename was last
975 accessed. The description will be verbose or brief depending
976 on the brief argument.
979 :meth:`get_file_raw_atime`,
980 :meth:`get_file_atime_age_seconds`,
981 :meth:`get_file_atime_as_datetime`,
982 :meth:`get_file_atime_timedelta`,
983 :meth:`get_file_raw_ctime`,
984 :meth:`get_file_raw_mtime`,
985 :meth:`get_file_raw_timestamps`
986 :meth:`set_file_raw_atime`,
987 :meth:`set_file_raw_atime_and_mtime`
989 return describe_file_timestamp(filename, lambda x: x.st_atime, brief=brief)
992 def describe_file_ctime(filename: str, *, brief=False) -> Optional[str]:
993 """Describes a file's creation time.
996 filename: the file whose ctime should be described.
997 brief: if True, describe ctime briefly.
1000 A string that represents how long ago filename was created.
1001 The description will be verbose or brief depending
1002 on the brief argument.
1005 :meth:`get_file_raw_atime`,
1006 :meth:`get_file_raw_ctime`,
1007 :meth:`get_file_ctime_age_seconds`,
1008 :meth:`get_file_ctime_as_datetime`,
1009 :meth:`get_file_ctime_timedelta`,
1010 :meth:`get_file_raw_mtime`,
1011 :meth:`get_file_raw_timestamps`
1013 return describe_file_timestamp(filename, lambda x: x.st_ctime, brief=brief)
1016 def describe_file_mtime(filename: str, *, brief=False) -> Optional[str]:
1017 """Describes how long ago a file was modified.
1020 filename: the file whose mtime should be described.
1021 brief: if True, describe mtime briefly.
1024 A string that represents how long ago filename was last
1025 modified. The description will be verbose or brief depending
1026 on the brief argument.
1029 :meth:`get_file_raw_atime`,
1030 :meth:`get_file_raw_ctime`,
1031 :meth:`get_file_raw_mtime`,
1032 :meth:`get_file_mtime_age_seconds`,
1033 :meth:`get_file_mtime_as_datetime`,
1034 :meth:`get_file_mtime_timedelta`,
1035 :meth:`get_file_raw_timestamps`,
1036 :meth:`set_file_raw_atime`,
1037 :meth:`set_file_raw_atime_and_mtime`
1039 return describe_file_timestamp(filename, lambda x: x.st_mtime, brief=brief)
1042 def touch_file(filename: str, *, mode: Optional[int] = 0o666):
1043 """Like unix "touch" command's semantics: update the timestamp
1044 of a file to the current time if the file exists. Create the
1045 file if it doesn't exist.
1048 filename: the filename
1049 mode: the mode to create the file with
1053 The default creation mode is 0x666 which is world readable
1054 and writable. Override this by passing in your own mode
1055 parameter if desired.
1057 See also :meth:`set_file_raw_atime`, :meth:`set_file_raw_atime_and_mtime`,
1058 :meth:`set_file_raw_mtime`, :meth:`create_path_if_not_exist`
1060 pathlib.Path(filename, mode=mode).touch()
1063 def expand_globs(in_filename: str):
1065 Expands shell globs (* and ? wildcards) to the matching files.
1068 in_filename: the filepath to be expanded. May contain '*' and '?'
1069 globbing characters.
1072 A Generator that yields filenames that match the input pattern.
1074 See also :meth:`get_files`, :meth:`get_files_recursive`.
1076 for filename in glob.glob(in_filename):
1080 def get_files(directory: str):
1081 """Returns the files in a directory as a generator.
1084 directory: the directory to list files under.
1087 A generator that yields all files in the input directory.
1089 See also :meth:`expand_globs`, :meth:`get_files_recursive`,
1090 :meth:`get_matching_files`.
1092 for filename in os.listdir(directory):
1093 full_path = join(directory, filename)
1094 if isfile(full_path) and exists(full_path):
1098 def get_matching_files(directory: str, glob: str):
1100 Returns the subset of files whose name matches a glob.
1103 directory: the directory to match files within.
1104 glob: the globbing pattern (may include '*' and '?') to
1105 use when matching files.
1108 A generator that yields filenames in directory that match
1109 the given glob pattern.
1111 See also :meth:`get_files`, :meth:`expand_globs`.
1113 for filename in get_files(directory):
1114 if fnmatch.fnmatch(filename, glob):
1118 def get_directories(directory: str):
1120 Returns the subdirectories in a directory as a generator.
1123 directory: the directory to list subdirectories within.
1126 A generator that yields all subdirectories within the given
1129 See also :meth:`get_files`, :meth:`get_files_recursive`.
1131 for d in os.listdir(directory):
1132 full_path = join(directory, d)
1133 if not isfile(full_path) and exists(full_path):
1137 def get_files_recursive(directory: str):
1139 Find the files and directories under a root recursively.
1142 directory: the root directory under which to list subdirectories
1146 A generator that yields all directories and files beneath the input
1149 See also :meth:`get_files`, :meth:`get_matching_files`,
1150 :meth:`get_matching_files_recursive`
1152 for filename in get_files(directory):
1154 for subdir in get_directories(directory):
1155 for file_or_directory in get_files_recursive(subdir):
1156 yield file_or_directory
1159 def get_matching_files_recursive(directory: str, glob: str):
1161 Returns the subset of files whose name matches a glob under a root recursively.
1164 directory: the root under which to search
1165 glob: a globbing pattern that describes the subset of files and directories
1166 to return. May contain '?' and '*'.
1169 A generator that yields all files and directories under the given root
1170 directory that match the given globbing pattern.
1172 See also :meth:`get_files_recursive`.
1174 for filename in get_files_recursive(directory):
1175 if fnmatch.fnmatch(filename, glob):
1179 class FileWriter(contextlib.AbstractContextManager):
1180 """A helper that writes a file to a temporary location and then
1181 moves it atomically to its ultimate destination on close.
1183 Example usage. Creates a temporary file that is populated by the
1184 print statements within the context. Until the context is exited,
1185 the true destination file does not exist so no reader of it can
1186 see partial writes due to buffering or code timing. Once the
1187 context is exited, the file is moved from its temporary location
1188 to its permanent location by a call to `/bin/mv` which should be
1191 with FileWriter('/home/bob/foobar.txt') as w:
1192 print("This is a test!", file=w)
1194 print("This is only a test...", file=w)
1197 def __init__(self, filename: str) -> None:
1200 filename: the ultimate destination file we want to populate.
1201 On exit, the file will be atomically created.
1203 self.filename = filename
1205 self.tempfile = f"{filename}-{uuid}.tmp"
1206 self.handle: Optional[TextIO] = None
1208 def __enter__(self) -> TextIO:
1209 assert not does_path_exist(self.tempfile)
1210 self.handle = open(self.tempfile, mode="w")
1213 def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
1214 if self.handle is not None:
1216 cmd = f"/bin/mv -f {self.tempfile} {self.filename}"
1217 ret = os.system(cmd)
1219 raise Exception(f"{cmd} failed, exit value {ret>>8}!")
1223 class CreateFileWithMode(contextlib.AbstractContextManager):
1224 """This helper context manager can be used instead of the typical
1225 pattern for creating a file if you want to ensure that the file
1226 created is a particular permission mode upon creation.
1228 Python's open doesn't support this; you need to set the os.umask
1229 and then create a descriptor to open via os.open, see below.
1232 >>> filename = f'/tmp/CreateFileWithModeTest.{os.getpid()}'
1233 >>> with CreateFileWithMode(filename, mode=0o600) as wf:
1234 ... print('This is a test', file=wf)
1235 >>> result = os.stat(filename)
1237 Note: there is a high order bit set in this that is S_IFREG indicating
1238 that the file is a "normal file". Clear it with the mask.
1240 >>> print(f'{result.st_mode & 0o7777:o}')
1242 >>> with open(filename, 'r') as rf:
1243 ... contents = rf.read()
1246 >>> remove(filename)
1249 def __init__(self, filename: str, mode=0o600) -> None:
1252 filename: path of the file to create. It must not already
1253 exist or we raise an Exception.
1254 mode: the UNIX-style octal mode with which to create the
1257 self.filename = filename
1258 self.mode = mode & 0o7777
1259 self.handle: Optional[TextIO] = None
1260 self.old_umask = os.umask(0)
1262 def __enter__(self) -> TextIO:
1263 if does_file_exist(self.filename):
1265 f"{self.filename} already exists; it must not to use this class"
1267 descriptor = os.open(
1269 flags=(os.O_WRONLY | os.O_CREAT | os.O_TRUNC),
1272 self.handle = open(descriptor, "w")
1275 def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
1276 os.umask(self.old_umask)
1277 if self.handle is not None:
1282 if __name__ == "__main__":