From 84f53cd94d18a8d239216704a6a33c7dbf2fc6b9 Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Thu, 13 Oct 2022 11:44:10 -0700 Subject: [PATCH] Improve docs. --- docs/pyutils.collectionz.rst | 22 ++++ docs/pyutils.compress.rst | 5 + docs/pyutils.datetimez.rst | 28 +++++ docs/pyutils.files.rst | 24 ++++ docs/pyutils.parallelize.rst | 28 +++++ docs/pyutils.rst | 160 ++++++++++++++++++++++++ examples/README | 3 +- examples/reminder/.gitignore | 1 + src/pyutils/ansi.py | 39 +++--- src/pyutils/argparse_utils.py | 112 ++++++++++++++--- src/pyutils/datetimez/datetime_utils.py | 18 ++- src/pyutils/decorator_utils.py | 5 + 12 files changed, 405 insertions(+), 40 deletions(-) create mode 100644 examples/reminder/.gitignore diff --git a/docs/pyutils.collectionz.rst b/docs/pyutils.collectionz.rst index c342450..e0632af 100644 --- a/docs/pyutils.collectionz.rst +++ b/docs/pyutils.collectionz.rst @@ -7,6 +7,11 @@ Submodules pyutils.collectionz.bidict module --------------------------------- +The bidict.BiDict class is a bidirectional dictionary. It maps each +key to a value in constant time and each value back to the one or more +keys it is associated with in constant time. It does this by simply +storing the data twice. + .. automodule:: pyutils.collectionz.bidict :members: :undoc-members: @@ -15,6 +20,8 @@ pyutils.collectionz.bidict module pyutils.collectionz.bst module ------------------------------ +The bst.BinarySearchTree class is a binary search tree container. + .. automodule:: pyutils.collectionz.bst :members: :undoc-members: @@ -23,6 +30,14 @@ pyutils.collectionz.bst module pyutils.collectionz.shared\_dict module --------------------------------------- +The shared\_dict.SharedDict class is a normal python dictionary that +can be accessed safely in parallel from multiple threads or processes +without (external) locking by using Multiprocessing.SharedMemory. It +uses internal locking and rewrites the shared memory region as it is +changed so it is slower than a normal dict. It also does not grow +dynamically; the creator of the shared\_dict must declare a maximum +size. + .. automodule:: pyutils.collectionz.shared_dict :members: :undoc-members: @@ -31,6 +46,13 @@ pyutils.collectionz.shared\_dict module pyutils.collectionz.trie module ------------------------------- +The trie.Trie class is a Trie or prefix tree. It can be used with +arbitrary sequences as keys and stores its values in a tree with paths +determined by the sequence determined by each key. Thus, it can +determine whether a value is contained in the tree via a simple +traversal in linear time and can also check whether a key-prefix is +present in the tree in linear time. + .. automodule:: pyutils.collectionz.trie :members: :undoc-members: diff --git a/docs/pyutils.compress.rst b/docs/pyutils.compress.rst index e07a5e9..7957494 100644 --- a/docs/pyutils.compress.rst +++ b/docs/pyutils.compress.rst @@ -7,6 +7,11 @@ Submodules pyutils.compress.letter\_compress module ---------------------------------------- +This is a simple, honestly, toy compression scheme that uses a custom +alphabet of 32 characters which can each be represented in six bits +instead of eight. It therefore reduces the size of data composed of +only those letters by 25% without loss. + .. automodule:: pyutils.compress.letter_compress :members: :undoc-members: diff --git a/docs/pyutils.datetimez.rst b/docs/pyutils.datetimez.rst index 167a22c..3b6cf28 100644 --- a/docs/pyutils.datetimez.rst +++ b/docs/pyutils.datetimez.rst @@ -7,6 +7,8 @@ Submodules pyutils.datetimez.constants module ---------------------------------- +A set of date and time related constants. + .. automodule:: pyutils.datetimez.constants :members: :undoc-members: @@ -15,6 +17,16 @@ pyutils.datetimez.constants module pyutils.datetimez.dateparse\_utils module ----------------------------------------- +The dateparse\_utils.DateParser class uses an English language grammar +(see dateparse\_utils.g4) to parse free form English text into a Python +datetime. It can handle somewhat complex constructs such as: "20 days +from next Wed at 3pm", "last Christmas", and "The 2nd Sunday in May, +2022". See the dateparse_utils_test.py for more examples. + +This code is used by other code in the pyutils library; for example, +when using argparse_utils.py to pass an argument of type datetime it +allows the user to use free form english expressions. + .. automodule:: pyutils.datetimez.dateparse_utils :members: :undoc-members: @@ -23,6 +35,9 @@ pyutils.datetimez.dateparse\_utils module pyutils.datetimez.dateparse\_utilsLexer module ---------------------------------------------- +This code is auto-generated by ANTLR from the dateparse\_utils.g4 +grammar. + .. automodule:: pyutils.datetimez.dateparse_utilsLexer :members: :undoc-members: @@ -31,6 +46,9 @@ pyutils.datetimez.dateparse\_utilsLexer module pyutils.datetimez.dateparse\_utilsListener module ------------------------------------------------- +This code is auto-generated by ANTLR from the dateparse\_utils.g4 +grammar. + .. automodule:: pyutils.datetimez.dateparse_utilsListener :members: :undoc-members: @@ -39,6 +57,9 @@ pyutils.datetimez.dateparse\_utilsListener module pyutils.datetimez.dateparse\_utilsParser module ----------------------------------------------- +This code is auto-generated by ANTLR from the dateparse\_utils.g4 +grammar. + .. automodule:: pyutils.datetimez.dateparse_utilsParser :members: :undoc-members: @@ -47,6 +68,11 @@ pyutils.datetimez.dateparse\_utilsParser module pyutils.datetimez.datetime\_utils module ---------------------------------------- +This is a set of utilities for dealing with Python datetimes and +dates. It supports operations such as checking timezones, +manipulating timezones, easy formatting, and using offsets with +datetimes. + .. automodule:: pyutils.datetimez.datetime_utils :members: :undoc-members: @@ -55,6 +81,8 @@ pyutils.datetimez.datetime\_utils module Module contents --------------- +This module contains utilities for dealing with Python datetimes. + .. automodule:: pyutils.datetimez :members: :undoc-members: diff --git a/docs/pyutils.files.rst b/docs/pyutils.files.rst index 6878963..c1152d8 100644 --- a/docs/pyutils.files.rst +++ b/docs/pyutils.files.rst @@ -7,6 +7,15 @@ Submodules pyutils.files.directory\_filter module -------------------------------------- +This module contains two classes meant to help reduce unnecessary disk +I/O operations: + +The first determines when the contents of a file held in memory are +identical to the file copy already on disk. The second is basically +the same except for the caller need not indicate the name of the disk +file because it will check the memory file's signature against a set +of signatures of all files in a particular directory on disk. + .. automodule:: pyutils.files.directory_filter :members: :undoc-members: @@ -15,6 +24,11 @@ pyutils.files.directory\_filter module pyutils.files.file\_utils module -------------------------------- +This is a grab bag of file-related utilities. It has code to, for example, +read files transforming the text as its read, normalize pathnames, strip +extensions, read and manipulate atimes/mtimes/ctimes, compute a signature +based on a file's contents, traverse the file system recursively, etc... + .. automodule:: pyutils.files.file_utils :members: :undoc-members: @@ -23,6 +37,14 @@ pyutils.files.file\_utils module pyutils.files.lockfile module ----------------------------- +This is a lockfile implementation I created for use with cronjobs on +my machine to prevent multiple copies of a job from running in +parallel. When one job is running this code keeps a file on disk to +indicate a lock is held. Other copies will fail to start if they +detect this lock until the lock is released. There are provisions in +the code for timing out locks, cleaning up a lock when a signal is +received, gracefully retrying lock acquisition on failure, etc... + .. automodule:: pyutils.files.lockfile :members: :undoc-members: @@ -31,6 +53,8 @@ pyutils.files.lockfile module Module contents --------------- +This module contains utilities for dealing with files on disk. + .. automodule:: pyutils.files :members: :undoc-members: diff --git a/docs/pyutils.parallelize.rst b/docs/pyutils.parallelize.rst index c53882d..32cdc25 100644 --- a/docs/pyutils.parallelize.rst +++ b/docs/pyutils.parallelize.rst @@ -7,6 +7,8 @@ Submodules pyutils.parallelize.deferred\_operand module -------------------------------------------- +DeferredOperand is the base class for SmartFuture. + .. automodule:: pyutils.parallelize.deferred_operand :members: :undoc-members: @@ -15,6 +17,13 @@ pyutils.parallelize.deferred\_operand module pyutils.parallelize.executors module ------------------------------------ +This module defines three executors: one for threads in the same +process, one for separate processes on the same machine and the third +for separate processes on remote machines. Each can be used via the +@parallelize decorator. These executor pools are automatically +cleaned up at program exit. + + .. automodule:: pyutils.parallelize.executors :members: :undoc-members: @@ -23,6 +32,8 @@ pyutils.parallelize.executors module pyutils.parallelize.parallelize module -------------------------------------- +This module defines a decorator that can be used for simple parallelization. + .. automodule:: pyutils.parallelize.parallelize :members: :undoc-members: @@ -31,6 +42,16 @@ pyutils.parallelize.parallelize module pyutils.parallelize.smart\_future module ---------------------------------------- +Defines a SmartFuture class that is part of the parallelization +framework. A SmartFuture is a kind of Future (i.e. a representation +of the result of asynchronous processing that may know its value or +not depending on whether the asynchronous operation has completed). +Whereas normal Python Futures must be waited on or resolved manually, +a SmartFuture automatically waits for its result to be known as soon +as it is utilized in an expression that demands its value. + +Also contains some utilility code for waiting for one/many futures. + .. automodule:: pyutils.parallelize.smart_future :members: :undoc-members: @@ -39,6 +60,8 @@ pyutils.parallelize.smart\_future module pyutils.parallelize.thread\_utils module ---------------------------------------- +Simple utils that deal with threads. + .. automodule:: pyutils.parallelize.thread_utils :members: :undoc-members: @@ -47,6 +70,11 @@ pyutils.parallelize.thread\_utils module Module contents --------------- +This module contains a framework for easy Python parallelization. To +see an example of how it is used, look at examples/wordle/... + +This module also contains some utilities that deal with parallelization. + .. automodule:: pyutils.parallelize :members: :undoc-members: diff --git a/docs/pyutils.rst b/docs/pyutils.rst index d8ba243..b7e0783 100644 --- a/docs/pyutils.rst +++ b/docs/pyutils.rst @@ -1,6 +1,45 @@ pyutils package =============== +When I was writing little tools in Python and found myself implementing +a generally useful pattern I stuffed it into a local library. That +library grew into pyutils: a set of collections, helpers and utilities +that I find useful and hope you will too. + +Code is under `src/pyutils/`. Most code includes documentation and inline +doctests. + +Unit and integration tests are under `tests/*`. To run all tests:: + + cd tests/ + ./run_tests.py --all [--coverage] + +See the README under `tests/` and the code of `run_tests.py` for more +options / information. + +This package generates Sphinx docs which are available at: + + https://wannabe.guru.org/pydocs/pyutils/pyutils.html + +Package code is checked into a local git server and available to clone +from git at https://wannabe.guru.org/git/pyutils.git or from a web browser +at: + + https://wannabe.guru.org/gitweb/?p=pyutils.git;a=summary + +For a long time this was just a local library on my machine that my +tools imported but I've now decided to release it on PyPi. Early +development happened in a different git repo: + + https://wannabe.guru.org/gitweb/?p=python_utils.git;a=summary + +I hope you find this useful. LICENSE and NOTICE describe reusing it +and where everything came from. Drop me a line if you are using this, +find a bug, or have a question: + + --Scott Gasch (scott.gasch@gmail.com) + + Subpackages ----------- @@ -22,6 +61,14 @@ Submodules pyutils.ansi module ------------------- +This file mainly contains code for changing the nature of text printed +to the console via ANSI escape sequences. e.g. it can be used to emit +text that is bolded, underlined, italicised, colorized, etc... + +It also contains a colorizing context that will apply color patterns +based on regular expressions to any data emitted to stdout that may be +useful in adding color to other programs' outputs, for instance. + .. automodule:: pyutils.ansi :members: :undoc-members: @@ -30,6 +77,16 @@ pyutils.ansi module pyutils.argparse\_utils module ------------------------------ +I use the Python internal `argparse` module for commandline argument +parsing but found it lacking in some ways. This module contains code to +fill those gaps. It include stuff like: + + - An `argparse.Action` to create pairs of flags such as + `--feature` and `--no_feature`. + - A helper to parse and validate bools, IP addresses, MAC + addresses, filenames, percentages, dates, datetimes, and + durations passed as flags. + .. automodule:: pyutils.argparse_utils :members: :undoc-members: @@ -38,6 +95,17 @@ pyutils.argparse\_utils module pyutils.bootstrap module ------------------------ +Bootstrap module defines a decorator meant to wrap your main function. +This decorator will do several things for you: + + - If your code uses the :file:`config.py` module (see below), it invokes + `parse` automatically to initialize `config.config` from commandline + flags, environment variables, or other sources. + - It will optionally break into pdb in response to an unhandled + Exception at the top level of your code. + - It initializes logging for your program (see :file:`logging.py`). + - It can optionally run a code and/or memory profiler on your code. + .. automodule:: pyutils.bootstrap :members: :undoc-members: @@ -46,6 +114,22 @@ pyutils.bootstrap module pyutils.config module --------------------- +This module reads the program's configuration parameters from: the +commandline (using argparse), environment variables and/or a shared +zookeeper-based configuration. It stores this configuration state in +a dict-like structure that can be queried by your code at runtime. + +It handles creating a nice `--help` message for your code. + +It can optionally react to dynamic config changes and change state at +runtime (iff you name your flag with the *dynamic_* prefix and are +using zookeeper-based configs). + +It can optionally save and retrieve sets of arguments from files on +the local disk or on zookeeper. + +All of my examples use this as does the pyutils library itself. + .. automodule:: pyutils.config :members: :undoc-members: @@ -54,6 +138,8 @@ pyutils.config module pyutils.decorator\_utils module ------------------------------- +This is a grab bag of decorators. + .. automodule:: pyutils.decorator_utils :members: :undoc-members: @@ -62,6 +148,8 @@ pyutils.decorator\_utils module pyutils.dict\_utils module -------------------------- +A bunch of helpers for dealing with Python dicts. + .. automodule:: pyutils.dict_utils :members: :undoc-members: @@ -70,6 +158,8 @@ pyutils.dict\_utils module pyutils.exec\_utils module -------------------------- +Helper code for dealing with subprocesses. + .. automodule:: pyutils.exec_utils :members: :undoc-members: @@ -78,6 +168,8 @@ pyutils.exec\_utils module pyutils.function\_utils module ------------------------------ +Helper util for dealing with functions. + .. automodule:: pyutils.function_utils :members: :undoc-members: @@ -86,6 +178,8 @@ pyutils.function\_utils module pyutils.id\_generator module ---------------------------- +Generate unique identifiers. + .. automodule:: pyutils.id_generator :members: :undoc-members: @@ -94,6 +188,9 @@ pyutils.id\_generator module pyutils.iter\_utils module -------------------------- +Iterator utilities including a :class:PeekingIterator, :class:PushbackIterator, +and :class:SamplingIterator. + .. automodule:: pyutils.iter_utils :members: :undoc-members: @@ -102,6 +199,8 @@ pyutils.iter\_utils module pyutils.list\_utils module -------------------------- +Utilities for dealing with Python lists. + .. automodule:: pyutils.list_utils :members: :undoc-members: @@ -110,6 +209,20 @@ pyutils.list\_utils module pyutils.logging\_utils module ----------------------------- +This is a module that offers an opinionated take on how whole program logging +should be initialized and controlled. It uses standard Python logging but gives +you control, via commandline config, to: + + - Set the logging level of the program including overriding the + logging level for individual modules, + - Define the logging message format including easily adding a + PID/TID marker on all messages to help with multithreaded debugging, + - Control the destination (file, sys.stderr, syslog) of messages, + - Control the facility and logging level used with syslog, + - Squelch repeated messages, + - Log probalistically, + - Clear rogue logging handlers added by other imports. + .. automodule:: pyutils.logging_utils :members: :undoc-members: @@ -118,6 +231,11 @@ pyutils.logging\_utils module pyutils.math\_utils module -------------------------- +Helper utilities that are "mathy" such as a :class:NumericPopulation that +makes population summary statistics available to your code quickly, GCD +computation, literate float truncation, percentage <-> multiplier, prime +number determination, etc... + .. automodule:: pyutils.math_utils :members: :undoc-members: @@ -126,6 +244,8 @@ pyutils.math\_utils module pyutils.misc\_utils module -------------------------- +Miscellaneous utilities: are we running as root, and is a debugger attached? + .. automodule:: pyutils.misc_utils :members: :undoc-members: @@ -134,6 +254,11 @@ pyutils.misc\_utils module pyutils.persistent module ------------------------- +Persistent defines a class hierarchy and decorator for creating +singleton classes that (optionally/conditionally) load their state +from some external location and (optionally/conditionally) stave their +state to an external location at shutdown. + .. automodule:: pyutils.persistent :members: :undoc-members: @@ -142,6 +267,10 @@ pyutils.persistent module pyutils.remote\_worker module ----------------------------- +This module defines a helper that is invoked by the remote executor to +run pickled code on a remote machine. It is used by code marked with +@parallelize(method=Method.REMOTE) in the parallelize framework. + .. automodule:: pyutils.remote_worker :members: :undoc-members: @@ -150,6 +279,14 @@ pyutils.remote\_worker module pyutils.state\_tracker module ----------------------------- +This module defines several classes (:class:StateTracker, +:class:AutomaticStateTracker, and +:class:WaitableAutomaticStateTracker) that can be used as base +classes. These class patterns are meant to encapsulate and represent +state that dynamically changes. These classes update their state +(either automatically or when invoked to poll) and allow their callers +to wait on state changes. + .. automodule:: pyutils.state_tracker :members: :undoc-members: @@ -158,6 +295,9 @@ pyutils.state\_tracker module pyutils.stopwatch module ------------------------ +This is a stopwatch context that just times how long something took to +execute. + .. automodule:: pyutils.stopwatch :members: :undoc-members: @@ -166,6 +306,10 @@ pyutils.stopwatch module pyutils.string\_utils module ---------------------------- +A bunch of utilities for dealing with strings. Based on a really great +starting library from Davide Zanotti, I've added a pile of other string +functions so hopefully it will handle all of your string-needs. + .. automodule:: pyutils.string_utils :members: :undoc-members: @@ -174,6 +318,16 @@ pyutils.string\_utils module pyutils.text\_utils module -------------------------- +Utilities for dealing with and creating text chunks. For example: + + - Make a bar graph, + - make a spark line, + - left, right, center, justify text, + - word wrap text, + - indent text, + - create a header line, + - draw a box around some text. + .. automodule:: pyutils.text_utils :members: :undoc-members: @@ -182,6 +336,8 @@ pyutils.text\_utils module pyutils.unittest\_utils module ------------------------------ +Utilities to support smarter unit tests. + .. automodule:: pyutils.unittest_utils :members: :undoc-members: @@ -190,6 +346,8 @@ pyutils.unittest\_utils module pyutils.unscrambler module -------------------------- +Unscramble scrambled English words quickly. + .. automodule:: pyutils.unscrambler :members: :undoc-members: @@ -198,6 +356,8 @@ pyutils.unscrambler module pyutils.zookeeper module ------------------------ +A helper module for dealing with Zookeeper that adds some functionality. + .. automodule:: pyutils.zookeeper :members: :undoc-members: diff --git a/examples/README b/examples/README index 5d02a87..c14e863 100644 --- a/examples/README +++ b/examples/README @@ -1,2 +1,3 @@ Stuff under here is example code that uses pyutils library routines and -is meant to just be illustrative and fun. +is meant to just be illustrative and fun. Each should be runnable as-is +if you have pyutils installed. Use the --help flag for more info. diff --git a/examples/reminder/.gitignore b/examples/reminder/.gitignore new file mode 100644 index 0000000..0a79c83 --- /dev/null +++ b/examples/reminder/.gitignore @@ -0,0 +1 @@ +.reminder_cache diff --git a/src/pyutils/ansi.py b/src/pyutils/ansi.py index 1d45d3b..b885a40 100755 --- a/src/pyutils/ansi.py +++ b/src/pyutils/ansi.py @@ -4,7 +4,7 @@ """A bunch of color names mapped into RGB tuples and some methods for setting the text color, background, etc... using ANSI escape -sequences. +sequences. See: https://en.wikipedia.org/wiki/ANSI_escape_code. """ import contextlib @@ -22,8 +22,6 @@ from pyutils import logging_utils, string_utils logger = logging.getLogger(__name__) -# https://en.wikipedia.org/wiki/ANSI_escape_code - COLOR_NAMES_TO_RGB: Dict[str, Tuple[int, int, int]] = { "abbey": (0x4C, 0x4F, 0x56), @@ -1673,37 +1671,37 @@ COLOR_NAMES_TO_RGB: Dict[str, Tuple[int, int, int]] = { def clear() -> str: """Clear screen ANSI escape sequence""" - return "" + return "\x1B[H\x1B[2J" def clear_screen() -> str: """Clear screen ANSI escape sequence""" - return "" + return "\x1B[H\x1B[2J" def clear_line() -> str: """Clear the current line ANSI escape sequence""" - return "\r" + return "\x1B[2K\r" def reset() -> str: """Reset text attributes to 'normal'""" - return "" + return "\x1B[m" def normal() -> str: """Reset text attributes to 'normal'""" - return "" + return "\x1B[m" def bold() -> str: """Set text to bold""" - return "" + return "\x1B[1m" def italic() -> str: """Set text to italic""" - return "" + return "\x1B[3m" def italics() -> str: @@ -1713,12 +1711,12 @@ def italics() -> str: def underline() -> str: """Set text to underline""" - return "" + return "\x1B[4m" def strikethrough() -> str: """Set text to strikethrough""" - return "" + return "\x1B[9m" def strike_through() -> str: @@ -1756,7 +1754,7 @@ def fg_16color(red: int, green: int, blue: int) -> str: bright_count += 1 if bright_count > 1: code += 60 - return f"[{code}m" + return f"\x1B[{code}m" def bg_16color(red: int, green: int, blue: int) -> str: @@ -1771,7 +1769,7 @@ def bg_16color(red: int, green: int, blue: int) -> str: bright_count += 1 if bright_count > 1: code += 60 - return f"[{code}m" + return f"\x1B[{code}m" def _pixel_to_216color(n: int) -> int: @@ -1794,7 +1792,7 @@ def fg_216color(red: int, green: int, blue: int) -> str: g = _pixel_to_216color(green) b = _pixel_to_216color(blue) code = 16 + r * 36 + g * 6 + b - return f"[38;5;{code}m" + return f"\x1B[38;5;{code}m" def bg_216color(red: int, green: int, blue: int) -> str: @@ -1803,17 +1801,17 @@ def bg_216color(red: int, green: int, blue: int) -> str: g = _pixel_to_216color(green) b = _pixel_to_216color(blue) code = 16 + r * 36 + g * 6 + b - return f"[48;5;{code}m" + return f"\x1B[48;5;{code}m" def fg_24bit(red: int, green: int, blue: int) -> str: """Set foreground using 24bit color mode""" - return f"[38;2;{red};{green};{blue}m" + return f"\x1B[38;2;{red};{green};{blue}m" def bg_24bit(red: int, green: int, blue: int) -> str: """Set background using 24bit color mode""" - return f"[48;2;{red};{green};{blue}m" + return f"\x1B[48;2;{red};{green};{blue}m" def _find_color_by_name(name: str) -> Tuple[int, int, int]: @@ -1889,6 +1887,8 @@ def fg( def reset_fg(): + """Returns: an ANSI escape code to reset just the foreground color while + preserving the background color and any other formatting (bold, italics, etc...)""" return '\033[39m' @@ -1974,6 +1974,9 @@ def bg( force_16color: force bg to use 16 color mode force_216color: force bg to use 216 color mode + Returns: + A string containing the requested escape sequence + >>> import string_utils as su >>> su.to_base64(bg("red")) # b'\x1b[48;5;196m' b'G1s0ODs1OzE5Nm0=\\n' diff --git a/src/pyutils/argparse_utils.py b/src/pyutils/argparse_utils.py index 3b466b0..daca1df 100644 --- a/src/pyutils/argparse_utils.py +++ b/src/pyutils/argparse_utils.py @@ -74,7 +74,15 @@ class ActionNoYes(argparse.Action): def valid_bool(v: Any) -> bool: """ - If the string is a valid bool, return its value. + If the string is a valid bool, return its value. Sample usage:: + + args.add_argument( + '--auto', + type=argparse_utils.valid_bool, + default=None, + metavar='True|False', + help='Use your best judgement about --primary and --secondary', + ) >>> valid_bool(True) True @@ -110,7 +118,15 @@ def valid_bool(v: Any) -> bool: def valid_ip(ip: str) -> str: """ If the string is a valid IPv4 address, return it. Otherwise raise - an ArgumentTypeError. + an ArgumentTypeError. Sample usage:: + + group.add_argument( + "-i", + "--ip_address", + metavar="TARGET_IP_ADDRESS", + help="Target IP Address", + type=argparse_utils.valid_ip, + ) >>> valid_ip("1.2.3.4") '1.2.3.4' @@ -134,7 +150,15 @@ def valid_ip(ip: str) -> str: def valid_mac(mac: str) -> str: """ If the string is a valid MAC address, return it. Otherwise raise - an ArgumentTypeError. + an ArgumentTypeError. Sample usage:: + + group.add_argument( + "-m", + "--mac", + metavar="MAC_ADDRESS", + help="Target MAC Address", + type=argparse_utils.valid_mac, + ) >>> valid_mac('12:23:3A:4F:55:66') '12:23:3A:4F:55:66' @@ -160,8 +184,15 @@ def valid_mac(mac: str) -> str: def valid_percentage(num: str) -> float: """ - If the string is a valid percentage, return it. Otherwise raise - an ArgumentTypeError. + If the string is a valid (0 <= n <= 100) percentage, return it. + Otherwise raise an ArgumentTypeError. Sample usage:: + + args.add_argument( + '--percent_change', + type=argparse_utils.valid_percentage, + default=0, + help='The percent change (0<=n<=100) of foobar', + ) >>> valid_percentage("15%") 15.0 @@ -187,7 +218,15 @@ def valid_percentage(num: str) -> float: def valid_filename(filename: str) -> str: """ If the string is a valid filename, return it. Otherwise raise - an ArgumentTypeError. + an ArgumentTypeError. Sample usage:: + + args.add_argument( + '--network_mac_addresses_file', + default='/home/scott/bin/network_mac_addresses.txt', + metavar='FILENAME', + help='Location of the network_mac_addresses file (must exist!).', + type=argparse_utils.valid_filename, + ) >>> valid_filename('/tmp') '/tmp' @@ -208,14 +247,23 @@ def valid_filename(filename: str) -> str: def valid_date(txt: str) -> datetime.date: """If the string is a valid date, return it. Otherwise raise - an ArgumentTypeError. + an ArgumentTypeError. Sample usage:: + + cfg.add_argument( + "--date", + nargs=1, + type=argparse_utils.valid_date, + metavar="DATE STRING", + default=None + ) >>> valid_date('6/5/2021') datetime.date(2021, 6, 5) - # Note: dates like 'next wednesday' work fine, they are just - # hard to test for without knowing when the testcase will be - # executed... + Note: dates like 'next wednesday' work fine, they are just + hard to test for without knowing when the testcase will be + executed... + >>> valid_date('next wednesday') # doctest: +ELLIPSIS -ANYTHING- """ @@ -231,14 +279,26 @@ def valid_date(txt: str) -> datetime.date: def valid_datetime(txt: str) -> datetime.datetime: """If the string is a valid datetime, return it. Otherwise raise - an ArgumentTypeError. + an ArgumentTypeError. Sample usage:: + + cfg.add_argument( + "--override_timestamp", + nargs=1, + type=argparse_utils.valid_datetime, + help="Don't use the current datetime, override to argument.", + metavar="DATE/TIME STRING", + default=None, + ) >>> valid_datetime('6/5/2021 3:01:02') datetime.datetime(2021, 6, 5, 3, 1, 2) - # Again, these types of expressions work fine but are - # difficult to test with doctests because the answer is - # relative to the time the doctest is executed. + Because this thing uses an English date-expression parsing grammar + internally, much more complex datetimes can be expressed in free form. + See: `tests/datetimez/dateparse_utils_test.py` for examples. These + are not included in here because they are hard to write valid doctests + for! + >>> valid_datetime('next christmas at 4:15am') # doctest: +ELLIPSIS -ANYTHING- """ @@ -254,21 +314,33 @@ def valid_datetime(txt: str) -> datetime.datetime: def valid_duration(txt: str) -> datetime.timedelta: """If the string is a valid time duration, return a - datetime.timedelta representing the period of time. Otherwise - maybe raise an ArgumentTypeError or potentially just treat the - time window as zero in length. + datetime.timedelta representing the period of time. + Sample usage:: + + cfg.add_argument( + '--ip_cache_max_staleness', + type=argparse_utils.valid_duration, + default=datetime.timedelta(seconds=60 * 60 * 12), + metavar='DURATION', + help='Max acceptable age of the IP address cache' + ) >>> valid_duration('3m') datetime.timedelta(seconds=180) - >>> valid_duration('your mom') - datetime.timedelta(0) + >>> valid_duration('3 days, 2 hours') + datetime.timedelta(days=3, seconds=7200) + + >>> valid_duration('a little while') + Traceback (most recent call last): + ... + argparse.ArgumentTypeError: a little while is not a valid duration. """ from pyutils.datetimez.datetime_utils import parse_duration try: - secs = parse_duration(txt) + secs = parse_duration(txt, raise_on_error=True) return datetime.timedelta(seconds=secs) except Exception as e: logger.exception(e) diff --git a/src/pyutils/datetimez/datetime_utils.py b/src/pyutils/datetimez/datetime_utils.py index 6026d9a..ed76c9e 100644 --- a/src/pyutils/datetimez/datetime_utils.py +++ b/src/pyutils/datetimez/datetime_utils.py @@ -804,7 +804,7 @@ def minute_number_to_time_string(minute_num: MinuteOfDay) -> str: return f"{hour:2}:{minute:02}{ampm}" -def parse_duration(duration: str) -> int: +def parse_duration(duration: str, raise_on_error=False) -> int: """ Parse a duration in string form into a delta seconds. @@ -820,9 +820,25 @@ def parse_duration(duration: str) -> int: >>> parse_duration('3min 2sec') 182 + >>> parse_duration('recent') + 0 + + >>> parse_duration('recent', raise_on_error=True) + Traceback (most recent call last): + ... + ValueError: recent is not a valid duration. + """ if duration.isdigit(): return int(duration) + + m = re.match( + r'(\d+ *d[ays]*)* *(\d+ *h[ours]*)* *(\d+ *m[inutes]*)* *(\d+ *[seconds]*)', + duration, + ) + if not m and raise_on_error: + raise ValueError(f'{duration} is not a valid duration.') + seconds = 0 m = re.search(r'(\d+) *d[ays]*', duration) if m is not None: diff --git a/src/pyutils/decorator_utils.py b/src/pyutils/decorator_utils.py index e8d2249..30b1bfb 100644 --- a/src/pyutils/decorator_utils.py +++ b/src/pyutils/decorator_utils.py @@ -488,6 +488,11 @@ def deprecated(func): """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used. + + >>> @deprecated + ... def foo() -> None: + ... pass + >>> foo() # prints + logs "Call to deprecated function foo" """ @functools.wraps(func) -- 2.45.2