Improve docs.

diff --git a/docs/pyutils.collectionz.rst b/docs/pyutils.collectionz.rst
index c3424503628c6c817a46203a42aefa903851ae4d..e0632afdbc3be35fc16657532065ed07646b57f4 100644 (file)
--- a/docs/pyutils.collectionz.rst
+++ b/docs/pyutils.collectionz.rst
@@ -7,6 +7,11 @@ Submodules
 pyutils.collectionz.bidict module
 ---------------------------------
 
 pyutils.collectionz.bidict module
 ---------------------------------
 

+The bidict.BiDict class is a bidirectional dictionary.  It maps each
+key to a value in constant time and each value back to the one or more
+keys it is associated with in constant time.  It does this by simply
+storing the data twice.
+

 .. automodule:: pyutils.collectionz.bidict
    :members:
    :undoc-members:
 .. automodule:: pyutils.collectionz.bidict
    :members:
    :undoc-members:


@@ -15,6 +20,8 @@ pyutils.collectionz.bidict module
 pyutils.collectionz.bst module
 ------------------------------
 
 pyutils.collectionz.bst module
 ------------------------------
 

+The bst.BinarySearchTree class is a binary search tree container.
+

 .. automodule:: pyutils.collectionz.bst
    :members:
    :undoc-members:
 .. automodule:: pyutils.collectionz.bst
    :members:
    :undoc-members:


@@ -23,6 +30,14 @@ pyutils.collectionz.bst module
 pyutils.collectionz.shared\_dict module
 ---------------------------------------
 
 pyutils.collectionz.shared\_dict module
 ---------------------------------------
 

+The shared\_dict.SharedDict class is a normal python dictionary that
+can be accessed safely in parallel from multiple threads or processes
+without (external) locking by using Multiprocessing.SharedMemory.  It
+uses internal locking and rewrites the shared memory region as it is
+changed so it is slower than a normal dict.  It also does not grow
+dynamically; the creator of the shared\_dict must declare a maximum
+size.
+

 .. automodule:: pyutils.collectionz.shared_dict
    :members:
    :undoc-members:
 .. automodule:: pyutils.collectionz.shared_dict
    :members:
    :undoc-members:


@@ -31,6 +46,13 @@ pyutils.collectionz.shared\_dict module
 pyutils.collectionz.trie module
 -------------------------------
 
 pyutils.collectionz.trie module
 -------------------------------
 

+The trie.Trie class is a Trie or prefix tree.  It can be used with
+arbitrary sequences as keys and stores its values in a tree with paths
+determined by the sequence determined by each key.  Thus, it can
+determine whether a value is contained in the tree via a simple
+traversal in linear time and can also check whether a key-prefix is
+present in the tree in linear time.
+

 .. automodule:: pyutils.collectionz.trie
    :members:
    :undoc-members:
 .. automodule:: pyutils.collectionz.trie
    :members:
    :undoc-members:

diff --git a/docs/pyutils.compress.rst b/docs/pyutils.compress.rst
index e07a5e97e2b0f745abc0dcabf9ada5776830c9b3..7957494e773138987ff211aed2045132a97bb944 100644 (file)
--- a/docs/pyutils.compress.rst
+++ b/docs/pyutils.compress.rst
@@ -7,6 +7,11 @@ Submodules
 pyutils.compress.letter\_compress module
 ----------------------------------------
 
 pyutils.compress.letter\_compress module
 ----------------------------------------
 

+This is a simple, honestly, toy compression scheme that uses a custom
+alphabet of 32 characters which can each be represented in six bits
+instead of eight.  It therefore reduces the size of data composed of
+only those letters by 25% without loss.
+

 .. automodule:: pyutils.compress.letter_compress
    :members:
    :undoc-members:
 .. automodule:: pyutils.compress.letter_compress
    :members:
    :undoc-members:

diff --git a/docs/pyutils.datetimez.rst b/docs/pyutils.datetimez.rst
index 167a22cadca925f62dd27f06b592b05dfcd1f0a0..3b6cf28ca751be45b58d3bfdf005ed803b88b20d 100644 (file)
--- a/docs/pyutils.datetimez.rst
+++ b/docs/pyutils.datetimez.rst
@@ -7,6 +7,8 @@ Submodules
 pyutils.datetimez.constants module
 ----------------------------------
 
 pyutils.datetimez.constants module
 ----------------------------------
 

+A set of date and time related constants.
+

 .. automodule:: pyutils.datetimez.constants
    :members:
    :undoc-members:
 .. automodule:: pyutils.datetimez.constants
    :members:
    :undoc-members:


@@ -15,6 +17,16 @@ pyutils.datetimez.constants module
 pyutils.datetimez.dateparse\_utils module
 -----------------------------------------
 
 pyutils.datetimez.dateparse\_utils module
 -----------------------------------------
 

+The dateparse\_utils.DateParser class uses an English language grammar
+(see dateparse\_utils.g4) to parse free form English text into a Python
+datetime.  It can handle somewhat complex constructs such as: "20 days
+from next Wed at 3pm", "last Christmas", and "The 2nd Sunday in May,
+2022".  See the dateparse_utils_test.py for more examples.
+
+This code is used by other code in the pyutils library; for example,
+when using argparse_utils.py to pass an argument of type datetime it
+allows the user to use free form english expressions.
+

 .. automodule:: pyutils.datetimez.dateparse_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.datetimez.dateparse_utils
    :members:
    :undoc-members:


@@ -23,6 +35,9 @@ pyutils.datetimez.dateparse\_utils module
 pyutils.datetimez.dateparse\_utilsLexer module
 ----------------------------------------------
 
 pyutils.datetimez.dateparse\_utilsLexer module
 ----------------------------------------------
 

+This code is auto-generated by ANTLR from the dateparse\_utils.g4
+grammar.
+

 .. automodule:: pyutils.datetimez.dateparse_utilsLexer
    :members:
    :undoc-members:
 .. automodule:: pyutils.datetimez.dateparse_utilsLexer
    :members:
    :undoc-members:


@@ -31,6 +46,9 @@ pyutils.datetimez.dateparse\_utilsLexer module
 pyutils.datetimez.dateparse\_utilsListener module
 -------------------------------------------------
 
 pyutils.datetimez.dateparse\_utilsListener module
 -------------------------------------------------
 

+This code is auto-generated by ANTLR from the dateparse\_utils.g4
+grammar.
+

 .. automodule:: pyutils.datetimez.dateparse_utilsListener
    :members:
    :undoc-members:
 .. automodule:: pyutils.datetimez.dateparse_utilsListener
    :members:
    :undoc-members:


@@ -39,6 +57,9 @@ pyutils.datetimez.dateparse\_utilsListener module
 pyutils.datetimez.dateparse\_utilsParser module
 -----------------------------------------------
 
 pyutils.datetimez.dateparse\_utilsParser module
 -----------------------------------------------
 

+This code is auto-generated by ANTLR from the dateparse\_utils.g4
+grammar.
+

 .. automodule:: pyutils.datetimez.dateparse_utilsParser
    :members:
    :undoc-members:
 .. automodule:: pyutils.datetimez.dateparse_utilsParser
    :members:
    :undoc-members:


@@ -47,6 +68,11 @@ pyutils.datetimez.dateparse\_utilsParser module
 pyutils.datetimez.datetime\_utils module
 ----------------------------------------
 
 pyutils.datetimez.datetime\_utils module
 ----------------------------------------
 

+This is a set of utilities for dealing with Python datetimes and
+dates.  It supports operations such as checking timezones,
+manipulating timezones, easy formatting, and using offsets with
+datetimes.
+

 .. automodule:: pyutils.datetimez.datetime_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.datetimez.datetime_utils
    :members:
    :undoc-members:


@@ -55,6 +81,8 @@ pyutils.datetimez.datetime\_utils module
 Module contents
 ---------------
 
 Module contents
 ---------------
 

+This module contains utilities for dealing with Python datetimes.
+

 .. automodule:: pyutils.datetimez
    :members:
    :undoc-members:
 .. automodule:: pyutils.datetimez
    :members:
    :undoc-members:

diff --git a/docs/pyutils.files.rst b/docs/pyutils.files.rst
index 6878963cc3051f9c066713d4b934e4c18a415cd2..c1152d8e3ea17c8e9a9dbb803c3c34dfaa15ccd6 100644 (file)
--- a/docs/pyutils.files.rst
+++ b/docs/pyutils.files.rst
@@ -7,6 +7,15 @@ Submodules
 pyutils.files.directory\_filter module
 --------------------------------------
 
 pyutils.files.directory\_filter module
 --------------------------------------
 

+This module contains two classes meant to help reduce unnecessary disk
+I/O operations:
+
+The first determines when the contents of a file held in memory are
+identical to the file copy already on disk.  The second is basically
+the same except for the caller need not indicate the name of the disk
+file because it will check the memory file's signature against a set
+of signatures of all files in a particular directory on disk.
+

 .. automodule:: pyutils.files.directory_filter
    :members:
    :undoc-members:
 .. automodule:: pyutils.files.directory_filter
    :members:
    :undoc-members:


@@ -15,6 +24,11 @@ pyutils.files.directory\_filter module
 pyutils.files.file\_utils module
 --------------------------------
 
 pyutils.files.file\_utils module
 --------------------------------
 

+This is a grab bag of file-related utilities.  It has code to, for example,
+read files transforming the text as its read, normalize pathnames, strip
+extensions, read and manipulate atimes/mtimes/ctimes, compute a signature
+based on a file's contents, traverse the file system recursively, etc...
+

 .. automodule:: pyutils.files.file_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.files.file_utils
    :members:
    :undoc-members:


@@ -23,6 +37,14 @@ pyutils.files.file\_utils module
 pyutils.files.lockfile module
 -----------------------------
 
 pyutils.files.lockfile module
 -----------------------------
 

+This is a lockfile implementation I created for use with cronjobs on
+my machine to prevent multiple copies of a job from running in
+parallel.  When one job is running this code keeps a file on disk to
+indicate a lock is held.  Other copies will fail to start if they
+detect this lock until the lock is released.  There are provisions in
+the code for timing out locks, cleaning up a lock when a signal is
+received, gracefully retrying lock acquisition on failure, etc...
+

 .. automodule:: pyutils.files.lockfile
    :members:
    :undoc-members:
 .. automodule:: pyutils.files.lockfile
    :members:
    :undoc-members:


@@ -31,6 +53,8 @@ pyutils.files.lockfile module
 Module contents
 ---------------
 
 Module contents
 ---------------
 

+This module contains utilities for dealing with files on disk.
+

 .. automodule:: pyutils.files
    :members:
    :undoc-members:
 .. automodule:: pyutils.files
    :members:
    :undoc-members:

diff --git a/docs/pyutils.parallelize.rst b/docs/pyutils.parallelize.rst
index c53882d361bb8b0872e7e2b7155ab01fa0b7b099..32cdc255a0753b79f53dc9d4128d9caba7ee60d0 100644 (file)
--- a/docs/pyutils.parallelize.rst
+++ b/docs/pyutils.parallelize.rst
@@ -7,6 +7,8 @@ Submodules
 pyutils.parallelize.deferred\_operand module
 --------------------------------------------
 
 pyutils.parallelize.deferred\_operand module
 --------------------------------------------
 

+DeferredOperand is the base class for SmartFuture.
+

 .. automodule:: pyutils.parallelize.deferred_operand
    :members:
    :undoc-members:
 .. automodule:: pyutils.parallelize.deferred_operand
    :members:
    :undoc-members:


@@ -15,6 +17,13 @@ pyutils.parallelize.deferred\_operand module
 pyutils.parallelize.executors module
 ------------------------------------
 
 pyutils.parallelize.executors module
 ------------------------------------
 

+This module defines three executors: one for threads in the same
+process, one for separate processes on the same machine and the third
+for separate processes on remote machines.  Each can be used via the
+@parallelize decorator.  These executor pools are automatically
+cleaned up at program exit.
+
+

 .. automodule:: pyutils.parallelize.executors
    :members:
    :undoc-members:
 .. automodule:: pyutils.parallelize.executors
    :members:
    :undoc-members:


@@ -23,6 +32,8 @@ pyutils.parallelize.executors module
 pyutils.parallelize.parallelize module
 --------------------------------------
 
 pyutils.parallelize.parallelize module
 --------------------------------------
 

+This module defines a decorator that can be used for simple parallelization.
+

 .. automodule:: pyutils.parallelize.parallelize
    :members:
    :undoc-members:
 .. automodule:: pyutils.parallelize.parallelize
    :members:
    :undoc-members:


@@ -31,6 +42,16 @@ pyutils.parallelize.parallelize module
 pyutils.parallelize.smart\_future module
 ----------------------------------------
 
 pyutils.parallelize.smart\_future module
 ----------------------------------------
 

+Defines a SmartFuture class that is part of the parallelization
+framework.  A SmartFuture is a kind of Future (i.e. a representation
+of the result of asynchronous processing that may know its value or
+not depending on whether the asynchronous operation has completed).
+Whereas normal Python Futures must be waited on or resolved manually,
+a SmartFuture automatically waits for its result to be known as soon
+as it is utilized in an expression that demands its value.
+
+Also contains some utilility code for waiting for one/many futures.
+

 .. automodule:: pyutils.parallelize.smart_future
    :members:
    :undoc-members:
 .. automodule:: pyutils.parallelize.smart_future
    :members:
    :undoc-members:


@@ -39,6 +60,8 @@ pyutils.parallelize.smart\_future module
 pyutils.parallelize.thread\_utils module
 ----------------------------------------
 
 pyutils.parallelize.thread\_utils module
 ----------------------------------------
 

+Simple utils that deal with threads.
+

 .. automodule:: pyutils.parallelize.thread_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.parallelize.thread_utils
    :members:
    :undoc-members:


@@ -47,6 +70,11 @@ pyutils.parallelize.thread\_utils module
 Module contents
 ---------------
 
 Module contents
 ---------------
 

+This module contains a framework for easy Python parallelization.  To
+see an example of how it is used, look at examples/wordle/...
+
+This module also contains some utilities that deal with parallelization.
+

 .. automodule:: pyutils.parallelize
    :members:
    :undoc-members:
 .. automodule:: pyutils.parallelize
    :members:
    :undoc-members:

diff --git a/docs/pyutils.rst b/docs/pyutils.rst
index d8ba243646941be641eab803986f4fe38ec43fb8..b7e07836efc09fd501f8748b37698422d4fcaf65 100644 (file)
--- a/docs/pyutils.rst
+++ b/docs/pyutils.rst
@@ -1,6 +1,45 @@
 pyutils package
 ===============
 
 pyutils package
 ===============
 

+When I was writing little tools in Python and found myself implementing
+a generally useful pattern I stuffed it into a local library.  That
+library grew into pyutils: a set of collections, helpers and utilities
+that I find useful and hope you will too.
+
+Code is under `src/pyutils/`.  Most code includes documentation and inline
+doctests.
+
+Unit and integration tests are under `tests/*`.  To run all tests::
+
+    cd tests/
+    ./run_tests.py --all [--coverage]
+
+See the README under `tests/` and the code of `run_tests.py` for more
+options / information.
+
+This package generates Sphinx docs which are available at:
+
+    https://wannabe.guru.org/pydocs/pyutils/pyutils.html
+
+Package code is checked into a local git server and available to clone
+from git at https://wannabe.guru.org/git/pyutils.git or from a web browser
+at:
+
+    https://wannabe.guru.org/gitweb/?p=pyutils.git;a=summary
+
+For a long time this was just a local library on my machine that my
+tools imported but I've now decided to release it on PyPi.  Early
+development happened in a different git repo:
+
+    https://wannabe.guru.org/gitweb/?p=python_utils.git;a=summary
+
+I hope you find this useful.  LICENSE and NOTICE describe reusing it
+and where everything came from.  Drop me a line if you are using this,
+find a bug, or have a question:
+
+  --Scott Gasch ([email protected])
+
+

 Subpackages
 -----------
 
 Subpackages
 -----------
 


@@ -22,6 +61,14 @@ Submodules
 pyutils.ansi module
 -------------------
 
 pyutils.ansi module
 -------------------
 

+This file mainly contains code for changing the nature of text printed
+to the console via ANSI escape sequences.  e.g. it can be used to emit
+text that is bolded, underlined, italicised, colorized, etc...
+
+It also contains a colorizing context that will apply color patterns
+based on regular expressions to any data emitted to stdout that may be
+useful in adding color to other programs' outputs, for instance.
+

 .. automodule:: pyutils.ansi
    :members:
    :undoc-members:
 .. automodule:: pyutils.ansi
    :members:
    :undoc-members:


@@ -30,6 +77,16 @@ pyutils.ansi module
 pyutils.argparse\_utils module
 ------------------------------
 
 pyutils.argparse\_utils module
 ------------------------------
 

+I use the Python internal `argparse` module for commandline argument
+parsing but found it lacking in some ways.  This module contains code to
+fill those gaps.  It include stuff like:
+
+    - An `argparse.Action` to create pairs of flags such as
+      `--feature` and `--no_feature`.
+    - A helper to parse and validate bools, IP addresses, MAC
+      addresses, filenames, percentages, dates, datetimes, and
+      durations passed as flags.
+

 .. automodule:: pyutils.argparse_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.argparse_utils
    :members:
    :undoc-members:


@@ -38,6 +95,17 @@ pyutils.argparse\_utils module
 pyutils.bootstrap module
 ------------------------
 
 pyutils.bootstrap module
 ------------------------
 

+Bootstrap module defines a decorator meant to wrap your main function.
+This decorator will do several things for you:
+
+    - If your code uses the :file:`config.py` module (see below), it invokes
+      `parse` automatically to initialize `config.config` from commandline
+      flags, environment variables, or other sources.
+    - It will optionally break into pdb in response to an unhandled
+      Exception at the top level of your code.
+    - It initializes logging for your program (see :file:`logging.py`).
+    - It can optionally run a code and/or memory profiler on your code.
+

 .. automodule:: pyutils.bootstrap
    :members:
    :undoc-members:
 .. automodule:: pyutils.bootstrap
    :members:
    :undoc-members:


@@ -46,6 +114,22 @@ pyutils.bootstrap module
 pyutils.config module
 ---------------------
 
 pyutils.config module
 ---------------------
 

+This module reads the program's configuration parameters from: the
+commandline (using argparse), environment variables and/or a shared
+zookeeper-based configuration.  It stores this configuration state in
+a dict-like structure that can be queried by your code at runtime.
+
+It handles creating a nice `--help` message for your code.
+
+It can optionally react to dynamic config changes and change state at
+runtime (iff you name your flag with the *dynamic_* prefix and are
+using zookeeper-based configs).
+
+It can optionally save and retrieve sets of arguments from files on
+the local disk or on zookeeper.
+
+All of my examples use this as does the pyutils library itself.
+

 .. automodule:: pyutils.config
    :members:
    :undoc-members:
 .. automodule:: pyutils.config
    :members:
    :undoc-members:


@@ -54,6 +138,8 @@ pyutils.config module
 pyutils.decorator\_utils module
 -------------------------------
 
 pyutils.decorator\_utils module
 -------------------------------
 

+This is a grab bag of decorators.
+

 .. automodule:: pyutils.decorator_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.decorator_utils
    :members:
    :undoc-members:


@@ -62,6 +148,8 @@ pyutils.decorator\_utils module
 pyutils.dict\_utils module
 --------------------------
 
 pyutils.dict\_utils module
 --------------------------
 

+A bunch of helpers for dealing with Python dicts.
+

 .. automodule:: pyutils.dict_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.dict_utils
    :members:
    :undoc-members:


@@ -70,6 +158,8 @@ pyutils.dict\_utils module
 pyutils.exec\_utils module
 --------------------------
 
 pyutils.exec\_utils module
 --------------------------
 

+Helper code for dealing with subprocesses.
+

 .. automodule:: pyutils.exec_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.exec_utils
    :members:
    :undoc-members:


@@ -78,6 +168,8 @@ pyutils.exec\_utils module
 pyutils.function\_utils module
 ------------------------------
 
 pyutils.function\_utils module
 ------------------------------
 

+Helper util for dealing with functions.
+

 .. automodule:: pyutils.function_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.function_utils
    :members:
    :undoc-members:


@@ -86,6 +178,8 @@ pyutils.function\_utils module
 pyutils.id\_generator module
 ----------------------------
 
 pyutils.id\_generator module
 ----------------------------
 

+Generate unique identifiers.
+

 .. automodule:: pyutils.id_generator
    :members:
    :undoc-members:
 .. automodule:: pyutils.id_generator
    :members:
    :undoc-members:


@@ -94,6 +188,9 @@ pyutils.id\_generator module
 pyutils.iter\_utils module
 --------------------------
 
 pyutils.iter\_utils module
 --------------------------
 

+Iterator utilities including a :class:PeekingIterator, :class:PushbackIterator,
+and :class:SamplingIterator.
+

 .. automodule:: pyutils.iter_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.iter_utils
    :members:
    :undoc-members:


@@ -102,6 +199,8 @@ pyutils.iter\_utils module
 pyutils.list\_utils module
 --------------------------
 
 pyutils.list\_utils module
 --------------------------
 

+Utilities for dealing with Python lists.
+

 .. automodule:: pyutils.list_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.list_utils
    :members:
    :undoc-members:


@@ -110,6 +209,20 @@ pyutils.list\_utils module
 pyutils.logging\_utils module
 -----------------------------
 
 pyutils.logging\_utils module
 -----------------------------
 

+This is a module that offers an opinionated take on how whole program logging
+should be initialized and controlled.  It uses standard Python logging but gives
+you control, via commandline config, to:
+
+    - Set the logging level of the program including overriding the
+      logging level for individual modules,
+    - Define the logging message format including easily adding a
+      PID/TID marker on all messages to help with multithreaded debugging,
+    - Control the destination (file, sys.stderr, syslog) of messages,
+    - Control the facility and logging level used with syslog,
+    - Squelch repeated messages,
+    - Log probalistically,
+    - Clear rogue logging handlers added by other imports.
+

 .. automodule:: pyutils.logging_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.logging_utils
    :members:
    :undoc-members:


@@ -118,6 +231,11 @@ pyutils.logging\_utils module
 pyutils.math\_utils module
 --------------------------
 
 pyutils.math\_utils module
 --------------------------
 

+Helper utilities that are "mathy" such as a :class:NumericPopulation that
+makes population summary statistics available to your code quickly, GCD
+computation, literate float truncation, percentage <-> multiplier, prime
+number determination, etc...
+

 .. automodule:: pyutils.math_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.math_utils
    :members:
    :undoc-members:


@@ -126,6 +244,8 @@ pyutils.math\_utils module
 pyutils.misc\_utils module
 --------------------------
 
 pyutils.misc\_utils module
 --------------------------
 

+Miscellaneous utilities: are we running as root, and is a debugger attached?
+

 .. automodule:: pyutils.misc_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.misc_utils
    :members:
    :undoc-members:


@@ -134,6 +254,11 @@ pyutils.misc\_utils module
 pyutils.persistent module
 -------------------------
 
 pyutils.persistent module
 -------------------------
 

+Persistent defines a class hierarchy and decorator for creating
+singleton classes that (optionally/conditionally) load their state
+from some external location and (optionally/conditionally) stave their
+state to an external location at shutdown.
+

 .. automodule:: pyutils.persistent
    :members:
    :undoc-members:
 .. automodule:: pyutils.persistent
    :members:
    :undoc-members:


@@ -142,6 +267,10 @@ pyutils.persistent module
 pyutils.remote\_worker module
 -----------------------------
 
 pyutils.remote\_worker module
 -----------------------------
 

+This module defines a helper that is invoked by the remote executor to
+run pickled code on a remote machine.  It is used by code marked with
+@parallelize(method=Method.REMOTE) in the parallelize framework.
+

 .. automodule:: pyutils.remote_worker
    :members:
    :undoc-members:
 .. automodule:: pyutils.remote_worker
    :members:
    :undoc-members:


@@ -150,6 +279,14 @@ pyutils.remote\_worker module
 pyutils.state\_tracker module
 -----------------------------
 
 pyutils.state\_tracker module
 -----------------------------
 

+This module defines several classes (:class:StateTracker,
+:class:AutomaticStateTracker, and
+:class:WaitableAutomaticStateTracker) that can be used as base
+classes.  These class patterns are meant to encapsulate and represent
+state that dynamically changes.  These classes update their state
+(either automatically or when invoked to poll) and allow their callers
+to wait on state changes.
+

 .. automodule:: pyutils.state_tracker
    :members:
    :undoc-members:
 .. automodule:: pyutils.state_tracker
    :members:
    :undoc-members:


@@ -158,6 +295,9 @@ pyutils.state\_tracker module
 pyutils.stopwatch module
 ------------------------
 
 pyutils.stopwatch module
 ------------------------
 

+This is a stopwatch context that just times how long something took to
+execute.
+

 .. automodule:: pyutils.stopwatch
    :members:
    :undoc-members:
 .. automodule:: pyutils.stopwatch
    :members:
    :undoc-members:


@@ -166,6 +306,10 @@ pyutils.stopwatch module
 pyutils.string\_utils module
 ----------------------------
 
 pyutils.string\_utils module
 ----------------------------
 

+A bunch of utilities for dealing with strings.  Based on a really great
+starting library from Davide Zanotti, I've added a pile of other string
+functions so hopefully it will handle all of your string-needs.
+

 .. automodule:: pyutils.string_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.string_utils
    :members:
    :undoc-members:


@@ -174,6 +318,16 @@ pyutils.string\_utils module
 pyutils.text\_utils module
 --------------------------
 
 pyutils.text\_utils module
 --------------------------
 

+Utilities for dealing with and creating text chunks.  For example:
+
+    - Make a bar graph,
+    - make a spark line,
+    - left, right, center, justify text,
+    - word wrap text,
+    - indent text,
+    - create a header line,
+    - draw a box around some text.
+

 .. automodule:: pyutils.text_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.text_utils
    :members:
    :undoc-members:


@@ -182,6 +336,8 @@ pyutils.text\_utils module
 pyutils.unittest\_utils module
 ------------------------------
 
 pyutils.unittest\_utils module
 ------------------------------
 

+Utilities to support smarter unit tests.
+

 .. automodule:: pyutils.unittest_utils
    :members:
    :undoc-members:
 .. automodule:: pyutils.unittest_utils
    :members:
    :undoc-members:


@@ -190,6 +346,8 @@ pyutils.unittest\_utils module
 pyutils.unscrambler module
 --------------------------
 
 pyutils.unscrambler module
 --------------------------
 

+Unscramble scrambled English words quickly.
+

 .. automodule:: pyutils.unscrambler
    :members:
    :undoc-members:
 .. automodule:: pyutils.unscrambler
    :members:
    :undoc-members:


@@ -198,6 +356,8 @@ pyutils.unscrambler module
 pyutils.zookeeper module
 ------------------------
 
 pyutils.zookeeper module
 ------------------------
 

+A helper module for dealing with Zookeeper that adds some functionality.
+

 .. automodule:: pyutils.zookeeper
    :members:
    :undoc-members:
 .. automodule:: pyutils.zookeeper
    :members:
    :undoc-members:

diff --git a/examples/README b/examples/README
index 5d02a8756590010173eff49c2e352cac641bf473..c14e8639d481567bd234a96bff174c19138b6187 100644 (file)
--- a/examples/README
+++ b/examples/README
@@ -1,2 +1,3 @@
 Stuff under here is example code that uses pyutils library routines and
 Stuff under here is example code that uses pyutils library routines and

-is meant to just be illustrative and fun.
+is meant to just be illustrative and fun.  Each should be runnable as-is
+if you have pyutils installed.  Use the --help flag for more info.

diff --git a/examples/reminder/.gitignore b/examples/reminder/.gitignore
new file mode 100644 (file)
index 0000000..0a79c83
--- /dev/null
+++ b/examples/reminder/.gitignore
@@ -0,0 +1 @@
+.reminder_cache

diff --git a/src/pyutils/ansi.py b/src/pyutils/ansi.py
index 1d45d3b944c7babeaf4c91eca199747af5726d60..b885a406648f7db63ab53e3fca9a85ccb78fb0c7 100755 (executable)
--- a/src/pyutils/ansi.py
+++ b/src/pyutils/ansi.py
@@ -4,7 +4,7 @@
 
 """A bunch of color names mapped into RGB tuples and some methods for
 setting the text color, background, etc... using ANSI escape
 
 """A bunch of color names mapped into RGB tuples and some methods for
 setting the text color, background, etc... using ANSI escape

-sequences.
+sequences.  See: https://en.wikipedia.org/wiki/ANSI_escape_code.

 """
 
 import contextlib
 """
 
 import contextlib


@@ -22,8 +22,6 @@ from pyutils import logging_utils, string_utils
 
 logger = logging.getLogger(__name__)
 
 
 logger = logging.getLogger(__name__)
 

-# https://en.wikipedia.org/wiki/ANSI_escape_code
-

 
 COLOR_NAMES_TO_RGB: Dict[str, Tuple[int, int, int]] = {
     "abbey": (0x4C, 0x4F, 0x56),
 
 COLOR_NAMES_TO_RGB: Dict[str, Tuple[int, int, int]] = {
     "abbey": (0x4C, 0x4F, 0x56),


@@ -1673,37 +1671,37 @@ COLOR_NAMES_TO_RGB: Dict[str, Tuple[int, int, int]] = {
 
 def clear() -> str:
     """Clear screen ANSI escape sequence"""
 
 def clear() -> str:
     """Clear screen ANSI escape sequence"""

-    return "\e[H\e[2J"
+    return "\x1B[H\x1B[2J"

 
 
 def clear_screen() -> str:
     """Clear screen ANSI escape sequence"""
 
 
 def clear_screen() -> str:
     """Clear screen ANSI escape sequence"""

-    return "\e[H\e[2J"
+    return "\x1B[H\x1B[2J"

 
 
 def clear_line() -> str:
     """Clear the current line ANSI escape sequence"""
 
 
 def clear_line() -> str:
     """Clear the current line ANSI escape sequence"""

-    return "\e[2K\r"
+    return "\x1B[2K\r"

 
 
 def reset() -> str:
     """Reset text attributes to 'normal'"""
 
 
 def reset() -> str:
     """Reset text attributes to 'normal'"""

-    return "\e[m"
+    return "\x1B[m"

 
 
 def normal() -> str:
     """Reset text attributes to 'normal'"""
 
 
 def normal() -> str:
     """Reset text attributes to 'normal'"""

-    return "\e[m"
+    return "\x1B[m"

 
 
 def bold() -> str:
     """Set text to bold"""
 
 
 def bold() -> str:
     """Set text to bold"""

-    return "\e[1m"
+    return "\x1B[1m"

 
 
 def italic() -> str:
     """Set text to italic"""
 
 
 def italic() -> str:
     """Set text to italic"""

-    return "\e[3m"
+    return "\x1B[3m"

 
 
 def italics() -> str:
 
 
 def italics() -> str:


@@ -1713,12 +1711,12 @@ def italics() -> str:
 
 def underline() -> str:
     """Set text to underline"""
 
 def underline() -> str:
     """Set text to underline"""

-    return "\e[4m"
+    return "\x1B[4m"

 
 
 def strikethrough() -> str:
     """Set text to strikethrough"""
 
 
 def strikethrough() -> str:
     """Set text to strikethrough"""

-    return "\e[9m"
+    return "\x1B[9m"

 
 
 def strike_through() -> str:
 
 
 def strike_through() -> str:


@@ -1756,7 +1754,7 @@ def fg_16color(red: int, green: int, blue: int) -> str:
         bright_count += 1
     if bright_count > 1:
         code += 60
         bright_count += 1
     if bright_count > 1:
         code += 60

-    return f"\e[{code}m"
+    return f"\x1B[{code}m"

 
 
 def bg_16color(red: int, green: int, blue: int) -> str:
 
 
 def bg_16color(red: int, green: int, blue: int) -> str:


@@ -1771,7 +1769,7 @@ def bg_16color(red: int, green: int, blue: int) -> str:
         bright_count += 1
     if bright_count > 1:
         code += 60
         bright_count += 1
     if bright_count > 1:
         code += 60

-    return f"\e[{code}m"
+    return f"\x1B[{code}m"

 
 
 def _pixel_to_216color(n: int) -> int:
 
 
 def _pixel_to_216color(n: int) -> int:


@@ -1794,7 +1792,7 @@ def fg_216color(red: int, green: int, blue: int) -> str:
     g = _pixel_to_216color(green)
     b = _pixel_to_216color(blue)
     code = 16 + r * 36 + g * 6 + b
     g = _pixel_to_216color(green)
     b = _pixel_to_216color(blue)
     code = 16 + r * 36 + g * 6 + b

-    return f"\e[38;5;{code}m"
+    return f"\x1B[38;5;{code}m"

 
 
 def bg_216color(red: int, green: int, blue: int) -> str:
 
 
 def bg_216color(red: int, green: int, blue: int) -> str:


@@ -1803,17 +1801,17 @@ def bg_216color(red: int, green: int, blue: int) -> str:
     g = _pixel_to_216color(green)
     b = _pixel_to_216color(blue)
     code = 16 + r * 36 + g * 6 + b
     g = _pixel_to_216color(green)
     b = _pixel_to_216color(blue)
     code = 16 + r * 36 + g * 6 + b

-    return f"\e[48;5;{code}m"
+    return f"\x1B[48;5;{code}m"

 
 
 def fg_24bit(red: int, green: int, blue: int) -> str:
     """Set foreground using 24bit color mode"""
 
 
 def fg_24bit(red: int, green: int, blue: int) -> str:
     """Set foreground using 24bit color mode"""

-    return f"\e[38;2;{red};{green};{blue}m"
+    return f"\x1B[38;2;{red};{green};{blue}m"

 
 
 def bg_24bit(red: int, green: int, blue: int) -> str:
     """Set background using 24bit color mode"""
 
 
 def bg_24bit(red: int, green: int, blue: int) -> str:
     """Set background using 24bit color mode"""

-    return f"\e[48;2;{red};{green};{blue}m"
+    return f"\x1B[48;2;{red};{green};{blue}m"

 
 
 def _find_color_by_name(name: str) -> Tuple[int, int, int]:
 
 
 def _find_color_by_name(name: str) -> Tuple[int, int, int]:


@@ -1889,6 +1887,8 @@ def fg(
 
 
 def reset_fg():
 
 
 def reset_fg():

+    """Returns: an ANSI escape code to reset just the foreground color while
+    preserving the background color and any other formatting (bold, italics, etc...)"""

     return '\033[39m'
 
 
     return '\033[39m'
 
 


@@ -1974,6 +1974,9 @@ def bg(
         force_16color: force bg to use 16 color mode
         force_216color: force bg to use 216 color mode
 
         force_16color: force bg to use 16 color mode
         force_216color: force bg to use 216 color mode
 

+    Returns:
+        A string containing the requested escape sequence
+

     >>> import string_utils as su
     >>> su.to_base64(bg("red"))    # b'\x1b[48;5;196m'
     b'G1s0ODs1OzE5Nm0=\\n'
     >>> import string_utils as su
     >>> su.to_base64(bg("red"))    # b'\x1b[48;5;196m'
     b'G1s0ODs1OzE5Nm0=\\n'

diff --git a/src/pyutils/argparse_utils.py b/src/pyutils/argparse_utils.py
index 3b466b0d93da43bcf69c98215fc993bb121871e6..daca1df10ef66ee913b0afa6e6ff8e226e6aa22e 100644 (file)
--- a/src/pyutils/argparse_utils.py
+++ b/src/pyutils/argparse_utils.py
@@ -74,7 +74,15 @@ class ActionNoYes(argparse.Action):
 
 def valid_bool(v: Any) -> bool:
     """
 
 def valid_bool(v: Any) -> bool:
     """

-    If the string is a valid bool, return its value.
+    If the string is a valid bool, return its value.  Sample usage::
+
+        args.add_argument(
+            '--auto',
+            type=argparse_utils.valid_bool,
+            default=None,
+            metavar='True|False',
+            help='Use your best judgement about --primary and --secondary',
+        )

 
     >>> valid_bool(True)
     True
 
     >>> valid_bool(True)
     True


@@ -110,7 +118,15 @@ def valid_bool(v: Any) -> bool:
 def valid_ip(ip: str) -> str:
     """
     If the string is a valid IPv4 address, return it.  Otherwise raise
 def valid_ip(ip: str) -> str:
     """
     If the string is a valid IPv4 address, return it.  Otherwise raise

-    an ArgumentTypeError.
+    an ArgumentTypeError.  Sample usage::
+
+        group.add_argument(
+            "-i",
+            "--ip_address",
+            metavar="TARGET_IP_ADDRESS",
+            help="Target IP Address",
+            type=argparse_utils.valid_ip,
+        )

 
     >>> valid_ip("1.2.3.4")
     '1.2.3.4'
 
     >>> valid_ip("1.2.3.4")
     '1.2.3.4'


@@ -134,7 +150,15 @@ def valid_ip(ip: str) -> str:
 def valid_mac(mac: str) -> str:
     """
     If the string is a valid MAC address, return it.  Otherwise raise
 def valid_mac(mac: str) -> str:
     """
     If the string is a valid MAC address, return it.  Otherwise raise

-    an ArgumentTypeError.
+    an ArgumentTypeError.  Sample usage::
+
+        group.add_argument(
+            "-m",
+            "--mac",
+            metavar="MAC_ADDRESS",
+            help="Target MAC Address",
+            type=argparse_utils.valid_mac,
+        )

 
     >>> valid_mac('12:23:3A:4F:55:66')
     '12:23:3A:4F:55:66'
 
     >>> valid_mac('12:23:3A:4F:55:66')
     '12:23:3A:4F:55:66'


@@ -160,8 +184,15 @@ def valid_mac(mac: str) -> str:
 
 def valid_percentage(num: str) -> float:
     """
 
 def valid_percentage(num: str) -> float:
     """

-    If the string is a valid percentage, return it.  Otherwise raise
-    an ArgumentTypeError.
+    If the string is a valid (0 <= n <= 100) percentage, return it.
+    Otherwise raise an ArgumentTypeError.  Sample usage::
+
+        args.add_argument(
+            '--percent_change',
+            type=argparse_utils.valid_percentage,
+            default=0,
+            help='The percent change (0<=n<=100) of foobar',
+        )

 
     >>> valid_percentage("15%")
     15.0
 
     >>> valid_percentage("15%")
     15.0


@@ -187,7 +218,15 @@ def valid_percentage(num: str) -> float:
 def valid_filename(filename: str) -> str:
     """
     If the string is a valid filename, return it.  Otherwise raise
 def valid_filename(filename: str) -> str:
     """
     If the string is a valid filename, return it.  Otherwise raise

-    an ArgumentTypeError.
+    an ArgumentTypeError.  Sample usage::
+
+        args.add_argument(
+            '--network_mac_addresses_file',
+            default='/home/scott/bin/network_mac_addresses.txt',
+            metavar='FILENAME',
+            help='Location of the network_mac_addresses file (must exist!).',
+            type=argparse_utils.valid_filename,
+        )

 
     >>> valid_filename('/tmp')
     '/tmp'
 
     >>> valid_filename('/tmp')
     '/tmp'


@@ -208,14 +247,23 @@ def valid_filename(filename: str) -> str:
 
 def valid_date(txt: str) -> datetime.date:
     """If the string is a valid date, return it.  Otherwise raise
 
 def valid_date(txt: str) -> datetime.date:
     """If the string is a valid date, return it.  Otherwise raise

-    an ArgumentTypeError.
+    an ArgumentTypeError.  Sample usage::
+
+        cfg.add_argument(
+            "--date",
+            nargs=1,
+            type=argparse_utils.valid_date,
+            metavar="DATE STRING",
+            default=None
+        )

 
     >>> valid_date('6/5/2021')
     datetime.date(2021, 6, 5)
 
 
     >>> valid_date('6/5/2021')
     datetime.date(2021, 6, 5)
 

-    # Note: dates like 'next wednesday' work fine, they are just
-    # hard to test for without knowing when the testcase will be
-    # executed...
+    Note: dates like 'next wednesday' work fine, they are just
+    hard to test for without knowing when the testcase will be
+    executed...
+

     >>> valid_date('next wednesday') # doctest: +ELLIPSIS
     -ANYTHING-
     """
     >>> valid_date('next wednesday') # doctest: +ELLIPSIS
     -ANYTHING-
     """


@@ -231,14 +279,26 @@ def valid_date(txt: str) -> datetime.date:
 
 def valid_datetime(txt: str) -> datetime.datetime:
     """If the string is a valid datetime, return it.  Otherwise raise
 
 def valid_datetime(txt: str) -> datetime.datetime:
     """If the string is a valid datetime, return it.  Otherwise raise

-    an ArgumentTypeError.
+    an ArgumentTypeError.  Sample usage::
+
+        cfg.add_argument(
+            "--override_timestamp",
+            nargs=1,
+            type=argparse_utils.valid_datetime,
+            help="Don't use the current datetime, override to argument.",
+            metavar="DATE/TIME STRING",
+            default=None,
+        )

 
     >>> valid_datetime('6/5/2021 3:01:02')
     datetime.datetime(2021, 6, 5, 3, 1, 2)
 
 
     >>> valid_datetime('6/5/2021 3:01:02')
     datetime.datetime(2021, 6, 5, 3, 1, 2)
 

-    # Again, these types of expressions work fine but are
-    # difficult to test with doctests because the answer is
-    # relative to the time the doctest is executed.
+    Because this thing uses an English date-expression parsing grammar
+    internally, much more complex datetimes can be expressed in free form.
+    See: `tests/datetimez/dateparse_utils_test.py` for examples.  These
+    are not included in here because they are hard to write valid doctests
+    for!
+

     >>> valid_datetime('next christmas at 4:15am') # doctest: +ELLIPSIS
     -ANYTHING-
     """
     >>> valid_datetime('next christmas at 4:15am') # doctest: +ELLIPSIS
     -ANYTHING-
     """


@@ -254,21 +314,33 @@ def valid_datetime(txt: str) -> datetime.datetime:
 
 def valid_duration(txt: str) -> datetime.timedelta:
     """If the string is a valid time duration, return a
 
 def valid_duration(txt: str) -> datetime.timedelta:
     """If the string is a valid time duration, return a

-    datetime.timedelta representing the period of time.  Otherwise
-    maybe raise an ArgumentTypeError or potentially just treat the
-    time window as zero in length.
+    datetime.timedelta representing the period of time.
+    Sample usage::
+
+        cfg.add_argument(
+            '--ip_cache_max_staleness',
+            type=argparse_utils.valid_duration,
+            default=datetime.timedelta(seconds=60 * 60 * 12),
+            metavar='DURATION',
+            help='Max acceptable age of the IP address cache'
+        )

 
     >>> valid_duration('3m')
     datetime.timedelta(seconds=180)
 
 
     >>> valid_duration('3m')
     datetime.timedelta(seconds=180)
 

-    >>> valid_duration('your mom')
-    datetime.timedelta(0)
+    >>> valid_duration('3 days, 2 hours')
+    datetime.timedelta(days=3, seconds=7200)
+
+    >>> valid_duration('a little while')
+    Traceback (most recent call last):
+    ...
+    argparse.ArgumentTypeError: a little while is not a valid duration.

 
     """
     from pyutils.datetimez.datetime_utils import parse_duration
 
     try:
 
     """
     from pyutils.datetimez.datetime_utils import parse_duration
 
     try:

-        secs = parse_duration(txt)
+        secs = parse_duration(txt, raise_on_error=True)

         return datetime.timedelta(seconds=secs)
     except Exception as e:
         logger.exception(e)
         return datetime.timedelta(seconds=secs)
     except Exception as e:
         logger.exception(e)

diff --git a/src/pyutils/datetimez/datetime_utils.py b/src/pyutils/datetimez/datetime_utils.py
index 6026d9af4f27b6fe19c1b5b66e8d0fe64c476b9c..ed76c9e2407f72c4ad4dffd9666a945f7df7b444 100644 (file)
--- a/src/pyutils/datetimez/datetime_utils.py
+++ b/src/pyutils/datetimez/datetime_utils.py
@@ -804,7 +804,7 @@ def minute_number_to_time_string(minute_num: MinuteOfDay) -> str:
     return f"{hour:2}:{minute:02}{ampm}"
 
 
     return f"{hour:2}:{minute:02}{ampm}"
 
 

-def parse_duration(duration: str) -> int:
+def parse_duration(duration: str, raise_on_error=False) -> int:

     """
     Parse a duration in string form into a delta seconds.
 
     """
     Parse a duration in string form into a delta seconds.
 


@@ -820,9 +820,25 @@ def parse_duration(duration: str) -> int:
     >>> parse_duration('3min 2sec')
     182
 
     >>> parse_duration('3min 2sec')
     182
 

+    >>> parse_duration('recent')
+    0
+
+    >>> parse_duration('recent', raise_on_error=True)
+    Traceback (most recent call last):
+    ...
+    ValueError: recent is not a valid duration.
+

     """
     if duration.isdigit():
         return int(duration)
     """
     if duration.isdigit():
         return int(duration)

+
+    m = re.match(
+        r'(\d+ *d[ays]*)* *(\d+ *h[ours]*)* *(\d+ *m[inutes]*)* *(\d+ *[seconds]*)',
+        duration,
+    )
+    if not m and raise_on_error:
+        raise ValueError(f'{duration} is not a valid duration.')
+

     seconds = 0
     m = re.search(r'(\d+) *d[ays]*', duration)
     if m is not None:
     seconds = 0
     m = re.search(r'(\d+) *d[ays]*', duration)
     if m is not None:

diff --git a/src/pyutils/decorator_utils.py b/src/pyutils/decorator_utils.py
index e8d2249c2b1c785b51ec5e3a02e3417f09e12a89..30b1bfb6f756324981563e6d7345dd81d2e25cde 100644 (file)
--- a/src/pyutils/decorator_utils.py
+++ b/src/pyutils/decorator_utils.py
@@ -488,6 +488,11 @@ def deprecated(func):
     """This is a decorator which can be used to mark functions
     as deprecated. It will result in a warning being emitted
     when the function is used.
     """This is a decorator which can be used to mark functions
     as deprecated. It will result in a warning being emitted
     when the function is used.

+
+    >>> @deprecated
+    ... def foo() -> None:
+    ...     pass
+    >>> foo()   # prints + logs "Call to deprecated function foo"

     """
 
     @functools.wraps(func)
     """
 
     @functools.wraps(func)