3 # © Copyright 2021-2022, Scott Gasch
5 """A :class:`Persistent` is just a class with a load and save method. This
6 module defines the :class:`Persistent` base and a decorator that can be used to
7 create a persistent singleton that autoloads and autosaves."""
15 from abc import ABC, abstractmethod
16 from typing import Any, Optional
18 from overrides import overrides
20 from pyutils.files import file_utils
22 logger = logging.getLogger(__name__)
25 class Persistent(ABC):
27 A base class of an object with a load/save method. Classes that are
28 decorated with :code:`@persistent_autoloaded_singleton` should subclass
29 this and implement their :meth:`save` and :meth:`load` methods.
33 def save(self) -> bool:
35 Save this thing somewhere that you'll remember when someone calls
36 :meth:`load` later on in a way that makes sense to your code.
43 """Load this thing from somewhere and give back an instance which
44 will become the global singleton and which may (see
45 below) be saved (via :meth:`save`) at program exit time.
47 Oh, in case this is handy, here's a reminder how to write a
48 factory method that doesn't call the c'tor in python::
51 def load_from_somewhere(cls, somewhere):
52 # Note: __new__ does not call __init__.
53 obj = cls.__new__(cls)
55 # Don't forget to call any polymorphic base class initializers
56 super(MyClass, obj).__init__()
58 # Load the piece(s) of obj that you want to from somewhere.
59 obj._state = load_from_somewhere(somewhere)
65 class FileBasedPersistent(Persistent):
66 """A Persistent that uses a file to save/load data and knows the conditions
67 under which the state should be saved/loaded."""
71 def get_filename() -> str:
72 """Since this class saves/loads to/from a file, what's its full path?"""
77 def should_we_save_data(filename: str) -> bool:
82 def should_we_load_data(filename: str) -> bool:
86 def get_persistent_data(self) -> Any:
90 class PicklingFileBasedPersistent(FileBasedPersistent):
96 @persistent.persistent_autoloaded_singleton()
97 class MyClass(persistent.PicklingFileBasedPersistent):
98 def __init__(self, data: Whatever):
99 #initialize youself from data
103 def get_filename() -> str:
104 return "/path/to/where/you/want/to/save/data.bin"
108 def should_we_save_data(filename: str) -> bool:
109 return true_if_we_should_save_the_data_this_time()
113 def should_we_load_data(filename: str) -> bool:
114 return persistent.was_file_written_within_n_seconds(whatever)
116 # Persistent will handle the plumbing to instantiate your class from its
117 # persisted state iff the should_we_load_data says it's ok to. It will
118 # also persist the current in memory state to disk at program exit iff
119 # the should_we_save_data methods says to.
126 def load(cls) -> Optional[Any]:
127 filename = cls.get_filename()
128 if cls.should_we_load_data(filename):
129 logger.debug('Attempting to load state from %s', filename)
130 assert file_utils.file_is_readable(filename)
135 with open(filename, 'rb') as rf:
136 data = pickle.load(rf)
139 except Exception as e:
140 raise Exception(f'Failed to load {filename}.') from e
144 def save(self) -> bool:
145 filename = self.get_filename()
146 if self.should_we_save_data(filename):
147 logger.debug('Trying to save state in %s', filename)
151 with open(filename, 'wb') as wf:
152 pickle.dump(self.get_persistent_data(), wf, pickle.HIGHEST_PROTOCOL)
154 except Exception as e:
155 raise Exception(f'Failed to save to {filename}.') from e
159 class JsonFileBasedPersistent(FileBasedPersistent):
165 @persistent.persistent_autoloaded_singleton()
166 class MyClass(persistent.JsonFileBasedPersistent):
167 def __init__(self, data: Whatever):
168 #initialize youself from data
172 def get_filename() -> str:
173 return "/path/to/where/you/want/to/save/data.json"
177 def should_we_save_data(filename: str) -> bool:
178 return true_if_we_should_save_the_data_this_time()
182 def should_we_load_data(filename: str) -> bool:
183 return persistent.was_file_written_within_n_seconds(whatever)
185 # Persistent will handle the plumbing to instantiate your class from its
186 # persisted state iff the should_we_load_data says it's ok to. It will
187 # also persist the current in memory state to disk at program exit iff
188 # the should_we_save_data methods says to.
195 def load(cls) -> Any:
196 filename = cls.get_filename()
197 if cls.should_we_load_data(filename):
198 logger.debug('Trying to load state from %s', filename)
202 with open(filename, 'r') as rf:
203 lines = rf.readlines()
205 # This is probably bad... but I like comments
206 # in config files and JSON doesn't support them. So
207 # pre-process the buffer to remove comments thus
208 # allowing people to add them.
211 line = re.sub(r'#.*$', '', line)
214 json_dict = json.loads(buf)
215 return cls(json_dict)
217 except Exception as e:
219 raise Exception(f'Failed to load {filename}.') from e
223 def save(self) -> bool:
224 filename = self.get_filename()
225 if self.should_we_save_data(filename):
226 logger.debug('Trying to save state in %s', filename)
230 json_blob = json.dumps(self.get_persistent_data())
231 with open(filename, 'w') as wf:
232 wf.writelines(json_blob)
234 except Exception as e:
235 raise Exception(f'Failed to save to {filename}.') from e
239 def was_file_written_today(filename: str) -> bool:
240 """Convenience wrapper around :meth:`was_file_written_within_n_seconds`.
243 filename: filename to check
246 True if filename was written today.
249 >>> filename = f'/tmp/testing_persistent_py_{os.getpid()}'
250 >>> os.system(f'touch {filename}')
252 >>> was_file_written_today(filename)
254 >>> os.system(f'touch -d 1974-04-15T01:02:03.99 {filename}')
256 >>> was_file_written_today(filename)
258 >>> os.system(f'/bin/rm -f {filename}')
260 >>> was_file_written_today(filename)
264 if not file_utils.does_file_exist(filename):
267 mtime = file_utils.get_file_mtime_as_datetime(filename)
268 assert mtime is not None
269 now = datetime.datetime.now()
270 return mtime.month == now.month and mtime.day == now.day and mtime.year == now.year
273 def was_file_written_within_n_seconds(
277 """Helper for determining persisted state staleness.
280 filename: the filename to check
281 limit_seconds: how fresh, in seconds, it must be
284 True if filename was written within the past limit_seconds
285 or False otherwise (or on error).
288 >>> filename = f'/tmp/testing_persistent_py_{os.getpid()}'
289 >>> os.system(f'touch {filename}')
291 >>> was_file_written_within_n_seconds(filename, 60)
295 >>> was_file_written_within_n_seconds(filename, 2)
297 >>> os.system(f'/bin/rm -f {filename}')
299 >>> was_file_written_within_n_seconds(filename, 60)
303 if not file_utils.does_file_exist(filename):
306 mtime = file_utils.get_file_mtime_as_datetime(filename)
307 assert mtime is not None
308 now = datetime.datetime.now()
309 return (now - mtime).total_seconds() <= limit_seconds
312 class PersistAtShutdown(enum.Enum):
314 An enum to describe the conditions under which state is persisted
315 to disk. This is passed as an argument to the decorator below and
316 is used to indicate when to call :meth:`save` on a :class:`Persistent`
319 * NEVER: never call :meth:`save`
320 * IF_NOT_LOADED: call :meth:`save` as long as we did not successfully
321 :meth:`load` its state.
322 * ALWAYS: always call :meth:`save`
330 class persistent_autoloaded_singleton(object):
331 """A decorator that can be applied to a :class:`Persistent` subclass
332 (i.e. a class with :meth:`save` and :meth:`load` methods. The
333 decorator will intercept attempts to instantiate the class via
334 it's c'tor and, instead, invoke the class' :meth:`load` to give it a
335 chance to read state from somewhere persistent (disk, db,
336 whatever). Subsequent calls to construt instances of the wrapped
337 class will return a single, global instance (i.e. the wrapped
338 class is a singleton).
340 If :meth:`load` fails (returns None), the c'tor is invoked with the
341 original args as a fallback.
343 Based upon the value of the optional argument
344 :code:`persist_at_shutdown` argument, (NEVER, IF_NOT_LOADED,
345 ALWAYS), the :meth:`save` method of the class will be invoked just
346 before program shutdown to give the class a chance to save its
350 The implementations of :meth:`save` and :meth:`load` and where the
351 class persists its state are details left to the :class:`Persistent`
352 implementation. Essentially this decorator just handles the
353 plumbing of calling your save/load and appropriate times and
354 creates a transparent global singleton whose state can be
355 persisted between runs.
362 persist_at_shutdown: PersistAtShutdown = PersistAtShutdown.IF_NOT_LOADED,
364 self.persist_at_shutdown = persist_at_shutdown
367 def __call__(self, cls: Persistent):
368 @functools.wraps(cls) # type: ignore
369 def _load(*args, **kwargs):
371 # If class has already been loaded, act like a singleton
372 # and return a reference to the one and only instance in
374 if self.instance is not None:
376 'Returning already instantiated singleton instance of %s.',
381 # Otherwise, try to load it from persisted state.
383 logger.debug('Attempting to load %s from persisted state.', cls.__name__)
384 self.instance = cls.load()
385 if not self.instance:
386 msg = 'Loading from cache failed.'
388 logger.debug('Attempting to instantiate %s directly.', cls.__name__)
389 self.instance = cls(*args, **kwargs)
392 'Class %s was loaded from persisted state successfully.',
397 assert self.instance is not None
399 if self.persist_at_shutdown is PersistAtShutdown.ALWAYS or (
401 and self.persist_at_shutdown is PersistAtShutdown.IF_NOT_LOADED
404 'Scheduling a deferred called to save at process shutdown time.'
406 atexit.register(self.instance.save)
412 if __name__ == '__main__':