3 # © Copyright 2021-2022, Scott Gasch
5 """A :class:`Persistent` is just a class with a load and save method. This
6 module defines the :class:`Persistent` base and a decorator that can be used to
7 create a persistent singleton that autoloads and autosaves."""
15 from abc import ABC, abstractmethod
16 from typing import Any, Optional
18 from overrides import overrides
20 from pyutils.files import file_utils
22 logger = logging.getLogger(__name__)
25 class Persistent(ABC):
27 A base class of an object with a load/save method. Classes that are
28 decorated with :code:`@persistent_autoloaded_singleton` should subclass
29 this and implement their :meth:`save` and :meth:`load` methods.
33 def save(self) -> bool:
35 Save this thing somewhere that you'll remember when someone calls
36 :meth:`load` later on in a way that makes sense to your code.
43 """Load this thing from somewhere and give back an instance which
44 will become the global singleton and which may (see
45 below) be saved (via :meth:`save`) at program exit time.
47 Oh, in case this is handy, here's a reminder how to write a
48 factory method that doesn't call the c'tor in python::
51 def load_from_somewhere(cls, somewhere):
52 # Note: __new__ does not call __init__.
53 obj = cls.__new__(cls)
55 # Don't forget to call any polymorphic base class initializers
56 super(MyClass, obj).__init__()
58 # Load the piece(s) of obj that you want to from somewhere.
59 obj._state = load_from_somewhere(somewhere)
65 class FileBasedPersistent(Persistent):
66 """A Persistent that uses a file to save/load data and knows the conditions
67 under which the state should be saved/loaded."""
71 def get_filename() -> str:
72 """Since this class saves/loads to/from a file, what's its full path?"""
77 def should_we_save_data(filename: str) -> bool:
82 def should_we_load_data(filename: str) -> bool:
86 def get_persistent_data(self) -> Any:
90 class PicklingFileBasedPersistent(FileBasedPersistent):
92 A class that stores its state in a file as pickled python objects.
98 @persistent.persistent_autoloaded_singleton()
99 class MyClass(persistent.PicklingFileBasedPersistent):
100 def __init__(self, data: Whatever):
101 #initialize youself from data
105 def get_filename() -> str:
106 return "/path/to/where/you/want/to/save/data.bin"
110 def should_we_save_data(filename: str) -> bool:
111 return true_if_we_should_save_the_data_this_time()
115 def should_we_load_data(filename: str) -> bool:
116 return persistent.was_file_written_within_n_seconds(whatever)
118 # Persistent will handle the plumbing to instantiate your class from its
119 # persisted state iff the should_we_load_data says it's ok to. It will
120 # also persist the current in memory state to disk at program exit iff
121 # the should_we_save_data methods says to.
128 def load(cls) -> Optional[Any]:
129 filename = cls.get_filename()
130 if cls.should_we_load_data(filename):
131 logger.debug('Attempting to load state from %s', filename)
132 assert file_utils.file_is_readable(filename)
137 with open(filename, 'rb') as rf:
138 data = pickle.load(rf)
141 except Exception as e:
142 raise Exception(f'Failed to load {filename}.') from e
146 def save(self) -> bool:
147 filename = self.get_filename()
148 if self.should_we_save_data(filename):
149 logger.debug('Trying to save state in %s', filename)
153 with open(filename, 'wb') as wf:
154 pickle.dump(self.get_persistent_data(), wf, pickle.HIGHEST_PROTOCOL)
156 except Exception as e:
157 raise Exception(f'Failed to save to {filename}.') from e
161 class JsonFileBasedPersistent(FileBasedPersistent):
163 A class that stores its state in a JSON format file.
169 @persistent.persistent_autoloaded_singleton()
170 class MyClass(persistent.JsonFileBasedPersistent):
171 def __init__(self, data: Whatever):
172 #initialize youself from data
176 def get_filename() -> str:
177 return "/path/to/where/you/want/to/save/data.json"
181 def should_we_save_data(filename: str) -> bool:
182 return true_if_we_should_save_the_data_this_time()
186 def should_we_load_data(filename: str) -> bool:
187 return persistent.was_file_written_within_n_seconds(whatever)
189 # Persistent will handle the plumbing to instantiate your class from its
190 # persisted state iff the should_we_load_data says it's ok to. It will
191 # also persist the current in memory state to disk at program exit iff
192 # the should_we_save_data methods says to.
199 def load(cls) -> Any:
200 filename = cls.get_filename()
201 if cls.should_we_load_data(filename):
202 logger.debug('Trying to load state from %s', filename)
206 with open(filename, 'r') as rf:
207 lines = rf.readlines()
209 # This is probably bad... but I like comments
210 # in config files and JSON doesn't support them. So
211 # pre-process the buffer to remove comments thus
212 # allowing people to add them.
215 line = re.sub(r'#.*$', '', line)
218 json_dict = json.loads(buf)
219 return cls(json_dict)
221 except Exception as e:
223 raise Exception(f'Failed to load {filename}.') from e
227 def save(self) -> bool:
228 filename = self.get_filename()
229 if self.should_we_save_data(filename):
230 logger.debug('Trying to save state in %s', filename)
234 json_blob = json.dumps(self.get_persistent_data())
235 with open(filename, 'w') as wf:
236 wf.writelines(json_blob)
238 except Exception as e:
239 raise Exception(f'Failed to save to {filename}.') from e
243 def was_file_written_today(filename: str) -> bool:
244 """Convenience wrapper around :meth:`was_file_written_within_n_seconds`.
247 filename: filename to check
250 True if filename was written today.
253 >>> filename = f'/tmp/testing_persistent_py_{os.getpid()}'
254 >>> os.system(f'touch {filename}')
256 >>> was_file_written_today(filename)
258 >>> os.system(f'touch -d 1974-04-15T01:02:03.99 {filename}')
260 >>> was_file_written_today(filename)
262 >>> os.system(f'/bin/rm -f {filename}')
264 >>> was_file_written_today(filename)
268 if not file_utils.does_file_exist(filename):
271 mtime = file_utils.get_file_mtime_as_datetime(filename)
272 assert mtime is not None
273 now = datetime.datetime.now()
274 return mtime.month == now.month and mtime.day == now.day and mtime.year == now.year
277 def was_file_written_within_n_seconds(
281 """Helper for determining persisted state staleness.
284 filename: the filename to check
285 limit_seconds: how fresh, in seconds, it must be
288 True if filename was written within the past limit_seconds
289 or False otherwise (or on error).
292 >>> filename = f'/tmp/testing_persistent_py_{os.getpid()}'
293 >>> os.system(f'touch {filename}')
295 >>> was_file_written_within_n_seconds(filename, 60)
299 >>> was_file_written_within_n_seconds(filename, 2)
301 >>> os.system(f'/bin/rm -f {filename}')
303 >>> was_file_written_within_n_seconds(filename, 60)
307 if not file_utils.does_file_exist(filename):
310 mtime = file_utils.get_file_mtime_as_datetime(filename)
311 assert mtime is not None
312 now = datetime.datetime.now()
313 return (now - mtime).total_seconds() <= limit_seconds
316 class PersistAtShutdown(enum.Enum):
318 An enum to describe the conditions under which state is persisted
319 to disk. This is passed as an argument to the decorator below and
320 is used to indicate when to call :meth:`save` on a :class:`Persistent`
323 * NEVER: never call :meth:`save`
324 * IF_NOT_LOADED: call :meth:`save` as long as we did not successfully
325 :meth:`load` its state.
326 * ALWAYS: always call :meth:`save`
334 class persistent_autoloaded_singleton(object):
335 """A decorator that can be applied to a :class:`Persistent` subclass
336 (i.e. a class with :meth:`save` and :meth:`load` methods. The
337 decorator will intercept attempts to instantiate the class via
338 it's c'tor and, instead, invoke the class' :meth:`load` to give it a
339 chance to read state from somewhere persistent (disk, db,
340 whatever). Subsequent calls to construt instances of the wrapped
341 class will return a single, global instance (i.e. the wrapped
342 class is a singleton).
344 If :meth:`load` fails (returns None), the c'tor is invoked with the
345 original args as a fallback.
347 Based upon the value of the optional argument
348 :code:`persist_at_shutdown` argument, (NEVER, IF_NOT_LOADED,
349 ALWAYS), the :meth:`save` method of the class will be invoked just
350 before program shutdown to give the class a chance to save its
354 The implementations of :meth:`save` and :meth:`load` and where the
355 class persists its state are details left to the :class:`Persistent`
356 implementation. Essentially this decorator just handles the
357 plumbing of calling your save/load and appropriate times and
358 creates a transparent global singleton whose state can be
359 persisted between runs.
366 persist_at_shutdown: PersistAtShutdown = PersistAtShutdown.IF_NOT_LOADED,
368 self.persist_at_shutdown = persist_at_shutdown
371 def __call__(self, cls: Persistent):
372 @functools.wraps(cls) # type: ignore
373 def _load(*args, **kwargs):
375 # If class has already been loaded, act like a singleton
376 # and return a reference to the one and only instance in
378 if self.instance is not None:
380 'Returning already instantiated singleton instance of %s.',
385 # Otherwise, try to load it from persisted state.
387 logger.debug('Attempting to load %s from persisted state.', cls.__name__)
388 self.instance = cls.load()
389 if not self.instance:
390 msg = 'Loading from cache failed.'
392 logger.debug('Attempting to instantiate %s directly.', cls.__name__)
393 self.instance = cls(*args, **kwargs)
396 'Class %s was loaded from persisted state successfully.',
401 assert self.instance is not None
403 if self.persist_at_shutdown is PersistAtShutdown.ALWAYS or (
405 and self.persist_at_shutdown is PersistAtShutdown.IF_NOT_LOADED
408 'Scheduling a deferred called to save at process shutdown time.'
410 atexit.register(self.instance.save)
416 if __name__ == '__main__':