import pickle
from contextlib import contextmanager
from functools import wraps
-from multiprocessing import shared_memory, Lock
+from multiprocessing import shared_memory, RLock
from typing import (
Any,
Dict,
class SharedDict(object):
NULL_BYTE = b'\x00'
- MPLOCK = Lock()
+ MPLOCK = RLock()
def __init__(
self,
name: str,
- size: int,
+ size_bytes: Optional[int] = None,
) -> None:
super().__init__()
+ self.name = name
self._serializer = PickleSerializer()
- self.shared_memory = self._get_or_create_memory_block(name, size)
+ self.shared_memory = self._get_or_create_memory_block(name, size_bytes)
self._ensure_memory_initialization()
+ self.lock = RLock()
+
+ def get_name(self):
+ return self.name
def _get_or_create_memory_block(
- self, name: str, size: int
+ self,
+ name: str,
+ size_bytes: Optional[int] = None,
) -> shared_memory.SharedMemory:
try:
return shared_memory.SharedMemory(name=name)
except FileNotFoundError:
- return shared_memory.SharedMemory(name=name, create=True, size=size)
+ assert size_bytes
+ return shared_memory.SharedMemory(name=name, create=True, size=size_bytes)
def _ensure_memory_initialization(self):
memory_is_empty = (
return
self.shared_memory.unlink()
- @synchronized(MPLOCK)
def clear(self) -> None:
self._save_memory({})
with self._modify_db() as db:
return db.popitem()
- @synchronized(MPLOCK)
@contextmanager
def _modify_db(self) -> Generator:
- db = self._read_memory()
- yield db
- self._save_memory(db)
+ with SharedDict.MPLOCK:
+ db = self._read_memory()
+ yield db
+ self._save_memory(db)
def __getitem__(self, key: str) -> Any:
return self._read_memory()[key]
return db.setdefault(key, default)
def _save_memory(self, db: Dict[str, Any]) -> None:
- data = self._serializer.dumps(db)
- try:
- self.shared_memory.buf[: len(data)] = data
- except ValueError as exc:
- raise ValueError("exceeds available storage") from exc
+ with SharedDict.MPLOCK:
+ data = self._serializer.dumps(db)
+ try:
+ self.shared_memory.buf[: len(data)] = data
+ except ValueError as exc:
+ raise ValueError("exceeds available storage") from exc
def _read_memory(self) -> Dict[str, Any]:
- return self._serializer.loads(self.shared_memory.buf.tobytes())
+ with SharedDict.MPLOCK:
+ return self._serializer.loads(self.shared_memory.buf.tobytes())
if __name__ == '__main__':