From a9544f025acaf321c8bd90cb48fffb03a34800b6 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Fri, 28 Jun 2024 09:26:13 -0700 Subject: [PATCH 1/2] Externalize the size limit for LMDB map size for Baser to an environment variable. (#810) Signed-off-by: pfeairheller --- src/keri/db/basing.py | 6 ++++++ src/keri/db/dbing.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d4263fc91..090b357d4 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -494,6 +494,9 @@ def reopenDB(db, clear=False, **kwa): db.close(clear=clear) +KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE" + + class Baser(dbing.LMDBer): """ Baser sets up named sub databases with Keri Event Logs within main database @@ -773,6 +776,9 @@ def __init__(self, headDirPath=None, reopen=False, **kwa): self._kevers = dbdict() self._kevers.db = self # assign db for read through cache of kevers + if (mapSize := os.getenv(KERIBaserMapSizeKey)) is not None: + self.MapSize = int(mapSize) + super(Baser, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) @property diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index d3831e214..bd2db031f 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -306,6 +306,7 @@ class LMDBer(filing.Filer): TempSuffix = "_test" Perm = stat.S_ISVTX | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # 0o1700==960 MaxNamedDBs = 96 + MapSize = 104857600 def __init__(self, readonly=False, **kwa): @@ -380,7 +381,7 @@ def reopen(self, readonly=False, **kwa): # open lmdb major database instance # creates files data.mdb and lock.mdb in .dbDirPath - self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600, + self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=self.MapSize, mode=self.perm, readonly=self.readonly) self.opened = True if opened and self.env else False return self.opened From 750a46202a63e3f6565282f62f469c3bbc81528a Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Mon, 8 Jul 2024 18:04:54 -0400 Subject: [PATCH 2/2] Fixed KERI_BASER_MAP_SIZE not integer bug. (#812) * Fixed KERI_BASER_MAP_SIZE not integer bug. Added log message and exception check for when KERI_BASER_MAP_SIZE not an integer. Signed-off-by: Charles Lanahan * Rethrow exception per PR request to fail fast. Updated log message to reflect the issue --------- Signed-off-by: Charles Lanahan --- src/keri/db/basing.py | 6 +++++- tests/db/test_basing.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 090b357d4..d1c83563f 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -777,7 +777,11 @@ def __init__(self, headDirPath=None, reopen=False, **kwa): self._kevers.db = self # assign db for read through cache of kevers if (mapSize := os.getenv(KERIBaserMapSizeKey)) is not None: - self.MapSize = int(mapSize) + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error("KERI_BASER_MAP_SIZE must be an integer value >1!") + raise super(Baser, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index b067985dc..87428731e 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -2273,6 +2273,22 @@ def test_group_members(): """End Test""" +def test_KERI_BASER_MAP_SIZE_handles_bad_values(caplog): + # Base case works because of above tests, they will all break if happy path + # is broken. We'll just test some unhappy values. + + # Pytest will fail if any exceptions raised here. + os.environ["KERI_BASER_MAP_SIZE"] = "foo" # Not an int + err_msg = "KERI_BASER_MAP_SIZE must be an integer value > 1!" + with pytest.raises(ValueError): + Baser(reopen=False, temp=True) + assert err_msg in caplog.messages + os.environ["KERI_BASER_MAP_SIZE"] = "1.0" # Not an int + with pytest.raises(ValueError): + Baser(reopen=False, temp=True) + assert err_msg in caplog.messages + os.environ.pop("KERI_BASER_MAP_SIZE") + if __name__ == "__main__": test_baser()