Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix lmdb limit issue #4

Merged
merged 2 commits into from
Aug 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/keri/db/basing.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,9 @@ def reopenDB(db, clear=False, **kwa):
db.close(clear=clear)


KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE"


class Baser(dbing.LMDBer):
"""
Baser sets up named sub databases with Keri Event Logs within main database
Expand Down Expand Up @@ -773,6 +776,13 @@ def __init__(self, headDirPath=None, reopen=False, **kwa):
self._kevers = dbdict()
self._kevers.db = self # assign db for read through cache of kevers

if (mapSize := os.getenv(KERIBaserMapSizeKey)) is not None:
try:
self.MapSize = int(mapSize)
except ValueError:
logger.error("KERI_BASER_MAP_SIZE must be an integer value >1!")
raise

super(Baser, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa)

@property
Expand Down
3 changes: 2 additions & 1 deletion src/keri/db/dbing.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,7 @@ class LMDBer(filing.Filer):
TempSuffix = "_test"
Perm = stat.S_ISVTX | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # 0o1700==960
MaxNamedDBs = 96
MapSize = 104857600


def __init__(self, readonly=False, **kwa):
Expand Down Expand Up @@ -380,7 +381,7 @@ def reopen(self, readonly=False, **kwa):

# open lmdb major database instance
# creates files data.mdb and lock.mdb in .dbDirPath
self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600,
self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=self.MapSize,
mode=self.perm, readonly=self.readonly)
self.opened = True if opened and self.env else False
return self.opened
Expand Down
16 changes: 16 additions & 0 deletions tests/db/test_basing.py
Original file line number Diff line number Diff line change
Expand Up @@ -2273,6 +2273,22 @@ def test_group_members():

"""End Test"""

def test_KERI_BASER_MAP_SIZE_handles_bad_values(caplog):
# Base case works because of above tests, they will all break if happy path
# is broken. We'll just test some unhappy values.

# Pytest will fail if any exceptions raised here.
os.environ["KERI_BASER_MAP_SIZE"] = "foo" # Not an int
err_msg = "KERI_BASER_MAP_SIZE must be an integer value > 1!"
with pytest.raises(ValueError):
Baser(reopen=False, temp=True)
assert err_msg in caplog.messages
os.environ["KERI_BASER_MAP_SIZE"] = "1.0" # Not an int
with pytest.raises(ValueError):
Baser(reopen=False, temp=True)
assert err_msg in caplog.messages
os.environ.pop("KERI_BASER_MAP_SIZE")


if __name__ == "__main__":
test_baser()
Expand Down
Loading