Skip to content

Commit

Permalink
Removed LedgerEntry SQL code
Browse files Browse the repository at this point in the history
  • Loading branch information
@SirTyson
SirTyson committed Dec 18, 2024
1 parent 70b2228 commit 09bb518
Show file tree
Hide file tree
Showing 26 changed files with 145 additions and 4,560 deletions.
170 changes: 74 additions & 96 deletions src/bucket/test/BucketManagerTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,112 +189,90 @@ TEST_CASE("skip list", "[bucket][bucketmanager]")

TEST_CASE_VERSIONS("bucketmanager ownership", "[bucket][bucketmanager]")
{
auto test = [&](bool bucketListDB) {
VirtualClock clock;
Config cfg = getTestConfig();

// Make sure all Buckets serialize indexes to disk for test
cfg.BUCKETLIST_DB_INDEX_CUTOFF = 0;
cfg.MANUAL_CLOSE = false;

if (bucketListDB)
{
// Enable BucketListDB with persistent indexes
cfg.NODE_IS_VALIDATOR = false;
cfg.FORCE_SCP = false;
}

for_versions_with_differing_bucket_logic(cfg, [&](Config const& cfg) {
Application::pointer app = createTestApplication(clock, cfg);

std::vector<LedgerEntry> live(
LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
{CONFIG_SETTING}, 10));
std::vector<LedgerKey> dead{};

std::shared_ptr<LiveBucket> b1;

{
std::shared_ptr<LiveBucket> b2 = LiveBucket::fresh(
app->getBucketManager(), getAppLedgerVersion(app), {}, live,
dead, /*countMergeEvents=*/true, clock.getIOContext(),
/*doFsync=*/true);
b1 = b2;

// Bucket is referenced by b1, b2 and the BucketManager.
CHECK(b1.use_count() == 3);

std::shared_ptr<LiveBucket> b3 = LiveBucket::fresh(
app->getBucketManager(), getAppLedgerVersion(app), {}, live,
dead, /*countMergeEvents=*/true, clock.getIOContext(),
/*doFsync=*/true);
std::shared_ptr<LiveBucket> b4 = LiveBucket::fresh(
app->getBucketManager(), getAppLedgerVersion(app), {}, live,
dead, /*countMergeEvents=*/true, clock.getIOContext(),
/*doFsync=*/true);
// Bucket is referenced by b1, b2, b3, b4 and the BucketManager.
CHECK(b1.use_count() == 5);
}

// Take pointer by reference to not mess up use_count()
auto dropBucket = [&](std::shared_ptr<LiveBucket>& b) {
std::string filename = b->getFilename().string();
std::string indexFilename =
app->getBucketManager().bucketIndexFilename(b->getHash());
CHECK(fs::exists(filename));
if (bucketListDB)
{
CHECK(fs::exists(indexFilename));
}
VirtualClock clock;
Config cfg = getTestConfig();

b.reset();
app->getBucketManager().forgetUnreferencedBuckets();
CHECK(!fs::exists(filename));
CHECK(!fs::exists(indexFilename));
};
// Make sure all Buckets serialize indexes to disk for test
cfg.BUCKETLIST_DB_INDEX_CUTOFF = 0;
cfg.MANUAL_CLOSE = false;

// Bucket is now only referenced by b1 and the BucketManager.
CHECK(b1.use_count() == 2);
for_versions_with_differing_bucket_logic(cfg, [&](Config const& cfg) {
Application::pointer app = createTestApplication(clock, cfg);

// Drop bucket ourselves then purge bucketManager.
dropBucket(b1);
std::vector<LedgerEntry> live(
LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
{CONFIG_SETTING}, 10));
std::vector<LedgerKey> dead{};

// Try adding a bucket to the BucketManager's bucketlist
auto& bl = app->getBucketManager().getLiveBucketList();
bl.addBatch(*app, 1, getAppLedgerVersion(app), {}, live, dead);
clearFutures(app, bl);
b1 = bl.getLevel(0).getCurr();
std::shared_ptr<LiveBucket> b1;

// Bucket should be referenced by bucketlist itself, BucketManager
// cache and b1.
CHECK(b1.use_count() == 3);
{
std::shared_ptr<LiveBucket> b2 = LiveBucket::fresh(
app->getBucketManager(), getAppLedgerVersion(app), {}, live,
dead, /*countMergeEvents=*/true, clock.getIOContext(),
/*doFsync=*/true);
b1 = b2;

// This shouldn't change if we forget unreferenced buckets since
// it's referenced by bucketlist.
app->getBucketManager().forgetUnreferencedBuckets();
// Bucket is referenced by b1, b2 and the BucketManager.
CHECK(b1.use_count() == 3);

// But if we mutate the curr bucket of the bucketlist, it should.
live[0] = LedgerTestUtils::generateValidLedgerEntryWithExclusions(
{CONFIG_SETTING});
bl.addBatch(*app, 1, getAppLedgerVersion(app), {}, live, dead);
clearFutures(app, bl);
CHECK(b1.use_count() == 2);

// Drop it again.
dropBucket(b1);
});
};
std::shared_ptr<LiveBucket> b3 = LiveBucket::fresh(
app->getBucketManager(), getAppLedgerVersion(app), {}, live,
dead, /*countMergeEvents=*/true, clock.getIOContext(),
/*doFsync=*/true);
std::shared_ptr<LiveBucket> b4 = LiveBucket::fresh(
app->getBucketManager(), getAppLedgerVersion(app), {}, live,
dead, /*countMergeEvents=*/true, clock.getIOContext(),
/*doFsync=*/true);
// Bucket is referenced by b1, b2, b3, b4 and the BucketManager.
CHECK(b1.use_count() == 5);
}

SECTION("BucketListDB")
{
test(true);
}
// Take pointer by reference to not mess up use_count()
auto dropBucket = [&](std::shared_ptr<LiveBucket>& b) {
std::string filename = b->getFilename().string();
std::string indexFilename =
app->getBucketManager().bucketIndexFilename(b->getHash());
CHECK(fs::exists(filename));
CHECK(fs::exists(indexFilename));

SECTION("SQL")
{
test(false);
}
b.reset();
app->getBucketManager().forgetUnreferencedBuckets();
CHECK(!fs::exists(filename));
CHECK(!fs::exists(indexFilename));
};

// Bucket is now only referenced by b1 and the BucketManager.
CHECK(b1.use_count() == 2);

// Drop bucket ourselves then purge bucketManager.
dropBucket(b1);

// Try adding a bucket to the BucketManager's bucketlist
auto& bl = app->getBucketManager().getLiveBucketList();
bl.addBatch(*app, 1, getAppLedgerVersion(app), {}, live, dead);
clearFutures(app, bl);
b1 = bl.getLevel(0).getCurr();

// Bucket should be referenced by bucketlist itself, BucketManager
// cache and b1.
CHECK(b1.use_count() == 3);

// This shouldn't change if we forget unreferenced buckets since
// it's referenced by bucketlist.
app->getBucketManager().forgetUnreferencedBuckets();
CHECK(b1.use_count() == 3);

// But if we mutate the curr bucket of the bucketlist, it should.
live[0] = LedgerTestUtils::generateValidLedgerEntryWithExclusions(
{CONFIG_SETTING});
bl.addBatch(*app, 1, getAppLedgerVersion(app), {}, live, dead);
clearFutures(app, bl);
CHECK(b1.use_count() == 2);

// Drop it again.
dropBucket(b1);
});
}

TEST_CASE("bucketmanager missing buckets fail", "[bucket][bucketmanager]")
Expand Down
3 changes: 2 additions & 1 deletion src/catchup/ApplyBucketsWork.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@ ApplyBucketsWork::doReset()
// The current size of this set is 1.6 million during BucketApply
// (as of 12/20/23). There's not a great way to estimate this, so
// reserving with some extra wiggle room
mSeenKeys.reserve(2'000'000);
static const size_t estimatedOfferKeyCount = 2'000'000;
mSeenKeys.reserve(estimatedOfferKeyCount);

auto addBucket = [this](std::shared_ptr<LiveBucket> const& bucket) {
if (bucket->getSize() > 0)
Expand Down
36 changes: 0 additions & 36 deletions src/database/Database.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,48 +252,12 @@ Database::upgradeToCurrentSchema()
putSchemaVersion(vers);
}

// Tx meta column no longer supported
dropTxMetaIfExists();
maybeUpgradeToBucketListDB();

CLOG_INFO(Database, "DB schema is in current version");
releaseAssert(vers == SCHEMA_VERSION);
}

void
Database::dropTxMetaIfExists()
{
int txMetaExists{};
std::string selectStr;
if (isSqlite())
{
selectStr = "SELECT EXISTS ("
"SELECT 1 "
"FROM pragma_table_info('txhistory') "
"WHERE name = 'txmeta');";
}
else
{
selectStr = "SELECT EXISTS ("
"SELECT 1 "
"FROM information_schema.columns "
"WHERE "
"table_name = 'txhistory' AND "
"column_name = 'txmeta');";
}

auto& st = getPreparedStatement(selectStr).statement();
st.exchange(soci::into(txMetaExists));
st.define_and_bind();
st.execute(true);

if (txMetaExists)
{
CLOG_INFO(Database, "Dropping txmeta column from txhistory table");
getSession() << "ALTER TABLE txhistory DROP COLUMN txmeta;";
}
}

void
Database::maybeUpgradeToBucketListDB()
{
Expand Down
Loading

0 comments on commit 09bb518

Please sign in to comment.