Skip to content

Commit

Permalink
Some Soroban loadgen/test fixes.
Browse files Browse the repository at this point in the history 
- Increase the refundable fee for Wasm uploads
- Revert the accidental default instructions change
- Add some fixes to the loadgen test to (hopefully) ensure there are no failures
  • Loading branch information
@dmkozh
dmkozh committed Dec 17, 2024
1 parent 73fa340 commit 363ca1a
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 96 deletions.
8 changes: 4 additions & 4 deletions Builds/VisualStudio/stellar-core.vcxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -616,8 +616,6 @@ exit /b 0
<ClCompile Include="..\..\src\invariant\test\SponsorshipCountIsValidTests.cpp" />
<ClCompile Include="..\..\src\ledger\CheckpointRange.cpp" />
<ClCompile Include="..\..\src\ledger\FlushAndRotateMetaDebugWork.cpp" />
<ClCompile Include="..\..\src\ledger\InMemoryLedgerTxn.cpp" />
<ClCompile Include="..\..\src\ledger\InMemoryLedgerTxnRoot.cpp" />
<ClCompile Include="..\..\src\ledger\InternalLedgerEntry.cpp" />
<ClCompile Include="..\..\src\ledger\LedgerCloseMetaFrame.cpp" />
<ClCompile Include="..\..\src\ledger\LedgerHeaderUtils.cpp" />
Expand All @@ -639,6 +637,8 @@ exit /b 0
<ClCompile Include="..\..\src\ledger\LedgerTxnTrustLineSQL.cpp" />
<ClCompile Include="..\..\src\ledger\LedgerTypeUtils.cpp" />
<ClCompile Include="..\..\src\ledger\NetworkConfig.cpp" />
<ClCompile Include="..\..\src\ledger\test\InMemoryLedgerTxn.cpp" />
<ClCompile Include="..\..\src\ledger\test\InMemoryLedgerTxnRoot.cpp" />
<ClCompile Include="..\..\src\ledger\test\LedgerCloseMetaStreamTests.cpp" />
<ClCompile Include="..\..\src\ledger\test\LedgerHeaderTests.cpp" />
<ClCompile Include="..\..\src\ledger\test\LedgerTests.cpp" />
Expand Down Expand Up @@ -1068,8 +1068,6 @@ exit /b 0
<ClInclude Include="..\..\src\invariant\test\InvariantTestUtils.h" />
<ClInclude Include="..\..\src\ledger\CheckpointRange.h" />
<ClInclude Include="..\..\src\ledger\FlushAndRotateMetaDebugWork.h" />
<ClInclude Include="..\..\src\ledger\InMemoryLedgerTxn.h" />
<ClInclude Include="..\..\src\ledger\InMemoryLedgerTxnRoot.h" />
<ClInclude Include="..\..\src\ledger\InternalLedgerEntry.h" />
<ClInclude Include="..\..\src\ledger\LedgerCloseMetaFrame.h" />
<ClInclude Include="..\..\src\ledger\LedgerHashUtils.h" />
Expand All @@ -1085,6 +1083,8 @@ exit /b 0
<ClInclude Include="..\..\src\ledger\LedgerTypeUtils.h" />
<ClInclude Include="..\..\src\ledger\NetworkConfig.h" />
<ClInclude Include="..\..\src\ledger\NonSociRelatedException.h" />
<ClInclude Include="..\..\src\ledger\test\InMemoryLedgerTxn.h" />
<ClInclude Include="..\..\src\ledger\test\InMemoryLedgerTxnRoot.h" />
<ClInclude Include="..\..\src\ledger\test\LedgerTestUtils.h" />
<ClInclude Include="..\..\src\ledger\SorobanMetrics.h" />
<ClInclude Include="..\..\src\ledger\TrustLineWrapper.h" />
Expand Down
24 changes: 12 additions & 12 deletions Builds/VisualStudio/stellar-core.vcxproj.filters
View file
Original file line number Diff line number Diff line change
Expand Up @@ -900,12 +900,6 @@
<ClCompile Include="..\..\src\ledger\FlushAndRotateMetaDebugWork.cpp">
<Filter>ledger</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ledger\InMemoryLedgerTxn.cpp">
<Filter>ledger</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ledger\InMemoryLedgerTxnRoot.cpp">
<Filter>ledger</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ledger\InternalLedgerEntry.cpp">
<Filter>ledger</Filter>
</ClCompile>
Expand Down Expand Up @@ -1401,6 +1395,12 @@
<ClCompile Include="..\..\src\bucket\BucketUtils.cpp">
<Filter>bucket</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ledger\test\InMemoryLedgerTxn.cpp">
<Filter>ledger\tests</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ledger\test\InMemoryLedgerTxnRoot.cpp">
<Filter>ledger\tests</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\lib\util\cpptoml.h">
Expand Down Expand Up @@ -2096,12 +2096,6 @@
<ClInclude Include="..\..\src\ledger\FlushAndRotateMetaDebugWork.h">
<Filter>ledger</Filter>
</ClInclude>
<ClInclude Include="..\..\src\ledger\InMemoryLedgerTxn.h">
<Filter>ledger</Filter>
</ClInclude>
<ClInclude Include="..\..\src\ledger\InMemoryLedgerTxnRoot.h">
<Filter>ledger</Filter>
</ClInclude>
<ClInclude Include="..\..\src\ledger\InternalLedgerEntry.h">
<Filter>ledger</Filter>
</ClInclude>
Expand Down Expand Up @@ -2449,6 +2443,12 @@
<ClInclude Include="..\..\src\bucket\BucketUtils.h">
<Filter>bucket</Filter>
</ClInclude>
<ClInclude Include="..\..\src\ledger\test\InMemoryLedgerTxn.h">
<Filter>ledger\tests</Filter>
</ClInclude>
<ClInclude Include="..\..\src\ledger\test\InMemoryLedgerTxnRoot.h">
<Filter>ledger\tests</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="..\..\AUTHORS" />
Expand Down
29 changes: 15 additions & 14 deletions src/simulation/TxGenerator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ constexpr uint32_t DEFAULT_WASM_BYTES = 35 * 1024;
constexpr uint32_t DEFAULT_NUM_DATA_ENTRIES = 2;
constexpr uint32_t DEFAULT_IO_KILOBYTES = 1;
constexpr uint32_t DEFAULT_TX_SIZE_BYTES = 256;
constexpr uint64_t DEFAULT_INSTRUCTIONS = 28'000'000;
constexpr uint32_t DEFAULT_INSTRUCTIONS = 28'000'000;

// Sample from a discrete distribution of `values` with weights `weights`.
// Returns `defaultValue` if `values` is empty.
Expand Down Expand Up @@ -373,16 +373,16 @@ TxGenerator::invokeSorobanLoadTransaction(
// instruction count is not perfect. Some TXs will fail due to exceeding
// resource limitations, but failures will be rare and those failures
// will happen at apply time, so they will still generate significant load.
uint64_t const baseInstructionCount = 1'500'000;
uint64_t const instructionsPerGuestCycle = 80;
uint64_t const instructionsPerHostCycle = 5030;
uint32_t const baseInstructionCount = 1'500'000;
uint32_t const instructionsPerGuestCycle = 80;
uint32_t const instructionsPerHostCycle = 5030;

// Very rough estimates.
uint64_t const instructionsPerKbWritten = 50000;
uint32_t const instructionsPerKbWritten = 50000;

// instructionsPerPaddingByte is just a value we know works. We use an auth
// payload as padding, so it consumes instructions on the host side.
uint64_t const instructionsPerPaddingByte = 100;
uint32_t const instructionsPerPaddingByte = 100;

SorobanResources resources;
resources.footprint.readOnly = instance.readOnlyKeys;
Expand Down Expand Up @@ -451,9 +451,10 @@ TxGenerator::invokeSorobanLoadTransaction(
instructionsPerPaddingByte * paddingBytes;

// Pick random number of cycles between bounds
uint64_t targetInstructions = sampleDiscrete(
appCfg.LOADGEN_INSTRUCTIONS_FOR_TESTING,
appCfg.LOADGEN_INSTRUCTIONS_DISTRIBUTION_FOR_TESTING, 0u);
uint32_t targetInstructions =
sampleDiscrete(appCfg.LOADGEN_INSTRUCTIONS_FOR_TESTING,
appCfg.LOADGEN_INSTRUCTIONS_DISTRIBUTION_FOR_TESTING,
DEFAULT_INSTRUCTIONS);

// Factor in instructions for storage
targetInstructions = baseInstructionCount + instructionsForStorageAndAuth >=
Expand All @@ -463,12 +464,12 @@ TxGenerator::invokeSorobanLoadTransaction(
instructionsForStorageAndAuth;

// Randomly select a number of guest cycles
uint64_t guestCyclesMax = targetInstructions / instructionsPerGuestCycle;
uint64_t guestCycles = rand_uniform<uint64_t>(0, guestCyclesMax);
uint32_t guestCyclesMax = targetInstructions / instructionsPerGuestCycle;
uint32_t guestCycles = rand_uniform<uint64_t>(0, guestCyclesMax);

// Rest of instructions consumed by host cycles
targetInstructions -= guestCycles * instructionsPerGuestCycle;
uint64_t hostCycles = targetInstructions / instructionsPerHostCycle;
uint32_t hostCycles = targetInstructions / instructionsPerHostCycle;

auto guestCyclesU64 = makeU64(guestCycles);
auto hostCyclesU64 = makeU64(hostCycles);
Expand All @@ -490,7 +491,7 @@ TxGenerator::invokeSorobanLoadTransaction(

increaseOpSize(op, paddingBytes);

int64_t instructionCount =
uint32_t instructionCount =
baseInstructionCount + hostCycles * instructionsPerHostCycle +
guestCycles * instructionsPerGuestCycle + instructionsForStorageAndAuth;
resources.instructions = instructionCount;
Expand Down Expand Up @@ -995,7 +996,7 @@ TxGenerator::sorobanRandomWasmTransaction(uint32_t ledgerNum,
int64_t resourceFee = sorobanResourceFee(
mApp, resources, 5000 + static_cast<size_t>(wasmSize), 100);
// Roughly cover the rent fee.
resourceFee += 100000;
resourceFee += 1'000'000;
auto tx = sorobanTransactionFrameFromOps(mApp.getNetworkID(), *account,
{uploadOp}, {}, resources,
inclusionFee, resourceFee);
Expand Down
72 changes: 6 additions & 66 deletions src/simulation/test/LoadGeneratorTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,7 +456,7 @@ TEST_CASE("generate soroban load", "[loadgen][soroban]")
cfg.mTxMaxSizeBytes * cfg.mLedgerMaxTxCount;
},
simulation);
auto const numInstances = 10;
auto const numInstances = nAccounts;
auto const numSorobanTxs = 100;

numTxsBefore = getSuccessfulTxCount();
Expand Down Expand Up @@ -490,8 +490,7 @@ TEST_CASE("generate soroban load", "[loadgen][soroban]")
/* txRate */ 1);

invokeLoadCfg.getMutSorobanConfig().nInstances = numInstances;
constexpr int maxInvokeFail = 10;
invokeLoadCfg.setMinSorobanPercentSuccess(100 - maxInvokeFail);
invokeLoadCfg.setMinSorobanPercentSuccess(100);

loadGen.generateLoad(invokeLoadCfg);
completeCount = complete.count();
Expand All @@ -506,15 +505,8 @@ TEST_CASE("generate soroban load", "[loadgen][soroban]")
{"ledger", "apply-soroban", "success"});
auto& txsFailed = node->getMetrics().NewCounter(
{"ledger", "apply-soroban", "failure"});

// Because we can't preflight TXs, some invocations will fail due to too
// few resources. This is expected, as our instruction counts are
// approximations. The following checks will make sure all set up
// phases succeeded, so only the invoke phase may have acceptable failed
// TXs
REQUIRE(txsSucceeded.count() >
numTxsBefore + numSorobanTxs - maxInvokeFail);
REQUIRE(txsFailed.count() < maxInvokeFail);
REQUIRE(txsSucceeded.count() == numTxsBefore + numSorobanTxs);
REQUIRE(txsFailed.count() == 0);
}

auto instanceKeys = loadGen.getContractInstanceKeysForTesting();
Expand Down Expand Up @@ -576,16 +568,7 @@ TEST_CASE("generate soroban load", "[loadgen][soroban]")
constexpr uint32_t uploadWeight = 5;
mixCfg.sorobanUploadWeight = uploadWeight;

// Because we can't preflight TXs, some invocations will fail due to too
// few resources. This is expected, as our instruction counts are
// approximations. Additionally, many upload transactions will fail as
// they are likely to generate invalid wasm. Therefore, we check that
// all but `maxInvokeFail + 1.5 * uploadWeight` transactions succeed. In
// case the random sampling produces more upload transactions than
// expected, we allow for a 50% margin of error on the number of upload
// transactions.
constexpr int maxSorobanFail = 1.5 * uploadWeight + maxInvokeFail;
mixLoadCfg.setMinSorobanPercentSuccess(100 - maxSorobanFail);
mixLoadCfg.setMinSorobanPercentSuccess(100);

loadGen.generateLoad(mixLoadCfg);
auto numSuccessBefore = getSuccessfulTxCount();
Expand All @@ -601,54 +584,11 @@ TEST_CASE("generate soroban load", "[loadgen][soroban]")
// Check results
for (auto node : nodes)
{
auto& totalSucceeded =
node->getMetrics().NewCounter({"ledger", "apply", "success"});
auto& totalFailed =
node->getMetrics().NewCounter({"ledger", "apply", "failure"});
auto& sorobanSucceeded = node->getMetrics().NewCounter(
{"ledger", "apply-soroban", "success"});
auto& sorobanFailed = node->getMetrics().NewCounter(
{"ledger", "apply-soroban", "failure"});

// Total number of classic transactions
int64_t classicTotal =
totalSucceeded.count() + totalFailed.count() -
sorobanSucceeded.count() - sorobanFailed.count();

// All classic transaction should succeed
REQUIRE(totalSucceeded.count() - sorobanSucceeded.count() ==
classicTotal);
// All failures should be soroban failures)
REQUIRE(totalFailed.count() == sorobanFailed.count());

// Check soroban results
REQUIRE(sorobanSucceeded.count() > numSuccessBefore + numMixedTxs -
classicTotal -
maxSorobanFail);
REQUIRE(sorobanFailed.count() <= maxSorobanFail + numFailedBefore);
REQUIRE(totalFailed.count() == 0);
}
}

// Test minimum percent success with too many transactions that fail to
// apply by requiring a 100% success rate for SOROBAN_UPLOAD mode
SECTION("Too many failed transactions")
{
auto uploadFailCfg = GeneratedLoadConfig::txLoad(
LoadGenMode::SOROBAN_UPLOAD, nAccounts, numSorobanTxs,
/* txRate */ 1);

// Set success percentage to 100% and leave other parameters at default.
uploadFailCfg.setMinSorobanPercentSuccess(100);

// LoadGen should fail
loadGen.generateLoad(uploadFailCfg);
auto& fail =
app.getMetrics().NewMeter({"loadgen", "run", "failed"}, "run");
auto failCount = fail.count();
simulation->crankUntil([&]() { return fail.count() == failCount + 1; },
300 * Herder::EXP_LEDGER_TIMESPAN_SECONDS,
false);
}
}

TEST_CASE("Multi-op pretend transactions are valid", "[loadgen]")
Expand Down

0 comments on commit 363ca1a

Please sign in to comment.