Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make persistent storage synchronization interval configurable #1285

Open
wants to merge 3 commits into
base: openssl
Choose a base branch
from
Open

Make persistent storage synchronization interval configurable #1285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3fd6931
Make persistent storage synchronization interval configurable
hakunamtu Jan 16, 2019
6c79ccd
Periodic obsolete profiles cleanup
hakunamtu Jan 20, 2019
ad437ad
use cached base64 ident hash
hakunamtu Mar 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions libi2pd/Config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ namespace config {
options_description persist("Network information persisting options");
persist.add_options()
("persist.profiles", value<bool>()->default_value(true), "Persist peer profiles (default: true)")
("persist.syncinterval", value<unsigned>()->default_value(60), "Peer profiles and NetDb persistent storage sync interval in seconds (0 - save only on exit)")
;

m_OptionsDesc
Expand Down
95 changes: 74 additions & 21 deletions libi2pd/NetDb.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ namespace data
{
NetDb netdb;

NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat"), m_PersistProfiles (true), m_HiddenMode(false)
NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat"), m_PersistSyncInterval(60), m_PersistProfiles (true), m_HiddenMode(false)
{
}

Expand All @@ -49,6 +49,7 @@ namespace data
Reseed ();

i2p::config::GetOption("persist.profiles", m_PersistProfiles);
i2p::config::GetOption("persist.syncinterval", m_PersistSyncInterval);

m_IsRunning = true;
m_Thread = new std::thread (std::bind (&NetDb::Run, this));
Expand All @@ -61,8 +62,10 @@ namespace data
if (m_PersistProfiles)
for (auto& it: m_RouterInfos)
it.second->SaveProfile ();
SaveUpdated ();
DeleteObsoleteProfiles ();
m_RouterInfos.clear ();
m_UnsavedProfiles.clear ();
m_Floodfills.clear ();
if (m_Thread)
{
Expand All @@ -79,7 +82,7 @@ namespace data

void NetDb::Run ()
{
uint32_t lastSave = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0;
uint32_t lastSave = 0, lastLeasesetsManage = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0, lastProfilesCleanup = 0;
while (m_IsRunning)
{
try
Expand Down Expand Up @@ -123,13 +126,19 @@ namespace data
m_Requests.ManageRequests ();
lastManageRequest = ts;
}
if (ts - lastSave >= 60) // save routers, manage leasesets and validate subscriptions every minute
if (ts - lastLeasesetsManage >= 60) // manage leasesets and validate subscriptions every minute
{
if (lastSave)
if (lastLeasesetsManage)
{
SaveUpdated ();
RemoveExpired ();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removing every minute?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As the old SaveUpdated did.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It must be with interval as specified in this config parameter. Otherwise you might end up with empty netdb quickly

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for the long delay. As far as I can see, doing it less frequently allows netdb to bloat uncontrollably. One apparent issue that needs to be fixed is that recently updated (IsUpdated() == true) but now unreachable records are deleted from netdb, which deviates from existing behaviour. Otherwise netdb service does the same thing, except for less frequent disk writes. If there is something else I fail to see, please, tell me.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also persisting netDb on exit does not work with the default systemd unit's stop signal - SIGQUIT.

ManageLeaseSets ();
}
lastLeasesetsManage = ts;
}
if (ts - lastSave >= m_PersistSyncInterval && m_PersistSyncInterval > 0) // save routers
{
if (lastSave)
SaveUpdated ();
lastSave = ts;
}
if (ts - lastDestinationCleanup >= i2p::garlic::INCOMING_TAGS_EXPIRATION_TIMEOUT)
Expand Down Expand Up @@ -162,6 +171,11 @@ namespace data
lastExploratory = ts;
}
}
if (ts - lastProfilesCleanup >= PEER_PROFILE_EXPIRATION_TIMEOUT * 3600)
{
RemoveObsoleteProfiles ();
lastProfilesCleanup = ts;
}
}
catch (std::exception& ex)
{
Expand Down Expand Up @@ -229,6 +243,13 @@ namespace data
}
if (inserted)
{
{
std::unique_lock<std::mutex> l(m_UnsavedProfilesMutex);
auto it = m_UnsavedProfiles.find (ident);
if (it != m_UnsavedProfiles.end ())
r->SetProfile(it->second);
}

LogPrint (eLogInfo, "NetDb: RouterInfo added: ", ident.ToBase64());
if (r->IsFloodfill () && r->IsReachable ()) // floodfill must be reachable
{
Expand Down Expand Up @@ -489,6 +510,7 @@ namespace data
// make sure we cleanup netDb from previous attempts
m_RouterInfos.clear ();
m_Floodfills.clear ();
m_UnsavedProfiles.clear ();

m_LastLoad = i2p::util::GetSecondsSinceEpoch();
std::vector<std::string> files;
Expand All @@ -501,16 +523,7 @@ namespace data

void NetDb::SaveUpdated ()
{
int updatedCount = 0, deletedCount = 0;
auto total = m_RouterInfos.size ();
uint64_t expirationTimeout = NETDB_MAX_EXPIRATION_TIMEOUT*1000LL;
uint64_t ts = i2p::util::GetMillisecondsSinceEpoch();
// routers don't expire if less than 90 or uptime is less than 1 hour
bool checkForExpiration = total > NETDB_MIN_ROUTERS && ts > (i2p::context.GetStartupTime () + 600)*1000LL; // 10 minutes
if (checkForExpiration && ts > (i2p::context.GetStartupTime () + 3600)*1000LL) // 1 hour
expirationTimeout = i2p::context.IsFloodfill () ? NETDB_FLOODFILL_EXPIRATION_TIMEOUT*1000LL :
NETDB_MIN_EXPIRATION_TIMEOUT*1000LL + (NETDB_MAX_EXPIRATION_TIMEOUT - NETDB_MIN_EXPIRATION_TIMEOUT)*1000LL*NETDB_MIN_ROUTERS/total;

int updatedCount = 0;
for (auto& it: m_RouterInfos)
{
std::string ident = it.second->GetIdentHashBase64();
Expand All @@ -519,11 +532,37 @@ namespace data
{
it.second->SaveToFile (path);
it.second->SetUpdated (false);
it.second->SetUnreachable (false);
it.second->DeleteBuffer ();
updatedCount++;
continue;
}
} // m_RouterInfos iteration

if (updatedCount > 0)
LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers");

for (auto& it: m_UnsavedProfiles)
it.second->Save (it.first);
{
std::unique_lock<std::mutex> l(m_UnsavedProfilesMutex);
m_UnsavedProfiles.clear ();
}
}

void NetDb::RemoveExpired ()
{
auto total = m_RouterInfos.size ();
int deletedCount = 0;

uint64_t expirationTimeout = NETDB_MAX_EXPIRATION_TIMEOUT*1000LL;
uint64_t ts = i2p::util::GetMillisecondsSinceEpoch();
// routers don't expire if less than 90 or uptime is less than 1 hour
bool checkForExpiration = total > NETDB_MIN_ROUTERS && ts > (i2p::context.GetStartupTime () + 600)*1000LL; // 10 minutes
if (checkForExpiration && ts > (i2p::context.GetStartupTime () + 3600)*1000LL) // 1 hour
expirationTimeout = i2p::context.IsFloodfill () ? NETDB_FLOODFILL_EXPIRATION_TIMEOUT*1000LL :
NETDB_MIN_EXPIRATION_TIMEOUT*1000LL + (NETDB_MAX_EXPIRATION_TIMEOUT - NETDB_MIN_EXPIRATION_TIMEOUT)*1000LL*NETDB_MIN_ROUTERS/total;

for (auto& it: m_RouterInfos)
{
// find & mark expired routers
if (it.second->UsesIntroducer ())
{
Expand All @@ -537,14 +576,12 @@ namespace data
if (it.second->IsUnreachable ())
{
// delete RI file
m_Storage.Remove(ident);
m_Storage.Remove (it.second->GetIdentHashBase64 ());
deletedCount++;
if (total - deletedCount < NETDB_MIN_ROUTERS) checkForExpiration = false;
}
} // m_RouterInfos iteration

if (updatedCount > 0)
LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers");
if (deletedCount > 0)
{
LogPrint (eLogInfo, "NetDb: deleting ", deletedCount, " unreachable routers");
Expand All @@ -555,7 +592,8 @@ namespace data
{
if (it->second->IsUnreachable ())
{
if (m_PersistProfiles) it->second->SaveProfile ();
if (m_PersistProfiles && it->second->HasProfile ())
m_UnsavedProfiles[it->second->GetIdentHash ()] = it->second->GetProfile ();
it = m_RouterInfos.erase (it);
continue;
}
Expand All @@ -574,6 +612,21 @@ namespace data
}
}

void NetDb::RemoveObsoleteProfiles ()
{
DeleteObsoleteProfiles ();
auto now = boost::posix_time::second_clock::local_time ();
{
std::unique_lock<std::mutex> l(m_UnsavedProfilesMutex);
for (auto it = m_UnsavedProfiles.begin (); it != m_UnsavedProfiles.end ();)
if ((now - it->second->GetLastUpdateTime ()).hours () >= PEER_PROFILE_EXPIRATION_TIMEOUT)
it = m_UnsavedProfiles.erase (it);
else
++it;
}
LogPrint (eLogInfo, "NetDb: obsolete profiles were removed");
}

void NetDb::RequestDestination (const IdentHash& destination, RequestedDestination::RequestComplete requestComplete)
{
auto dest = m_Requests.CreateRequest (destination, false, requestComplete); // non-exploratory
Expand Down
5 changes: 5 additions & 0 deletions libi2pd/NetDb.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ namespace data
void Load ();
bool LoadRouterInfo (const std::string & path);
void SaveUpdated ();
void RemoveExpired ();
void RemoveObsoleteProfiles ();
void Run (); // exploratory thread
void Explore (int numDestinations);
void Publish ();
Expand All @@ -129,6 +131,8 @@ namespace data
std::map<IdentHash, std::shared_ptr<LeaseSet> > m_LeaseSets;
mutable std::mutex m_RouterInfosMutex;
std::map<IdentHash, std::shared_ptr<RouterInfo> > m_RouterInfos;
mutable std::mutex m_UnsavedProfilesMutex;
std::map<IdentHash, std::shared_ptr<RouterProfile>> m_UnsavedProfiles;
mutable std::mutex m_FloodfillsMutex;
std::list<std::shared_ptr<RouterInfo> > m_Floodfills;

Expand All @@ -145,6 +149,7 @@ namespace data
friend class NetDbRequests;
NetDbRequests m_Requests;

unsigned m_PersistSyncInterval;
bool m_PersistProfiles;

/** router info we are bootstrapping from or nullptr if we are not currently doing that*/
Expand Down
2 changes: 2 additions & 0 deletions libi2pd/Profiling.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ namespace data
RouterProfile ();
RouterProfile& operator= (const RouterProfile& ) = default;

boost::posix_time::ptime GetLastUpdateTime () const { return m_LastUpdateTime; };

void Save (const IdentHash& identHash);
void Load (const IdentHash& identHash);

Expand Down
2 changes: 2 additions & 0 deletions libi2pd/RouterInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,8 @@ namespace data
bool SaveToFile (const std::string& fullPath);

std::shared_ptr<RouterProfile> GetProfile () const;
bool HasProfile () const { return static_cast<bool>(m_Profile); };
void SetProfile (const std::shared_ptr<RouterProfile>& profile) { m_Profile = profile; };
void SaveProfile () { if (m_Profile) m_Profile->Save (GetIdentHash ()); };

void Update (const uint8_t * buf, int len);
Expand Down