From 3fd69311cb14fef23e3d0b1c7ee15103e590ff7c Mon Sep 17 00:00:00 2001 From: hakunamtu Date: Wed, 16 Jan 2019 22:30:25 +0300 Subject: [PATCH 1/3] Make persistent storage synchronization interval configurable --- libi2pd/Config.cpp | 1 + libi2pd/NetDb.cpp | 75 +++++++++++++++++++++++++++++++------------- libi2pd/NetDb.hpp | 4 +++ libi2pd/RouterInfo.h | 2 ++ 4 files changed, 61 insertions(+), 21 deletions(-) diff --git a/libi2pd/Config.cpp b/libi2pd/Config.cpp index 89531f5a8ee..32bc815c566 100644 --- a/libi2pd/Config.cpp +++ b/libi2pd/Config.cpp @@ -256,6 +256,7 @@ namespace config { options_description persist("Network information persisting options"); persist.add_options() ("persist.profiles", value()->default_value(true), "Persist peer profiles (default: true)") + ("persist.syncinterval", value()->default_value(60), "Peer profiles and NetDb persistent storage sync interval in seconds (0 - save only on exit)") ; m_OptionsDesc diff --git a/libi2pd/NetDb.cpp b/libi2pd/NetDb.cpp index 714c327da0e..7a54e6062eb 100644 --- a/libi2pd/NetDb.cpp +++ b/libi2pd/NetDb.cpp @@ -26,7 +26,7 @@ namespace data { NetDb netdb; - NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat"), m_PersistProfiles (true), m_HiddenMode(false) + NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat"), m_PersistSyncInterval(60), m_PersistProfiles (true), m_HiddenMode(false) { } @@ -49,6 +49,7 @@ namespace data Reseed (); i2p::config::GetOption("persist.profiles", m_PersistProfiles); + i2p::config::GetOption("persist.syncinterval", m_PersistSyncInterval); m_IsRunning = true; m_Thread = new std::thread (std::bind (&NetDb::Run, this)); @@ -61,8 +62,10 @@ namespace data if (m_PersistProfiles) for (auto& it: m_RouterInfos) it.second->SaveProfile (); + SaveUpdated (); DeleteObsoleteProfiles (); m_RouterInfos.clear (); + m_UnsavedProfiles.clear (); m_Floodfills.clear (); if (m_Thread) { @@ -79,7 +82,7 @@ namespace data void NetDb::Run () { - uint32_t lastSave = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0; + uint32_t lastSave = 0, lastLeasesetsManage = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0; while (m_IsRunning) { try @@ -123,13 +126,19 @@ namespace data m_Requests.ManageRequests (); lastManageRequest = ts; } - if (ts - lastSave >= 60) // save routers, manage leasesets and validate subscriptions every minute + if (ts - lastLeasesetsManage >= 60) // manage leasesets and validate subscriptions every minute { - if (lastSave) + if (lastLeasesetsManage) { - SaveUpdated (); + RemoveExpired (); ManageLeaseSets (); } + lastLeasesetsManage = ts; + } + if (ts - lastSave >= m_PersistSyncInterval && m_PersistSyncInterval > 0) // save routers + { + if (lastSave) + SaveUpdated (); lastSave = ts; } if (ts - lastDestinationCleanup >= i2p::garlic::INCOMING_TAGS_EXPIRATION_TIMEOUT) @@ -229,6 +238,13 @@ namespace data } if (inserted) { + { + std::unique_lock l(m_UnsavedProfilesMutex); + auto it = m_UnsavedProfiles.find (ident); + if (it != m_UnsavedProfiles.end ()) + r->SetProfile(it->second); + } + LogPrint (eLogInfo, "NetDb: RouterInfo added: ", ident.ToBase64()); if (r->IsFloodfill () && r->IsReachable ()) // floodfill must be reachable { @@ -489,6 +505,7 @@ namespace data // make sure we cleanup netDb from previous attempts m_RouterInfos.clear (); m_Floodfills.clear (); + m_UnsavedProfiles.clear (); m_LastLoad = i2p::util::GetSecondsSinceEpoch(); std::vector files; @@ -501,16 +518,7 @@ namespace data void NetDb::SaveUpdated () { - int updatedCount = 0, deletedCount = 0; - auto total = m_RouterInfos.size (); - uint64_t expirationTimeout = NETDB_MAX_EXPIRATION_TIMEOUT*1000LL; - uint64_t ts = i2p::util::GetMillisecondsSinceEpoch(); - // routers don't expire if less than 90 or uptime is less than 1 hour - bool checkForExpiration = total > NETDB_MIN_ROUTERS && ts > (i2p::context.GetStartupTime () + 600)*1000LL; // 10 minutes - if (checkForExpiration && ts > (i2p::context.GetStartupTime () + 3600)*1000LL) // 1 hour - expirationTimeout = i2p::context.IsFloodfill () ? NETDB_FLOODFILL_EXPIRATION_TIMEOUT*1000LL : - NETDB_MIN_EXPIRATION_TIMEOUT*1000LL + (NETDB_MAX_EXPIRATION_TIMEOUT - NETDB_MIN_EXPIRATION_TIMEOUT)*1000LL*NETDB_MIN_ROUTERS/total; - + int updatedCount = 0; for (auto& it: m_RouterInfos) { std::string ident = it.second->GetIdentHashBase64(); @@ -519,11 +527,37 @@ namespace data { it.second->SaveToFile (path); it.second->SetUpdated (false); - it.second->SetUnreachable (false); it.second->DeleteBuffer (); updatedCount++; - continue; } + } // m_RouterInfos iteration + + if (updatedCount > 0) + LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers"); + + for (auto& it: m_UnsavedProfiles) + it.second->Save (it.first); + { + std::unique_lock l(m_UnsavedProfilesMutex); + m_UnsavedProfiles.clear (); + } + } + + void NetDb::RemoveExpired () + { + auto total = m_RouterInfos.size (); + int deletedCount = 0; + + uint64_t expirationTimeout = NETDB_MAX_EXPIRATION_TIMEOUT*1000LL; + uint64_t ts = i2p::util::GetMillisecondsSinceEpoch(); + // routers don't expire if less than 90 or uptime is less than 1 hour + bool checkForExpiration = total > NETDB_MIN_ROUTERS && ts > (i2p::context.GetStartupTime () + 600)*1000LL; // 10 minutes + if (checkForExpiration && ts > (i2p::context.GetStartupTime () + 3600)*1000LL) // 1 hour + expirationTimeout = i2p::context.IsFloodfill () ? NETDB_FLOODFILL_EXPIRATION_TIMEOUT*1000LL : + NETDB_MIN_EXPIRATION_TIMEOUT*1000LL + (NETDB_MAX_EXPIRATION_TIMEOUT - NETDB_MIN_EXPIRATION_TIMEOUT)*1000LL*NETDB_MIN_ROUTERS/total; + + for (auto& it: m_RouterInfos) + { // find & mark expired routers if (it.second->UsesIntroducer ()) { @@ -537,14 +571,12 @@ namespace data if (it.second->IsUnreachable ()) { // delete RI file - m_Storage.Remove(ident); + m_Storage.Remove (it.first.ToBase64 ()); deletedCount++; if (total - deletedCount < NETDB_MIN_ROUTERS) checkForExpiration = false; } } // m_RouterInfos iteration - if (updatedCount > 0) - LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers"); if (deletedCount > 0) { LogPrint (eLogInfo, "NetDb: deleting ", deletedCount, " unreachable routers"); @@ -555,7 +587,8 @@ namespace data { if (it->second->IsUnreachable ()) { - if (m_PersistProfiles) it->second->SaveProfile (); + if (m_PersistProfiles && it->second->HasProfile ()) + m_UnsavedProfiles[it->second->GetIdentHash ()] = it->second->GetProfile (); it = m_RouterInfos.erase (it); continue; } diff --git a/libi2pd/NetDb.hpp b/libi2pd/NetDb.hpp index 94ed2ed2445..3e814746306 100644 --- a/libi2pd/NetDb.hpp +++ b/libi2pd/NetDb.hpp @@ -109,6 +109,7 @@ namespace data void Load (); bool LoadRouterInfo (const std::string & path); void SaveUpdated (); + void RemoveExpired (); void Run (); // exploratory thread void Explore (int numDestinations); void Publish (); @@ -129,6 +130,8 @@ namespace data std::map > m_LeaseSets; mutable std::mutex m_RouterInfosMutex; std::map > m_RouterInfos; + mutable std::mutex m_UnsavedProfilesMutex; + std::map> m_UnsavedProfiles; mutable std::mutex m_FloodfillsMutex; std::list > m_Floodfills; @@ -145,6 +148,7 @@ namespace data friend class NetDbRequests; NetDbRequests m_Requests; + unsigned m_PersistSyncInterval; bool m_PersistProfiles; /** router info we are bootstrapping from or nullptr if we are not currently doing that*/ diff --git a/libi2pd/RouterInfo.h b/libi2pd/RouterInfo.h index f66a73fabd8..e90f0e2fa18 100644 --- a/libi2pd/RouterInfo.h +++ b/libi2pd/RouterInfo.h @@ -194,6 +194,8 @@ namespace data bool SaveToFile (const std::string& fullPath); std::shared_ptr GetProfile () const; + bool HasProfile () const { return static_cast(m_Profile); }; + void SetProfile (const std::shared_ptr& profile) { m_Profile = profile; }; void SaveProfile () { if (m_Profile) m_Profile->Save (GetIdentHash ()); }; void Update (const uint8_t * buf, int len); From 6c79ccd1dc6a4879c6d634851356b4b5f26d9be9 Mon Sep 17 00:00:00 2001 From: hakunamtu Date: Sun, 20 Jan 2019 21:49:26 +0300 Subject: [PATCH 2/3] Periodic obsolete profiles cleanup --- libi2pd/NetDb.cpp | 22 +++++++++++++++++++++- libi2pd/NetDb.hpp | 1 + libi2pd/Profiling.h | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/libi2pd/NetDb.cpp b/libi2pd/NetDb.cpp index 7a54e6062eb..599643777b7 100644 --- a/libi2pd/NetDb.cpp +++ b/libi2pd/NetDb.cpp @@ -82,7 +82,7 @@ namespace data void NetDb::Run () { - uint32_t lastSave = 0, lastLeasesetsManage = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0; + uint32_t lastSave = 0, lastLeasesetsManage = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0, lastProfilesCleanup = 0; while (m_IsRunning) { try @@ -171,6 +171,11 @@ namespace data lastExploratory = ts; } } + if (ts - lastProfilesCleanup >= PEER_PROFILE_EXPIRATION_TIMEOUT * 3600) + { + RemoveObsoleteProfiles (); + lastProfilesCleanup = ts; + } } catch (std::exception& ex) { @@ -607,6 +612,21 @@ namespace data } } + void NetDb::RemoveObsoleteProfiles () + { + DeleteObsoleteProfiles (); + auto now = boost::posix_time::second_clock::local_time (); + { + std::unique_lock l(m_UnsavedProfilesMutex); + for (auto it = m_UnsavedProfiles.begin (); it != m_UnsavedProfiles.end ();) + if ((now - it->second->GetLastUpdateTime ()).hours () >= PEER_PROFILE_EXPIRATION_TIMEOUT) + it = m_UnsavedProfiles.erase (it); + else + ++it; + } + LogPrint (eLogInfo, "NetDb: obsolete profiles were removed"); + } + void NetDb::RequestDestination (const IdentHash& destination, RequestedDestination::RequestComplete requestComplete) { auto dest = m_Requests.CreateRequest (destination, false, requestComplete); // non-exploratory diff --git a/libi2pd/NetDb.hpp b/libi2pd/NetDb.hpp index 3e814746306..568899c1f87 100644 --- a/libi2pd/NetDb.hpp +++ b/libi2pd/NetDb.hpp @@ -110,6 +110,7 @@ namespace data bool LoadRouterInfo (const std::string & path); void SaveUpdated (); void RemoveExpired (); + void RemoveObsoleteProfiles (); void Run (); // exploratory thread void Explore (int numDestinations); void Publish (); diff --git a/libi2pd/Profiling.h b/libi2pd/Profiling.h index 4ba6702f10f..708cae455bd 100644 --- a/libi2pd/Profiling.h +++ b/libi2pd/Profiling.h @@ -29,6 +29,8 @@ namespace data RouterProfile (); RouterProfile& operator= (const RouterProfile& ) = default; + boost::posix_time::ptime GetLastUpdateTime () const { return m_LastUpdateTime; }; + void Save (const IdentHash& identHash); void Load (const IdentHash& identHash); From ad437add24b8b8a419cd6bccc990045c25b20035 Mon Sep 17 00:00:00 2001 From: hakunamtu Date: Sun, 3 Mar 2019 21:04:19 +0300 Subject: [PATCH 3/3] use cached base64 ident hash --- libi2pd/NetDb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libi2pd/NetDb.cpp b/libi2pd/NetDb.cpp index 599643777b7..414f7dec0cc 100644 --- a/libi2pd/NetDb.cpp +++ b/libi2pd/NetDb.cpp @@ -576,7 +576,7 @@ namespace data if (it.second->IsUnreachable ()) { // delete RI file - m_Storage.Remove (it.first.ToBase64 ()); + m_Storage.Remove (it.second->GetIdentHashBase64 ()); deletedCount++; if (total - deletedCount < NETDB_MIN_ROUTERS) checkForExpiration = false; }