diff --git a/curvefs/src/client/common/common.cpp b/curvefs/src/client/common/common.cpp index b50898a630..f21ba12de5 100644 --- a/curvefs/src/client/common/common.cpp +++ b/curvefs/src/client/common/common.cpp @@ -80,6 +80,7 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) { const char kCurveFsWarmupOpAdd[] = "add"; const char kCurveFsWarmupOpCancel[] = "cancel"; +const char kCurveFsWarmupOpCheck[] = "check"; const char kCurveFsWarmupTypeList[] = "list"; const char kCurveFsWarmupTypeSingle[] = "single"; @@ -89,6 +90,8 @@ WarmupOpType GetWarmupOpType(const std::string& op) { ret = WarmupOpType::kWarmupOpAdd; } else if (op == kCurveFsWarmupOpCancel) { ret = WarmupOpType::kWarmupOpCancel; + } else if (op == kCurveFsWarmupOpCheck) { + ret = WarmupOpType::kWarmupOpCheck; } return ret; } diff --git a/curvefs/src/client/common/common.h b/curvefs/src/client/common/common.h index a76312b117..d343244573 100644 --- a/curvefs/src/client/common/common.h +++ b/curvefs/src/client/common/common.h @@ -68,11 +68,13 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype); constexpr size_t kMinWarmupOpArgsNum = 1; constexpr size_t kWarmupAddArgsNum = 6; constexpr size_t kWarmupCancelArgsNum = 2; +constexpr size_t kWarmupCheckArgsNum = 6; enum class WarmupOpType { kWarmupOpUnknown = 0, kWarmupOpAdd = 1, kWarmupOpCancel = 2, + kWarmupOpCheck = 3, }; WarmupOpType GetWarmupOpType(const std::string& op); diff --git a/curvefs/src/client/curve_fuse_op.cpp b/curvefs/src/client/curve_fuse_op.cpp index cd2128d805..f6501898c0 100644 --- a/curvefs/src/client/curve_fuse_op.cpp +++ b/curvefs/src/client/curve_fuse_op.cpp @@ -23,7 +23,10 @@ #include "curvefs/src/client/curve_fuse_op.h" +#include + #include +#include #include #include #include @@ -66,6 +69,7 @@ using ::curvefs::client::common::WarmupStorageType; using ::curvefs::client::filesystem::AttrOut; using ::curvefs::client::filesystem::EntryOut; using ::curvefs::client::filesystem::FileOut; +using ::curvefs::client::filesystem::IsCheckXAttr; using ::curvefs::client::filesystem::IsListWarmupXAttr; using ::curvefs::client::filesystem::IsWarmupXAttr; using ::curvefs::client::filesystem::StrAttr; @@ -237,16 +241,18 @@ void UnInitFuseClient() { int AddWarmupTask(curvefs::client::common::WarmupType type, fuse_ino_t key, const std::string& path, curvefs::client::common::WarmupStorageType storageType, - const std::string& mount_point, const std::string& root) { + const std::string& mount_point, const std::string& root, + bool check = false) { int ret = 0; bool result = true; switch (type) { case curvefs::client::common::WarmupType::kWarmupTypeList: - result = g_ClientInstance->PutWarmFilelistTask(key, storageType, path, - mount_point, root); - break; + result = g_ClientInstance->PutWarmFilelistTask( + key, storageType, path, mount_point, root, check); + break; case curvefs::client::common::WarmupType::kWarmupTypeSingle: - result = g_ClientInstance->PutWarmFileTask(key, path, storageType); + result = + g_ClientInstance->PutWarmFileTask(key, path, storageType, check); break; default: // not support add warmup type (warmup single file/dir or filelist) @@ -291,6 +297,18 @@ void QueryWarmupTask(fuse_ino_t key, std::string *data) { VLOG(9) << "Warmup [" << key << "]" << *data; } +void QueryCheckCachedTask(fuse_ino_t key, std::string* data) { + WarmupProgress progress; + bool ret = g_ClientInstance->GetCheckCachedProgress(key, &progress); + if (!ret) { + *data = "no check task or not warmup yet"; + } else { + *data = + fmt::format("{}/{}", progress.GetFinished(), progress.GetTotal()); + } + VLOG(9) << "check cached [" << key << "]" << *data; +} + void ListWarmupTasks(std::string* data) { WarmupProgress progress; std::unordered_map filepath2progress; @@ -344,9 +362,15 @@ int Warmup(fuse_ino_t key, const char* name, const std::string& values) { } int ret = 0; + bool check = false; + if (curvefs::client::common::GetWarmupOpType(warmupOpType) == + curvefs::client::common::WarmupOpType::kWarmupOpCheck) { + check = true; + } switch (curvefs::client::common::GetWarmupOpType(warmupOpType)) { - case curvefs::client::common::WarmupOpType::kWarmupOpAdd: { + case curvefs::client::common::WarmupOpType::kWarmupOpCheck: + case curvefs::client::common::WarmupOpType::kWarmupOpAdd : { if (opTypePath.size() != curvefs::client::common::kWarmupAddArgsNum) { LOG(ERROR) @@ -373,7 +397,7 @@ int Warmup(fuse_ino_t key, const char* name, const std::string& values) { ret = AddWarmupTask( curvefs::client::common::GetWarmupType(warmupDataType), key, entryFilePathInClient, storageType, mountPointInCurvefs, - rootPathInCurvefs); + rootPathInCurvefs, check); break; } case curvefs::client::common::WarmupOpType::kWarmupOpCancel: { @@ -446,6 +470,17 @@ void QueryWarmup(fuse_req_t req, fuse_ino_t ino, size_t size) { return fs->ReplyBuffer(req, data.data(), data.length()); } +void QueryCheckCached(fuse_req_t req, fuse_ino_t ino, size_t size) { + auto fs = Client()->GetFileSystem(); + + std::string data; + QueryCheckCachedTask(ino, &data); + if (size == 0) { + return fs->ReplyXattr(req, data.length()); + } + return fs->ReplyBuffer(req, data.data(), data.length()); +} + void ListWarmup(fuse_req_t req, size_t size) { auto fs = Client()->GetFileSystem(); @@ -944,6 +979,8 @@ void FuseOpGetXattr(fuse_req_t req, return ListWarmup(req, size); } else if (IsWarmupXAttr(name)) { return QueryWarmup(req, ino, size); + } else if (IsCheckXAttr(name)) { + return QueryCheckCached(req, ino, size); } rc = Client()->FuseOpGetXattr(req, ino, name, &value, size); diff --git a/curvefs/src/client/filesystem/xattr.h b/curvefs/src/client/filesystem/xattr.h index 26bc3a6c2e..deb2889220 100644 --- a/curvefs/src/client/filesystem/xattr.h +++ b/curvefs/src/client/filesystem/xattr.h @@ -45,6 +45,7 @@ const char XATTR_DIR_RFBYTES[] = "curve.dir.rfbytes"; const char XATTR_DIR_PREFIX[] = "curve.dir"; const char XATTR_WARMUP_OP[] = "curvefs.warmup.op"; const char XATTR_WARMUP_OP_LIST[] = "curvefs.warmup.op.list"; +const char XATTR_WARMUP_OP_CHECK[] = "curvefs.warmup.check"; inline bool IsSpecialXAttr(const std::string& key) { static std::map xattrs { @@ -65,6 +66,10 @@ inline bool IsWarmupXAttr(const std::string& key) { return key == XATTR_WARMUP_OP; } +inline bool IsCheckXAttr(const std::string& key) { + return key == XATTR_WARMUP_OP_CHECK; +} + inline bool IsListWarmupXAttr(const std::string& key) { return key == XATTR_WARMUP_OP_LIST; } diff --git a/curvefs/src/client/fuse_client.h b/curvefs/src/client/fuse_client.h index 13adc3c591..fc22ffb1d1 100644 --- a/curvefs/src/client/fuse_client.h +++ b/curvefs/src/client/fuse_client.h @@ -312,18 +312,18 @@ class FuseClient { bool PutWarmFilelistTask(fuse_ino_t key, common::WarmupStorageType type, const std::string& path, const std::string& mount_point, - const std::string& root) { + const std::string& root, bool check = false) { if (fsInfo_->fstype() == FSType::TYPE_S3) { return warmupManager_->AddWarmupFilelist(key, type, path, - mount_point, root); + mount_point, root, check); } // only support s3 return true; } - bool PutWarmFileTask(fuse_ino_t key, const std::string &path, - common::WarmupStorageType type) { + bool PutWarmFileTask(fuse_ino_t key, const std::string& path, + common::WarmupStorageType type, bool check = false) { if (fsInfo_->fstype() == FSType::TYPE_S3) { - return warmupManager_->AddWarmupFile(key, path, type); + return warmupManager_->AddWarmupFile(key, path, type, check); } // only support s3 return true; } @@ -342,6 +342,14 @@ class FuseClient { return false; } + bool GetCheckCachedProgress(fuse_ino_t key, + warmup::WarmupProgress* progress) { + if (fsInfo_->fstype() == FSType::TYPE_S3) { + return warmupManager_->QueryCheckCachedProgress(key, progress); + } + return false; + } + bool GetAllWarmupProgress(Filepath2WarmupProgressMap* filepath2progress) { if (fsInfo_->fstype() == FSType::TYPE_S3) { return warmupManager_->ListWarmupProgress(filepath2progress); diff --git a/curvefs/src/client/kvclient/BUILD b/curvefs/src/client/kvclient/BUILD index 202b6b5743..c4e25971b4 100644 --- a/curvefs/src/client/kvclient/BUILD +++ b/curvefs/src/client/kvclient/BUILD @@ -35,5 +35,6 @@ cc_library( "//src/common:curve_common", "//src/common:curve_s3_adapter", "@libmemcached", + "@fmt//:fmt", ], ) diff --git a/curvefs/src/client/kvclient/kvclient.h b/curvefs/src/client/kvclient/kvclient.h index a96c733762..41ad176349 100644 --- a/curvefs/src/client/kvclient/kvclient.h +++ b/curvefs/src/client/kvclient/kvclient.h @@ -24,6 +24,7 @@ #include +#include #include namespace curvefs { @@ -54,6 +55,8 @@ class KVClient { virtual bool Get(const std::string& key, char* value, uint64_t offset, uint64_t length, std::string* errorlog, uint64_t* actLength, memcached_return_t* retCod) = 0; + + virtual bool Exist(const std::string& key) = 0; }; } // namespace client diff --git a/curvefs/src/client/kvclient/kvclient_manager.cpp b/curvefs/src/client/kvclient/kvclient_manager.cpp index c87630758b..84d5c22a77 100644 --- a/curvefs/src/client/kvclient/kvclient_manager.cpp +++ b/curvefs/src/client/kvclient/kvclient_manager.cpp @@ -128,5 +128,12 @@ int KVClientManager::GetKvCache( return 0; } +void KVClientManager::Exist(std::shared_ptr task) { + threadPool_.Enqueue([task, this]() { + task->res = client_->Exist(task->key); + OnReturn(&kvClientManagerMetric_->exist, task); + }); +} + } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/kvclient/kvclient_manager.h b/curvefs/src/client/kvclient/kvclient_manager.h index bad99193a8..5b42141b68 100644 --- a/curvefs/src/client/kvclient/kvclient_manager.h +++ b/curvefs/src/client/kvclient/kvclient_manager.h @@ -46,9 +46,7 @@ namespace client { class KVClientManager; struct SetKVCacheTask; struct GetKVCacheTask; - -class GetKvCacheContext; -class SetKvCacheContext; +struct ExistKVCacheTask; using curve::common::GetObjectAsyncContext; using curve::common::TaskThreadPool; @@ -58,6 +56,8 @@ using SetKVCacheDone = std::function&)>; using GetKVCacheDone = std::function&)>; +using ExistKVCacheDone = + std::function&)>; struct SetKVCacheTask { std::string key; @@ -79,7 +79,7 @@ struct SetKVCacheTask { }; struct GetKVCacheTask { - const std::string& key; + std::string key; char* value; uint64_t offset; uint64_t valueLength; @@ -101,11 +101,21 @@ struct GetKVCacheTask { timer(butil::Timer::STARTED) {} }; -using GetKvCacheCallBack = - std::function&)>; +struct ExistKVCacheTask { + std::string key; + bool res; + uint64_t length = 0; // useless,just for OnReturn + ExistKVCacheDone done; + butil::Timer timer; -using SetKvCacheCallBack = - std::function&)>; + explicit ExistKVCacheTask( + const std::string& k, + ExistKVCacheDone done = [](const std::shared_ptr&) {}) + : key(k), + res(false), + done(std::move(done)), + timer(butil::Timer::STARTED) {} +}; struct KvCacheContext { std::string key; @@ -117,17 +127,6 @@ struct KvCacheContext { uint64_t startTime; }; -struct GetKvCacheContext : KvCacheContext { - char* value; - bool res; - GetKvCacheCallBack cb; -}; - -struct SetKvCacheContext : KvCacheContext { - const char* value; - SetKvCacheCallBack cb; -}; - class KVClientManager { public: KVClientManager() = default; @@ -146,6 +145,8 @@ class KVClientManager { void Get(std::shared_ptr task); + void Exist(std::shared_ptr task); + KVClientManagerMetric* GetMetricForTesting() { return kvClientManagerMetric_.get(); } diff --git a/curvefs/src/client/kvclient/memcache_client.cpp b/curvefs/src/client/kvclient/memcache_client.cpp index 92376e79c2..84e3bc93ea 100644 --- a/curvefs/src/client/kvclient/memcache_client.cpp +++ b/curvefs/src/client/kvclient/memcache_client.cpp @@ -22,10 +22,44 @@ #include "curvefs/src/client/kvclient/memcache_client.h" +#include +#include + +#include "src/client/client_metric.h" + namespace curvefs { namespace client { thread_local memcached_st* tcli = nullptr; +bool MemCachedClient::Exist(const std::string& key) { + // https://awesomized.github.io/libmemcached/libmemcached/memcached_exist.html?highlight=exist#_CPPv415memcached_existP12memcached_stPcP6size_t + memcached_return_t ue; + size_t value_length = 0; + uint64_t start = butil::cpuwide_time_us(); + if (nullptr == tcli) { + LOG(ERROR) << "create tcli"; + tcli = memcached_clone(nullptr, client_); + } + ue = memcached_exist(tcli, key.c_str(), key.length()); + if (ue == MEMCACHED_SUCCESS) { + curve::client::CollectMetrics(&metric_->exist, 0, + butil::cpuwide_time_us() - start); + return true; + } + + if (ue == MEMCACHED_NOTFOUND) { + curve::client::CollectMetrics(&metric_->exist, 0, + butil::cpuwide_time_us() - start); + } else { + std::string errorlog = ResError(ue); + LOG(ERROR) << "Exist key = " << key << " error = " << errorlog; + metric_->exist.eps.count << 1; + } + memcached_free(tcli); + tcli = nullptr; + return false; +} + } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/kvclient/memcache_client.h b/curvefs/src/client/kvclient/memcache_client.h index d481f5e5e4..d9f916f5e3 100644 --- a/curvefs/src/client/kvclient/memcache_client.h +++ b/curvefs/src/client/kvclient/memcache_client.h @@ -23,8 +23,10 @@ #ifndef CURVEFS_SRC_CLIENT_KVCLIENT_MEMCACHE_CLIENT_H_ #define CURVEFS_SRC_CLIENT_KVCLIENT_MEMCACHE_CLIENT_H_ +#include #include #include +#include #include #include @@ -90,7 +92,7 @@ class MemCachedClient : public KVClient { bool Init(const MemcacheClusterInfo& kvcachecluster, const std::string& fsName) { metric_ = absl::make_unique(fsName); - client_ = memcached(nullptr, 0); + // client_ = memcached(nullptr, 0); for (int i = 0; i < kvcachecluster.servers_size(); i++) { if (!AddServer(kvcachecluster.servers(i).ip(), @@ -146,7 +148,7 @@ class MemCachedClient : public KVClient { uint32_t flags = 0; size_t value_length = 0; memcached_return_t ue; - char *res = memcached_get(tcli, key.c_str(), key.length(), + char* res = memcached_get(tcli, key.c_str(), key.length(), &value_length, &flags, &ue); if (actLength != nullptr) { (*actLength) = value_length; @@ -178,6 +180,9 @@ class MemCachedClient : public KVClient { return false; } + // get key exist or not + bool Exist(const std::string& key); + // transform the res to a error string const std::string ResError(const memcached_return_t res) { return memcached_strerror(nullptr, res); diff --git a/curvefs/src/client/metric/client_metric.h b/curvefs/src/client/metric/client_metric.h index bfbf0f3373..2809d341a8 100644 --- a/curvefs/src/client/metric/client_metric.h +++ b/curvefs/src/client/metric/client_metric.h @@ -311,6 +311,7 @@ struct KVClientManagerMetric { std::string fsName; InterfaceMetric get; InterfaceMetric set; + InterfaceMetric exist; // kvcache count bvar::Adder count; // kvcache hit @@ -323,6 +324,7 @@ struct KVClientManagerMetric { : prefix + curve::common::ToHexString(this)), get(prefix, fsName + "_get"), set(prefix, fsName + "_set"), + exist(prefix, fsName + "_exist"), count(prefix, fsName + "_count"), hit(prefix, fsName + "_hit"), miss(prefix, fsName + "_miss") {} @@ -334,12 +336,14 @@ struct MemcacheClientMetric { std::string fsName; InterfaceMetric get; InterfaceMetric set; + InterfaceMetric exist; explicit MemcacheClientMetric(const std::string& name = "") : fsName(!name.empty() ? name : prefix + curve::common::ToHexString(this)), get(prefix, fsName + "_get"), - set(prefix, fsName + "_set") {} + set(prefix, fsName + "_set"), + exist(prefix, fsName + "_exist"){} }; struct S3ChunkInfoMetric { diff --git a/curvefs/src/client/s3/disk_cache_manager.cpp b/curvefs/src/client/s3/disk_cache_manager.cpp index 235989ca87..38e37bbcb2 100644 --- a/curvefs/src/client/s3/disk_cache_manager.cpp +++ b/curvefs/src/client/s3/disk_cache_manager.cpp @@ -403,7 +403,7 @@ void DiskCacheManager::TrimCache() { waitIntervalSec_.Init(FLAGS_diskTrimCheckIntervalSec * 1000); // trim will start after get the disk size while (!IsDiskUsedInited()) { - if (!isRunning_) { + if (!isRunning_.load()) { return; } waitIntervalSec_.WaitForNextExcution(); @@ -506,7 +506,7 @@ int DiskCacheManager::TrimStop() { } if (isRunning_.exchange(false)) { LOG(INFO) << "stop DiskCacheManager trim thread..."; - isRunning_ = false; + // isRunning_ = false; // useless? waitIntervalSec_.StopWait(); backEndThread_.join(); LOG(INFO) << "stop DiskCacheManager trim thread ok."; diff --git a/curvefs/src/client/warmup/warmup_manager.cpp b/curvefs/src/client/warmup/warmup_manager.cpp index bcc9fb9d73..7be3f0f8cd 100644 --- a/curvefs/src/client/warmup/warmup_manager.cpp +++ b/curvefs/src/client/warmup/warmup_manager.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -55,24 +56,27 @@ using curve::common::WriteLockGuard; #define ROOT_PATH_NAME "/" -static bool pass_uint32(const char*, uint32_t) { return true; } +static bool pass_uint32(const char*, uint32_t) { + return true; +} DEFINE_uint32(warmupMaxSymLink, 1 << 2, "The maximum number of times to parse sym link"); DEFINE_validator(warmupMaxSymLink, &pass_uint32); +DEFINE_uint32(warmupDoneTimeoutSecond, 5, + "Timeout after completion of warmup task"); +DEFINE_validator(warmupDoneTimeoutSecond, &pass_uint32); -bool WarmupManagerS3Impl::AddWarmupFilelist(fuse_ino_t key, - WarmupStorageType type, - const std::string& path, - const std::string& mount_point, - const std::string& root) { +bool WarmupManagerS3Impl::AddWarmupFilelist( + fuse_ino_t key, WarmupStorageType type, const std::string& path, + const std::string& mount_point, const std::string& root, bool check) { if (!mounted_.load(std::memory_order_acquire)) { LOG(ERROR) << "not mounted"; return false; } // add warmup Progress WriteLockGuard lock(inode2ProgressMutex_); - if (AddWarmupProcessLocked(key, path, type)) { - LOG(INFO) << "add warmup list task:" << key; + if (AddWarmupProcessLocked(key, path, type, check)) { + LOG(INFO) << "add warmup list task:" << key << " is check: " << check; WriteLockGuard lock(warmupFilelistDequeMutex_); auto iter = FindWarmupFilelistByKeyLocked(key); if (iter == warmupFilelistDeque_.end()) { @@ -91,15 +95,15 @@ bool WarmupManagerS3Impl::AddWarmupFilelist(fuse_ino_t key, } bool WarmupManagerS3Impl::AddWarmupFile(fuse_ino_t key, const std::string& path, - WarmupStorageType type) { + WarmupStorageType type, bool check) { if (!mounted_.load(std::memory_order_acquire)) { LOG(ERROR) << "not mounted"; return false; } // add warmup Progress WriteLockGuard lock(inode2ProgressMutex_); - if (AddWarmupProcessLocked(key, path, type)) { - LOG(INFO) << "add warmup single task:" << key; + if (AddWarmupProcessLocked(key, path, type, check)) { + LOG(INFO) << "add warmup single task:" << key << " is check: " << check; FetchDentryEnqueue(key, path); } return true; @@ -402,6 +406,28 @@ void WarmupManagerS3Impl::TravelChunks( fuse_ino_t key, fuse_ino_t ino, const S3ChunkInfoMapType& s3ChunkInfoMap) { VLOG(9) << "travel chunk start: " << ino << ", size: " << s3ChunkInfoMap.size(); + std::function>&)> + taskObjectsFunc; + { + ReadLockGuard lock(inode2ProgressMutex_); + auto iter = FindWarmupProgressByKeyLocked(key); + if (iter->second.IsCachedTask()) { + taskObjectsFunc = + [this]( + fuse_ino_t key, + const std::list>& list) { + CheckCachedAllObjs(key, list); + }; + } else { + taskObjectsFunc = + [this]( + fuse_ino_t key, + const std::list>& list) { + WarmUpAllObjs(key, list); + }; + } + } for (auto const& infoIter : s3ChunkInfoMap) { VLOG(9) << "travel chunk: " << infoIter.first; std::list> prefetchObjs; @@ -415,8 +441,8 @@ void WarmupManagerS3Impl::TravelChunks( LOG(ERROR) << "no such warmup progress: " << key; } } - auto task = [this, key, prefetchObjs]() { - WarmUpAllObjs(key, prefetchObjs); + auto task = [key, prefetchObjs, taskObjectsFunc]() { + taskObjectsFunc(key, prefetchObjs); }; AddFetchS3objectsTask(key, task); } @@ -656,11 +682,28 @@ void WarmupManagerS3Impl::ScanCleanFetchS3ObjectsPool() { void WarmupManagerS3Impl::ScanCleanWarmupProgress() { // clean done warmupProgress - ReadLockGuard lock(inode2ProgressMutex_); + WriteLockGuard lock(inode2ProgressMutex_); for (auto iter = inode2Progress_.begin(); iter != inode2Progress_.end();) { if (ProgressDone(iter->first)) { - LOG(INFO) << "warmup task: " << iter->first << " done!"; - iter = inode2Progress_.erase(iter); + if (!iter->second.IsCachedTask() || + (iter->second.IsDoneTimerStart() && + !iter->second.IsQuerying() && + iter->second.GetDoneTimerSecond() >= + FLAGS_warmupDoneTimeoutSecond)) { + // Tasks that are not cache query can be deleted + // The task of check-cache can be deleted if it times out + // and is not in the query state. + LOG(INFO) << "warmup task: " << iter->first << " done! " + << iter->second.ToString() << " " + << " is check: " << iter->second.IsCachedTask(); + iter = inode2Progress_.erase(iter); + } else if (!iter->second.IsDoneTimerStart()) { + // check task is done and not start timer + // don't delete util turn to not cached task or outtime + iter->second.DoneTimerStart(); + // start the timer + iter->second.SignalCheckTask(); + } } else { ++iter; } @@ -910,6 +953,56 @@ bool WarmupManagerS3Impl::GetInodeSubPathParent( } } +void WarmupManagerS3Impl::CheckCachedAllObjs( + fuse_ino_t key, + const std::list>& prefetchObjs) { + std::function checkFunc; + auto checkDisk = [this, key](const std::string& name) { + if (s3Adaptor_->GetDiskCacheManager()->IsCached(name)) { + ReadLockGuard lock(inode2ProgressMutex_); + auto iterProgress = FindWarmupProgressByKeyLocked(key); + if (iterProgress != inode2Progress_.end()) { + iterProgress->second.FinishedPlusOne(); + } + } + }; + + ExistKVCacheDone cb = + [this, key](const std::shared_ptr& context) { + if (context->res == true) { + ReadLockGuard lock(inode2ProgressMutex_); + auto iterProgress = FindWarmupProgressByKeyLocked(key); + if (iterProgress != inode2Progress_.end()) { + iterProgress->second.FinishedPlusOne(); + } + } + }; + auto checkKvCache = [this, cb](const std::string& name) { + std::shared_ptr task = + std::make_shared(name, cb); + kvClientManager_->Exist(task); + }; + { + ReadLockGuard lock(inode2ProgressMutex_); + auto iterProgress = FindWarmupProgressByKeyLocked(key); + switch (iterProgress->second.GetStorageType()) { + case curvefs::client::common::WarmupStorageType:: + kWarmupStorageTypeDisk: + checkFunc = checkDisk; + break; + case curvefs::client::common::WarmupStorageType:: + kWarmupStorageTypeKvClient: + checkFunc = checkKvCache; + break; + default: + break; + } + } + for (auto const& obj : prefetchObjs) { + checkFunc(obj.first); + } +} + } // namespace warmup } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/warmup/warmup_manager.h b/curvefs/src/client/warmup/warmup_manager.h index f55752c801..b71283372f 100644 --- a/curvefs/src/client/warmup/warmup_manager.h +++ b/curvefs/src/client/warmup/warmup_manager.h @@ -23,6 +23,7 @@ #ifndef CURVEFS_SRC_CLIENT_WARMUP_WARMUP_MANAGER_H_ #define CURVEFS_SRC_CLIENT_WARMUP_WARMUP_MANAGER_H_ +#include #include #include #include @@ -117,17 +118,35 @@ class WarmupProgress { public: explicit WarmupProgress(WarmupStorageType type = curvefs::client::common:: WarmupStorageType::kWarmupStorageTypeUnknown, - std::string filePath = "") + std::string filePath = "", bool cachedTask = false) : total_(0), finished_(0), storageType_(type), - filePathInClient_(filePath) {} + filePathInClient_(filePath), + cachedTask_(cachedTask), + checkTaskDone_(1), + doneTimerStarted_(false), + querying_(false) {} - WarmupProgress(const WarmupProgress& wp) + explicit WarmupProgress(WarmupProgress&& wp) noexcept : total_(wp.total_), finished_(wp.finished_), storageType_(wp.storageType_), - filePathInClient_(wp.filePathInClient_) {} + filePathInClient_(std::move(wp.filePathInClient_)), + cachedTask_(wp.cachedTask_), + checkTaskDone_(1), + doneTimerStarted_(wp.doneTimerStarted_.load()), + querying_(wp.querying_.load()) {} + + explicit WarmupProgress(const WarmupProgress& wp) noexcept + : total_(wp.total_), + finished_(wp.finished_), + storageType_(wp.storageType_), + filePathInClient_(wp.filePathInClient_), + cachedTask_(wp.cachedTask_), + checkTaskDone_(1), + doneTimerStarted_(wp.doneTimerStarted_.load()), + querying_(wp.querying_.load()) {} void AddTotal(uint64_t add) { std::lock_guard lock(totalMutex_); @@ -162,6 +181,40 @@ class WarmupProgress { ",finished:" + std::to_string(finished_); } + bool IsCachedTask() const { + return cachedTask_; + } + + void SignalCheckTask() { + checkTaskDone_.Signal(); + } + + void WaitCheckTask() { + checkTaskDone_.Wait(); + } + + void DoneTimerStart() { + doneTimerStarted_.store(true); + doneTimer_.start(); + } + + uint32_t GetDoneTimerSecond() { + doneTimer_.stop(); + return doneTimer_.s_elapsed(); + } + + bool IsDoneTimerStart() { + return doneTimerStarted_.load(); + } + + bool IsQuerying() { + return querying_.load(); + } + + void UpdateQuerying(bool start) { + querying_.store(start); + } + std::string GetFilePathInClient() { return filePathInClient_; } WarmupStorageType GetStorageType() { return storageType_; } @@ -173,6 +226,11 @@ class WarmupProgress { std::mutex finishedMutex_; WarmupStorageType storageType_; std::string filePathInClient_; + bool cachedTask_; // Is it a task to check whether it is cached + CountDownEvent checkTaskDone_; + butil::Timer doneTimer_; // time after recording is completed + std::atomic doneTimerStarted_; + std::atomic querying_; // Being queried, avoid being deleted }; using FuseOpReadFunctionType = @@ -215,10 +273,10 @@ class WarmupManager { virtual bool AddWarmupFilelist(fuse_ino_t key, WarmupStorageType type, const std::string& path, const std::string& mount_point, - const std::string& root) = 0; + const std::string& root, + bool check = false) = 0; virtual bool AddWarmupFile(fuse_ino_t key, const std::string& path, - WarmupStorageType type) = 0; - + WarmupStorageType type, bool check = false) = 0; virtual bool CancelWarmupFileOrFilelist(fuse_ino_t key) = 0; virtual bool CancelWarmupDependentQueue(fuse_ino_t key) = 0; @@ -262,6 +320,32 @@ class WarmupManager { return ret; } + /** + * @brief + * + * @param key + * @param progress + * @return true + * @return false no this check task or has warmup task + */ + bool QueryCheckCachedProgress(fuse_ino_t key, WarmupProgress* progress) { + bool ret = true; + std::unordered_map::iterator iter; + { + ReadLockGuard lock(inode2ProgressMutex_); + iter = FindWarmupProgressByKeyLocked(key); + if (iter == inode2Progress_.end() || !iter->second.IsCachedTask()) { + // no this check + return false; + } + } + iter->second.WaitCheckTask(); + *progress = iter->second; + // can delete iter after querying + iter->second.UpdateQuerying(false); + return ret; + } + bool ListWarmupProgress(Filepath2WarmupProgressMap* filepath2progress) { ReadLockGuard lock(inode2ProgressMutex_); @@ -282,13 +366,15 @@ class WarmupManager { /** * @brief Add warmupProcess * - * @return true - * @return false warmupProcess has been added + * @return */ virtual bool AddWarmupProcessLocked(fuse_ino_t key, const std::string& path, - WarmupStorageType type) { - auto retPg = inode2Progress_.emplace(key, WarmupProgress(type, path)); - return retPg.second; + WarmupStorageType type, + bool cachedTask = false) { + // WriteLockGuard lock(inode2ProgressMutex_); + return inode2Progress_ + .emplace(key, WarmupProgress(type, path, cachedTask)) + .second; } virtual bool CancelWarmupProcess(fuse_ino_t key) { @@ -369,10 +455,10 @@ class WarmupManagerS3Impl : public WarmupManager { bool AddWarmupFilelist(fuse_ino_t key, WarmupStorageType type, const std::string& path, const std::string& mount_point, - const std::string& root) override; + const std::string& root, + bool cache = false) override; bool AddWarmupFile(fuse_ino_t key, const std::string& path, - WarmupStorageType type) override; - + WarmupStorageType type, bool cache = false) override; bool CancelWarmupFileOrFilelist(fuse_ino_t key) override; bool CancelWarmupDependentQueue(fuse_ino_t key) override; @@ -489,6 +575,9 @@ class WarmupManagerS3Impl : public WarmupManager { void WarmUpAllObjs( fuse_ino_t key, const std::list>& prefetchObjs); + void CheckCachedAllObjs( + fuse_ino_t key, + const std::list>& prefetchObjs); /** * @brief Whether the warmup task[key] is completed (or terminated) diff --git a/curvefs/test/client/mock_kvclient.h b/curvefs/test/client/mock_kvclient.h index 8ce35e08c8..b255327ecc 100644 --- a/curvefs/test/client/mock_kvclient.h +++ b/curvefs/test/client/mock_kvclient.h @@ -42,6 +42,7 @@ class MockKVClient : public KVClient { MOCK_METHOD7(Get, bool(const std::string&, char*, uint64_t, uint64_t, std::string*, uint64_t*, memcached_return_t* retCod)); + MOCK_METHOD1(Exist, bool(const std::string&)); }; } // namespace client diff --git a/curvefs/test/client/test_disk_cache_manager.cpp b/curvefs/test/client/test_disk_cache_manager.cpp index df83daa0b4..8fa94485de 100644 --- a/curvefs/test/client/test_disk_cache_manager.cpp +++ b/curvefs/test/client/test_disk_cache_manager.cpp @@ -397,19 +397,19 @@ TEST_F(TestDiskCacheManager, TrimRun_1) { } TEST_F(TestDiskCacheManager, TrimCache_2) { - struct statfs stat; - stat.f_frsize = 1; - stat.f_blocks = 1; - stat.f_bfree = 0; - stat.f_bavail = 0; + struct statfs stat1; + stat1.f_frsize = 1; + stat1.f_blocks = 1; + stat1.f_bfree = 0; + stat1.f_bavail = 0; + struct statfs stat2; + stat2.f_frsize = 1; + stat2.f_blocks = 1; + stat2.f_bfree = 2; + stat2.f_bavail = 101; EXPECT_CALL(*wrapper, statfs(NotNull(), _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(0))); - stat.f_frsize = 1; - stat.f_blocks = 1; - stat.f_bfree = 2; - stat.f_bavail = 101; - EXPECT_CALL(*wrapper, statfs(NotNull(), _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(0))); + .WillOnce(DoAll(SetArgPointee<1>(stat1), Return(0))) + .WillRepeatedly(DoAll(SetArgPointee<1>(stat2), Return(0))); std::string buf = "test"; EXPECT_CALL(*diskCacheWrite_, GetCacheIoFullDir()) .WillRepeatedly(Return(buf)); @@ -419,7 +419,8 @@ TEST_F(TestDiskCacheManager, TrimCache_2) { option.diskCacheOpt.cacheDir = "/tmp"; option.diskCacheOpt.trimCheckIntervalSec = 1; option.objectPrefix = 0; - EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(-1)); + EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())) + .WillRepeatedly(Return(-1)); EXPECT_CALL(*wrapper, mkdir(_, _)).WillOnce(Return(-1)); diskCacheManager_->Init(client_, option); diskCacheManager_->InitMetrics("test", s3Metric_); @@ -430,19 +431,19 @@ TEST_F(TestDiskCacheManager, TrimCache_2) { } TEST_F(TestDiskCacheManager, TrimCache_4) { - struct statfs stat; - stat.f_frsize = 1; - stat.f_blocks = 1; - stat.f_bfree = 0; - stat.f_bavail = 0; + struct statfs stat1; + stat1.f_frsize = 1; + stat1.f_blocks = 1; + stat1.f_bfree = 0; + stat1.f_bavail = 0; + struct statfs stat2; + stat2.f_frsize = 1; + stat2.f_blocks = 1; + stat2.f_bfree = 2; + stat2.f_bavail = 101; EXPECT_CALL(*wrapper, statfs(NotNull(), _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(0))); - stat.f_frsize = 1; - stat.f_blocks = 1; - stat.f_bfree = 2; - stat.f_bavail = 101; - EXPECT_CALL(*wrapper, statfs(NotNull(), _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(0))); + .WillOnce(DoAll(SetArgPointee<1>(stat1), Return(0))) + .WillRepeatedly(DoAll(SetArgPointee<1>(stat2), Return(0))); std::string buf = "test"; EXPECT_CALL(*diskCacheWrite_, GetCacheIoFullDir()) .WillRepeatedly(Return(buf)); @@ -454,7 +455,8 @@ TEST_F(TestDiskCacheManager, TrimCache_4) { S3ClientAdaptorOption option; option.diskCacheOpt.cacheDir = "/tmp"; option.diskCacheOpt.trimCheckIntervalSec = 1; - EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(-1)); + EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())) + .WillRepeatedly(Return(-1)); EXPECT_CALL(*wrapper, mkdir(_, _)).WillOnce(Return(-1)); option.objectPrefix = 0; diskCacheManager_->Init(client_, option); @@ -466,19 +468,19 @@ TEST_F(TestDiskCacheManager, TrimCache_4) { } TEST_F(TestDiskCacheManager, TrimCache_5) { - struct statfs stat; - stat.f_frsize = 1; - stat.f_blocks = 1; - stat.f_bfree = 0; - stat.f_bavail = 0; - EXPECT_CALL(*wrapper, statfs(NotNull(), _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(0))); - stat.f_frsize = 1; - stat.f_blocks = 1; - stat.f_bfree = 2; - stat.f_bavail = 101; + struct statfs stat1; + stat1.f_frsize = 1; + stat1.f_blocks = 1; + stat1.f_bfree = 0; + stat1.f_bavail = 0; + struct statfs stat2; + stat2.f_frsize = 1; + stat2.f_blocks = 1; + stat2.f_bfree = 2; + stat2.f_bavail = 101; EXPECT_CALL(*wrapper, statfs(NotNull(), _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(0))); + .WillOnce(DoAll(SetArgPointee<1>(stat1), Return(0))) + .WillRepeatedly(DoAll(SetArgPointee<1>(stat2), Return(0))); std::string buf = "test"; EXPECT_CALL(*diskCacheWrite_, GetCacheIoFullDir()) @@ -492,7 +494,8 @@ TEST_F(TestDiskCacheManager, TrimCache_5) { option.diskCacheOpt.cacheDir = "/tmp"; option.diskCacheOpt.trimCheckIntervalSec = 1; option.objectPrefix = 0; - EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(-1)); + EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())) + .WillRepeatedly(Return(-1)); EXPECT_CALL(*wrapper, mkdir(_, _)).WillOnce(Return(-1)); diskCacheManager_->Init(client_, option); diskCacheManager_->InitMetrics("test", s3Metric_); diff --git a/tools-v2/internal/utils/string.go b/tools-v2/internal/utils/string.go index 92060c760a..a40275a73c 100644 --- a/tools-v2/internal/utils/string.go +++ b/tools-v2/internal/utils/string.go @@ -195,3 +195,12 @@ func StringList2Uint32List(strList []string) ([]uint32, error) { } return retList, nil } + +func IsNumeric(s string) bool { + for _, r := range s { + if !IsDigit(r) { + return false + } + } + return true +} diff --git a/tools-v2/pkg/cli/command/curvefs/warmup/check/check.go b/tools-v2/pkg/cli/command/curvefs/warmup/check/check.go new file mode 100644 index 0000000000..421c2700d7 --- /dev/null +++ b/tools-v2/pkg/cli/command/curvefs/warmup/check/check.go @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: CurveCli + * Created Date: 2023-11-07 + * Author: chengyi (Cyber-SiKu) + */ + +package check + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + mountinfo "github.com/cilium/cilium/pkg/mountinfo" + cmderror "github.com/opencurve/curve/tools-v2/internal/error" + cobrautil "github.com/opencurve/curve/tools-v2/internal/utils" + basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command" + "github.com/opencurve/curve/tools-v2/pkg/config" + "github.com/opencurve/curve/tools-v2/pkg/output" + "github.com/pkg/xattr" + "github.com/spf13/cobra" + "golang.org/x/sys/unix" +) + +const ( + checkExample = `$ curve fs warmup check --filelist /mnt/warmup/0809.list # warmup the file(dir) saved in /mnt/warmup/0809.list +$ curve fs warmup check /mnt/warmup # warmup all files in /mnt/warmup` +) + +const ( + CURVEFS_WARMUP_OP_XATTR = "curvefs.warmup.op" + CURVEFS_WARMUP_CEHCK_XATTR = "curvefs.warmup.check" + CURVEFS_WARMUP_OP_CHECK_SINGLE = "check\nsingle\n%s\n%s\n%s\n%s" + CURVEFS_WARMUP_OP_CHECK_LIST = "check\nlist\n%s\n%s\n%s\n%s" +) + +var STORAGE_TYPE = map[string]string{ + "disk": "disk", + "mem": "kvclient", +} + +type CheckCommand struct { + basecmd.FinalCurveCmd + Mountpoint *mountinfo.MountInfo + Path string // path in user system + CurvefsPath string // path in curvefs + Single bool // warmup a single file or directory + StorageType string // warmup storage type +} + +var _ basecmd.FinalCurveCmdFunc = (*CheckCommand)(nil) // check interface + +func NewAddWarmupCommand() *CheckCommand { + cCmd := &CheckCommand{ + FinalCurveCmd: basecmd.FinalCurveCmd{ + Use: "check", + Short: "Check if the files/dir are all in the cache", + Example: checkExample, + }, + } + basecmd.NewFinalCurveCli(&cCmd.FinalCurveCmd, cCmd) + return cCmd +} + +func NewCheckCommand() *cobra.Command { + return NewAddWarmupCommand().Cmd +} + +func (cCmd *CheckCommand) AddFlags() { + config.AddFileListOptionFlag(cCmd.Cmd) + config.AddStorageOptionFlag(cCmd.Cmd) +} + +func (cCmd *CheckCommand) Init(cmd *cobra.Command, args []string) error { + // check has curvefs mountpoint + mountpoints, err := cobrautil.GetCurveFSMountPoints() + if err.TypeCode() != cmderror.CODE_SUCCESS { + return err.ToError() + } else if len(mountpoints) == 0 { + return errors.New("no curvefs mountpoint found") + } + + // check args + cCmd.Single = false + fileList := config.GetFileListOptionFlag(cCmd.Cmd) + if fileList == "" && len(args) == 0 { + cmd.SilenceUsage = false + return fmt.Errorf("no --filelist or file(dir) specified") + } else if fileList != "" { + cCmd.Path = fileList + } else { + cCmd.Path = args[0] + cCmd.Single = true + } + + // check file is exist + info, errStat := os.Stat(cCmd.Path) + if errStat != nil { + if os.IsNotExist(errStat) { + return fmt.Errorf("[%s]: no such file or directory", cCmd.Path) + } else { + return fmt.Errorf("stat [%s] fail: %s", cCmd.Path, errStat.Error()) + } + } else if !cCmd.Single && info.IsDir() { + // --filelist must be a file + return fmt.Errorf("[%s]: must be a file", cCmd.Path) + } + + cCmd.Mountpoint = nil + for _, mountpoint := range mountpoints { + absPath, _ := filepath.Abs(cCmd.Path) + rel, err := filepath.Rel(mountpoint.MountPoint, absPath) + if err == nil && !strings.HasPrefix(rel, "..") { + // found the mountpoint + if cCmd.Mountpoint == nil || + len(cCmd.Mountpoint.MountPoint) < len(mountpoint.MountPoint) { + // Prevent the curvefs directory from being mounted under the curvefs directory + // /a/b/c: + // test-1 mount in /a + // test-1 mount in /a/b + // warmup /a/b/c. + cCmd.Mountpoint = mountpoint + cCmd.CurvefsPath = cobrautil.Path2CurvefsPath(cCmd.Path, mountpoint) + } + } + } + if cCmd.Mountpoint == nil { + return fmt.Errorf("[%s] is not saved in curvefs", cCmd.Path) + } + + // check storage type + cCmd.StorageType = STORAGE_TYPE[config.GetStorageFlag(cCmd.Cmd)] + if cCmd.StorageType == "" { + return fmt.Errorf("[%s] is not support storage type", cCmd.StorageType) + } + + cCmd.SetHeader([]string{cobrautil.ROW_PATH, cobrautil.ROW_RESULT}) + return nil +} + +func (cCmd *CheckCommand) Print(cmd *cobra.Command, args []string) error { + return output.FinalCmdOutput(&cCmd.FinalCurveCmd, cCmd) +} + +func (cCmd *CheckCommand) verifyFilelist() *cmderror.CmdError { + data, err := ioutil.ReadFile(cCmd.Path) + if err != nil { + readErr := cmderror.ErrReadFile() + readErr.Format(cCmd.Path, err.Error()) + return readErr + } + + lines := strings.Split(string(data), "\n") + + verifyFailMsg := "" + var verifyReplaceErr error + for i, line := range lines { + if line == "" { + continue + } + rel, err := filepath.Rel(cCmd.Mountpoint.MountPoint, line) + if err != nil || strings.HasPrefix(rel, "..") { + verifyReplaceErr = err + verifyFailMsg += fmt.Sprintf("line %d: [%s:%s] is not saved in curvefs\n", i + 1, cCmd.Path, line) + } + } + + if verifyReplaceErr != nil { + verifyErr := cmderror.ErrVerifyError() + verifyErr.Format(verifyFailMsg, verifyReplaceErr.Error()) + return verifyErr + } + return cmderror.ErrSuccess() +} + +func (cCmd *CheckCommand) RunCommand(cmd *cobra.Command, args []string) error { + checkAttr := CURVEFS_WARMUP_OP_CHECK_SINGLE + if !cCmd.Single { + verifyErr := cCmd.verifyFilelist() + if verifyErr.TypeCode() != cmderror.CODE_SUCCESS { + return verifyErr.ToError() + } + checkAttr = CURVEFS_WARMUP_OP_CHECK_LIST + } + values := fmt.Sprintf(checkAttr, cCmd.CurvefsPath, cCmd.StorageType, cCmd.Mountpoint.MountPoint, cCmd.Mountpoint.Root) + err := xattr.Set(cCmd.Path, CURVEFS_WARMUP_OP_XATTR, []byte(values)) + if err == unix.ENOTSUP || err == unix.EOPNOTSUPP { + return fmt.Errorf("filesystem does not support extended attributes") + } else if err != nil { + setErr := cmderror.ErrSetxattr() + setErr.Format(CURVEFS_WARMUP_OP_XATTR, err.Error()) + return setErr.ToError() + } + result, err := xattr.Get(cCmd.Path, CURVEFS_WARMUP_CEHCK_XATTR) + if err != nil { + return err + } + cCmd.TableNew.Append([]string{cCmd.Path, string(result)}) + return nil +} + +func (cCmd *CheckCommand) ResultPlainOutput() error { + return output.FinalCmdOutputPlain(&cCmd.FinalCurveCmd) +} diff --git a/tools-v2/pkg/cli/command/curvefs/warmup/warmup.go b/tools-v2/pkg/cli/command/curvefs/warmup/warmup.go index 2250ce232a..06722f5b05 100644 --- a/tools-v2/pkg/cli/command/curvefs/warmup/warmup.go +++ b/tools-v2/pkg/cli/command/curvefs/warmup/warmup.go @@ -26,6 +26,7 @@ import ( basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/warmup/add" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/warmup/cancel" + "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/warmup/check" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/warmup/list" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/warmup/query" "github.com/spf13/cobra" @@ -43,6 +44,7 @@ func (warmupCmd *WarmupCommand) AddSubCommands() { query.NewQueryCommand(), cancel.NewCancelCommand(), list.NewListCommand(), + check.NewCheckCommand(), ) }