Skip to content

Commit

Permalink
Fix format
Browse files Browse the repository at this point in the history
  • Loading branch information
acvictor committed Mar 30, 2024
1 parent b3e697a commit d365076
Show file tree
Hide file tree
Showing 6 changed files with 27 additions and 14 deletions.
11 changes: 7 additions & 4 deletions velox/connectors/hive/FileHandle.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ std::string groupName(const std::string& filename) {
} // namespace

std::shared_ptr<FileHandle> FileHandleGenerator::operator()(
std::tuple<const std::string, const std::string, const std::string> params) {
std::tuple<const std::string, const std::string, const std::string>
params) {
// We have seen cases where drivers are stuck when creating file handles.
// Adding a trace here to spot this more easily in future.
process::TraceContext trace("FileHandleGenerator::operator()");
Expand All @@ -46,10 +47,12 @@ std::shared_ptr<FileHandle> FileHandleGenerator::operator()(
filesystems::FileOptions options;
options.values["fileSize"] = std::get<1>(params);
// Add length and modification time here to create a unique handle?
fileHandle->file = filesystems::getFileSystem(std::get<0>(params), properties_)
->openFileForRead(std::get<0>(params), options);
fileHandle->file =
filesystems::getFileSystem(std::get<0>(params), properties_)
->openFileForRead(std::get<0>(params), options);
fileHandle->uuid = StringIdLease(fileIds(), std::get<0>(params));
fileHandle->groupId = StringIdLease(fileIds(), groupName(std::get<0>(params)));
fileHandle->groupId =
StringIdLease(fileIds(), groupName(std::get<0>(params)));
VLOG(1) << "Generating file handle for: " << std::get<0>(params)
<< " uuid: " << fileHandle->uuid.id();
}
Expand Down
3 changes: 2 additions & 1 deletion velox/connectors/hive/FileHandle.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ class FileHandleGenerator {
FileHandleGenerator(std::shared_ptr<const Config> properties)
: properties_(std::move(properties)) {}
std::shared_ptr<FileHandle> operator()(
std::tuple<const std::string, const std::string, const std::string> params);
std::tuple<const std::string, const std::string, const std::string>
params);

private:
const std::shared_ptr<const Config> properties_;
Expand Down
8 changes: 6 additions & 2 deletions velox/connectors/hive/HiveConnector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,12 @@ HiveConnector::HiveConnector(
hiveConfig_(std::make_shared<HiveConfig>(config)),
fileHandleFactory_(
hiveConfig_->isFileHandleCacheEnabled()
? std::make_unique<
SimpleLRUCache<std::tuple<const std::string, const std::string, const std::string>, std::shared_ptr<FileHandle>>>(
? std::make_unique<SimpleLRUCache<
std::tuple<
const std::string,
const std::string,
const std::string>,
std::shared_ptr<FileHandle>>>(
hiveConfig_->numCacheFileHandles())
: nullptr,
std::make_unique<FileHandleGenerator>(config)),
Expand Down
11 changes: 7 additions & 4 deletions velox/connectors/hive/SplitReader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -147,17 +147,20 @@ void SplitReader::prepareSplit(
std::shared_ptr<FileHandle> fileHandle;
try {
auto fileSizeIter = hiveSplit_->infoColumns.find(kFileSize);
auto fileModificationTimeIter = hiveSplit_->infoColumns.find(kModificationTime);
auto fileModificationTimeIter =
hiveSplit_->infoColumns.find(kModificationTime);
if (fileSizeIter != hiveSplit_->infoColumns.end() &&
fileModificationTimeIter != hiveSplit_->infoColumns.end()) {
fileHandle = fileHandleFactory_
->generate(
std::make_tuple(hiveSplit_->filePath,
->generate(std::make_tuple(
hiveSplit_->filePath,
fileSizeIter->second,
fileModificationTimeIter->second))
.second;
} else {
fileHandle = fileHandleFactory_->generate(std::make_tuple(hiveSplit_->filePath, "", "")).second;
fileHandle = fileHandleFactory_
->generate(std::make_tuple(hiveSplit_->filePath, "", ""))
.second;
}
} catch (const VeloxRuntimeError& e) {
if (e.errorCode() == error_code::kFileNotFound &&
Expand Down
3 changes: 2 additions & 1 deletion velox/connectors/hive/iceberg/PositionalDeleteFileReader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ PositionalDeleteFileReader::PositionalDeleteFileReader(
deleteSplit_);

auto deleteFileHandle =
fileHandleFactory_->generate(make_tuple(deleteFile_.filePath, "", "")).second;
fileHandleFactory_->generate(make_tuple(deleteFile_.filePath, "", ""))
.second;
auto deleteFileInput = createBufferedInput(
*deleteFileHandle,
deleteReaderOpts,
Expand Down
5 changes: 3 additions & 2 deletions velox/connectors/hive/tests/FileHandleTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ TEST(FileHandleTest, localFile) {
}

FileHandleFactory factory(
std::make_unique<
SimpleLRUCache<std::tuple<const std::string, const std::string, const std::string>, std::shared_ptr<FileHandle>>>(1000),
std::make_unique<SimpleLRUCache<
std::tuple<const std::string, const std::string, const std::string>,
std::shared_ptr<FileHandle>>>(1000),
std::make_unique<FileHandleGenerator>());
auto fileHandle = factory.generate(std::make_tuple(filename, "", "")).second;
ASSERT_EQ(fileHandle->file->size(), 3);
Expand Down

0 comments on commit d365076

Please sign in to comment.