From 7bc3aa21dbe26b589e18caa440421d22a04d6ba1 Mon Sep 17 00:00:00 2001 From: Max Kojin Date: Mon, 4 Dec 2023 19:02:55 +0500 Subject: [PATCH 1/4] code sync https://github.com/osmandapp/OsmAnd/pull/10732/files?diff=split&w=0#diff-d612bfcae735526555e6a689087946d419c2ede0090a3d2ec83aa027791b46dcR616 --- src/Data/ObfAddressSectionReader_P.cpp | 6 +- src/Data/ObfPoiSectionReader_P.cpp | 108 +++++++++++++++------- src/Data/ObfPoiSectionReader_P.h | 2 +- src/Data/ObfReaderUtilities.cpp | 121 +++++++++++++------------ src/Data/ObfReaderUtilities.h | 12 +-- 5 files changed, 149 insertions(+), 100 deletions(-) diff --git a/src/Data/ObfAddressSectionReader_P.cpp b/src/Data/ObfAddressSectionReader_P.cpp index bc26cea27..f67274353 100644 --- a/src/Data/ObfAddressSectionReader_P.cpp +++ b/src/Data/ObfAddressSectionReader_P.cpp @@ -1084,8 +1084,12 @@ void OsmAnd::ObfAddressSectionReader_P::scanNameIndex( const auto length = ObfReaderUtilities::readBigEndianInt(cis); baseOffset = cis->CurrentPosition(); const auto oldLimit = cis->PushLimit(length); + + QVector charsList = {0}; + QList queries = {query}; + QList> lofsets = {intermediateOffsets}; - ObfReaderUtilities::scanIndexedStringTable(cis, query, intermediateOffsets, strictMatch); + ObfReaderUtilities::readIndexedStringTable(cis, queries, lofsets, charsList); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); diff --git a/src/Data/ObfPoiSectionReader_P.cpp b/src/Data/ObfPoiSectionReader_P.cpp index be08a7a73..87c6af339 100644 --- a/src/Data/ObfPoiSectionReader_P.cpp +++ b/src/Data/ObfPoiSectionReader_P.cpp @@ -21,7 +21,7 @@ #include "CollatorStringMatcher.h" #include -const int BUCKET_SEARCH_BY_NAME = 5; +const int BUCKET_SEARCH_BY_NAME = 15; // should be bigger 100? const int BASE_POI_SHIFT = 7; const int FINAL_POI_SHIFT = 5; const int BASE_POI_ZOOM = 31 - BASE_POI_SHIFT; @@ -1172,7 +1172,7 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenitiesByName( const auto offset = cis->CurrentPosition(); const auto oldLimit = cis->PushLimit(length); - scanNameIndex( + dataBoxesOffsetsSet = readPoiNameIndex( reader, query, dataBoxesOffsetsSet, @@ -1254,7 +1254,7 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenitiesByName( } } -void OsmAnd::ObfPoiSectionReader_P::scanNameIndex( +QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( const ObfReader_P& reader, const QString& query, QMap& outDataOffsets, @@ -1264,8 +1264,10 @@ void OsmAnd::ObfPoiSectionReader_P::scanNameIndex( { const auto cis = reader.getCodedInputStream().get(); - uint32_t baseOffset; - QVector intermediateOffsets; + QMap offsets = QMap(); + QList> listOffsets = QList>(); + QList> listOfSepOffsets = QList>(); + uint32_t offset = 0; for (;;) { @@ -1273,51 +1275,87 @@ void OsmAnd::ObfPoiSectionReader_P::scanNameIndex( switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) { case 0: - if (!ObfReaderUtilities::reachedDataEnd(cis)) - return; + return offsets; - return; case OBF::OsmAndPoiNameIndex::kTableFieldNumber: { const auto length = ObfReaderUtilities::readBigEndianInt(cis); - baseOffset = cis->CurrentPosition(); const auto oldLimit = cis->PushLimit(length); - - ObfReaderUtilities::scanIndexedStringTable(cis, query, intermediateOffsets); + offset = cis->CurrentPosition(); + + QList queries = QList(); + for (QString word : query.split(" ")) + { + if (word.trimmed().length() > 0) + queries.append(word.trimmed()); + } + + QVector charsList = QVector(); + listOffsets = QList>(); + while (listOffsets.size() < queries.size()) + { + charsList.append(0); + listOffsets.append(QVector()); + } + + ObfReaderUtilities::readIndexedStringTable(cis, queries, listOffsets, charsList); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); - - if (intermediateOffsets.isEmpty()) - { - cis->Skip(cis->BytesUntilLimit()); - return; - } break; } case OBF::OsmAndPoiNameIndex::kDataFieldNumber: { - std::sort(intermediateOffsets); - for (const auto& intermediateOffset : constOf(intermediateOffsets)) + if (!listOffsets.isEmpty()) { - cis->Seek(baseOffset + intermediateOffset); - - gpb::uint32 length; - cis->ReadVarint32(&length); - const auto oldLimit = cis->PushLimit(length); - - readNameIndexData( - reader, - outDataOffsets, - xy31, - bbox31, - tileFilter); - ObfReaderUtilities::ensureAllDataWasRead(cis); - - cis->PopLimit(oldLimit); + for (QVector dataOffsets : listOffsets) + { + QMap offsetMap = QMap(); + std::sort(dataOffsets); // 1104125 + for (const auto& dataOffset : constOf(dataOffsets)) + { + cis->Seek(dataOffset + offset); + + gpb::uint32 length; + cis->ReadVarint32(&length); + const auto oldLimit = cis->PushLimit(length); + + readNameIndexData( + reader, + offsetMap, + xy31, + bbox31, + tileFilter); + ObfReaderUtilities::ensureAllDataWasRead(cis); + + cis->PopLimit(oldLimit); + + // TODO: add to params req.isCancelled() ? + // if (req.isCancelled()) { + // codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); //cis->Skip(cis->BytesUntilLimit()); + // return offsets; + // } + } + listOfSepOffsets.append(offsetMap); + } + } + if (listOfSepOffsets.size() > 0) + { + offsets.insert(listOfSepOffsets[0]); + for (int j = 1; j < listOfSepOffsets.size(); j++) + { + QMap mp = listOfSepOffsets[j]; + for (auto chKey : offsets.keys()) + { + if (!mp.contains(chKey)) + { + offsets.remove(chKey); + } + } + } } cis->Skip(cis->BytesUntilLimit()); - return; + return offsets; } default: ObfReaderUtilities::skipUnknownField(cis, tag); diff --git a/src/Data/ObfPoiSectionReader_P.h b/src/Data/ObfPoiSectionReader_P.h index ab70f735d..45244792a 100644 --- a/src/Data/ObfPoiSectionReader_P.h +++ b/src/Data/ObfPoiSectionReader_P.h @@ -100,7 +100,7 @@ namespace OsmAnd const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, const std::shared_ptr& queryController); - static void scanNameIndex( + static QMap readPoiNameIndex( const ObfReader_P& reader, const QString& query, QMap& outDataOffsets, diff --git a/src/Data/ObfReaderUtilities.cpp b/src/Data/ObfReaderUtilities.cpp index 9df490717..862b582c5 100644 --- a/src/Data/ObfReaderUtilities.cpp +++ b/src/Data/ObfReaderUtilities.cpp @@ -84,16 +84,17 @@ void OsmAnd::ObfReaderUtilities::readStringTable(gpb::io::CodedInputStream* cis, } } -int OsmAnd::ObfReaderUtilities::scanIndexedStringTable( +void OsmAnd::ObfReaderUtilities::readIndexedStringTable( gpb::io::CodedInputStream* cis, - const QString& query, - QVector& outValues, - const bool strictMatch /*= false*/, - const QString& keysPrefix /*= QString::null*/, - const int matchedCharactersCount_ /*= 0*/) + const QList& queries, + QList>& listOffsets, + QVector& matchedCharacters, + const QString& prefix /*= QString::null*/ + ) { QString key; - auto matchedCharactersCount = matchedCharactersCount_; + QVector matched(matchedCharacters.size()); + bool shouldWeReadSubtable = false; for (;;) { @@ -101,78 +102,84 @@ int OsmAnd::ObfReaderUtilities::scanIndexedStringTable( switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) { case 0: - if (!ObfReaderUtilities::reachedDataEnd(cis)) - return matchedCharactersCount; - - return matchedCharactersCount; + return; + case OBF::IndexedStringTable::kKeyFieldNumber: { readQString(cis, key); - if (!keysPrefix.isEmpty()) - key.prepend(keysPrefix); - - bool matchesForward = false; - bool matchesBackward = false; - if (strictMatch) - matchesForward = key.startsWith(query, Qt::CaseInsensitive); - else - matchesForward = CollatorStringMatcher::cmatches(key, query, StringMatcherMode::CHECK_ONLY_STARTS_WITH); - - if (!matchesForward) - { - if (strictMatch) - matchesBackward = query.startsWith(key, Qt::CaseInsensitive); - else - matchesBackward = CollatorStringMatcher::cmatches(query, key, StringMatcherMode::CHECK_ONLY_STARTS_WITH); - } - - if (matchesForward) - { - if (query.length() > matchedCharactersCount) - { - matchedCharactersCount = query.length(); - outValues.clear(); - } - else if (query.length() < matchedCharactersCount) - { - key = QString::null; - } - } - else if (matchesBackward) + if (!prefix.isEmpty()) + key.prepend(prefix); + + shouldWeReadSubtable = false; + for (int i = 0; i < queries.size(); i++) { - if (key.length() > matchedCharactersCount) + int charMatches = matchedCharacters[i]; + QString query = queries[i]; + matched[i] = false; + if (query.isNull()) + continue; + + // check query is part of key (the best matching) + if (CollatorStringMatcher::cmatches(key, query, StringMatcherMode::CHECK_ONLY_STARTS_WITH)) { - matchedCharactersCount = key.length(); - outValues.clear(); + if (query.length() >= charMatches) + { + if (query.length() > charMatches) + { + matchedCharacters[i] = query.length(); + listOffsets[i].clear(); + } + matched[i] = true; + } } - else if (key.length() < matchedCharactersCount) + // check key is part of query + else if (CollatorStringMatcher::cmatches(query, key, StringMatcherMode::CHECK_ONLY_STARTS_WITH)) { - key = QString::null; + if (key.length() >= charMatches) + { + if (key.length() > charMatches) + { + matchedCharacters[i] = key.length(); + listOffsets[i].clear(); + } + matched[i] = true; + } } - } - else - { - key = QString::null; + shouldWeReadSubtable |= matched[i]; } break; } case OBF::IndexedStringTable::kValFieldNumber: { const auto value = readBigEndianInt(cis); - - if (!key.isNull()) - outValues.push_back(value); + for (int i = 0; i < queries.size(); i++) + { + if (matched[i]) + listOffsets[i].push_back(value); + } break; } case OBF::IndexedStringTable::kSubtablesFieldNumber: { const auto length = ObfReaderUtilities::readLength(cis); const auto oldLimit = cis->PushLimit(length); - - if (!key.isNull()) - matchedCharactersCount = scanIndexedStringTable(cis, query, outValues, strictMatch, key, matchedCharactersCount); + + if (shouldWeReadSubtable && !key.isNull()) + { + QList subqueries = queries; + + // reset query so we don't search what was not matched + for (int i = 0; i < queries.size(); i++) + { + if (!matched[i]) + subqueries[i] = QString::null; + } + readIndexedStringTable(cis, subqueries, listOffsets, matchedCharacters, key); + } else + { cis->Skip(cis->BytesUntilLimit()); + } ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); diff --git a/src/Data/ObfReaderUtilities.h b/src/Data/ObfReaderUtilities.h index 3b302d902..beb8182ab 100644 --- a/src/Data/ObfReaderUtilities.h +++ b/src/Data/ObfReaderUtilities.h @@ -32,13 +32,13 @@ namespace OsmAnd static uint32_t readBigEndianInt(gpb::io::CodedInputStream* cis); static uint32_t readLength(gpb::io::CodedInputStream* cis); static void readStringTable(gpb::io::CodedInputStream* cis, QStringList& stringTableOut); - static int scanIndexedStringTable( + static void readIndexedStringTable( gpb::io::CodedInputStream* cis, - const QString& query, - QVector& outValues, - const bool strictMatch = false, - const QString& keysPrefix = QString(), - const int matchedCharactersCount = 0); + const QList& queries, + QList>& listOffsets, + QVector& matchedCharacters, + const QString& prefix = QString()); + static void readTileBox(gpb::io::CodedInputStream* cis, AreaI& outArea); static void skipUnknownField(gpb::io::CodedInputStream* cis, int tag); From 0cf379b1489e6c34b0a70b63590e384071fa89eb Mon Sep 17 00:00:00 2001 From: Max Kojin Date: Tue, 5 Dec 2023 06:42:38 +0500 Subject: [PATCH 2/4] sync search --- include/OsmAndCore/Data/ObfPoiSectionReader.h | 2 +- include/OsmAndCore/ObfDataInterface.h | 2 +- src/Data/ObfPoiSectionReader.cpp | 4 +- src/Data/ObfPoiSectionReader_P.cpp | 233 ++++++++++++------ src/Data/ObfPoiSectionReader_P.h | 23 +- src/ObfDataInterface.cpp | 4 +- src/Search/AmenitiesByNameSearch.cpp | 4 +- 7 files changed, 187 insertions(+), 85 deletions(-) diff --git a/include/OsmAndCore/Data/ObfPoiSectionReader.h b/include/OsmAndCore/Data/ObfPoiSectionReader.h index 2161c8d2b..4aef01487 100644 --- a/include/OsmAndCore/Data/ObfPoiSectionReader.h +++ b/include/OsmAndCore/Data/ObfPoiSectionReader.h @@ -53,7 +53,7 @@ namespace OsmAnd const ObfPoiSectionReader::VisitorFunction visitor = nullptr, const std::shared_ptr& queryController = nullptr); - static void scanAmenitiesByName( + static void searchAmenitiyByName( const std::shared_ptr& reader, const std::shared_ptr& section, const QString& query, diff --git a/include/OsmAndCore/ObfDataInterface.h b/include/OsmAndCore/ObfDataInterface.h index 534124a28..62d291d4b 100644 --- a/include/OsmAndCore/ObfDataInterface.h +++ b/include/OsmAndCore/ObfDataInterface.h @@ -100,7 +100,7 @@ namespace OsmAnd const ObfPoiSectionReader::VisitorFunction visitor = nullptr, const std::shared_ptr& queryController = nullptr); - bool scanAmenitiesByName( + bool searchAmenitiesByName( const QString& query, QList< std::shared_ptr >* outAmenities, const PointI* const xy31 = nullptr, diff --git a/src/Data/ObfPoiSectionReader.cpp b/src/Data/ObfPoiSectionReader.cpp index a2225c424..59de1200f 100644 --- a/src/Data/ObfPoiSectionReader.cpp +++ b/src/Data/ObfPoiSectionReader.cpp @@ -52,7 +52,7 @@ void OsmAnd::ObfPoiSectionReader::loadAmenities( queryController); } -void OsmAnd::ObfPoiSectionReader::scanAmenitiesByName( +void OsmAnd::ObfPoiSectionReader::searchAmenitiyByName( const std::shared_ptr& reader, const std::shared_ptr& section, const QString& query, QList< std::shared_ptr >* outAmenities, @@ -63,7 +63,7 @@ void OsmAnd::ObfPoiSectionReader::scanAmenitiesByName( const ObfPoiSectionReader::VisitorFunction visitor /*= nullptr*/, const std::shared_ptr& queryController /*= nullptr*/) { - ObfPoiSectionReader_P::scanAmenitiesByName( + ObfPoiSectionReader_P::searchAmenitiyByName( *reader->_p, section, query, diff --git a/src/Data/ObfPoiSectionReader_P.cpp b/src/Data/ObfPoiSectionReader_P.cpp index 87c6af339..1b64d744b 100644 --- a/src/Data/ObfPoiSectionReader_P.cpp +++ b/src/Data/ObfPoiSectionReader_P.cpp @@ -717,6 +717,111 @@ bool OsmAnd::ObfPoiSectionReader_P::scanTileForMatchingCategories( } } +void OsmAnd::ObfPoiSectionReader_P::readPoiData( + const ObfReader_P& reader, + const std::shared_ptr& section, + QList< std::shared_ptr >* outAmenities, + const QString& query, + const AreaI* const bbox31, + const TileAcceptorFunction tileFilter, + const ZoomLevel zoomFilter, + const std::shared_ptr> pTilesToSkip, + const QSet* const categoriesFilter, + const ObfPoiSectionReader::VisitorFunction visitor, + const std::shared_ptr& queryController) +{ + const auto cis = reader.getCodedInputStream().get(); + + auto zoom = InvalidZoomLevel; + auto tileId = TileId::zero(); + const CollatorStringMatcher matcher(query, StringMatcherMode::CHECK_STARTS_FROM_SPACE); + + for (;;) + { + const auto t = cis->ReadTag(); + const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); + switch (tag) + { + case 0: + return; + case OBF::OsmAndPoiBoxData::kXFieldNumber: + cis->ReadVarint32(reinterpret_cast(&tileId.x)); + break; + case OBF::OsmAndPoiBoxData::kZoomFieldNumber: + cis->ReadVarint32(reinterpret_cast(&zoom)); + break; + case OBF::OsmAndPoiBoxData::kYFieldNumber: + cis->ReadVarint32(reinterpret_cast(&tileId.y)); + break; + case OBF::OsmAndPoiBoxData::kPoiDataFieldNumber: + { + gpb::uint32 length; + cis->ReadVarint32(&length); + const auto oldLimit = cis->PushLimit(length); + + std::shared_ptr am; + readAmenity( + reader, + section, + am, + query, + tileId, + zoom, + bbox31, + categoriesFilter, + queryController); + + ObfReaderUtilities::ensureAllDataWasRead(cis); + cis->PopLimit(oldLimit); + + if (am != nullptr) + { + bool matches = matcher.matches(am->getName(QStringLiteral(""), false).toLower()) || matcher.matches(am->getName(QStringLiteral("en"), true).toLower()); + if (!matches) + { + for (const auto &s : am->localizedNames) + { + matches = matcher.matches(s.toLower()); + if (matches) + break; + } + if (!matches) + { + const auto values = am->getDecodedValuesHash(); + for (const auto &key : values.keys()) + { + if (!key.contains(QStringLiteral("_name")) && + key != QString("brand")) + { + continue; + } + matches = matcher.matches(values.value(key)); + if (matches) + break; + } + } + } + if (matches) + { + if (!visitor || visitor(am)) + { + if (outAmenities) + outAmenities->push_back(qMove(am)); + } + } + } + break; + } + default: + { + ObfReaderUtilities::skipUnknownField(cis, t); + break; + } + } + } +} + + bool OsmAnd::ObfPoiSectionReader_P::readAmenitiesDataBox( const ObfReader_P& reader, const std::shared_ptr& section, @@ -1142,7 +1247,7 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenity( } } -void OsmAnd::ObfPoiSectionReader_P::readAmenitiesByName( +void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( const ObfReader_P& reader, const std::shared_ptr& section, const QString& query, @@ -1155,21 +1260,19 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenitiesByName( const std::shared_ptr& queryController) { const auto cis = reader.getCodedInputStream().get(); + const auto indexOffset = cis->TotalBytesRead(); QMap dataBoxesOffsetsSet; for (;;) { - const auto tag = cis->ReadTag(); - switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) + const auto t = cis->ReadTag(); + const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); + switch (tag) { case 0: - if (!ObfReaderUtilities::reachedDataEnd(cis)) - return; - return; case OBF::OsmAndPoiIndex::kNameIndexFieldNumber: { const auto length = ObfReaderUtilities::readBigEndianInt(cis); - const auto offset = cis->CurrentPosition(); const auto oldLimit = cis->PushLimit(length); dataBoxesOffsetsSet = readPoiNameIndex( @@ -1184,45 +1287,44 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenitiesByName( cis->PopLimit(oldLimit); break; } - case OBF::OsmAndPoiIndex::kBoxesFieldNumber: - ObfReaderUtilities::skipUnknownField(cis, tag); - break; case OBF::OsmAndPoiIndex::kPoiDataFieldNumber: { auto offKeys = dataBoxesOffsetsSet.keys(); - std::sort(offKeys, - [&dataBoxesOffsetsSet] - (const uint32_t& offset1, const uint32_t& offset2) -> bool - { - return dataBoxesOffsetsSet[offset1] < dataBoxesOffsetsSet[offset2]; - }); - - int p = BUCKET_SEARCH_BY_NAME * 3; - if (p < offKeys.length()) + if (dataBoxesOffsetsSet.size() > 0) { - for (int i = p + BUCKET_SEARCH_BY_NAME; ; i += BUCKET_SEARCH_BY_NAME) + std::sort(offKeys, + [&dataBoxesOffsetsSet] + (const uint32_t& offset1, const uint32_t& offset2) -> bool + { + return dataBoxesOffsetsSet[offset1] < dataBoxesOffsetsSet[offset2]; + }); + + int p = BUCKET_SEARCH_BY_NAME * 3; + if (p < offKeys.length()) { - if (i > offKeys.length()) + for (int i = p + BUCKET_SEARCH_BY_NAME; ; i += BUCKET_SEARCH_BY_NAME) { - std::sort(offKeys.begin() + p, offKeys.end()); - break; - } - else - { - std::sort(offKeys.begin() + p, offKeys.begin() + i); + if (i > offKeys.length()) + { + std::sort(offKeys.begin() + p, offKeys.end()); + break; + } + else + { + std::sort(offKeys.begin() + p, offKeys.begin() + i); + } + p = i; } - p = i; } } for (const auto dataOffset : offKeys) { - cis->Seek(section->offset + dataOffset); + cis->Seek(dataOffset + indexOffset); const auto length = ObfReaderUtilities::readBigEndianInt(cis); - const auto offset = cis->CurrentPosition(); const auto oldLimit = cis->PushLimit(length); - - readAmenitiesDataBox( + + readPoiData( reader, section, outAmenities, @@ -1248,7 +1350,7 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenitiesByName( return; } default: - ObfReaderUtilities::skipUnknownField(cis, tag); + ObfReaderUtilities::skipUnknownField(cis, t); break; } } @@ -1271,8 +1373,9 @@ QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( for (;;) { - const auto tag = cis->ReadTag(); - switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) + const auto t = cis->ReadTag(); + const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); + switch (tag) { case 0: return offsets; @@ -1281,7 +1384,7 @@ QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( { const auto length = ObfReaderUtilities::readBigEndianInt(cis); const auto oldLimit = cis->PushLimit(length); - offset = cis->CurrentPosition(); + offset = cis->TotalBytesRead(); QList queries = QList(); for (QString word : query.split(" ")) @@ -1320,7 +1423,7 @@ QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( cis->ReadVarint32(&length); const auto oldLimit = cis->PushLimit(length); - readNameIndexData( + readPoiNameIndexData( reader, offsetMap, xy31, @@ -1347,7 +1450,7 @@ QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( QMap mp = listOfSepOffsets[j]; for (auto chKey : offsets.keys()) { - if (!mp.contains(chKey)) + if (!mp.keys().contains(chKey)) { offsets.remove(chKey); } @@ -1364,7 +1467,7 @@ QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( } } -void OsmAnd::ObfPoiSectionReader_P::readNameIndexData( +void OsmAnd::ObfPoiSectionReader_P::readPoiNameIndexData( const ObfReader_P& reader, QMap& outDataOffsets, const PointI* const xy31, @@ -1375,13 +1478,11 @@ void OsmAnd::ObfPoiSectionReader_P::readNameIndexData( for (;;) { - const auto tag = cis->ReadTag(); - switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) + const auto t = cis->ReadTag(); + const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); + switch (tag) { case 0: - if (!ObfReaderUtilities::reachedDataEnd(cis)) - return; - return; case OBF::OsmAndPoiNameIndex_OsmAndPoiNameIndexData::kAtomsFieldNumber: { @@ -1389,7 +1490,7 @@ void OsmAnd::ObfPoiSectionReader_P::readNameIndexData( cis->ReadVarint32(&length); const auto oldLimit = cis->PushLimit(length); - readNameIndexDataAtom( + readPoiNameIndexDataAtom( reader, outDataOffsets, xy31, @@ -1407,7 +1508,7 @@ void OsmAnd::ObfPoiSectionReader_P::readNameIndexData( } } -void OsmAnd::ObfPoiSectionReader_P::readNameIndexDataAtom( +void OsmAnd::ObfPoiSectionReader_P::readPoiNameIndexDataAtom( const ObfReader_P& reader, QMap& outDataOffsets, const PointI* const xy31, @@ -1416,17 +1517,15 @@ void OsmAnd::ObfPoiSectionReader_P::readNameIndexDataAtom( { const auto cis = reader.getCodedInputStream().get(); auto tileId = TileId::zero(); - auto zoom = MinZoomLevel; + auto zoom = ZoomLevel15; for (;;) { - const auto tag = cis->ReadTag(); - switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) + const auto t = cis->ReadTag(); + const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); + switch (tag) { case 0: - if (!ObfReaderUtilities::reachedDataEnd(cis)) - return; - return; case OBF::OsmAndPoiNameIndexDataAtom::kXFieldNumber: cis->ReadVarint32(reinterpret_cast(&tileId.x)); @@ -1439,27 +1538,14 @@ void OsmAnd::ObfPoiSectionReader_P::readNameIndexDataAtom( break; case OBF::OsmAndPoiNameIndexDataAtom::kShiftToFieldNumber: { - const auto dataOffset = ObfReaderUtilities::readBigEndianInt(cis); - - bool accept = true; - - if (accept && tileFilter) - accept = tileFilter(tileId, zoom); - PointI position31; - uint32_t d = 0; - if (accept && bbox31) - { - position31.x = tileId.x << (31 - zoom); - position31.y = tileId.y << (31 - zoom); - accept = bbox31->contains(position31); - } - - if (accept) + position31.x = tileId.x << (31 - zoom); + position31.y = tileId.y << (31 - zoom); + const auto shift = ObfReaderUtilities::readBigEndianInt(cis); + if (bbox31->contains(position31)) { - if (xy31) - d = qAbs(xy31->x - position31.x) + qAbs(xy31->y - position31.y); - outDataOffsets.insert(dataOffset, d); + uint32_t d = qAbs(xy31->x - position31.x) + qAbs(xy31->y - position31.y); + outDataOffsets.insert(shift, d); } break; } @@ -1524,7 +1610,8 @@ void OsmAnd::ObfPoiSectionReader_P::loadAmenities( cis->PopLimit(oldLimit); } -void OsmAnd::ObfPoiSectionReader_P::scanAmenitiesByName( +// searchPoiByName +void OsmAnd::ObfPoiSectionReader_P::searchAmenitiyByName( const ObfReader_P& reader, const std::shared_ptr& section, const QString& query, @@ -1544,7 +1631,7 @@ void OsmAnd::ObfPoiSectionReader_P::scanAmenitiesByName( auto oldLimit = cis->PushLimit(section->length); cis->Skip(section->nameIndexInnerOffset); - readAmenitiesByName( + searchPoiByName( reader, section, query, diff --git a/src/Data/ObfPoiSectionReader_P.h b/src/Data/ObfPoiSectionReader_P.h index 45244792a..7b8ad16b1 100644 --- a/src/Data/ObfPoiSectionReader_P.h +++ b/src/Data/ObfPoiSectionReader_P.h @@ -14,6 +14,7 @@ #include "DataCommonTypes.h" #include "ObfPoiSectionReader.h" #include "ObfPoiSectionInfo.h" +#include namespace OsmAnd { @@ -89,7 +90,7 @@ namespace OsmAnd const ObfReader_P& reader, const QSet& categories); - static void readAmenitiesByName( + static void searchPoiByName( const ObfReader_P& reader, const std::shared_ptr& section, const QString& query, @@ -100,6 +101,7 @@ namespace OsmAnd const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, const std::shared_ptr& queryController); + static QMap readPoiNameIndex( const ObfReader_P& reader, const QString& query, @@ -107,18 +109,31 @@ namespace OsmAnd const PointI* const xy31, const AreaI* const bbox31, const TileAcceptorFunction tileFilter); - static void readNameIndexData( + static void readPoiNameIndexData( const ObfReader_P& reader, QMap& outDataOffsets, const PointI* const xy31, const AreaI* const bbox31, const TileAcceptorFunction tileFilter); - static void readNameIndexDataAtom( + static void readPoiNameIndexDataAtom( const ObfReader_P& reader, QMap& outDataOffsets, const PointI* const xy31, const AreaI* const bbox31, const TileAcceptorFunction tileFilter); + + static void readPoiData( + const ObfReader_P& reader, + const std::shared_ptr& section, + QList< std::shared_ptr >* outAmenities, + const QString& query, + const AreaI* const bbox31, + const TileAcceptorFunction tileFilter, + const ZoomLevel zoomFilter, + const std::shared_ptr> pTilesToSkip, + const QSet* const categoriesFilter, + const ObfPoiSectionReader::VisitorFunction visitor, + const std::shared_ptr& queryController); static bool readAmenitiesDataBox( const ObfReader_P& reader, @@ -166,7 +181,7 @@ namespace OsmAnd const ObfPoiSectionReader::VisitorFunction visitor, const std::shared_ptr& queryController); - static void scanAmenitiesByName( + static void searchAmenitiyByName( const ObfReader_P& reader, const std::shared_ptr& section, const QString& query, diff --git a/src/ObfDataInterface.cpp b/src/ObfDataInterface.cpp index fec4ffadd..2d84dea6c 100644 --- a/src/ObfDataInterface.cpp +++ b/src/ObfDataInterface.cpp @@ -575,7 +575,7 @@ bool OsmAnd::ObfDataInterface::loadAmenities( return true; } -bool OsmAnd::ObfDataInterface::scanAmenitiesByName( +bool OsmAnd::ObfDataInterface::searchAmenitiesByName( const QString& query, QList< std::shared_ptr >* outAmenities, const PointI* const xy31 /*= nullptr*/, @@ -685,7 +685,7 @@ bool OsmAnd::ObfDataInterface::scanAmenitiesByName( } } - OsmAnd::ObfPoiSectionReader::scanAmenitiesByName( + OsmAnd::ObfPoiSectionReader::searchAmenitiyByName( obfReader, poiSection, query, diff --git a/src/Search/AmenitiesByNameSearch.cpp b/src/Search/AmenitiesByNameSearch.cpp index 2bf9be08d..a177dac6e 100644 --- a/src/Search/AmenitiesByNameSearch.cpp +++ b/src/Search/AmenitiesByNameSearch.cpp @@ -34,7 +34,7 @@ void OsmAnd::AmenitiesByNameSearch::performSearch( return true; }; - dataInterface->scanAmenitiesByName( + dataInterface->searchAmenitiesByName( criteria.name, nullptr, criteria.xy31.getValuePtrOrNullptr(), @@ -76,7 +76,7 @@ void OsmAnd::AmenitiesByNameSearch::performTravelGuidesSearch( return true; }; - dataInterface->scanAmenitiesByName( + dataInterface->searchAmenitiesByName( criteria.name, nullptr, criteria.xy31.getValuePtrOrNullptr(), From 338249e21d139b9d9032386cff1b5758117d18c3 Mon Sep 17 00:00:00 2001 From: Max Kojin Date: Wed, 6 Dec 2023 21:57:08 +0500 Subject: [PATCH 3/4] code sync --- src/Data/ObfPoiSectionReader_P.cpp | 157 ++++++++++++++++++----------- src/Data/ObfPoiSectionReader_P.h | 9 +- src/Data/ObfReaderUtilities.cpp | 2 +- 3 files changed, 104 insertions(+), 64 deletions(-) diff --git a/src/Data/ObfPoiSectionReader_P.cpp b/src/Data/ObfPoiSectionReader_P.cpp index 1b64d744b..1c0efaa88 100644 --- a/src/Data/ObfPoiSectionReader_P.cpp +++ b/src/Data/ObfPoiSectionReader_P.cpp @@ -736,7 +736,7 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiData( auto tileId = TileId::zero(); const CollatorStringMatcher matcher(query, StringMatcherMode::CHECK_STARTS_FROM_SPACE); - for (;;) + while (true) { const auto t = cis->ReadTag(); const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); @@ -760,7 +760,9 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiData( const auto oldLimit = cis->PushLimit(length); std::shared_ptr am; - readAmenity( + +// readPoiPoint() + readPoiPoint( reader, section, am, @@ -803,11 +805,16 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiData( } if (matches) { - if (!visitor || visitor(am)) - { - if (outAmenities) - outAmenities->push_back(qMove(am)); - } + if (visitor) + visitor(am); + if (outAmenities) + outAmenities->push_back(qMove(am)); + +// if (!visitor || visitor(am)) +// { +// if (outAmenities) +// outAmenities->push_back(qMove(am)); +// } } } break; @@ -893,7 +900,7 @@ bool OsmAnd::ObfPoiSectionReader_P::readAmenitiesDataBox( const auto oldLimit = cis->PushLimit(length); std::shared_ptr amenity; - readAmenity( + readPoiPoint( reader, section, amenity, @@ -951,7 +958,7 @@ bool OsmAnd::ObfPoiSectionReader_P::readAmenitiesDataBox( } } -void OsmAnd::ObfPoiSectionReader_P::readAmenity( +void OsmAnd::ObfPoiSectionReader_P::readPoiPoint( const ObfReader_P& reader, const std::shared_ptr& section, std::shared_ptr& outAmenity, @@ -981,7 +988,7 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenity( const CollatorStringMatcher matcher(query, StringMatcherMode::CHECK_STARTS_FROM_SPACE); uint32_t precisionXY = 0; - for (;;) + while (true) { const auto t = cis->ReadTag(); const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); @@ -990,6 +997,10 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenity( cis->Skip(cis->BytesUntilLimit()); return; } +// if (amenityType == null && (tag > OsmandOdb.OsmAndPoiBoxDataAtom.CATEGORIES_FIELD_NUMBER || tag == 0)) { +// codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); +// return null; +// } switch (tag) { case 0: @@ -1111,13 +1122,6 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenity( } break; } - case OBF::OsmAndPoiBoxDataAtom::kCategoriesFieldNumber: - { - ObfPoiCategoryId categoryId; - cis->ReadVarint32(reinterpret_cast(&categoryId)); - categories.push_back(categoryId); - break; - } case OBF::OsmAndPoiBoxDataAtom::kSubcategoriesFieldNumber: { if (!categoriesFilterChecked && @@ -1142,26 +1146,6 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenity( intValues.insert(subtypeIndex, QVariant(intValue)); break; } - case OBF::OsmAndPoiBoxDataAtom::kNameFieldNumber: - ObfReaderUtilities::readQString(cis, nativeName); - break; - case OBF::OsmAndPoiBoxDataAtom::kNameEnFieldNumber: - { - QString name; - ObfReaderUtilities::readQString(cis, name); - localizedNames.insert(QLatin1String("en"), name); - - break; - } - case OBF::OsmAndPoiBoxDataAtom::kIdFieldNumber: - { - gpb::uint64 rawId; - cis->ReadVarint64(&rawId); - id = ObfObjectId::generateUniqueId(rawId, baseOffset, section); - - autogenerateId = false; - break; - } case OBF::OsmAndPoiBoxDataAtom::kTextCategoriesFieldNumber: { gpb::uint32 rawValue; @@ -1203,6 +1187,33 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenity( break; } + case OBF::OsmAndPoiBoxDataAtom::kCategoriesFieldNumber: + { + ObfPoiCategoryId categoryId; + cis->ReadVarint32(reinterpret_cast(&categoryId)); + categories.push_back(categoryId); + break; + } + case OBF::OsmAndPoiBoxDataAtom::kIdFieldNumber: + { + gpb::uint64 rawId; + cis->ReadVarint64(&rawId); + id = ObfObjectId::generateUniqueId(rawId, baseOffset, section); + + autogenerateId = false; + break; + } + case OBF::OsmAndPoiBoxDataAtom::kNameFieldNumber: + ObfReaderUtilities::readQString(cis, nativeName); + break; + case OBF::OsmAndPoiBoxDataAtom::kNameEnFieldNumber: + { + QString name; + ObfReaderUtilities::readQString(cis, name); + localizedNames.insert(QLatin1String("en"), name); + + break; + } case OBF::OsmAndPoiBoxDataAtom::kOpeningHoursFieldNumber: { QString valueString; @@ -1250,7 +1261,7 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenity( void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( const ObfReader_P& reader, const std::shared_ptr& section, - const QString& query, + const QString& nameQuery, QList< std::shared_ptr >* outAmenities, const PointI* const xy31, const AreaI* const bbox31, @@ -1260,9 +1271,12 @@ void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( const std::shared_ptr& queryController) { const auto cis = reader.getCodedInputStream().get(); + QMap offsets; + QString query = normalizeSearchPoiByNameQuery(nameQuery); + auto time = QDateTime::currentMSecsSinceEpoch(); const auto indexOffset = cis->TotalBytesRead(); - QMap dataBoxesOffsetsSet; - for (;;) + + while (true) { const auto t = cis->ReadTag(); const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); @@ -1274,14 +1288,15 @@ void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( { const auto length = ObfReaderUtilities::readBigEndianInt(cis); const auto oldLimit = cis->PushLimit(length); - - dataBoxesOffsetsSet = readPoiNameIndex( + // here offsets are sorted by distance + offsets = readPoiNameIndex( reader, query, - dataBoxesOffsetsSet, + offsets, xy31, bbox31, - tileFilter); + tileFilter, + queryController); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); @@ -1289,14 +1304,15 @@ void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( } case OBF::OsmAndPoiIndex::kPoiDataFieldNumber: { - auto offKeys = dataBoxesOffsetsSet.keys(); - if (dataBoxesOffsetsSet.size() > 0) + // also offsets can be randomly skipped by limit + auto offKeys = offsets.keys(); + if (offsets.size() > 0) { std::sort(offKeys, - [&dataBoxesOffsetsSet] + [&offsets] (const uint32_t& offset1, const uint32_t& offset2) -> bool { - return dataBoxesOffsetsSet[offset1] < dataBoxesOffsetsSet[offset2]; + return offsets[offset1] < offsets[offset2]; }); int p = BUCKET_SEARCH_BY_NAME * 3; @@ -1318,6 +1334,8 @@ void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( } } + LogPrintf(OsmAnd::LogSeverityLevel::Info, "Searched poi structure in %d ms. Found %d subtrees", QDateTime::currentMSecsSinceEpoch() - time, offKeys.size()); + for (const auto dataOffset : offKeys) { cis->Seek(dataOffset + indexOffset); @@ -1341,10 +1359,14 @@ void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( cis->PopLimit(oldLimit); if (queryController && queryController->isAborted()) { - cis->Skip(cis->BytesUntilLimit()); return; } } + + if (outAmenities) + LogPrintf(OsmAnd::LogSeverityLevel::Info, "Whole poi by name search is done in %d ms. Found %d", QDateTime::currentMSecsSinceEpoch() - time, outAmenities->size()); + else + LogPrintf(OsmAnd::LogSeverityLevel::Info, "Whole poi by name search is done in %d ms.", QDateTime::currentMSecsSinceEpoch() - time); cis->Skip(cis->BytesUntilLimit()); return; @@ -1356,13 +1378,19 @@ void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( } } +QString OsmAnd::ObfPoiSectionReader_P::normalizeSearchPoiByNameQuery(QString query) +{ + return query.replace("\"", "").toLower(); +} + QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( const ObfReader_P& reader, const QString& query, QMap& outDataOffsets, const PointI* const xy31, const AreaI* const bbox31, - const TileAcceptorFunction tileFilter) + const TileAcceptorFunction tileFilter, + const std::shared_ptr& queryController) { const auto cis = reader.getCodedInputStream().get(); @@ -1371,7 +1399,7 @@ QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( QList> listOfSepOffsets = QList>(); uint32_t offset = 0; - for (;;) + while (true) { const auto t = cis->ReadTag(); const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); @@ -1386,6 +1414,7 @@ QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( const auto oldLimit = cis->PushLimit(length); offset = cis->TotalBytesRead(); + // TODO: implement Algorithms.splitByWordsLowercase() QList queries = QList(); for (QString word : query.split(" ")) { @@ -1432,12 +1461,11 @@ QMap OsmAnd::ObfPoiSectionReader_P::readPoiNameIndex( ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); - - // TODO: add to params req.isCancelled() ? - // if (req.isCancelled()) { - // codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); //cis->Skip(cis->BytesUntilLimit()); - // return offsets; - // } + if (queryController && queryController->isAborted()) + { + cis->Skip(cis->BytesUntilLimit()); + return offsets; + } } listOfSepOffsets.append(offsetMap); } @@ -1476,7 +1504,7 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiNameIndexData( { const auto cis = reader.getCodedInputStream().get(); - for (;;) + while (true) { const auto t = cis->ReadTag(); const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); @@ -1519,7 +1547,7 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiNameIndexDataAtom( auto tileId = TileId::zero(); auto zoom = ZoomLevel15; - for (;;) + while (true) { const auto t = cis->ReadTag(); const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); @@ -1610,7 +1638,7 @@ void OsmAnd::ObfPoiSectionReader_P::loadAmenities( cis->PopLimit(oldLimit); } -// searchPoiByName +// searchPoiByName() in Android void OsmAnd::ObfPoiSectionReader_P::searchAmenitiyByName( const ObfReader_P& reader, const std::shared_ptr& section, @@ -1623,6 +1651,15 @@ void OsmAnd::ObfPoiSectionReader_P::searchAmenitiyByName( const ObfPoiSectionReader::VisitorFunction visitor, const std::shared_ptr& queryController) { + if (query.isNull() || query.isEmpty()) + { + LogPrintf(OsmAnd::LogSeverityLevel::Error, "ERROR: ObfPoiSectionReader_P::searchAmenitiyByName - query is empty"); + return; + } + + // TODO: implement + // for (PoiRegion poiIndex : poiIndexes) { + ensureCategoriesLoaded(reader, section); ensureSubtypesLoaded(reader, section); diff --git a/src/Data/ObfPoiSectionReader_P.h b/src/Data/ObfPoiSectionReader_P.h index 7b8ad16b1..a43a04cd5 100644 --- a/src/Data/ObfPoiSectionReader_P.h +++ b/src/Data/ObfPoiSectionReader_P.h @@ -93,7 +93,7 @@ namespace OsmAnd static void searchPoiByName( const ObfReader_P& reader, const std::shared_ptr& section, - const QString& query, + const QString& nameQuery, QList< std::shared_ptr >* outAmenities, const PointI* const xy31, const AreaI* const bbox31, @@ -108,7 +108,8 @@ namespace OsmAnd QMap& outDataOffsets, const PointI* const xy31, const AreaI* const bbox31, - const TileAcceptorFunction tileFilter); + const TileAcceptorFunction tileFilter, + const std::shared_ptr& queryController); static void readPoiNameIndexData( const ObfReader_P& reader, QMap& outDataOffsets, @@ -122,6 +123,8 @@ namespace OsmAnd const AreaI* const bbox31, const TileAcceptorFunction tileFilter); + static QString normalizeSearchPoiByNameQuery(QString query); + static void readPoiData( const ObfReader_P& reader, const std::shared_ptr& section, @@ -147,7 +150,7 @@ namespace OsmAnd const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, const std::shared_ptr& queryController); - static void readAmenity( + static void readPoiPoint( const ObfReader_P& reader, const std::shared_ptr& section, std::shared_ptr& outAmenity, diff --git a/src/Data/ObfReaderUtilities.cpp b/src/Data/ObfReaderUtilities.cpp index 862b582c5..05d67b312 100644 --- a/src/Data/ObfReaderUtilities.cpp +++ b/src/Data/ObfReaderUtilities.cpp @@ -96,7 +96,7 @@ void OsmAnd::ObfReaderUtilities::readIndexedStringTable( QVector matched(matchedCharacters.size()); bool shouldWeReadSubtable = false; - for (;;) + while (true) { const auto tag = cis->ReadTag(); switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) From 30af2b3805044df68c8fe4ccdbbb22e1e83ee071 Mon Sep 17 00:00:00 2001 From: Max Kojin Date: Tue, 12 Dec 2023 16:19:56 +0500 Subject: [PATCH 4/4] code syncing --- include/OsmAndCore/Data/ObfPoiSectionReader.h | 8 +- src/Data/ObfPoiSectionReader.cpp | 12 +- src/Data/ObfPoiSectionReader_P.cpp | 158 +++++++++++++----- src/Data/ObfPoiSectionReader_P.h | 26 ++- 4 files changed, 144 insertions(+), 60 deletions(-) diff --git a/include/OsmAndCore/Data/ObfPoiSectionReader.h b/include/OsmAndCore/Data/ObfPoiSectionReader.h index 4aef01487..f8363af0e 100644 --- a/include/OsmAndCore/Data/ObfPoiSectionReader.h +++ b/include/OsmAndCore/Data/ObfPoiSectionReader.h @@ -10,6 +10,7 @@ #include #include #include +#include "binaryRead.h" namespace OsmAnd { @@ -19,6 +20,7 @@ namespace OsmAnd class ObfPoiSectionInfo; class Amenity; class IQueryController; + class MapDataObject; class OSMAND_CORE_API ObfPoiSectionReader { @@ -51,7 +53,8 @@ namespace OsmAnd const ZoomLevel zoomFilter = InvalidZoomLevel, const QSet* const categoriesFilter = nullptr, const ObfPoiSectionReader::VisitorFunction visitor = nullptr, - const std::shared_ptr& queryController = nullptr); + const std::shared_ptr& queryController = nullptr, + const std::shared_ptr>& req = nullptr); static void searchAmenitiyByName( const std::shared_ptr& reader, @@ -63,7 +66,8 @@ namespace OsmAnd const TileAcceptorFunction tileFilter = nullptr, const QSet* const categoriesFilter = nullptr, const ObfPoiSectionReader::VisitorFunction visitor = nullptr, - const std::shared_ptr& queryController = nullptr); + const std::shared_ptr& queryController = nullptr, + const std::shared_ptr>& req = nullptr); }; } diff --git a/src/Data/ObfPoiSectionReader.cpp b/src/Data/ObfPoiSectionReader.cpp index 59de1200f..eb139ae3a 100644 --- a/src/Data/ObfPoiSectionReader.cpp +++ b/src/Data/ObfPoiSectionReader.cpp @@ -38,7 +38,8 @@ void OsmAnd::ObfPoiSectionReader::loadAmenities( const ZoomLevel zoomFilter /*= InvalidZoomLevel*/, const QSet* const categoriesFilter /*= nullptr*/, const ObfPoiSectionReader::VisitorFunction visitor /*= nullptr*/, - const std::shared_ptr& queryController /*= nullptr*/) + const std::shared_ptr& queryController /*= nullptr*/, + const std::shared_ptr>& req /*= nullptr*/) { ObfPoiSectionReader_P::loadAmenities( *reader->_p, @@ -49,7 +50,8 @@ void OsmAnd::ObfPoiSectionReader::loadAmenities( zoomFilter, categoriesFilter, visitor, - queryController); + queryController, + req); } void OsmAnd::ObfPoiSectionReader::searchAmenitiyByName( @@ -61,7 +63,8 @@ void OsmAnd::ObfPoiSectionReader::searchAmenitiyByName( const TileAcceptorFunction tileFilter /*= nullptr*/, const QSet* const categoriesFilter /*= nullptr*/, const ObfPoiSectionReader::VisitorFunction visitor /*= nullptr*/, - const std::shared_ptr& queryController /*= nullptr*/) + const std::shared_ptr& queryController /*= nullptr*/, + const std::shared_ptr>& req /*= nullptr*/) { ObfPoiSectionReader_P::searchAmenitiyByName( *reader->_p, @@ -73,5 +76,6 @@ void OsmAnd::ObfPoiSectionReader::searchAmenitiyByName( tileFilter, categoriesFilter, visitor, - queryController); + queryController, + req); } diff --git a/src/Data/ObfPoiSectionReader_P.cpp b/src/Data/ObfPoiSectionReader_P.cpp index 1c0efaa88..5300552c5 100644 --- a/src/Data/ObfPoiSectionReader_P.cpp +++ b/src/Data/ObfPoiSectionReader_P.cpp @@ -393,7 +393,8 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenities( const ZoomLevel zoomFilter, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController) + const std::shared_ptr& queryController, + const std::shared_ptr>& req) { const auto cis = reader.getCodedInputStream().get(); @@ -480,7 +481,8 @@ void OsmAnd::ObfPoiSectionReader_P::readAmenities( tilesToSkip, categoriesFilter, visitor, - queryController); + queryController, + req); if (tilesToSkip && zoomFilter != InvalidZoomLevel && atLeastOneAccepted) tilesToSkip->insert(tileValue); @@ -728,7 +730,8 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiData( const std::shared_ptr> pTilesToSkip, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController) + const std::shared_ptr& queryController, + const std::shared_ptr>& req) { const auto cis = reader.getCodedInputStream().get(); @@ -761,7 +764,6 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiData( std::shared_ptr am; -// readPoiPoint() readPoiPoint( reader, section, @@ -771,7 +773,8 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiData( zoom, bbox31, categoriesFilter, - queryController); + queryController, + req); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); @@ -809,12 +812,6 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiData( visitor(am); if (outAmenities) outAmenities->push_back(qMove(am)); - -// if (!visitor || visitor(am)) -// { -// if (outAmenities) -// outAmenities->push_back(qMove(am)); -// } } } break; @@ -840,7 +837,8 @@ bool OsmAnd::ObfPoiSectionReader_P::readAmenitiesDataBox( const std::shared_ptr> pTilesToSkip, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController) + const std::shared_ptr& queryController, + const std::shared_ptr>& req) { const auto cis = reader.getCodedInputStream().get(); @@ -909,7 +907,8 @@ bool OsmAnd::ObfPoiSectionReader_P::readAmenitiesDataBox( zoom, bbox31, categoriesFilter, - queryController); + queryController, + req); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); @@ -958,7 +957,7 @@ bool OsmAnd::ObfPoiSectionReader_P::readAmenitiesDataBox( } } -void OsmAnd::ObfPoiSectionReader_P::readPoiPoint( +std::shared_ptr OsmAnd::ObfPoiSectionReader_P::readPoiPoint( const ObfReader_P& reader, const std::shared_ptr& section, std::shared_ptr& outAmenity, @@ -967,17 +966,26 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiPoint( const ZoomLevel boxZoom, const AreaI* const bbox31, const QSet* const categoriesFilter, - const std::shared_ptr& queryController) + const std::shared_ptr& queryController, + const std::shared_ptr>& req) { const auto cis = reader.getCodedInputStream().get(); const auto baseOffset = cis->CurrentPosition(); const auto subtypes = section->_p->_subtypes; + + std::shared_ptr am; + int32_t x = 0; + int32_t y = 0; + uint32_t precisionXY = 0; + bool hasLocation = false; + QString retValue = QString(); +// PoiCategory amenityType = null; + QList textTags = QList(); + // TODO: old vars. delete if not used ObfObjectId id; bool autogenerateId = true; - std::shared_ptr amenity; - PointI position31; QString nativeName; QHash localizedNames; QList categories; @@ -986,25 +994,74 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiPoint( QHash stringOrDataValues; auto categoriesFilterChecked = false; const CollatorStringMatcher matcher(query, StringMatcherMode::CHECK_STARTS_FROM_SPACE); - uint32_t precisionXY = 0; + while (true) { const auto t = cis->ReadTag(); const auto tag = gpb::internal::WireFormatLite::GetTagFieldNumber(t); + if (categories.size() == 0 && (tag > OBF::OsmAndPoiBoxDataAtom::kCategoriesFieldNumber || tag == 0)) { cis->Skip(cis->BytesUntilLimit()); - return; + return nullptr; } -// if (amenityType == null && (tag > OsmandOdb.OsmAndPoiBoxDataAtom.CATEGORIES_FIELD_NUMBER || tag == 0)) { -// codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); -// return null; -// } + // if (amenityType == null && (tag > OsmandOdb.OsmAndPoiBoxDataAtom.CATEGORIES_FIELD_NUMBER || tag == 0)) { + // codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); + // return null; + // } + switch (tag) { case 0: { + req->numberOfAcceptedObjects++; + + if (hasLocation) + { + if (precisionXY != 0) + { + std::pair xy = OsmAnd::Utilities::calculateFinalXYFromBaseAndPrecisionXY(BASE_POI_ZOOM, FINAL_POI_ZOOM, precisionXY, x >> BASE_POI_SHIFT, y >> BASE_POI_SHIFT, true); + auto x31 = xy.first << FINAL_POI_SHIFT; + auto y31 = xy.second << FINAL_POI_SHIFT; + am->position31.x = x31; + am->position31.y = y31; + } + else + { + am->position31.x = x; + am->position31.y = y; + } + } + else + { + return nullptr; + } + + if (req->radius > 0) + { + +// LatLon loc = am.getLocation(); +// List locs = req.tiles.get(req.getTileHashOnPath(loc.getLatitude(), loc.getLongitude())); +// if (locs == null) { +// return null; +// } +// AmenityRoutePoint arp = dist(am.getLocation(), locs, req.radius); +// if (arp == null) { +// return null; +// } else { +// am.setRoutePoint(arp); +// } + + //TODO: implement req parameter and continue. + } + //return am; + //case OsmandOdb.OsmAndPoiBoxDataAtom.DX_FIELD_NUMBER: + + + // ====================================================== + // old code. refactor. + if (!ObfReaderUtilities::reachedDataEnd(cis)) return; @@ -1070,28 +1127,28 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiPoint( return; } - if (!amenity) - amenity.reset(new Amenity(section)); + if (!am) + am.reset(new Amenity(section)); - amenity->nativeName = qMove(nativeName); - amenity->localizedNames = qMove(localizedNames); + am->nativeName = qMove(nativeName); + am->localizedNames = qMove(localizedNames); if (precisionXY > 0) { - int xBase = position31.x >> BASE_POI_SHIFT; - int yBase = position31.y >> BASE_POI_SHIFT; + int xBase = x >> BASE_POI_SHIFT; + int yBase = y >> BASE_POI_SHIFT; std::pair precisedXY = OsmAnd::Utilities::calculateFinalXYFromBaseAndPrecisionXY(BASE_POI_ZOOM, FINAL_POI_ZOOM, precisionXY, xBase, yBase, true); - position31.x = precisedXY.first << FINAL_POI_SHIFT; - position31.y = precisedXY.second << FINAL_POI_SHIFT; + x = precisedXY.first << FINAL_POI_SHIFT; + y = precisedXY.second << FINAL_POI_SHIFT; } - amenity->position31 = position31; - amenity->categories = qMove(categories); + //amenity->position31 = position31; + am->categories = qMove(categories); if (autogenerateId) - amenity->id = ObfObjectId::generateUniqueId(baseOffset, section); + am->id = ObfObjectId::generateUniqueId(baseOffset, section); else - amenity->id = id; - amenity->values = detachedOf(intValues).unite(stringOrDataValues); - amenity->evaluateTypes(); - outAmenity = amenity; + am->id = id; + am->values = detachedOf(intValues).unite(stringOrDataValues); + am->evaluateTypes(); + outAmenity = am; ////////////////////////////////////////////////////////////////////////// //if (amenity->id.getOsmId() == 582502308u) @@ -1106,15 +1163,18 @@ void OsmAnd::ObfPoiSectionReader_P::readPoiPoint( { const auto d = ObfReaderUtilities::readSInt32(cis); assert(d >= 0); - position31.x = ((boxTileId.x << (BASE_POI_ZOOM - boxZoom)) + d) << BASE_POI_SHIFT; + x = ((boxTileId.x << (BASE_POI_ZOOM - boxZoom)) + d) << BASE_POI_SHIFT; break; } case OBF::OsmAndPoiBoxDataAtom::kDyFieldNumber: { const auto d = ObfReaderUtilities::readSInt32(cis); assert(d >= 0); - position31.y = ((boxTileId.y << (BASE_POI_ZOOM - boxZoom)) + d) << BASE_POI_SHIFT; + y = ((boxTileId.y << (BASE_POI_ZOOM - boxZoom)) + d) << BASE_POI_SHIFT; + PointI position31; + position31.x = x; + position31.y = y; if (bbox31 && !bbox31->contains(position31)) { cis->Skip(cis->BytesUntilLimit()); @@ -1268,7 +1328,8 @@ void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( const TileAcceptorFunction tileFilter, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController) + const std::shared_ptr& queryController, + const std::shared_ptr>& req) { const auto cis = reader.getCodedInputStream().get(); QMap offsets; @@ -1353,7 +1414,8 @@ void OsmAnd::ObfPoiSectionReader_P::searchPoiByName( nullptr, categoriesFilter, visitor, - queryController); + queryController, + req); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); @@ -1613,7 +1675,8 @@ void OsmAnd::ObfPoiSectionReader_P::loadAmenities( const ZoomLevel zoomFilter, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController) + const std::shared_ptr& queryController, + const std::shared_ptr>& req) { ensureCategoriesLoaded(reader, section); ensureSubtypesLoaded(reader, section); @@ -1632,7 +1695,8 @@ void OsmAnd::ObfPoiSectionReader_P::loadAmenities( zoomFilter, categoriesFilter, visitor, - queryController); + queryController, + req); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); @@ -1649,7 +1713,8 @@ void OsmAnd::ObfPoiSectionReader_P::searchAmenitiyByName( const TileAcceptorFunction tileFilter, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController) + const std::shared_ptr& queryController, + const std::shared_ptr>& req) { if (query.isNull() || query.isEmpty()) { @@ -1678,7 +1743,8 @@ void OsmAnd::ObfPoiSectionReader_P::searchAmenitiyByName( tileFilter, categoriesFilter, visitor, - queryController); + queryController, + req); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); diff --git a/src/Data/ObfPoiSectionReader_P.h b/src/Data/ObfPoiSectionReader_P.h index a43a04cd5..d9d7b01d6 100644 --- a/src/Data/ObfPoiSectionReader_P.h +++ b/src/Data/ObfPoiSectionReader_P.h @@ -14,6 +14,7 @@ #include "DataCommonTypes.h" #include "ObfPoiSectionReader.h" #include "ObfPoiSectionInfo.h" +#include "binaryRead.h" #include namespace OsmAnd @@ -22,6 +23,7 @@ namespace OsmAnd class ObfPoiSectionInfo; class Amenity; class IQueryController; +// class MapDataObject; class ObfPoiSectionReader; class ObfPoiSectionReader_P Q_DECL_FINAL @@ -75,7 +77,8 @@ namespace OsmAnd const ZoomLevel zoomFilter, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController); + const std::shared_ptr& queryController, + const std::shared_ptr>& req); static bool scanTiles( const ObfReader_P& reader, QMap& outDataOffsetsMap, @@ -100,7 +103,8 @@ namespace OsmAnd const TileAcceptorFunction tileFilter, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController); + const std::shared_ptr& queryController, + const std::shared_ptr>& req); static QMap readPoiNameIndex( const ObfReader_P& reader, @@ -136,7 +140,8 @@ namespace OsmAnd const std::shared_ptr> pTilesToSkip, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController); + const std::shared_ptr& queryController, + const std::shared_ptr>& req); static bool readAmenitiesDataBox( const ObfReader_P& reader, @@ -149,8 +154,10 @@ namespace OsmAnd const std::shared_ptr> pTilesToSkip, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController); - static void readPoiPoint( + const std::shared_ptr& queryController, + const std::shared_ptr>& req); + + static std::shared_ptr readPoiPoint( const ObfReader_P& reader, const std::shared_ptr& section, std::shared_ptr& outAmenity, @@ -159,7 +166,8 @@ namespace OsmAnd const ZoomLevel boxZoom, const AreaI* const bbox31, const QSet* const categoriesFilter, - const std::shared_ptr& queryController); + const std::shared_ptr& queryController, + const std::shared_ptr>& req); public: static void loadCategories( const ObfReader_P& reader, @@ -182,7 +190,8 @@ namespace OsmAnd const ZoomLevel zoomFilter, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController); + const std::shared_ptr& queryController, + const std::shared_ptr>& req); static void searchAmenitiyByName( const ObfReader_P& reader, @@ -194,7 +203,8 @@ namespace OsmAnd const TileAcceptorFunction tileFilter, const QSet* const categoriesFilter, const ObfPoiSectionReader::VisitorFunction visitor, - const std::shared_ptr& queryController); + const std::shared_ptr& queryController, + const std::shared_ptr>& req); friend class OsmAnd::ObfReader_P; friend class OsmAnd::ObfPoiSectionReader;