diff --git a/include/attribute_store.h b/include/attribute_store.h index 7d3eafe6..26ca60ce 100644 --- a/include/attribute_store.h +++ b/include/attribute_store.h @@ -55,9 +55,9 @@ struct AttributeStore switch(lhs_id) { case Index::BOOL: return lhs.bool_value() < rhs.bool_value(); - case Index::FLOAT: + case Index::FLOAT: return lhs.float_value() < rhs.float_value(); - case Index::STRING: + case Index::STRING: return lhs.string_value() < rhs.string_value(); } @@ -111,7 +111,7 @@ struct AttributeStore key_value_set_ref_t empty_set() const { return new key_value_set(); } - key_value_set_ref_t store_set(key_value_set_ref_t attributes) { + key_value_set_ref_t store_set(key_value_set_ref_t attributes) { auto idx = attributes->values.size(); for(auto const &i: attributes->values) { boost::hash_combine(idx, i.minzoom); diff --git a/include/osm_lua_processing.h b/include/osm_lua_processing.h index 1485a0b2..a3cc60a4 100644 --- a/include/osm_lua_processing.h +++ b/include/osm_lua_processing.h @@ -157,6 +157,7 @@ class OsmLuaProcessing { void AttributeWithMinZoom(const std::string &key, const std::string &val, const char minzoom); void AttributeNumeric(const std::string &key, const float val); void AttributeNumericWithMinZoom(const std::string &key, const float val, const char minzoom); + void SetRank(const float val); void AttributeBoolean(const std::string &key, const bool val); void AttributeBooleanWithMinZoom(const std::string &key, const bool val, const char minzoom); void MinZoom(const double z); diff --git a/include/output_object.h b/include/output_object.h index c937162f..5d2cb8de 100644 --- a/include/output_object.h +++ b/include/output_object.h @@ -57,6 +57,7 @@ class OutputObject { unsigned minZoom : 4; AttributeStoreRef attributes; + float_t rankValue = std::numeric_limits::infinity(); void setZOrder(const ZOrder z) { #ifndef FLOAT_Z_ORDER @@ -75,9 +76,12 @@ class OutputObject { this->attributes = attributes; } + void setRankValue(const float_t value); + //\brief Write attribute key/value pairs (dictionary-encoded) void writeAttributes(std::vector *keyList, - std::vector *valueList, vector_tile::Tile_Feature *featurePtr, char zoom) const; + std::vector *valueList, vector_tile::Tile_Feature *featurePtr, + AttributeStoreRef extraAttributes, char zoom) const; /** * \brief Find a value in the value dictionary diff --git a/include/shared_data.h b/include/shared_data.h index e3553969..c3b566e7 100644 --- a/include/shared_data.h +++ b/include/shared_data.h @@ -31,6 +31,7 @@ struct LayerDef { bool indexed; std::string indexName; std::map attributeMap; + uint64_t rankMax; bool writeTo; }; @@ -51,6 +52,7 @@ class LayerDefinition { bool allSourceColumns, bool indexed, const std::string &indexName, + const uint64_t rankMax, const std::string &writeTo); std::vector getSortOrders(); rapidjson::Value serialiseToJSONValue(rapidjson::Document::AllocatorType &allocator) const; diff --git a/resources/config-openmaptiles.json b/resources/config-openmaptiles.json index 3cb1a3be..98007d5b 100644 --- a/resources/config-openmaptiles.json +++ b/resources/config-openmaptiles.json @@ -1,10 +1,10 @@ { "layers": { - "place": { "minzoom": 0, "maxzoom": 14 }, + "place": { "minzoom": 0, "maxzoom": 14, "rank_max": 32 }, "boundary": { "minzoom": 0, "maxzoom": 14, "simplify_below": 12, "simplify_level": 0.0003, "simplify_ratio": 2 }, - "poi": { "minzoom": 12, "maxzoom": 14 }, - "poi_detail": { "minzoom": 14, "maxzoom": 14, "write_to": "poi"}, + "poi": { "minzoom": 12, "maxzoom": 14, "rank_max": 64 }, + "poi_detail": { "minzoom": 14, "maxzoom": 14, "rank_max": 64, "write_to": "poi"}, "housenumber": { "minzoom": 14, "maxzoom": 14 }, diff --git a/resources/process-openmaptiles.lua b/resources/process-openmaptiles.lua index c121eb52..32a0a2b9 100644 --- a/resources/process-openmaptiles.lua +++ b/resources/process-openmaptiles.lua @@ -76,15 +76,14 @@ function node_function(node) -- we could potentially approximate it for cities based on the population tag local place = node:Find("place") if place ~= "" then - local rank = nil local mz = 13 local pop = tonumber(node:Find("population")) or 0 if place == "continent" then mz=0 elseif place == "country" then - if pop>50000000 then rank=1; mz=1 - elseif pop>20000000 then rank=2; mz=2 - else rank=3; mz=3 end + if pop>50000000 then mz=1 + elseif pop>20000000 then mz=2 + else mz=3 end elseif place == "state" then mz=4 elseif place == "city" then mz=5 elseif place == "town" and pop>8000 then mz=7 @@ -100,7 +99,7 @@ function node_function(node) node:Layer("place", false) node:Attribute("class", place) node:MinZoom(mz) - if rank then node:AttributeNumeric("rank", rank) end + node:SetRank(pop) if place=="country" then node:Attribute("iso_a2", node:Find("ISO3166-1:alpha2")) end SetNameAttributes(node) return @@ -579,7 +578,7 @@ function WritePOI(obj,class,subclass,rank) if rank>4 then layer="poi_detail" end obj:LayerAsCentroid(layer) SetNameAttributes(obj) - obj:AttributeNumeric("rank", rank) + obj:SetRank(rank) obj:Attribute("class", class) obj:Attribute("subclass", subclass) end diff --git a/src/osm_lua_processing.cpp b/src/osm_lua_processing.cpp index c69d55bc..704b628e 100644 --- a/src/osm_lua_processing.cpp +++ b/src/osm_lua_processing.cpp @@ -53,6 +53,7 @@ OsmLuaProcessing::OsmLuaProcessing( .addFunction("LayerAsCentroid", &OsmLuaProcessing::LayerAsCentroid) .addOverloadedFunctions("Attribute", &OsmLuaProcessing::Attribute, &OsmLuaProcessing::AttributeWithMinZoom) .addOverloadedFunctions("AttributeNumeric", &OsmLuaProcessing::AttributeNumeric, &OsmLuaProcessing::AttributeNumericWithMinZoom) + .addFunction("SetRank", &OsmLuaProcessing::SetRank) .addOverloadedFunctions("AttributeBoolean", &OsmLuaProcessing::AttributeBoolean, &OsmLuaProcessing::AttributeBooleanWithMinZoom) .addFunction("MinZoom", &OsmLuaProcessing::MinZoom) .addOverloadedFunctions("ZOrder", &OsmLuaProcessing::ZOrder, &OsmLuaProcessing::ZOrderWithScale) @@ -328,13 +329,13 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { if (geomType==POINT_) { Point p = Point(lon, latp); - if(!CorrectGeometry(p)) return; + if(!CorrectGeometry(p)) return; osmStore.store_point(osmStore.osm(), osmID, p); OutputObjectRef oo = osmMemTiles.CreateObject(OutputObjectOsmStorePoint(geomType, layers.layerMap[layerName], osmID, attributeStore.empty_set(), layerMinZoom)); outputs.push_back(std::make_pair(oo, attributeStore.empty_set())); - return; + return; } else if (geomType==POLYGON_) { // polygon @@ -358,7 +359,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { mp.push_back(p); } - if(!CorrectGeometry(mp)) return; + if(!CorrectGeometry(mp)) return; osmStore.store_multi_polygon(osmStore.osm(), osmID, mp); OutputObjectRef oo = osmMemTiles.CreateObject(OutputObjectOsmStoreMultiPolygon(geomType, @@ -476,6 +477,16 @@ void OsmLuaProcessing::AttributeNumericWithMinZoom(const string &key, const floa setVectorLayerMetadata(outputs.back().first->layer, key, 1); } +void OsmLuaProcessing::SetRank(const float val) { + if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } + + // First, set the attribute so it will be registered in the metadata + setVectorLayerMetadata(outputs.back().first->layer, "rank", 1); + + // Set the ranked attribute, used for relative processing during output. + outputs.back().first->setRankValue(val); +} + void OsmLuaProcessing::AttributeBoolean(const string &key, const bool val) { AttributeBooleanWithMinZoom(key,val,0); } void OsmLuaProcessing::AttributeBooleanWithMinZoom(const string &key, const bool val, const char minzoom) { if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } diff --git a/src/output_object.cpp b/src/output_object.cpp index dd249049..32e24caa 100644 --- a/src/output_object.cpp +++ b/src/output_object.cpp @@ -30,42 +30,49 @@ std::ostream& operator<<(std::ostream& os, OutputGeometryType geomType) return os; } +void OutputObject::setRankValue(const float_t value) { + rankValue = value; +} // Write attribute key/value pairs (dictionary-encoded) void OutputObject::writeAttributes( vector *keyList, vector *valueList, vector_tile::Tile_Feature *featurePtr, + AttributeStoreRef extraAttributes, char zoom) const { - for(auto const &it: attributes->values) { - if (it.minzoom > zoom) continue; - - // Look for key - std::string const &key = it.key; - auto kt = find(keyList->begin(), keyList->end(), key); - if (kt != keyList->end()) { - uint32_t subscript = kt - keyList->begin(); - featurePtr->add_tags(subscript); - } else { - uint32_t subscript = keyList->size(); - keyList->push_back(key); - featurePtr->add_tags(subscript); - } - - // Look for value - vector_tile::Tile_Value const &value = it.value; - int subscript = findValue(valueList, value); - if (subscript>-1) { - featurePtr->add_tags(subscript); - } else { - uint32_t subscript = valueList->size(); - valueList->push_back(value); - featurePtr->add_tags(subscript); - } + auto attributeList = { attributes, extraAttributes }; + for (auto const attributes: attributeList) { + for(auto const &it: attributes->values) { + if (it.minzoom > zoom) continue; + + // Look for key + std::string const &key = it.key; + auto kt = find(keyList->begin(), keyList->end(), key); + if (kt != keyList->end()) { + uint32_t subscript = kt - keyList->begin(); + featurePtr->add_tags(subscript); + } else { + uint32_t subscript = keyList->size(); + keyList->push_back(key); + featurePtr->add_tags(subscript); + } + + // Look for value + vector_tile::Tile_Value const &value = it.value; + int subscript = findValue(valueList, value); + if (subscript>-1) { + featurePtr->add_tags(subscript); + } else { + uint32_t subscript = valueList->size(); + valueList->push_back(value); + featurePtr->add_tags(subscript); + } - //if(value.has_string_value()) - // std::cout << "Write attr: " << key << " " << value.string_value() << std::endl; + //if(value.has_string_value()) + // std::cout << "Write attr: " << key << " " << value.string_value() << std::endl; + } } } diff --git a/src/shared_data.cpp b/src/shared_data.cpp index d4111d9f..e98b032d 100644 --- a/src/shared_data.cpp +++ b/src/shared_data.cpp @@ -25,13 +25,14 @@ uint LayerDefinition::addLayer(string name, uint minzoom, uint maxzoom, bool allSourceColumns, bool indexed, const std::string &indexName, + const uint64_t rankMax, const std::string &writeTo) { bool isWriteTo = !writeTo.empty(); LayerDef layer = { name, minzoom, maxzoom, simplifyBelow, simplifyLevel, simplifyLength, simplifyRatio, filterBelow, filterArea, combinePolygonsBelow, sortZOrderAscending, source, sourceColumns, allSourceColumns, indexed, indexName, - std::map(), isWriteTo }; + std::map(), rankMax, isWriteTo }; layers.push_back(layer); uint layerNum = layers.size()-1; layerMap[name] = layerNum; @@ -221,11 +222,17 @@ void Config::readConfig(rapidjson::Document &jsonConfig, bool &hasClippingBox, B } string indexName = it->value.HasMember("index_column") ? it->value["index_column"].GetString() : ""; + uint64_t rankMax = std::numeric_limits::max(); + if (it->value.HasMember("rank_max")) { + rankMax = it->value["rank_max"].GetUint64(); + if (rankMax == 0) rankMax = std::numeric_limits::max(); + } + layers.addLayer(layerName, minZoom, maxZoom, simplifyBelow, simplifyLevel, simplifyLength, simplifyRatio, filterBelow, filterArea, combinePolyBelow, sortZOrderAscending, source, sourceColumns, allSourceColumns, indexed, indexName, - writeTo); + rankMax, writeTo); cout << "Layer " << layerName << " (z" << minZoom << "-" << maxZoom << ")"; if (it->value.HasMember("write_to")) { cout << " -> " << it->value["write_to"].GetString(); } diff --git a/src/tile_worker.cpp b/src/tile_worker.cpp index 98a08ac3..bdf8ade3 100644 --- a/src/tile_worker.cpp +++ b/src/tile_worker.cpp @@ -1,6 +1,7 @@ /*! \file */ #include "tile_worker.h" #include +#include #include #include #include "helpers.h" @@ -134,14 +135,58 @@ void RemoveInnersBelowSize(MultiPolygon &g, double filterArea) { } } +void ProcessTileRelativeObjects(OutputObjectsConstIt ooSameLayerBegin, OutputObjectsConstIt ooSameLayerEnd, + std::map *ooRankMap) { + + std::vector tileRelativeObjectsForSort; + std::copy_if(ooSameLayerBegin, ooSameLayerEnd, std::back_inserter(tileRelativeObjectsForSort), + [](OutputObjectRef const &oo) -> bool { + // TODO: support multiple ranked attributes + return oo->rankValue < std::numeric_limits::infinity(); + }); + + uint64_t currentSortIdx = 0; + std::sort(tileRelativeObjectsForSort.begin(), tileRelativeObjectsForSort.end(), + [](const OutputObjectRef &lhs, const OutputObjectRef &rhs) -> bool { + return lhs->rankValue < rhs->rankValue; + }); + + for (auto jt = tileRelativeObjectsForSort.begin(); jt != tileRelativeObjectsForSort.end(); ++jt) { + // rank is a 1-indexed value + uint64_t index = distance(tileRelativeObjectsForSort.begin(), jt) + 1; + OutputObjectRef oo = *jt; + ooRankMap->emplace(oo->objectID & OSMID_MASK, index); + } +} + void ProcessObjects(OSMStore &osmStore, OutputObjectsConstIt ooSameLayerBegin, OutputObjectsConstIt ooSameLayerEnd, - class SharedData &sharedData, double simplifyLevel, double filterArea, bool combinePolygons, unsigned zoom, const TileBbox &bbox, + class SharedData &sharedData, double simplifyLevel, double filterArea, bool combinePolygons, unsigned zoom, + uint64_t rankMax, const TileBbox &bbox, vector_tile::Tile_Layer *vtLayer, vector &keyList, vector &valueList) { + std::map ooRankMap = std::map(); + ProcessTileRelativeObjects(ooSameLayerBegin, ooSameLayerEnd, &ooRankMap); + for (auto jt = ooSameLayerBegin; jt != ooSameLayerEnd; ++jt) { OutputObjectRef oo = *jt; if (zoom < oo->minZoom) { continue; } + AttributeStoreRef extraAttributes = new AttributeStore::key_value_set(); + + auto ooRank = ooRankMap.find(oo->objectID & OSMID_MASK); + if (ooRank != ooRankMap.end()) { + uint64_t rankInt = ooRank->second; + + if (rankInt > rankMax) { + // Do not write the object to the tile if it is greater than the maximum allowed rank in the layer config + continue; + } + + vector_tile::Tile_Value v; + v.set_uint_value(rankInt); + extraAttributes->values.emplace("rank", v, 0); + } + if (oo->geomType == POINT_) { vector_tile::Tile_Feature *featurePtr = vtLayer->add_features(); LatpLon pos = buildNodeGeometry(osmStore, *oo, bbox); @@ -151,7 +196,7 @@ void ProcessObjects(OSMStore &osmStore, OutputObjectsConstIt ooSameLayerBegin, O featurePtr->add_geometry((xy.second << 1) ^ (xy.second >> 31)); featurePtr->set_type(vector_tile::Tile_GeomType_POINT); - oo->writeAttributes(&keyList, &valueList, featurePtr, zoom); + oo->writeAttributes(&keyList, &valueList, featurePtr, extraAttributes, zoom); if (sharedData.config.includeID) { featurePtr->set_id(oo->objectID & OSMID_MASK); } } else { Geometry g; @@ -183,9 +228,8 @@ void ProcessObjects(OSMStore &osmStore, OutputObjectsConstIt ooSameLayerBegin, O WriteGeometryVisitor w(&bbox, featurePtr, simplifyLevel); boost::apply_visitor(w, g); if (featurePtr->geometry_size()==0) { vtLayer->mutable_features()->RemoveLast(); continue; } - oo->writeAttributes(&keyList, &valueList, featurePtr, zoom); + oo->writeAttributes(&keyList, &valueList, featurePtr, extraAttributes, zoom); if (sharedData.config.includeID) { featurePtr->set_id(oo->objectID & OSMID_MASK); } - } } } @@ -237,9 +281,10 @@ void ProcessLayer(OSMStore &osmStore, } auto ooListSameLayer = GetObjectsAtSubLayer(data, layerNum); + // Loop through output objects ProcessObjects(osmStore, ooListSameLayer.first, ooListSameLayer.second, sharedData, - simplifyLevel, filterArea, zoom < ld.combinePolygonsBelow, zoom, bbox, vtLayer, keyList, valueList); + simplifyLevel, filterArea, zoom < ld.combinePolygonsBelow, zoom, ld.rankMax, bbox, vtLayer, keyList, valueList); } if (verbose && std::time(0)-start>3) { std::cout << "Layer " << layerName << " at " << zoom << "/" << index.x << "/" << index.y << " took " << (std::time(0)-start) << " seconds" << std::endl;