gtfs-actions/generate-shapes
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: integrate libosmium for reading files

Browse source
This commit is contained in:
Vlad Vesa 2020-08-18 21:24:03 +03:00
parent 69dc466605
commit ccc89d574c
12 changed files with 766 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -4,6 +4,7 @@ CMakeCache.txt
*.a
cmake_install.cmake
*~
cmake-build-*
CMakeFiles
*.cfg
*.cmake

1
CMakeLists.txt
View file

@ -17,6 +17,7 @@ enable_testing()
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")
set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build")
find_package(Osmium 2.15 REQUIRED COMPONENTS pbf xml io)
find_package(OpenMP)
if (OPENMP_FOUND)

10
src/pfaedle/CMakeLists.txt
View file

@ -17,4 +17,12 @@ add_executable(pfaedle ${pfaedle_main})
add_library(pfaedle_dep ${pfaedle_SRC})
include_directories(pfaedle_dep PUBLIC ${PROJECT_SOURCE_DIR}/src/cppgtfs/src)
target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread)
if (OSMIUM_FOUND)
include_directories(${OSMIUM_INCLUDE_DIRS})
target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread
${OSMIUM_IO_LIBRARIES})
message("linking with osmium")
else ()
target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread)
endif()

57
src/pfaedle/PfaedleMain.cpp
View file

@ -1,18 +1,17 @@
// Copyright 2018, University of Freiburg,
// Chair of Algorithms and Data Structures.
// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
#include <limits.h>
#include <climits>
#include <pwd.h>
#include <signal.h>
#include <stdio.h>
#include <csignal>
#include <cstdio>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#include "ad/cppgtfs/Parser.h"
#include "ad/cppgtfs/Writer.h"
#include "pfaedle/config/ConfigReader.h"
@ -72,7 +71,7 @@ std::vector<std::string> getCfgPaths(const Config& cfg);
// _____________________________________________________________________________
int main(int argc, char** argv) {
// disable output buffering for standard output
setbuf(stdout, NULL);
setbuf(stdout, nullptr);
// initialize randomness
srand(time(NULL) + rand()); // NOLINT
@ -80,8 +79,7 @@ int main(int argc, char** argv) {
Config cfg;
MotConfigReader motCfgReader;
ConfigReader cr;
cr.read(&cfg, argc, argv);
ConfigReader::read(&cfg, argc, argv);
std::vector<pfaedle::gtfs::Feed> gtfs(cfg.feedPaths.size());
// feed containing the shapes in memory for evaluation
@ -102,7 +100,7 @@ int main(int argc, char** argv) {
exit(static_cast<int>(RetCode::NO_OSM_INPUT));
}
if (motCfgReader.getConfigs().size() == 0) {
if (motCfgReader.getConfigs().empty()) {
LOG(ERROR) << "No MOT configurations specified and no implicit "
"configurations found, see --help.";
exit(static_cast<int>(RetCode::NO_MOT_CFG));
@ -125,7 +123,7 @@ int main(int argc, char** argv) {
exit(static_cast<int>(RetCode::GTFS_PARSE_ERR));
}
if (!cfg.writeOverpass) LOG(INFO) << "Done.";
} else if (cfg.writeOsm.size() || cfg.writeOverpass) {
} else if (!cfg.writeOsm.empty() || cfg.writeOverpass) {
for (size_t i = 0; i < cfg.feedPaths.size(); i++) {
if (!cfg.writeOverpass)
LOG(INFO) << "Reading " << cfg.feedPaths[i] << " ...";
@ -147,10 +145,10 @@ int main(int argc, char** argv) {
LOG(DEBUG) << "Read " << motCfgReader.getConfigs().size()
<< " unique MOT configs.";
MOTs cmdCfgMots = cfg.mots;
pfaedle::gtfs::Trip* singleTrip = 0;
pfaedle::gtfs::Trip* singleTrip = nullptr;
if (cfg.shapeTripId.size()) {
if (!cfg.feedPaths.size()) {
if (!cfg.shapeTripId.empty()) {
if (cfg.feedPaths.empty()) {
std::cout << "No input feed specified, see --help" << std::endl;
exit(static_cast<int>(RetCode::NO_INPUT_FEED));
}
@ -161,7 +159,7 @@ int main(int argc, char** argv) {
}
}
if (cfg.writeOsm.size()) {
if (!cfg.writeOsm.empty()) {
LOG(INFO) << "Writing filtered XML to " << cfg.writeOsm << " ...";
BBoxIdx box(BOX_PADDING);
for (size_t i = 0; i < cfg.feedPaths.size(); i++) {
@ -200,20 +198,23 @@ int main(int argc, char** argv) {
}
osmBuilder.overpassQryWrite(&std::cout, opts, box);
exit(static_cast<int>(RetCode::SUCCESS));
} else if (!cfg.feedPaths.size()) {
} else if (cfg.feedPaths.empty()) {
std::cout << "No input feed specified, see --help" << std::endl;
exit(static_cast<int>(RetCode::NO_INPUT_FEED));
}
std::vector<double> dfBins;
auto dfBinStrings = util::split(std::string(cfg.evalDfBins), ',');
for (auto st : dfBinStrings) dfBins.push_back(atof(st.c_str()));
std::vector<double> dfBins(dfBinStrings.size());
for (const auto& st : dfBinStrings) {
dfBins.push_back(atof(st.c_str()));
}
Collector ecoll(cfg.evalPath, dfBins);
for (const auto& motCfg : motCfgReader.getConfigs()) {
std::string filePost;
auto usedMots = pfaedle::router::motISect(motCfg.mots, cmdCfgMots);
if (!usedMots.size()) continue;
if (usedMots.empty())
continue;
if (singleTrip && !usedMots.count(singleTrip->getRoute()->getType()))
continue;
if (motCfgReader.getConfigs().size() > 1)
@ -234,9 +235,10 @@ int main(int argc, char** argv) {
ShapeBuilder::getGtfsBox(&gtfs[0], cmdCfgMots, cfg.shapeTripId,
cfg.dropShapes, &box);
if (fStops.size())
if (!fStops.empty()) {
osmBuilder.read(cfg.osmPath, motCfg.osmBuildOpts, &graph, box,
cfg.gridSize, &fStops, &restr);
}
// TODO(patrick): move this somewhere else
for (auto& feedStop : fStops) {
@ -255,7 +257,7 @@ int main(int argc, char** argv) {
if (cfg.writeGraph) {
LOG(INFO) << "Outputting graph.json...";
util::geo::output::GeoGraphJsonOutput out;
mkdir(cfg.dbgOutputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
mkdir(cfg.dbgOutputPath.c_str(), 775);
std::ofstream fstr(cfg.dbgOutputPath + "/graph.json");
out.printLatLng(*shapeBuilder.getGraph(), fstr);
fstr.close();
@ -263,7 +265,7 @@ int main(int argc, char** argv) {
if (singleTrip) {
LOG(INFO) << "Outputting path.json...";
mkdir(cfg.dbgOutputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
mkdir(cfg.dbgOutputPath.c_str(), 775);
std::ofstream pstr(cfg.dbgOutputPath + "/path.json");
util::geo::output::GeoJsonOutput o(pstr);
@ -284,7 +286,7 @@ int main(int argc, char** argv) {
if (cfg.buildTransitGraph) {
util::geo::output::GeoGraphJsonOutput out;
LOG(INFO) << "Outputting trgraph" + filePost + ".json...";
mkdir(cfg.dbgOutputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
mkdir(cfg.dbgOutputPath.c_str(), 775);
std::ofstream fstr(cfg.dbgOutputPath + "/trgraph" + filePost + ".json");
out.printLatLng(ng, fstr);
fstr.close();
@ -296,11 +298,12 @@ int main(int argc, char** argv) {
}
}
if (cfg.evaluate) ecoll.printStats(&std::cout);
if (cfg.evaluate)
ecoll.printStats(&std::cout);
if (cfg.feedPaths.size()) {
if (!cfg.feedPaths.empty()) {
try {
mkdir(cfg.outputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
mkdir(cfg.outputPath.c_str(), 775);
LOG(INFO) << "Writing output GTFS to " << cfg.outputPath << " ...";
pfaedle::gtfs::Writer w;
w.write(&gtfs[0], cfg.outputPath);
@ -326,7 +329,9 @@ std::string getFileNameMotStr(const MOTs& mots) {
// _____________________________________________________________________________
std::vector<std::string> getCfgPaths(const Config& cfg) {
if (cfg.configPaths.size()) return cfg.configPaths;
if (!cfg.configPaths.empty()) {
return cfg.configPaths;
}
std::vector<std::string> ret;

10
src/pfaedle/config/ConfigReader.cpp
View file

@ -19,8 +19,11 @@ using std::string;
using std::exception;
using std::vector;
static const char* YEAR = __DATE__ + 7;
static const char* COPY =
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wstring-plus-int"
static auto YEAR = __DATE__ + 7;
#pragma clang diagnostic pop
static auto COPY =
"University of Freiburg - Chair of Algorithms and Data Structures";
static const char* AUTHORS = "Patrick Brosi <brosi@informatik.uni-freiburg.de>";
@ -148,8 +151,7 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
{0, 0, 0, 0}};
char c;
while ((c = getopt_long(argc, argv, ":o:hvi:c:x:Dm:g:X:T:d:p", ops, 0)) !=
-1) {
while ((c = getopt_long(argc, argv, ":o:hvi:c:x:Dm:g:X:T:d:p", ops, nullptr)) != -1) {
switch (c) {
case 1:
cfg->writeGraph = true;

5
src/pfaedle/osm/Osm.h
View file

@ -57,6 +57,11 @@ struct OsmNode {
struct Restriction {
osmid eFrom, eTo;
Restriction() = default;
Restriction(osmid from, osmid to):
eFrom(from),
eTo(to)
{}
};
typedef std::unordered_map<osmid, std::vector<Restriction>> RestrMap;

737
src/pfaedle/osm/OsmBuilder.cpp
View file

@ -2,7 +2,7 @@
// Chair of Algorithms and Data Structures.
// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
#include <float.h>
#include <cfloat>
#include <algorithm>
#include <exception>
#include <iostream>
@ -12,6 +12,13 @@
#include <string>
#include <utility>
#include <vector>
#include <osmium/io/any_input.hpp>
#include <osmium/util/file.hpp>
#include <osmium/geom/haversine.hpp>
#include <osmium/visitor.hpp>
#include <osmium/index/map/flex_mem.hpp>
#include "pfaedle/Def.h"
#include "pfaedle/osm/BBoxIdx.h"
#include "pfaedle/osm/Osm.h"
@ -47,18 +54,655 @@ using pfaedle::osm::EqSearch;
using pfaedle::osm::BlockSearch;
using ad::cppgtfs::gtfs::Stop;
class NodeHandler : public osmium::handler::Handler {
const pfaedle::osm::OsmFilter& filter;
const pfaedle::osm::BBoxIdx& bbox;
pfaedle::osm::OsmIdSet& bboxNodes;
pfaedle::osm::OsmIdSet& noHupNodes;
public:
NodeHandler(const pfaedle::osm::OsmFilter& filter,
const pfaedle::osm::BBoxIdx& bbox,
pfaedle::osm::OsmIdSet& bboxNodes,
pfaedle::osm::OsmIdSet& noHupNodes) :
filter(filter),
bbox(bbox),
bboxNodes(bboxNodes),
noHupNodes(noHupNodes)
{
}
void node(const osmium::Node& node) {
bool ignored = false;
for(const auto& tag: node.tags()) {
if (filter.nohup(tag.key(), tag.value())) {
noHupNodes.add(node.id());
ignored = true;
break;
}
}
if(!ignored) {
Point<double> point(node.location().lon(), node.location().lat());
if (bbox.contains(point)) {
bboxNodes.add(node.id());
}
}
}
};
class RelationHandler: public osmium::handler::Handler {
const pfaedle::osm::OsmFilter& filter;
const pfaedle::osm::BBoxIdx& bbox;
const pfaedle::osm::AttrKeySet& keepAttrs;
pfaedle::osm::RelLst& rels;
pfaedle::osm::RelMap& nodeRels;
pfaedle::osm::RelMap& wayRels;
pfaedle::osm::Restrictions& restrictions;
public:
RelationHandler(const pfaedle::osm::OsmFilter& filter,
const pfaedle::osm::BBoxIdx& bbox,
const pfaedle::osm::AttrKeySet& keepAttrs,
pfaedle::osm::RelLst& rels,
pfaedle::osm::RelMap& nodeRels,
pfaedle::osm::RelMap& wayRels,
pfaedle::osm::Restrictions& restrictions) :
filter(filter),
bbox(bbox),
keepAttrs(keepAttrs),
rels(rels),
nodeRels(nodeRels),
wayRels(wayRels),
restrictions(restrictions){
}
void relation(const osmium::Relation &relation) {
OsmRel rel;
rel.id = relation.id();
if(rel.id == 0) return;
for(const auto& tag: relation.tags()) {
if (keepAttrs.count(tag.key())) {
rel.attrs[tag.key()] = tag.value();
}
}
for(const auto& member: relation.members()) {
auto& obj = member.get_object();
if (member.type() == osmium::item_type::node) {
rel.nodes.push_back(obj.id());
rel.nodeRoles.emplace_back(member.role());
} else if (member.type() == osmium::item_type::way) {
rel.ways.push_back(obj.id());
rel.wayRoles.emplace_back(member.role());
}
}
for (auto id : rel.nodes) {
nodeRels[id].push_back(rels.rels.size() - 1);
}
for (auto id : rel.ways) {
wayRels[id].push_back(rels.rels.size() - 1);
}
uint64_t keepFlags = filter.keep(rel.attrs, pfaedle::osm::OsmFilter::REL);
uint64_t dropFlags = filter.drop(rel.attrs, pfaedle::osm::OsmFilter::REL);
if (rel.id && !rel.attrs.empty() && keepFlags && !dropFlags) {
rel.keepFlags = keepFlags;
rel.dropFlags = dropFlags;
}
rels.rels.push_back(rel.attrs);
if (rel.keepFlags & pfaedle::osm::REL_NO_DOWN) {
rels.flat.insert(rels.rels.size() - 1);
}
{
if (!rel.attrs.count("type")) return;
if (rel.attrs.find("type")->second != "restriction") return;
bool pos = filter.posRestr(rel.attrs);
bool neg = filter.negRestr(rel.attrs);
if (!pos && !neg) return;
uint64_t from = 0;
uint64_t to = 0;
uint64_t via = 0;
for (size_t i = 0; i < rel.ways.size(); i++) {
if (rel.wayRoles[i] == "from") {
if (from) return; // only one from member supported
from = rel.ways[i];
}
if (rel.wayRoles[i] == "to") {
if (to) return; // only one to member supported
to = rel.ways[i];
}
}
for (size_t i = 0; i < rel.nodes.size(); i++) {
if (rel.nodeRoles[i] == "via") {
via = rel.nodes[i];
break;
}
}
if (from && to && via) {
if (pos) {
restrictions.pos[via].emplace_back(from, to);
} else if (neg) {
restrictions.neg[via].emplace_back(from, to);
}
}
}
}
};
class WayHandler: public osmium::handler::Handler {
Graph &g;
const pfaedle::osm::RelLst &rels;
const pfaedle::osm::RelMap &wayRels;
const pfaedle::osm::OsmFilter &filter;
const pfaedle::osm::OsmIdSet &bBoxNodes;
pfaedle::osm::NIdMap &nodes;
pfaedle::osm::NIdMultMap &multiNodes;
const pfaedle::osm::OsmIdSet &noHupNodes;
const pfaedle::osm::AttrKeySet &keepAttrs;
const pfaedle::osm::Restrictions &rawRests;
pfaedle::osm::Restrictor &restor;
const pfaedle::osm::FlatRels &fl;
pfaedle::osm::EdgTracks &eTracks;
const pfaedle::osm::OsmReadOpts &opts;
std::map<TransitEdgeLine, TransitEdgeLine *> _lines;
std::map<size_t, TransitEdgeLine *> _relLines;
public:
WayHandler(Graph &g,
const pfaedle::osm::RelLst &rels,
const pfaedle::osm::RelMap &wayRels,
const pfaedle::osm::OsmFilter &filter,
const pfaedle::osm::OsmIdSet &bBoxNodes,
pfaedle::osm::NIdMap &nodes,
pfaedle::osm::NIdMultMap &multiNodes,
const pfaedle::osm::OsmIdSet &noHupNodes,
const pfaedle::osm::AttrKeySet &keepAttrs,
const pfaedle::osm::Restrictions &rawRests,
pfaedle::osm::Restrictor &restor,
const pfaedle::osm::FlatRels &fl,
pfaedle::osm::EdgTracks &eTracks,
const pfaedle::osm::OsmReadOpts &opts) :
g(g),
rels(rels),
wayRels(wayRels),
filter(filter),
bBoxNodes(bBoxNodes),
nodes(nodes),
multiNodes(multiNodes),
noHupNodes(noHupNodes),
keepAttrs(keepAttrs),
rawRests(rawRests),
restor(restor),
fl(fl),
eTracks(eTracks),
opts(opts) {
}
void way(const osmium::Way &way) {
OsmWay w;
w.id = way.id();
for (const auto &node: way.nodes()) {
w.nodes.emplace_back(node.ref());
}
for (const auto &tag: way.tags()) {
if (keepAttrs.count(tag.key())) {
w.attrs[tag.key()] = tag.value();
}
}
bool valid = false;
if (w.id && w.nodes.size() > 1 &&
(relKeep(w.id, wayRels, fl) || filter.keep(w.attrs, pfaedle::osm::OsmFilter::WAY)) &&
!filter.drop(w.attrs, pfaedle::osm::OsmFilter::WAY)) {
for (auto nid : w.nodes) {
if (bBoxNodes.has(nid)) {
valid = true;
break;
}
}
}
if (valid) {
Node *last = nullptr;
std::vector<TransitEdgeLine *> lines;
if (wayRels.count(w.id)) {
lines = getLines(wayRels.find(w.id)->second, rels, opts);
}
std::string track =
getAttrByFirstMatch(opts.edgePlatformRules, w.id, w.attrs, wayRels,
rels, opts.trackNormzer);
uint64_t lastnid = 0;
for (auto nid : w.nodes) {
Node *n = nullptr;
if (noHupNodes.has(nid)) {
n = g.addNd();
multiNodes[nid].insert(n);
} else if (!nodes.count(nid)) {
if (!bBoxNodes.has(nid)) {
continue;
}
n = g.addNd();
nodes[nid] = n;
} else {
n = nodes[nid];
}
if (last) {
auto e = g.addEdg(last, n, EdgePL());
if (!e)
continue;
processRestr(nid, w.id, rawRests, e, n, &restor);
processRestr(lastnid, w.id, rawRests, e, last, &restor);
e->pl().addLines(lines);
e->pl().setLvl(filter.level(w.attrs));
if (!track.empty()) {
eTracks[e] = track;
}
if (filter.oneway(w.attrs)) e->pl().setOneWay(1);
if (filter.onewayrev(w.attrs)) e->pl().setOneWay(2);
}
lastnid = nid;
last = n;
}
}
}
static bool relKeep(uint64_t id, const pfaedle::osm::RelMap &rels,
const pfaedle::osm::FlatRels &fl) {
auto it = rels.find(id);
if (it == rels.end()) return false;
for (auto relId : it->second) {
// as soon as any of this entities relations is not flat, return true
if (!fl.count(relId)) return true;
}
return false;
}
std::vector<TransitEdgeLine *> getLines(
const std::vector<size_t> &edgeRels, const pfaedle::osm::RelLst &rels,
const pfaedle::osm::OsmReadOpts &ops) {
std::vector<TransitEdgeLine *> ret;
for (size_t relId : edgeRels) {
TransitEdgeLine *elp = nullptr;
if (_relLines.count(relId)) {
elp = _relLines[relId];
} else {
TransitEdgeLine el;
bool found = false;
for (const auto &r : ops.relLinerules.sNameRule) {
for (const auto &relAttr : rels.rels[relId]) {
if (relAttr.first == r) {
el.shortName = ops.lineNormzer(pfxml::file::decode(relAttr.second));
if (!el.shortName.empty()) found = true;
}
}
if (found) break;
}
found = false;
for (const auto &r : ops.relLinerules.fromNameRule) {
for (const auto &relAttr : rels.rels[relId]) {
if (relAttr.first == r) {
el.fromStr = ops.statNormzer(pfxml::file::decode(relAttr.second));
if (!el.fromStr.empty()) found = true;
}
}
if (found) break;
}
found = false;
for (const auto &r : ops.relLinerules.toNameRule) {
for (const auto &relAttr : rels.rels[relId]) {
if (relAttr.first == r) {
el.toStr = ops.statNormzer(pfxml::file::decode(relAttr.second));
if (!el.toStr.empty()) found = true;
}
}
if (found) break;
}
if (!el.shortName.size() && !el.fromStr.size() && !el.toStr.size())
continue;
if (_lines.count(el)) {
elp = _lines[el];
_relLines[relId] = elp;
} else {
elp = new TransitEdgeLine(el);
_lines[el] = elp;
_relLines[relId] = elp;
}
}
ret.push_back(elp);
}
return ret;
}
static void processRestr(uint64_t nid, uint64_t wid,
const pfaedle::osm::Restrictions &rawRests,
Edge *e,
Node *n,
pfaedle::osm::Restrictor *restor) {
if (rawRests.pos.count(nid)) {
for (const auto &kv : rawRests.pos.find(nid)->second) {
if (kv.eFrom == wid) {
e->pl().setRestricted();
restor->add(e, kv.eTo, n, true);
} else if (kv.eTo == wid) {
e->pl().setRestricted();
restor->relax(wid, n, e);
}
}
}
if (rawRests.neg.count(nid)) {
for (const auto &kv : rawRests.neg.find(nid)->second) {
if (kv.eFrom == wid) {
e->pl().setRestricted();
restor->add(e, kv.eTo, n, false);
} else if (kv.eTo == wid) {
e->pl().setRestricted();
restor->relax(wid, n, e);
}
}
}
}
static std::string getAttr(const pfaedle::osm::DeepAttrRule &s, uint64_t id,
const pfaedle::osm::AttrMap &attrs,
const pfaedle::osm::RelMap &entRels,
const pfaedle::osm::RelLst &rels) {
if (s.relRule.kv.first.empty()) {
if (attrs.find(s.attr) != attrs.end()) {
return attrs.find(s.attr)->second;
}
} else {
if (entRels.count(id)) {
for (const auto &relId : entRels.find(id)->second) {
if (pfaedle::osm::OsmFilter::contained(rels.rels[relId], s.relRule.kv)) {
if (rels.rels[relId].count(s.attr)) {
return rels.rels[relId].find(s.attr)->second;
}
}
}
}
}
return "";
}
static std::string getAttrByFirstMatch(const pfaedle::osm::DeepAttrLst &rule, uint64_t id,
const pfaedle::osm::AttrMap &attrs,
const pfaedle::osm::RelMap &entRels,
const pfaedle::osm::RelLst &rels,
const Normalizer &norm) {
std::string ret;
for (const auto &s : rule) {
ret = norm(pfxml::file::decode(getAttr(s, id, attrs, entRels, rels)));
if (!ret.empty()) return ret;
}
return ret;
}
};
class NodeHandler2: public osmium::handler::Handler {
Graph &g;
const pfaedle::osm::RelLst &rels;
const pfaedle::osm::RelMap &nodeRels;
const pfaedle::osm::OsmFilter &filter;
const pfaedle::osm::OsmIdSet &bBoxNodes;
pfaedle::osm::NIdMap &nodes;
pfaedle::osm::NIdMultMap &multNodes;
pfaedle::osm::NodeSet &orphanStations;
const pfaedle::osm::AttrKeySet &keepAttrs;
const pfaedle::osm::FlatRels &fl;
const pfaedle::osm::OsmReadOpts &opts;
public:
NodeHandler2(Graph &g,
const pfaedle::osm::RelLst &rels,
const pfaedle::osm::RelMap &nodeRels,
const pfaedle::osm::OsmFilter &filter,
const pfaedle::osm::OsmIdSet &bBoxNodes,
pfaedle::osm::NIdMap &nodes,
pfaedle::osm::NIdMultMap &multNodes,
pfaedle::osm::NodeSet &orphanStations,
const pfaedle::osm::AttrKeySet &keepAttrs,
const pfaedle::osm::FlatRels &fl,
const pfaedle::osm::OsmReadOpts &opts) :
g(g),
rels(rels),
nodeRels(nodeRels),
filter(filter),
bBoxNodes(bBoxNodes),
nodes(nodes),
multNodes(multNodes),
orphanStations(orphanStations),
keepAttrs(keepAttrs),
fl(fl),
opts(opts) {
}
void node(const osmium::Node& node) {
OsmNode nd;
for(const auto& tag: node.tags())
{
if (keepAttrs.count(tag.key()))
nd.attrs[tag.key()] = tag.value();
}
nd.lat = node.location().lat();
nd.lng = node.location().lon();
nd.id = node.id();
bool valid = false;
if (nd.id &&
(nodes.count(nd.id) || multNodes.count(nd.id) ||
relKeep(nd.id, nodeRels, fl) || filter.keep(nd.attrs, pfaedle::osm::OsmFilter::NODE)) &&
(nodes.count(nd.id) || bBoxNodes.has(nd.id)) &&
(nodes.count(nd.id) || multNodes.count(nd.id) ||
!filter.drop(nd.attrs, pfaedle::osm::OsmFilter::NODE))) {
valid = true;
}
pfaedle::osm::StAttrGroups attrGroups;
if(valid)
{
Node* n = nullptr;
auto pos = util::geo::latLngToWebMerc<PFAEDLE_PRECISION>(nd.lat, nd.lng);
if (nodes.count(nd.id)) {
n = (nodes)[nd.id];
n->pl().setGeom(pos);
if (filter.station(nd.attrs)) {
auto si = getStatInfo(n, nd.id, pos, nd.attrs, &attrGroups, nodeRels,
rels, opts);
if (!si.isNull()) n->pl().setSI(si);
} else if (filter.blocker(nd.attrs)) {
n->pl().setBlocker();
}
} else if ((multNodes).count(nd.id)) {
for (auto* n : (multNodes)[nd.id]) {
n->pl().setGeom(pos);
if (filter.station(nd.attrs)) {
auto si = getStatInfo(n, nd.id, pos, nd.attrs, &attrGroups, nodeRels,
rels, opts);
if (!si.isNull()) n->pl().setSI(si);
} else if (filter.blocker(nd.attrs)) {
n->pl().setBlocker();
}
}
} else {
// these are nodes without any connected edges
if (filter.station(nd.attrs)) {
auto tmp = g.addNd(NodePL(pos));
auto si = getStatInfo(tmp, nd.id, pos, nd.attrs, &attrGroups, nodeRels,
rels, opts);
if (!si.isNull()) tmp->pl().setSI(si);
if (tmp->pl().getSI()) {
tmp->pl().getSI()->setIsFromOsm(false);
orphanStations.insert(tmp);
}
}
}
}
}
bool relKeep(uint64_t id, const pfaedle::osm::RelMap& rels, const pfaedle::osm::FlatRels& fl) const {
auto it = rels.find(id);
if (it == rels.end()) return false;
for (uint64_t relId : it->second) {
// as soon as any of this entities relations is not flat, return true
if (!fl.count(relId)) return true;
}
return false;
}
Nullable<StatInfo> getStatInfo(Node* node, uint64_t nid,
const POINT& pos, const pfaedle::osm::AttrMap& m,
pfaedle::osm::StAttrGroups* groups,
const pfaedle::osm::RelMap& nodeRels,
const pfaedle::osm::RelLst& rels,
const pfaedle::osm::OsmReadOpts& ops) const {
std::string platform;
std::vector<std::string> names;
names = getAttrMatchRanked(ops.statAttrRules.nameRule, nid, m, nodeRels, rels,
ops.statNormzer);
platform = getAttrByFirstMatch(ops.statAttrRules.platformRule, nid, m,
nodeRels, rels, ops.trackNormzer);
if (!names.size()) return Nullable<StatInfo>();
auto ret = StatInfo(names[0], platform, true);
#ifdef PFAEDLE_STATION_IDS
ret.setId(getAttrByFirstMatch(ops.statAttrRules.idRule, nid, m, nodeRels,
rels, ops.idNormzer));
#endif
for (size_t i = 1; i < names.size(); i++) ret.addAltName(names[i]);
bool groupFound = false;
for (const auto& rule : ops.statGroupNAttrRules) {
if (groupFound) break;
std::string ruleVal = getAttr(rule.attr, nid, m, nodeRels, rels);
if (!ruleVal.empty()) {
// check if a matching group exists
for (auto* group : (*groups)[rule.attr.attr][ruleVal]) {
if (groupFound) break;
for (const auto* member : group->getNodes()) {
if (webMercMeterDist(*member->pl().getGeom(), pos) <= rule.maxDist) {
// ok, group is matching
groupFound = true;
if (node) group->addNode(node);
ret.setGroup(group);
break;
}
}
}
}
}
if (!groupFound) {
for (const auto& rule : ops.statGroupNAttrRules) {
std::string ruleVal = getAttr(rule.attr, nid, m, nodeRels, rels);
if (!ruleVal.empty()) {
// add new group
auto* g = new StatGroup();
if (node) g->addNode(node);
ret.setGroup(g);
(*groups)[rule.attr.attr][ruleVal].push_back(g);
break;
}
}
}
return ret;
}
std::string getAttrByFirstMatch(const pfaedle::osm::DeepAttrLst& rule, uint64_t id,
const pfaedle::osm::AttrMap& attrs,
const pfaedle::osm::RelMap& entRels,
const pfaedle::osm::RelLst& rels,
const Normalizer& norm) const {
std::string ret;
for (const auto& s : rule) {
ret = norm(pfxml::file::decode(getAttr(s, id, attrs, entRels, rels)));
if (!ret.empty()) return ret;
}
return ret;
}
std::vector<std::string> getAttrMatchRanked(
const pfaedle::osm::DeepAttrLst& rule, uint64_t id, const pfaedle::osm::AttrMap& attrs,
const pfaedle::osm::RelMap& entRels, const pfaedle::osm::RelLst& rels, const Normalizer& norm) const {
std::vector<std::string> ret;
for (const auto& s : rule) {
std::string tmp =
norm(pfxml::file::decode(getAttr(s, id, attrs, entRels, rels)));
if (!tmp.empty()) ret.push_back(tmp);
}
return ret;
}
std::string getAttr(const pfaedle::osm::DeepAttrRule& s, uint64_t id,
const pfaedle::osm::AttrMap& attrs, const pfaedle::osm::RelMap& entRels,
const pfaedle::osm::RelLst& rels) const {
if (s.relRule.kv.first.empty()) {
if (attrs.find(s.attr) != attrs.end()) {
return attrs.find(s.attr)->second;
}
} else {
if (entRels.count(id)) {
for (const auto& relId : entRels.find(id)->second) {
if (pfaedle::osm::OsmFilter::contained(rels.rels[relId], s.relRule.kv)) {
if (rels.rels[relId].count(s.attr)) {
return rels.rels[relId].find(s.attr)->second;
}
}
}
}
}
return "";
}
};
// _____________________________________________________________________________
bool EqSearch::operator()(const Node* cand, const StatInfo* si) const {
if (orphanSnap && cand->pl().getSI() &&
(!cand->pl().getSI()->getGroup() ||
cand->pl().getSI()->getGroup()->getStops().size() == 0)) {
(!cand->pl().getSI()->getGroup() || cand->pl().getSI()->getGroup()->getStops().empty())) {
return true;
}
return cand->pl().getSI() && cand->pl().getSI()->simi(si) > minSimi;
}
// _____________________________________________________________________________
OsmBuilder::OsmBuilder() {}
OsmBuilder::OsmBuilder() = default;
// _____________________________________________________________________________
void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
@ -85,7 +729,20 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
OsmFilter filter(opts);
pfxml::file xml(path);
osmium::io::Reader reader{path, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way |osmium::osm_entity_bits::relation};
NodeHandler nodeHandler(filter, bbox, bboxNodes, noHupNodes);
RelationHandler relationHandler(filter, bbox, attrKeys[2],intmRels, nodeRels, wayRels, rawRests);
WayHandler wayHandler(*g, intmRels, wayRels, filter, bboxNodes, nodes, multNodes,
noHupNodes, attrKeys[1], rawRests, *res,
intmRels.flat, eTracks, opts);
osmium::apply(reader, nodeHandler, relationHandler, wayHandler);
osmium::io::Reader reader_nodes{path, osmium::osm_entity_bits::node};
NodeHandler2 nodeHandler2(*g, intmRels, nodeRels, filter, bboxNodes, nodes,
multNodes, orphanStations, attrKeys[0], intmRels.flat, opts);
osmium::apply(reader_nodes, nodeHandler2);
// we do four passes of the file here to be as memory creedy as possible:
// - the first pass collects all node IDs which are
@ -100,28 +757,6 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
// * collected as node ids in pass 1
// * match the filter criteria
// * have been used in a way in pass 3
LOG(VDEBUG) << "Reading bounding box nodes...";
skipUntil(&xml, "node");
pfxml::parser_state nodeBeg = xml.state();
pfxml::parser_state edgesBeg =
readBBoxNds(&xml, &bboxNodes, &noHupNodes, filter, bbox);
LOG(VDEBUG) << "Reading relations...";
skipUntil(&xml, "relation");
readRels(&xml, &intmRels, &nodeRels, &wayRels, filter, attrKeys[2],
&rawRests);
LOG(VDEBUG) << "Reading edges...";
xml.set_state(edgesBeg);
readEdges(&xml, g, intmRels, wayRels, filter, bboxNodes, &nodes, &multNodes,
noHupNodes, attrKeys[1], rawRests, res, intmRels.flat, &eTracks,
opts);
LOG(VDEBUG) << "Reading kept nodes...";
xml.set_state(nodeBeg);
readNodes(&xml, g, intmRels, nodeRels, filter, bboxNodes, &nodes,
&multNodes, &orphanStations, attrKeys[0], intmRels.flat, opts);
}
LOG(VDEBUG) << "OSM ID set lookups: " << osm::OsmIdSet::LOOKUPS
@ -361,7 +996,7 @@ void OsmBuilder::readWriteRels(pfxml::file* i, util::xml::XmlWriter* o,
}
}
if (realNodes.size() || realWays.size()) {
if (!realNodes.empty() || !realWays.empty()) {
o->openTag("relation", "id", std::to_string(rel.id));
for (size_t j = 0; j < realNodes.size(); j++) {
@ -425,8 +1060,7 @@ void OsmBuilder::readWriteWays(pfxml::file* i, util::xml::XmlWriter* o,
NodePL OsmBuilder::plFromGtfs(const Stop* s, const OsmReadOpts& ops) {
NodePL ret(
util::geo::latLngToWebMerc<PFAEDLE_PRECISION>(s->getLat(), s->getLng()),
StatInfo(ops.statNormzer(s->getName()),
ops.trackNormzer(s->getPlatformCode()), false));
StatInfo(ops.statNormzer(s->getName()), ops.trackNormzer(s->getPlatformCode()), false));
#ifdef PFAEDLE_STATION_IDS
// debug feature, store station id from GTFS
@ -464,7 +1098,8 @@ pfxml::parser_state OsmBuilder::readBBoxNds(pfxml::file* xml, OsmIdSet* nodes,
if (inNodeBlock) {
// block ended
if (strcmp(cur.name, "node")) return xml->state();
if (strcmp(cur.name, "node") != 0)
return xml->state();
double y = util::atof(cur.attrs.find("lat")->second, 7);
double x = util::atof(cur.attrs.find("lon")->second, 7);
@ -509,7 +1144,7 @@ OsmWay OsmBuilder::nextWayWithId(pfxml::file* xml, osmid wid,
// _____________________________________________________________________________
void OsmBuilder::skipUntil(pfxml::file* xml, const std::string& s) const {
while (xml->next() && strcmp(xml->get().name, s.c_str())) {
while (xml->next() && strcmp(xml->get().name, s.c_str()) != 0) {
}
}
@ -538,8 +1173,11 @@ OsmWay OsmBuilder::nextWay(pfxml::file* xml, const RelMap& wayRels,
do {
const pfxml::tag& cur = xml->get();
if (xml->level() == 2 || xml->level() == 0) {
if (keepWay(w, wayRels, filter, bBoxNodes, fl)) return w;
if (strcmp(cur.name, "way")) return OsmWay();
if (keepWay(w, wayRels, filter, bBoxNodes, fl))
return w;
if (strcmp(cur.name, "way") != 0)
return OsmWay();
w.id = util::atoul(cur.attrs.find("id")->second);
w.nodes.clear();
@ -551,13 +1189,16 @@ OsmWay OsmBuilder::nextWay(pfxml::file* xml, const RelMap& wayRels,
osmid nid = util::atoul(cur.attrs.find("ref")->second);
w.nodes.push_back(nid);
} else if (strcmp(cur.name, "tag") == 0) {
if (keepAttrs.count(cur.attrs.find("k")->second))
if (keepAttrs.count(cur.attrs.find("k")->second)) {
w.attrs[cur.attrs.find("k")->second] = cur.attrs.find("v")->second;
}
}
}
} while (xml->next());
if (keepWay(w, wayRels, filter, bBoxNodes, fl)) return w;
if (keepWay(w, wayRels, filter, bBoxNodes, fl))
return w;
return OsmWay();
}
@ -587,7 +1228,7 @@ void OsmBuilder::readEdges(pfxml::file* xml, const RelMap& wayRels,
while ((w = nextWay(xml, wayRels, filter, bBoxNodes, keepAttrs, flat)).id) {
ret->push_back(w.id);
for (auto n : w.nodes) {
(*nodes)[n] = 0;
(*nodes)[n] = nullptr;
}
}
}
@ -603,7 +1244,7 @@ void OsmBuilder::readEdges(pfxml::file* xml, Graph* g, const RelLst& rels,
const OsmReadOpts& opts) {
OsmWay w;
while ((w = nextWay(xml, wayRels, filter, bBoxNodes, keepAttrs, fl)).id) {
Node* last = 0;
Node* last = nullptr;
std::vector<TransitEdgeLine*> lines;
if (wayRels.count(w.id)) {
lines = getLines(wayRels.find(w.id)->second, rels, opts);
@ -614,7 +1255,7 @@ void OsmBuilder::readEdges(pfxml::file* xml, Graph* g, const RelLst& rels,
osmid lastnid = 0;
for (osmid nid : w.nodes) {
Node* n = 0;
Node* n = nullptr;
if (noHupNodes.has(nid)) {
n = g->addNd();
(*multiNodes)[nid].insert(n);
@ -808,7 +1449,7 @@ OsmRel OsmBuilder::nextRel(pfxml::file* xml, const OsmFilter& filter,
if (xml->level() == 2 || xml->level() == 0) {
uint64_t keepFlags = 0;
uint64_t dropFlags = 0;
if (rel.id && rel.attrs.size() &&
if (rel.id && !rel.attrs.empty() &&
(keepFlags = filter.keep(rel.attrs, OsmFilter::REL)) &&
!(dropFlags = filter.drop(rel.attrs, OsmFilter::REL))) {
rel.keepFlags = keepFlags;
@ -817,7 +1458,9 @@ OsmRel OsmBuilder::nextRel(pfxml::file* xml, const OsmFilter& filter,
}
// block ended
if (strcmp(cur.name, "relation")) return OsmRel();
if (strcmp(cur.name, "relation") != 0) {
return OsmRel();
}
rel.attrs.clear();
rel.nodes.clear();
@ -837,20 +1480,22 @@ OsmRel OsmBuilder::nextRel(pfxml::file* xml, const OsmFilter& filter,
// the bounding box!!!!
rel.nodes.push_back(id);
if (cur.attrs.count("role")) {
rel.nodeRoles.push_back(cur.attrs.find("role")->second);
rel.nodeRoles.emplace_back(cur.attrs.find("role")->second);
} else {
rel.nodeRoles.push_back("");
rel.nodeRoles.emplace_back("");
}
}
if (strcmp(cur.attrs.find("type")->second, "way") == 0) {
osmid id = util::atoul(cur.attrs.find("ref")->second);
rel.ways.push_back(id);
if (cur.attrs.count("role")) {
rel.wayRoles.push_back(cur.attrs.find("role")->second);
rel.wayRoles.emplace_back(cur.attrs.find("role")->second);
} else {
rel.wayRoles.push_back("");
rel.wayRoles.emplace_back("");
}
}
} else if (strcmp(cur.name, "tag") == 0) {
if (keepAttrs.count(cur.attrs.find("k")->second))
rel.attrs[cur.attrs.find("k")->second] = cur.attrs.find("v")->second;
@ -861,7 +1506,7 @@ OsmRel OsmBuilder::nextRel(pfxml::file* xml, const OsmFilter& filter,
// dont forget last relation
uint64_t keepFlags = 0;
uint64_t dropFlags = 0;
if (rel.id && rel.attrs.size() &&
if (rel.id && !rel.attrs.empty() &&
(keepFlags = filter.keep(rel.attrs, OsmFilter::REL)) &&
!(dropFlags = filter.drop(rel.attrs, OsmFilter::REL))) {
rel.keepFlags = keepFlags;

4
src/pfaedle/osm/OsmIdSet.cpp
View file

@ -23,7 +23,8 @@ size_t OsmIdSet::FLOOKUPS = 0;
// _____________________________________________________________________________
OsmIdSet::OsmIdSet()
: _closed(false),
: _element_size(0),
_closed(false),
_sorted(true),
_last(0),
_smallest(-1),
@ -50,6 +51,7 @@ void OsmIdSet::add(osmid id) {
if (_closed) throw std::exception();
diskAdd(id);
_element_size++;
if (_last > id) _sorted = false;
_last = id;
if (id < _smallest) _smallest = id;

5
src/pfaedle/osm/OsmIdSet.h
View file

@ -42,11 +42,16 @@ class OsmIdSet {
// Check if an OSM id is contained
bool has(osmid id) const;
size_t size() const {
return _element_size;
}
// Count the number of lookups and file lookups for debugging
static size_t LOOKUPS;
static size_t FLOOKUPS;
private:
mutable size_t _element_size;
std::string _tmpPath;
mutable bool _closed;
mutable int _file;

5
src/pfaedle/router/ShapeBuilder.cpp
View file

@ -15,6 +15,7 @@
#include <stdexcept>
#include <unordered_map>
#include <utility>
#include <random>
#include "ad/cppgtfs/gtfs/Feed.h"
#include "pfaedle/Def.h"
#include "pfaedle/eval/Collector.h"
@ -191,7 +192,9 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) {
}
// to avoid unfair load balance on threads
std::random_shuffle(clusters.begin(), clusters.end());
std::random_device rd;
std::mt19937 g(rd());
std::shuffle(clusters.begin(), clusters.end(), g);
size_t iters = EDijkstra::ITERS;
size_t totiters = EDijkstra::ITERS;

2
src/util/geo/output/GeoJsonOutput.cpp
View file

@ -17,7 +17,7 @@ GeoJsonOutput::GeoJsonOutput(std::ostream& str) : _wr(&str, 10, true) {
}
// _____________________________________________________________________________
GeoJsonOutput::GeoJsonOutput(std::ostream& str, json::Val attrs)
GeoJsonOutput::GeoJsonOutput(std::ostream& str, const json::Val& attrs)
: _wr(&str, 10, true) {
_wr.obj();
_wr.keyVal("type", "FeatureCollection");

2
src/util/geo/output/GeoJsonOutput.h
View file

@ -19,7 +19,7 @@ namespace output {
class GeoJsonOutput {
public:
GeoJsonOutput(std::ostream& str);
GeoJsonOutput(std::ostream& str, json::Val attrs);
GeoJsonOutput(std::ostream& str, const json::Val& attrs);
~GeoJsonOutput();
template <typename T>