From c25d174e60c4ae23d578af195ab921f2cdb875dd Mon Sep 17 00:00:00 2001 From: Patrick Brosi Date: Tue, 11 Jan 2022 17:32:50 +0100 Subject: [PATCH] add --osmfilter option --- src/pfaedle/PfaedleMain.cpp | 18 +++++++-- src/pfaedle/config/ConfigReader.cpp | 12 ++++-- src/pfaedle/config/MotConfigReader.cpp | 7 ++++ src/pfaedle/config/PfaedleConfig.h | 5 ++- src/pfaedle/osm/OsmBuilder.cpp | 44 ++++++++++++++++++++++ src/pfaedle/osm/OsmBuilder.h | 6 +++ src/pfaedle/router/Misc.h | 2 + src/pfaedle/router/ShapeBuilder.cpp | 23 +++++++++--- src/pfaedle/router/ShapeBuilder.h | 2 + src/pfaedle/router/Weights.cpp | 2 - src/shapevl/Collector.cpp | 51 +++++++++++++++++++++----- src/shapevl/Collector.h | 7 +++- src/shapevl/ShapevlMain.cpp | 35 ++++++++++++++++-- src/util/http/Server.cpp | 1 + 14 files changed, 185 insertions(+), 30 deletions(-) diff --git a/src/pfaedle/PfaedleMain.cpp b/src/pfaedle/PfaedleMain.cpp index 6b80cb1..4b4b872 100644 --- a/src/pfaedle/PfaedleMain.cpp +++ b/src/pfaedle/PfaedleMain.cpp @@ -109,7 +109,7 @@ int main(int argc, char** argv) { exit(static_cast(RetCode::MOT_CFG_PARSE_ERR)); } - if (cfg.osmPath.empty() && !cfg.writeOverpass) { + if (cfg.osmPath.empty() && !cfg.writeOverpass && !cfg.writeOsmfilter) { std::cerr << "No OSM input file specified (-x), see --help." << std::endl; exit(static_cast(RetCode::NO_OSM_INPUT)); } @@ -124,7 +124,7 @@ int main(int argc, char** argv) { if (cfg.feedPaths.size() == 1) { if (cfg.inPlace) cfg.outputPath = cfg.feedPaths[0]; - if (!cfg.writeOverpass) + if (!cfg.writeOverpass && !cfg.writeOsmfilter) LOG(INFO) << "Reading GTFS feed " << cfg.feedPaths[0] << " ..."; try { ad::cppgtfs::Parser p; @@ -136,7 +136,7 @@ int main(int argc, char** argv) { } } else if (cfg.writeOsm.size() || cfg.writeOverpass) { for (size_t i = 0; i < cfg.feedPaths.size(); i++) { - if (!cfg.writeOverpass) + if (!cfg.writeOverpass && !cfg.writeOsmfilter) LOG(INFO) << "Reading GTFS feed " << cfg.feedPaths[i] << " ..."; ad::cppgtfs::Parser p; try { @@ -218,6 +218,18 @@ int main(int argc, char** argv) { } osmBuilder.overpassQryWrite(&std::cout, opts, box); exit(static_cast(RetCode::SUCCESS)); + } else if (cfg.writeOsmfilter) { + BBoxIdx box(BOX_PADDING); + OsmBuilder osmBuilder; + std::vector opts; + for (const auto& o : motCfgReader.getConfigs()) { + if (std::find_first_of(o.mots.begin(), o.mots.end(), cmdCfgMots.begin(), + cmdCfgMots.end()) != o.mots.end()) { + opts.push_back(o.osmBuildOpts); + } + } + osmBuilder.osmfilterRuleWrite(&std::cout, opts, box); + exit(static_cast(RetCode::SUCCESS)); } else if (!cfg.feedPaths.size()) { std::cout << "No input feed specified, see --help" << std::endl; exit(static_cast(RetCode::NO_INPUT_FEED)); diff --git a/src/pfaedle/config/ConfigReader.cpp b/src/pfaedle/config/ConfigReader.cpp index 3075d46..1dbb34f 100644 --- a/src/pfaedle/config/ConfigReader.cpp +++ b/src/pfaedle/config/ConfigReader.cpp @@ -10,14 +10,15 @@ #include "pfaedle/Def.h" #include "pfaedle/_config.h" #include "pfaedle/config/ConfigReader.h" +#include "pfaedle/config/PfaedleConfig.h" #include "util/String.h" #include "util/geo/Geo.h" #include "util/log/Log.h" using pfaedle::config::ConfigReader; -using std::string; using std::exception; +using std::string; using std::vector; static const char* YEAR = &__DATE__[7]; @@ -32,8 +33,7 @@ void ConfigReader::help(const char* bin) { << " with geometry precision <" << PFDL_PREC_STR << ">)\n\n" << "(C) " << YEAR << " " << COPY << "\n" << "Authors: " << AUTHORS << "\n\n" - << "Usage: " << bin - << " -x \n\n" + << "Usage: " << bin << " -x \n\n" << "Allowed options:\n\n" << "General:\n" << std::setw(35) << " -v [ --version ]" @@ -93,6 +93,8 @@ void ConfigReader::help(const char* bin) { << " to /path.json\n" << std::setw(35) << " --overpass" << "Output overpass query for matching OSM data\n" + << std::setw(35) << " --osmfilter" + << "Output osmfilter filter rules for matching OSM data\n" << std::setw(35) << " --grid-size arg (=2000)" << "Approx. grid cell size in meters\n" << std::setw(35) << " --no-fast-hops" @@ -122,6 +124,7 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) { {"mots", required_argument, NULL, 'm'}, {"grid-size", required_argument, 0, 'g'}, {"overpass", no_argument, 0, 'a'}, + {"osmfilter", no_argument, 0, 'f'}, {"osm-out", required_argument, 0, 'X'}, {"trip-id", required_argument, 0, 'T'}, {"write-graph", no_argument, 0, 1}, @@ -194,6 +197,9 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) { case 'a': cfg->writeOverpass = true; break; + case 'f': + cfg->writeOsmfilter = true; + break; case 9: cfg->inPlace = true; break; diff --git a/src/pfaedle/config/MotConfigReader.cpp b/src/pfaedle/config/MotConfigReader.cpp index bf1b0cd..0db2cf2 100644 --- a/src/pfaedle/config/MotConfigReader.cpp +++ b/src/pfaedle/config/MotConfigReader.cpp @@ -47,6 +47,13 @@ void MotConfigReader::parse(const std::vector& paths, std::string secStr = sec.first; if (secStr.empty()) continue; + if (p.hasKey(secStr, "routing_emission_method")) { + cfg.routingOpts.emPenMethod = + p.getStr(secStr, "routing_emission_method"); + } else { + cfg.routingOpts.emPenMethod = "exp"; + } + if (p.hasKey(secStr, "routing_transition_method")) { cfg.routingOpts.transPenMethod = p.getStr(secStr, "routing_transition_method"); diff --git a/src/pfaedle/config/PfaedleConfig.h b/src/pfaedle/config/PfaedleConfig.h index b490ae7..8da2750 100644 --- a/src/pfaedle/config/PfaedleConfig.h +++ b/src/pfaedle/config/PfaedleConfig.h @@ -9,8 +9,8 @@ #include #include #include -#include "util/geo/Geo.h" #include "ad/cppgtfs/gtfs/Route.h" +#include "util/geo/Geo.h" namespace pfaedle { namespace config { @@ -28,6 +28,7 @@ struct Config { buildTransitGraph(false), useCaching(false), writeOverpass(false), + writeOsmfilter(false), inPlace(false), writeColors(false), noFastHops(false), @@ -53,6 +54,7 @@ struct Config { bool buildTransitGraph; bool useCaching; bool writeOverpass; + bool writeOsmfilter; bool inPlace; bool writeColors; bool noFastHops; @@ -76,6 +78,7 @@ struct Config { << "grid-size: " << gridSize << "\n" << "use-cache: " << useCaching << "\n" << "write-overpass: " << writeOverpass << "\n" + << "write-osmfilter: " << writeOsmfilter << "\n" << "inplace: " << inPlace << "\n" << "write-colors: " << writeColors << "\n" << "no-fast-hops: " << noFastHops << "\n" diff --git a/src/pfaedle/osm/OsmBuilder.cpp b/src/pfaedle/osm/OsmBuilder.cpp index 6b9e488..abc9530 100644 --- a/src/pfaedle/osm/OsmBuilder.cpp +++ b/src/pfaedle/osm/OsmBuilder.cpp @@ -183,6 +183,50 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts, LOG(DEBUG) << _lines.size() << " transit lines have been read."; } +// _____________________________________________________________________________ +void OsmBuilder::osmfilterRuleWrite(std::ostream* out, + const std::vector& opts, + const BBoxIdx& latLngBox) const { + UNUSED(latLngBox); + OsmIdSet bboxNodes, noHupNodes; + MultAttrMap emptyF; + + RelLst rels; + OsmIdList ways; + RelMap nodeRels, wayRels; + + NIdMap nodes; + + OsmFilter filter; + + AttrKeySet attrKeys[3] = {}; + + for (const OsmReadOpts& o : opts) { + filter = filter.merge(OsmFilter(o.keepFilter, o.dropFilter)); + getKeptAttrKeys(o, attrKeys); + } + + *out << "--keep=\n"; + + for (auto r : filter.getKeepRules()) { + for (auto val : r.second) { + *out << r.first << "="; + if (val.first != "*") *out << val.first; + *out << "\n"; + } + } + + *out << "\n"; + + *out << "--keep-tags=\n"; + + for (const auto& keys : attrKeys) { + for (auto val : keys) { + *out << val << "=\n"; + } + } +} + // _____________________________________________________________________________ void OsmBuilder::overpassQryWrite(std::ostream* out, const std::vector& opts, diff --git a/src/pfaedle/osm/OsmBuilder.h b/src/pfaedle/osm/OsmBuilder.h index 10029c4..9cf1505 100644 --- a/src/pfaedle/osm/OsmBuilder.h +++ b/src/pfaedle/osm/OsmBuilder.h @@ -92,6 +92,12 @@ class OsmBuilder { void overpassQryWrite(std::ostream* out, const std::vector& opts, const BBoxIdx& latLngBox) const; + // Based on the list of options, output an osmfilter configuration file + // to filter the data needed for routing + void osmfilterRuleWrite(std::ostream* out, + const std::vector& opts, + const BBoxIdx& latLngBox) const; + // Based on the list of options, read an OSM file from in and output an // OSM file to out which contains exactly the entities that are needed // from the file at in diff --git a/src/pfaedle/router/Misc.h b/src/pfaedle/router/Misc.h index 237c8b1..d96cb34 100644 --- a/src/pfaedle/router/Misc.h +++ b/src/pfaedle/router/Misc.h @@ -61,6 +61,7 @@ struct RoutingOpts { bool useStations; double transitionPen; std::string transPenMethod; + std::string emPenMethod; }; // _____________________________________________________________________________ @@ -80,6 +81,7 @@ inline bool operator==(const RoutingOpts& a, const RoutingOpts& b) { fabs(a.transitionPen - b.transitionPen) < 0.01 && fabs(a.nonStationPen - b.nonStationPen) < 0.01 && a.transPenMethod == b.transPenMethod && + a.emPenMethod == b.emPenMethod && a.useStations == b.useStations && a.popReachEdge == b.popReachEdge && a.noSelfHops == b.noSelfHops; } diff --git a/src/pfaedle/router/ShapeBuilder.cpp b/src/pfaedle/router/ShapeBuilder.cpp index 04ec816..abd1af5 100644 --- a/src/pfaedle/router/ShapeBuilder.cpp +++ b/src/pfaedle/router/ShapeBuilder.cpp @@ -206,7 +206,6 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const { // stations do not match, punish nameMatchPunish = _motCfg.routingOpts.stationUnmatchedPen; } - std::string platform = s->getPlatformCode(); if (!platform.empty() && !nd->pl().getSI()->getTrack().empty() && @@ -218,8 +217,7 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const { // don't snap to one way edges if (e->pl().oneWay() == 2) continue; ret.push_back({e, - mDist * _motCfg.routingOpts.stationDistPenFactor + - nameMatchPunish + trackMatchPunish, + emWeight(mDist) + nameMatchPunish + trackMatchPunish, 0, {}, 0, @@ -265,8 +263,7 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const { for (auto e : selected) { ret.push_back({e, - scores[e] * _motCfg.routingOpts.stationDistPenFactor + - _motCfg.routingOpts.nonStationPen, + emWeight(scores[e]) + _motCfg.routingOpts.nonStationPen, progrs[e], {}, 0, @@ -981,7 +978,7 @@ std::vector ShapeBuilder::getGeom( ret.push_back({hop.pointStart, *hop.end->getFrom()->pl().getGeom()}); } } else { - ret.push_back({hop.pointEnd, hop.pointStart}); + ret.push_back({hop.pointStart, hop.pointEnd}); } } else { const auto& l = getLine(hop, rAttrs, colors); @@ -1222,3 +1219,17 @@ uint32_t ShapeBuilder::getTextColor(uint32_t c) const { if (a < 140) return 0x00FFFFFF; return 0; } + +// _____________________________________________________________________________ +double ShapeBuilder::emWeight(double mDist) const { + if (_motCfg.routingOpts.emPenMethod == "exp") { + return mDist * _motCfg.routingOpts.stationDistPenFactor; + } + + if (_motCfg.routingOpts.emPenMethod == "norm") { + double s = mDist * _motCfg.routingOpts.stationDistPenFactor; + return 0.5 * s * s; + } + + return mDist; +} diff --git a/src/pfaedle/router/ShapeBuilder.h b/src/pfaedle/router/ShapeBuilder.h index 8087e84..fc176fc 100644 --- a/src/pfaedle/router/ShapeBuilder.h +++ b/src/pfaedle/router/ShapeBuilder.h @@ -119,6 +119,8 @@ class ShapeBuilder { std::map route(const TripTrie* trie, const EdgeCandMap& ecm, HopCache* hopCache) const; + double emWeight(double mDist) const; + void buildCandCache(const TripForests& clusters); void buildIndex(); diff --git a/src/pfaedle/router/Weights.cpp b/src/pfaedle/router/Weights.cpp index 70a3aef..750cad7 100644 --- a/src/pfaedle/router/Weights.cpp +++ b/src/pfaedle/router/Weights.cpp @@ -237,8 +237,6 @@ double DistDiffTransWeight::weight(uint32_t c, double d, double t0, double d0, const RoutingOpts& rOpts) { UNUSED(t0); UNUSED(c); - // double mean = 250; // expectation value of 250 meters for buses - // double lambda = 1.0 / mean; double w = fabs(d - d0); diff --git a/src/shapevl/Collector.cpp b/src/shapevl/Collector.cpp index 16f4401..d28fb7f 100644 --- a/src/shapevl/Collector.cpp +++ b/src/shapevl/Collector.cpp @@ -67,8 +67,15 @@ double Collector::add(const Trip* oldT, const Shape* oldS, const Trip* newT, std::vector newDists; LINE newL = getWebMercLine(newS, &newDists); - auto oldSegs = segmentize(oldT, oldL, oldDists); - auto newSegs = segmentize(newT, newL, newDists); + std::vector> newLenDists; + std::vector> oldLenDists; + + auto oldSegs = segmentize(oldT, oldL, oldDists, newLenDists); + auto newSegs = segmentize(newT, newL, newDists, oldLenDists); + + for (const auto& p : oldLenDists) { + _distDiffs.push_back(fabs(p.first - p.second)); + } // new lines build from cleaned-up shapes LINE oldLCut; @@ -77,8 +84,9 @@ double Collector::add(const Trip* oldT, const Shape* oldS, const Trip* newT, for (auto oldL : oldSegs) oldLCut.insert(oldLCut.end(), oldL.begin(), oldL.end()); - for (auto newL : newSegs) + for (auto newL : newSegs) { newLCut.insert(newLCut.end(), newL.begin(), newL.end()); + } // determine the scale factor between the distance in projected // coordinates and the real-world distance in meters @@ -150,8 +158,9 @@ double Collector::add(const Trip* oldT, const Shape* oldS, const Trip* newT, } // _____________________________________________________________________________ -std::vector Collector::segmentize(const Trip* t, const LINE& shape, - const std::vector& dists) { +std::vector Collector::segmentize( + const Trip* t, const LINE& shape, const std::vector& dists, + std::vector>& lenDist) { // The straightforward way to segmentize the shape would be to just cut it at // the exact measurements in stop_times.txt. We have tried that, but found // that it produces misleading results for the following reason: @@ -182,7 +191,7 @@ std::vector Collector::segmentize(const Trip* t, const LINE& shape, if (t->getStopTimes().size() < 2) return ret; POLYLINE pl(shape); - std::vector > cuts; + std::vector> cuts; size_t i = 0; for (auto st : t->getStopTimes()) { @@ -213,7 +222,17 @@ std::vector Collector::segmentize(const Trip* t, const LINE& shape, auto curLp = beforePl.projectOnAfter(cuts[i].first, lastLp.lastIndex); - ret.push_back(pl.getSegment(lastLp, curLp).getLine()); + auto curL = pl.getSegment(lastLp, curLp).getLine(); + + double dist = + util::geo::haversine(t->getStopTimes()[i - 1].getStop()->getLat(), + t->getStopTimes()[i - 1].getStop()->getLng(), + t->getStopTimes()[i].getStop()->getLat(), + t->getStopTimes()[i].getStop()->getLng()); + double len = util::geo::webMercLen(curL); + lenDist.push_back({dist, len}); + + ret.push_back(curL); lastLp = curLp; } @@ -365,13 +384,27 @@ void Collector::printStats(std::ostream* os) const { } // _____________________________________________________________________________ -util::json::Dict Collector::getJSONStats() const { - util::json::Dict stats = {}; +std::map Collector::getStats() { + std::map stats; + + if (_distDiffs.size()) { + auto i = _distDiffs.begin() + _distDiffs.size() / 2; + std::nth_element(_distDiffs.begin(), i, _distDiffs.end()); + + stats["median-dist-diff"] = *i; + } else { + stats["median-dist-diff"] = -1; + } stats["num-trips"] = _trips; stats["num-trips-matched"] = _results.size(); stats["num-trips-wo-shapes"] = _noOrigShp; stats["avg-fr"] = getAvgDist(); + if (_results.size()) { + stats["max-avg-frech-dist"] = (--_results.end())->getDist(); + } else { + stats["max-avg-frech-dist"] = -1; + } stats["an-0"] = (static_cast(_an0) / static_cast(_results.size())) * 100; stats["an-5"] = diff --git a/src/shapevl/Collector.h b/src/shapevl/Collector.h index ee17544..fe80007 100644 --- a/src/shapevl/Collector.h +++ b/src/shapevl/Collector.h @@ -59,7 +59,7 @@ class Collector { void printShortStats(std::ostream* os) const; // Get JSON stats - util::json::Dict getJSONStats() const; + std::map getStats(); // Print a CSV for the results to os void printCsv(std::ostream* os, const std::set& result) const; @@ -79,6 +79,8 @@ class Collector { size_t _trips; size_t _noOrigShp; + std::vector _distDiffs; + double _fdSum; size_t _unmatchedSegSum; double _unmatchedSegLengthSum; @@ -97,7 +99,8 @@ class Collector { const std::vector& b); static std::vector segmentize(const Trip* t, const LINE& shape, - const std::vector& dists); + const std::vector& dists, + std::vector>& lenDist); static std::vector getBins(double mind, double maxd, size_t steps); }; diff --git a/src/shapevl/ShapevlMain.cpp b/src/shapevl/ShapevlMain.cpp index c20b951..3f6815e 100644 --- a/src/shapevl/ShapevlMain.cpp +++ b/src/shapevl/ShapevlMain.cpp @@ -27,6 +27,7 @@ void printHelp(int argc, char** argv) { << "\nAllowed arguments:\n -g Ground truth GTFS file\n"; std::cout << " -s Only output summary\n"; std::cout << " --json Output JSON\n"; + std::cout << " --avg Take avg of all inputs (only for --json)\n"; std::cout << " -f Output full reports (per feed) to \n"; std::cout << " -m MOTs to match (GTFS MOT or string, default: all)\n"; @@ -87,6 +88,7 @@ int main(int argc, char** argv) { std::vector reportStreams; bool summarize = false; bool json = false; + bool avg = false; for (int i = 1; i < argc; i++) { std::string cur = argv[i]; @@ -103,6 +105,8 @@ int main(int argc, char** argv) { summarize = true; } else if (cur == "--json") { json = true; + } else if (cur == "--avg") { + avg = true; } else if (cur == "-f") { if (++i >= argc) { LOG(ERROR) << "Missing argument for full reports (-f)."; @@ -173,7 +177,11 @@ int main(int argc, char** argv) { util::json::Dict stats = {}; for (size_t i = 0; i < evalColls.size(); i++) { - stats[evlFeedPaths[i]] = evalColls[i].getJSONStats(); + util::json::Dict locStats = {}; + for (const auto& kv : evalColls[i].getStats()) { + locStats[kv.first] = kv.second; + } + stats[evlFeedPaths[i]] = locStats; } util::json::Dict jsonStats; @@ -183,9 +191,28 @@ int main(int argc, char** argv) { {"statistics", stats[evlFeedPaths[0]] }}; } else { - jsonStats = { - {"statistics", stats - }}; + if (avg) { + double count = evalColls.size(); + std::vector keys; + for (const auto& a : evalColls[0].getStats()) { + keys.push_back(a.first); + } + util::json::Dict avgStats; + for (const auto& k : keys) { + double sum = 0; + for (size_t i = 0; i < evalColls.size(); i++) { + sum += evalColls[i].getStats()[k]; + } + avgStats[k] = sum / count; + } + jsonStats = { + {"statistics", avgStats + }}; + } else { + jsonStats = { + {"statistics", stats + }}; + } } util::json::Writer wr(&std::cout, 10, true); diff --git a/src/util/http/Server.cpp b/src/util/http/Server.cpp index e124674..2742756 100644 --- a/src/util/http/Server.cpp +++ b/src/util/http/Server.cpp @@ -319,6 +319,7 @@ std::string HttpServer::compress(const std::string& str, std::string* enc) { *enc = "gzip"; return ret; #else + UNUSED(enc); return str; #endif }