add --osmfilter option

src/pfaedle/PfaedleMain.cpp

@@ -109,7 +109,7 @@ int main(int argc, char** argv) {
     exit(static_cast<int>(RetCode::MOT_CFG_PARSE_ERR));
   }
 
-  if (cfg.osmPath.empty() && !cfg.writeOverpass) {
+  if (cfg.osmPath.empty() && !cfg.writeOverpass && !cfg.writeOsmfilter) {
     std::cerr << "No OSM input file specified (-x), see --help." << std::endl;
     exit(static_cast<int>(RetCode::NO_OSM_INPUT));
   }
@@ -124,7 +124,7 @@ int main(int argc, char** argv) {
 
   if (cfg.feedPaths.size() == 1) {
     if (cfg.inPlace) cfg.outputPath = cfg.feedPaths[0];
-    if (!cfg.writeOverpass)
+    if (!cfg.writeOverpass && !cfg.writeOsmfilter)
       LOG(INFO) << "Reading GTFS feed " << cfg.feedPaths[0] << " ...";
     try {
       ad::cppgtfs::Parser p;
@@ -136,7 +136,7 @@ int main(int argc, char** argv) {
     }
   } else if (cfg.writeOsm.size() || cfg.writeOverpass) {
     for (size_t i = 0; i < cfg.feedPaths.size(); i++) {
-      if (!cfg.writeOverpass)
+      if (!cfg.writeOverpass && !cfg.writeOsmfilter)
         LOG(INFO) << "Reading GTFS feed " << cfg.feedPaths[i] << " ...";
       ad::cppgtfs::Parser p;
       try {
@@ -218,6 +218,18 @@ int main(int argc, char** argv) {
     }
     osmBuilder.overpassQryWrite(&std::cout, opts, box);
     exit(static_cast<int>(RetCode::SUCCESS));
+  } else if (cfg.writeOsmfilter) {
+    BBoxIdx box(BOX_PADDING);
+    OsmBuilder osmBuilder;
+    std::vector<pfaedle::osm::OsmReadOpts> opts;
+    for (const auto& o : motCfgReader.getConfigs()) {
+      if (std::find_first_of(o.mots.begin(), o.mots.end(), cmdCfgMots.begin(),
+                             cmdCfgMots.end()) != o.mots.end()) {
+        opts.push_back(o.osmBuildOpts);
+      }
+    }
+    osmBuilder.osmfilterRuleWrite(&std::cout, opts, box);
+    exit(static_cast<int>(RetCode::SUCCESS));
   } else if (!cfg.feedPaths.size()) {
     std::cout << "No input feed specified, see --help" << std::endl;
     exit(static_cast<int>(RetCode::NO_INPUT_FEED));

src/pfaedle/config/ConfigReader.cpp

@@ -10,14 +10,15 @@
 #include "pfaedle/Def.h"
 #include "pfaedle/_config.h"
 #include "pfaedle/config/ConfigReader.h"
+#include "pfaedle/config/PfaedleConfig.h"
 #include "util/String.h"
 #include "util/geo/Geo.h"
 #include "util/log/Log.h"
 
 using pfaedle::config::ConfigReader;
 
-using std::string;
 using std::exception;
+using std::string;
 using std::vector;
 
 static const char* YEAR = &__DATE__[7];
@@ -32,8 +33,7 @@ void ConfigReader::help(const char* bin) {
             << " with geometry precision <" << PFDL_PREC_STR << ">)\n\n"
             << "(C) " << YEAR << " " << COPY << "\n"
             << "Authors: " << AUTHORS << "\n\n"
-            << "Usage: " << bin
+            << "Usage: " << bin << " -x <OSM FILE> <GTFS FEED>\n\n"
-            << " -x <OSM FILE> <GTFS FEED>\n\n"
             << "Allowed options:\n\n"
             << "General:\n"
             << std::setw(35) << "  -v [ --version ]"
@@ -93,6 +93,8 @@ void ConfigReader::help(const char* bin) {
             << "  to <dbg-path>/path.json\n"
             << std::setw(35) << "  --overpass"
             << "Output overpass query for matching OSM data\n"
+            << std::setw(35) << "  --osmfilter"
+            << "Output osmfilter filter rules for matching OSM data\n"
             << std::setw(35) << "  --grid-size arg (=2000)"
             << "Approx. grid cell size in meters\n"
             << std::setw(35) << "  --no-fast-hops"
@@ -122,6 +124,7 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
                          {"mots", required_argument, NULL, 'm'},
                          {"grid-size", required_argument, 0, 'g'},
                          {"overpass", no_argument, 0, 'a'},
+                         {"osmfilter", no_argument, 0, 'f'},
                          {"osm-out", required_argument, 0, 'X'},
                          {"trip-id", required_argument, 0, 'T'},
                          {"write-graph", no_argument, 0, 1},
@@ -194,6 +197,9 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
       case 'a':
         cfg->writeOverpass = true;
         break;
+      case 'f':
+        cfg->writeOsmfilter = true;
+        break;
       case 9:
         cfg->inPlace = true;
         break;

src/pfaedle/config/MotConfigReader.cpp

@@ -47,6 +47,13 @@ void MotConfigReader::parse(const std::vector<std::string>& paths,
     std::string secStr = sec.first;
     if (secStr.empty()) continue;
 
+    if (p.hasKey(secStr, "routing_emission_method")) {
+      cfg.routingOpts.emPenMethod =
+          p.getStr(secStr, "routing_emission_method");
+    } else {
+      cfg.routingOpts.emPenMethod = "exp";
+    }
+
     if (p.hasKey(secStr, "routing_transition_method")) {
       cfg.routingOpts.transPenMethod =
           p.getStr(secStr, "routing_transition_method");

src/pfaedle/config/PfaedleConfig.h

@@ -9,8 +9,8 @@
 #include <sstream>
 #include <string>
 #include <vector>
-#include "util/geo/Geo.h"
 #include "ad/cppgtfs/gtfs/Route.h"
+#include "util/geo/Geo.h"
 
 namespace pfaedle {
 namespace config {
@@ -28,6 +28,7 @@ struct Config {
         buildTransitGraph(false),
         useCaching(false),
         writeOverpass(false),
+        writeOsmfilter(false),
         inPlace(false),
         writeColors(false),
         noFastHops(false),
@@ -53,6 +54,7 @@ struct Config {
   bool buildTransitGraph;
   bool useCaching;
   bool writeOverpass;
+  bool writeOsmfilter;
   bool inPlace;
   bool writeColors;
   bool noFastHops;
@@ -76,6 +78,7 @@ struct Config {
        << "grid-size: " << gridSize << "\n"
        << "use-cache: " << useCaching << "\n"
        << "write-overpass: " << writeOverpass << "\n"
+       << "write-osmfilter: " << writeOsmfilter << "\n"
        << "inplace: " << inPlace << "\n"
        << "write-colors: " << writeColors << "\n"
        << "no-fast-hops: " << noFastHops << "\n"

src/pfaedle/osm/OsmBuilder.cpp

@@ -183,6 +183,50 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
   LOG(DEBUG) << _lines.size() << " transit lines have been read.";
 }
 
+// _____________________________________________________________________________
+void OsmBuilder::osmfilterRuleWrite(std::ostream* out,
+                                  const std::vector<OsmReadOpts>& opts,
+                                  const BBoxIdx& latLngBox) const {
+  UNUSED(latLngBox);
+  OsmIdSet bboxNodes, noHupNodes;
+  MultAttrMap emptyF;
+
+  RelLst rels;
+  OsmIdList ways;
+  RelMap nodeRels, wayRels;
+
+  NIdMap nodes;
+
+  OsmFilter filter;
+
+  AttrKeySet attrKeys[3] = {};
+
+  for (const OsmReadOpts& o : opts) {
+    filter = filter.merge(OsmFilter(o.keepFilter, o.dropFilter));
+    getKeptAttrKeys(o, attrKeys);
+  }
+
+  *out << "--keep=\n";
+
+  for (auto r : filter.getKeepRules()) {
+    for (auto val : r.second) {
+      *out << r.first << "=";
+      if (val.first != "*") *out << val.first;
+      *out << "\n";
+    }
+  }
+
+  *out << "\n";
+
+  *out << "--keep-tags=\n";
+
+  for (const auto& keys : attrKeys) {
+    for (auto val : keys) {
+      *out << val << "=\n";
+    }
+  }
+}
+
 // _____________________________________________________________________________
 void OsmBuilder::overpassQryWrite(std::ostream* out,
                                   const std::vector<OsmReadOpts>& opts,

src/pfaedle/osm/OsmBuilder.h

@@ -92,6 +92,12 @@ class OsmBuilder {
   void overpassQryWrite(std::ostream* out, const std::vector<OsmReadOpts>& opts,
                         const BBoxIdx& latLngBox) const;
 
+  // Based on the list of options, output an osmfilter configuration file
+  // to filter the data needed for routing
+  void osmfilterRuleWrite(std::ostream* out,
+                          const std::vector<OsmReadOpts>& opts,
+                          const BBoxIdx& latLngBox) const;
+
   // Based on the list of options, read an OSM file from in and output an
   // OSM file to out which contains exactly the entities that are needed
   // from the file at in

src/pfaedle/router/Misc.h

@@ -61,6 +61,7 @@ struct RoutingOpts {
   bool useStations;
   double transitionPen;
   std::string transPenMethod;
+  std::string emPenMethod;
 };
 
 // _____________________________________________________________________________
@@ -80,6 +81,7 @@ inline bool operator==(const RoutingOpts& a, const RoutingOpts& b) {
          fabs(a.transitionPen - b.transitionPen) < 0.01 &&
          fabs(a.nonStationPen - b.nonStationPen) < 0.01 &&
          a.transPenMethod == b.transPenMethod &&
+         a.emPenMethod == b.emPenMethod &&
          a.useStations == b.useStations && a.popReachEdge == b.popReachEdge &&
          a.noSelfHops == b.noSelfHops;
 }

src/pfaedle/router/ShapeBuilder.cpp

@@ -206,7 +206,6 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const {
         // stations do not match, punish
         nameMatchPunish = _motCfg.routingOpts.stationUnmatchedPen;
       }
-
       std::string platform = s->getPlatformCode();
 
       if (!platform.empty() && !nd->pl().getSI()->getTrack().empty() &&
@@ -218,8 +217,7 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const {
         // don't snap to one way edges
         if (e->pl().oneWay() == 2) continue;
         ret.push_back({e,
-                       mDist * _motCfg.routingOpts.stationDistPenFactor +
+                       emWeight(mDist) + nameMatchPunish + trackMatchPunish,
-                           nameMatchPunish + trackMatchPunish,
                        0,
                        {},
                        0,
@@ -265,8 +263,7 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const {
 
   for (auto e : selected) {
     ret.push_back({e,
-                   scores[e] * _motCfg.routingOpts.stationDistPenFactor +
+                   emWeight(scores[e]) + _motCfg.routingOpts.nonStationPen,
-                       _motCfg.routingOpts.nonStationPen,
                    progrs[e],
                    {},
                    0,
@@ -981,7 +978,7 @@ std::vector<LINE> ShapeBuilder::getGeom(
           ret.push_back({hop.pointStart, *hop.end->getFrom()->pl().getGeom()});
         }
       } else {
-        ret.push_back({hop.pointEnd, hop.pointStart});
+        ret.push_back({hop.pointStart, hop.pointEnd});
       }
     } else {
       const auto& l = getLine(hop, rAttrs, colors);
@@ -1222,3 +1219,17 @@ uint32_t ShapeBuilder::getTextColor(uint32_t c) const {
   if (a < 140) return 0x00FFFFFF;
   return 0;
 }
+
+// _____________________________________________________________________________
+double ShapeBuilder::emWeight(double mDist) const {
+  if (_motCfg.routingOpts.emPenMethod == "exp") {
+    return mDist * _motCfg.routingOpts.stationDistPenFactor;
+  }
+
+  if (_motCfg.routingOpts.emPenMethod == "norm") {
+    double s = mDist * _motCfg.routingOpts.stationDistPenFactor;
+    return 0.5 * s * s;
+  }
+
+  return mDist;
+}

src/pfaedle/router/ShapeBuilder.h

@@ -119,6 +119,8 @@ class ShapeBuilder {
   std::map<size_t, router::EdgeListHops> route(const TripTrie* trie,
                                                const EdgeCandMap& ecm,
                                                HopCache* hopCache) const;
+  double emWeight(double mDist) const;
+
   void buildCandCache(const TripForests& clusters);
   void buildIndex();
 

src/pfaedle/router/Weights.cpp

@@ -237,8 +237,6 @@ double DistDiffTransWeight::weight(uint32_t c, double d, double t0, double d0,
                                    const RoutingOpts& rOpts) {
   UNUSED(t0);
   UNUSED(c);
-  // double mean = 250;  // expectation value of 250 meters for buses
-  // double lambda = 1.0 / mean;
 
   double w = fabs(d - d0);
 

src/shapevl/Collector.cpp

@@ -67,8 +67,15 @@ double Collector::add(const Trip* oldT, const Shape* oldS, const Trip* newT,
   std::vector<double> newDists;
   LINE newL = getWebMercLine(newS, &newDists);
 
-  auto oldSegs = segmentize(oldT, oldL, oldDists);
+  std::vector<std::pair<double, double>> newLenDists;
-  auto newSegs = segmentize(newT, newL, newDists);
+  std::vector<std::pair<double, double>> oldLenDists;
+
+  auto oldSegs = segmentize(oldT, oldL, oldDists, newLenDists);
+  auto newSegs = segmentize(newT, newL, newDists, oldLenDists);
+
+  for (const auto& p : oldLenDists) {
+    _distDiffs.push_back(fabs(p.first - p.second));
+  }
 
   // new lines build from cleaned-up shapes
   LINE oldLCut;
@@ -77,8 +84,9 @@ double Collector::add(const Trip* oldT, const Shape* oldS, const Trip* newT,
   for (auto oldL : oldSegs)
     oldLCut.insert(oldLCut.end(), oldL.begin(), oldL.end());
 
-  for (auto newL : newSegs)
+  for (auto newL : newSegs) {
     newLCut.insert(newLCut.end(), newL.begin(), newL.end());
+  }
 
   // determine the scale factor between the distance in projected
   // coordinates and the real-world distance in meters
@@ -150,8 +158,9 @@ double Collector::add(const Trip* oldT, const Shape* oldS, const Trip* newT,
 }
 
 // _____________________________________________________________________________
-std::vector<LINE> Collector::segmentize(const Trip* t, const LINE& shape,
+std::vector<LINE> Collector::segmentize(
-                                        const std::vector<double>& dists) {
+    const Trip* t, const LINE& shape, const std::vector<double>& dists,
+    std::vector<std::pair<double, double>>& lenDist) {
   // The straightforward way to segmentize the shape would be to just cut it at
   // the exact measurements in stop_times.txt. We have tried that, but found
   // that it produces misleading results for the following reason:
@@ -182,7 +191,7 @@ std::vector<LINE> Collector::segmentize(const Trip* t, const LINE& shape,
   if (t->getStopTimes().size() < 2) return ret;
 
   POLYLINE pl(shape);
-  std::vector<std::pair<POINT, double> > cuts;
+  std::vector<std::pair<POINT, double>> cuts;
 
   size_t i = 0;
   for (auto st : t->getStopTimes()) {
@@ -213,7 +222,17 @@ std::vector<LINE> Collector::segmentize(const Trip* t, const LINE& shape,
 
     auto curLp = beforePl.projectOnAfter(cuts[i].first, lastLp.lastIndex);
 
-    ret.push_back(pl.getSegment(lastLp, curLp).getLine());
+    auto curL = pl.getSegment(lastLp, curLp).getLine();
+
+    double dist =
+        util::geo::haversine(t->getStopTimes()[i - 1].getStop()->getLat(),
+                             t->getStopTimes()[i - 1].getStop()->getLng(),
+                             t->getStopTimes()[i].getStop()->getLat(),
+                             t->getStopTimes()[i].getStop()->getLng());
+    double len = util::geo::webMercLen(curL);
+    lenDist.push_back({dist, len});
+
+    ret.push_back(curL);
     lastLp = curLp;
   }
 
@@ -365,13 +384,27 @@ void Collector::printStats(std::ostream* os) const {
 }
 
 // _____________________________________________________________________________
-util::json::Dict Collector::getJSONStats() const {
+std::map<string, double> Collector::getStats() {
-  util::json::Dict stats = {};
+  std::map<string, double> stats;
+
+  if (_distDiffs.size()) {
+    auto i = _distDiffs.begin() + _distDiffs.size() / 2;
+    std::nth_element(_distDiffs.begin(), i, _distDiffs.end());
+
+    stats["median-dist-diff"] = *i;
+  } else {
+    stats["median-dist-diff"] = -1;
+  }
 
   stats["num-trips"] = _trips;
   stats["num-trips-matched"] = _results.size();
   stats["num-trips-wo-shapes"] = _noOrigShp;
   stats["avg-fr"] = getAvgDist();
+  if (_results.size()) {
+    stats["max-avg-frech-dist"] = (--_results.end())->getDist();
+  } else {
+    stats["max-avg-frech-dist"] = -1;
+  }
   stats["an-0"] =
       (static_cast<double>(_an0) / static_cast<double>(_results.size())) * 100;
   stats["an-5"] =

src/shapevl/Collector.h

@@ -59,7 +59,7 @@ class Collector {
   void printShortStats(std::ostream* os) const;
 
   // Get JSON stats
-  util::json::Dict getJSONStats() const;
+  std::map<string, double> getStats();
 
   // Print a CSV for the results to os
   void printCsv(std::ostream* os, const std::set<Result>& result) const;
@@ -79,6 +79,8 @@ class Collector {
   size_t _trips;
   size_t _noOrigShp;
 
+  std::vector<double> _distDiffs;
+
   double _fdSum;
   size_t _unmatchedSegSum;
   double _unmatchedSegLengthSum;
@@ -97,7 +99,8 @@ class Collector {
                                          const std::vector<LINE>& b);
 
   static std::vector<LINE> segmentize(const Trip* t, const LINE& shape,
-                                      const std::vector<double>& dists);
+                                      const std::vector<double>& dists,
+                                      std::vector<std::pair<double, double>>& lenDist);
 
   static std::vector<double> getBins(double mind, double maxd, size_t steps);
 };

src/shapevl/ShapevlMain.cpp

@@ -27,6 +27,7 @@ void printHelp(int argc, char** argv) {
       << "\nAllowed arguments:\n    -g <gtfs>     Ground truth GTFS file\n";
   std::cout << "     -s           Only output summary\n";
   std::cout << "     --json       Output JSON\n";
+  std::cout << "     --avg       Take avg of all inputs (only for --json)\n";
   std::cout << "     -f <folder>  Output full reports (per feed) to <folder>\n";
   std::cout
       << "     -m           MOTs to match (GTFS MOT or string, default: all)\n";
@@ -87,6 +88,7 @@ int main(int argc, char** argv) {
   std::vector<std::ofstream> reportStreams;
   bool summarize = false;
   bool json = false;
+  bool avg = false;
 
   for (int i = 1; i < argc; i++) {
     std::string cur = argv[i];
@@ -103,6 +105,8 @@ int main(int argc, char** argv) {
       summarize = true;
     } else if (cur == "--json") {
       json = true;
+    } else if (cur == "--avg") {
+      avg = true;
     } else if (cur == "-f") {
       if (++i >= argc) {
         LOG(ERROR) << "Missing argument for full reports (-f).";
@@ -173,7 +177,11 @@ int main(int argc, char** argv) {
     util::json::Dict stats = {};
 
     for (size_t i = 0; i < evalColls.size(); i++) {
-      stats[evlFeedPaths[i]] = evalColls[i].getJSONStats();
+      util::json::Dict locStats = {};
+      for (const auto& kv : evalColls[i].getStats()) {
+        locStats[kv.first] = kv.second;
+      }
+      stats[evlFeedPaths[i]] = locStats;
     }
 
     util::json::Dict jsonStats;
@@ -183,9 +191,28 @@ int main(int argc, char** argv) {
           {"statistics", stats[evlFeedPaths[0]]
       }};
     } else {
-      jsonStats = {
+      if (avg) {
-          {"statistics", stats
+        double count = evalColls.size();
-      }};
+        std::vector<std::string> keys;
+        for (const auto& a : evalColls[0].getStats()) {
+          keys.push_back(a.first);
+        }
+        util::json::Dict avgStats;
+        for (const auto& k : keys) {
+          double sum = 0;
+          for (size_t i = 0; i < evalColls.size(); i++) {
+            sum += evalColls[i].getStats()[k];
+          }
+          avgStats[k] = sum / count;
+        }
+        jsonStats = {
+            {"statistics", avgStats
+        }};
+      } else {
+        jsonStats = {
+            {"statistics", stats
+        }};
+      }
     }
 
     util::json::Writer wr(&std::cout, 10, true);

src/util/http/Server.cpp

@@ -319,6 +319,7 @@ std::string HttpServer::compress(const std::string& str, std::string* enc) {
   *enc = "gzip";
   return ret;
 #else
+  UNUSED(enc);
   return str;
 #endif
 }