gtfs-actions/generate-shapes
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add --osmfilter option

Browse source
This commit is contained in:
Patrick Brosi 2022-01-11 17:32:50 +01:00
parent 6473dcdb52
commit c25d174e60
14 changed files with 185 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/pfaedle/PfaedleMain.cpp
View file

@ -109,7 +109,7 @@ int main(int argc, char** argv) {
exit(static_cast<int>(RetCode::MOT_CFG_PARSE_ERR));
}
if (cfg.osmPath.empty() && !cfg.writeOverpass) {
if (cfg.osmPath.empty() && !cfg.writeOverpass && !cfg.writeOsmfilter) {
std::cerr << "No OSM input file specified (-x), see --help." << std::endl;
exit(static_cast<int>(RetCode::NO_OSM_INPUT));
}
@ -124,7 +124,7 @@ int main(int argc, char** argv) {
if (cfg.feedPaths.size() == 1) {
if (cfg.inPlace) cfg.outputPath = cfg.feedPaths[0];
if (!cfg.writeOverpass)
if (!cfg.writeOverpass && !cfg.writeOsmfilter)
LOG(INFO) << "Reading GTFS feed " << cfg.feedPaths[0] << " ...";
try {
ad::cppgtfs::Parser p;
@ -136,7 +136,7 @@ int main(int argc, char** argv) {
}
} else if (cfg.writeOsm.size() || cfg.writeOverpass) {
for (size_t i = 0; i < cfg.feedPaths.size(); i++) {
if (!cfg.writeOverpass)
if (!cfg.writeOverpass && !cfg.writeOsmfilter)
LOG(INFO) << "Reading GTFS feed " << cfg.feedPaths[i] << " ...";
ad::cppgtfs::Parser p;
try {
@ -218,6 +218,18 @@ int main(int argc, char** argv) {
}
osmBuilder.overpassQryWrite(&std::cout, opts, box);
exit(static_cast<int>(RetCode::SUCCESS));
} else if (cfg.writeOsmfilter) {
BBoxIdx box(BOX_PADDING);
OsmBuilder osmBuilder;
std::vector<pfaedle::osm::OsmReadOpts> opts;
for (const auto& o : motCfgReader.getConfigs()) {
if (std::find_first_of(o.mots.begin(), o.mots.end(), cmdCfgMots.begin(),
cmdCfgMots.end()) != o.mots.end()) {
opts.push_back(o.osmBuildOpts);
}
}
osmBuilder.osmfilterRuleWrite(&std::cout, opts, box);
exit(static_cast<int>(RetCode::SUCCESS));
} else if (!cfg.feedPaths.size()) {
std::cout << "No input feed specified, see --help" << std::endl;
exit(static_cast<int>(RetCode::NO_INPUT_FEED));

12
src/pfaedle/config/ConfigReader.cpp
View file

@ -10,14 +10,15 @@
#include "pfaedle/Def.h"
#include "pfaedle/_config.h"
#include "pfaedle/config/ConfigReader.h"
#include "pfaedle/config/PfaedleConfig.h"
#include "util/String.h"
#include "util/geo/Geo.h"
#include "util/log/Log.h"
using pfaedle::config::ConfigReader;
using std::string;
using std::exception;
using std::string;
using std::vector;
static const char* YEAR = &__DATE__[7];
@ -32,8 +33,7 @@ void ConfigReader::help(const char* bin) {
<< " with geometry precision <" << PFDL_PREC_STR << ">)\n\n"
<< "(C) " << YEAR << " " << COPY << "\n"
<< "Authors: " << AUTHORS << "\n\n"
<< "Usage: " << bin
<< " -x <OSM FILE> <GTFS FEED>\n\n"
<< "Usage: " << bin << " -x <OSM FILE> <GTFS FEED>\n\n"
<< "Allowed options:\n\n"
<< "General:\n"
<< std::setw(35) << " -v [ --version ]"
@ -93,6 +93,8 @@ void ConfigReader::help(const char* bin) {
<< " to <dbg-path>/path.json\n"
<< std::setw(35) << " --overpass"
<< "Output overpass query for matching OSM data\n"
<< std::setw(35) << " --osmfilter"
<< "Output osmfilter filter rules for matching OSM data\n"
<< std::setw(35) << " --grid-size arg (=2000)"
<< "Approx. grid cell size in meters\n"
<< std::setw(35) << " --no-fast-hops"
@ -122,6 +124,7 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
{"mots", required_argument, NULL, 'm'},
{"grid-size", required_argument, 0, 'g'},
{"overpass", no_argument, 0, 'a'},
{"osmfilter", no_argument, 0, 'f'},
{"osm-out", required_argument, 0, 'X'},
{"trip-id", required_argument, 0, 'T'},
{"write-graph", no_argument, 0, 1},
@ -194,6 +197,9 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
case 'a':
cfg->writeOverpass = true;
break;
case 'f':
cfg->writeOsmfilter = true;
break;
case 9:
cfg->inPlace = true;
break;

7
src/pfaedle/config/MotConfigReader.cpp
View file

@ -47,6 +47,13 @@ void MotConfigReader::parse(const std::vector<std::string>& paths,
std::string secStr = sec.first;
if (secStr.empty()) continue;
if (p.hasKey(secStr, "routing_emission_method")) {
cfg.routingOpts.emPenMethod =
p.getStr(secStr, "routing_emission_method");
} else {
cfg.routingOpts.emPenMethod = "exp";
}
if (p.hasKey(secStr, "routing_transition_method")) {
cfg.routingOpts.transPenMethod =
p.getStr(secStr, "routing_transition_method");

5
src/pfaedle/config/PfaedleConfig.h
View file

@ -9,8 +9,8 @@
#include <sstream>
#include <string>
#include <vector>
#include "util/geo/Geo.h"
#include "ad/cppgtfs/gtfs/Route.h"
#include "util/geo/Geo.h"
namespace pfaedle {
namespace config {
@ -28,6 +28,7 @@ struct Config {
buildTransitGraph(false),
useCaching(false),
writeOverpass(false),
writeOsmfilter(false),
inPlace(false),
writeColors(false),
noFastHops(false),
@ -53,6 +54,7 @@ struct Config {
bool buildTransitGraph;
bool useCaching;
bool writeOverpass;
bool writeOsmfilter;
bool inPlace;
bool writeColors;
bool noFastHops;
@ -76,6 +78,7 @@ struct Config {
<< "grid-size: " << gridSize << "\n"
<< "use-cache: " << useCaching << "\n"
<< "write-overpass: " << writeOverpass << "\n"
<< "write-osmfilter: " << writeOsmfilter << "\n"
<< "inplace: " << inPlace << "\n"
<< "write-colors: " << writeColors << "\n"
<< "no-fast-hops: " << noFastHops << "\n"

44
src/pfaedle/osm/OsmBuilder.cpp
View file

@ -183,6 +183,50 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
LOG(DEBUG) << _lines.size() << " transit lines have been read.";
}
// _____________________________________________________________________________
void OsmBuilder::osmfilterRuleWrite(std::ostream* out,
const std::vector<OsmReadOpts>& opts,
const BBoxIdx& latLngBox) const {
UNUSED(latLngBox);
OsmIdSet bboxNodes, noHupNodes;
MultAttrMap emptyF;
RelLst rels;
OsmIdList ways;
RelMap nodeRels, wayRels;
NIdMap nodes;
OsmFilter filter;
AttrKeySet attrKeys[3] = {};
for (const OsmReadOpts& o : opts) {
filter = filter.merge(OsmFilter(o.keepFilter, o.dropFilter));
getKeptAttrKeys(o, attrKeys);
}
*out << "--keep=\n";
for (auto r : filter.getKeepRules()) {
for (auto val : r.second) {
*out << r.first << "=";
if (val.first != "*") *out << val.first;
*out << "\n";
}
}
*out << "\n";
*out << "--keep-tags=\n";
for (const auto& keys : attrKeys) {
for (auto val : keys) {
*out << val << "=\n";
}
}
}
// _____________________________________________________________________________
void OsmBuilder::overpassQryWrite(std::ostream* out,
const std::vector<OsmReadOpts>& opts,

6
src/pfaedle/osm/OsmBuilder.h
View file

@ -92,6 +92,12 @@ class OsmBuilder {
void overpassQryWrite(std::ostream* out, const std::vector<OsmReadOpts>& opts,
const BBoxIdx& latLngBox) const;
// Based on the list of options, output an osmfilter configuration file
// to filter the data needed for routing
void osmfilterRuleWrite(std::ostream* out,
const std::vector<OsmReadOpts>& opts,
const BBoxIdx& latLngBox) const;
// Based on the list of options, read an OSM file from in and output an
// OSM file to out which contains exactly the entities that are needed
// from the file at in

2
src/pfaedle/router/Misc.h
View file

@ -61,6 +61,7 @@ struct RoutingOpts {
bool useStations;
double transitionPen;
std::string transPenMethod;
std::string emPenMethod;
};
// _____________________________________________________________________________
@ -80,6 +81,7 @@ inline bool operator==(const RoutingOpts& a, const RoutingOpts& b) {
fabs(a.transitionPen - b.transitionPen) < 0.01 &&
fabs(a.nonStationPen - b.nonStationPen) < 0.01 &&
a.transPenMethod == b.transPenMethod &&
a.emPenMethod == b.emPenMethod &&
a.useStations == b.useStations && a.popReachEdge == b.popReachEdge &&
a.noSelfHops == b.noSelfHops;
}

23
src/pfaedle/router/ShapeBuilder.cpp
View file

@ -206,7 +206,6 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const {
// stations do not match, punish
nameMatchPunish = _motCfg.routingOpts.stationUnmatchedPen;
}
std::string platform = s->getPlatformCode();
if (!platform.empty() && !nd->pl().getSI()->getTrack().empty() &&
@ -218,8 +217,7 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const {
// don't snap to one way edges
if (e->pl().oneWay() == 2) continue;
ret.push_back({e,
mDist * _motCfg.routingOpts.stationDistPenFactor +
nameMatchPunish + trackMatchPunish,
emWeight(mDist) + nameMatchPunish + trackMatchPunish,
0,
{},
0,
@ -265,8 +263,7 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const {
for (auto e : selected) {
ret.push_back({e,
scores[e] * _motCfg.routingOpts.stationDistPenFactor +
_motCfg.routingOpts.nonStationPen,
emWeight(scores[e]) + _motCfg.routingOpts.nonStationPen,
progrs[e],
{},
0,
@ -981,7 +978,7 @@ std::vector<LINE> ShapeBuilder::getGeom(
ret.push_back({hop.pointStart, *hop.end->getFrom()->pl().getGeom()});
}
} else {
ret.push_back({hop.pointEnd, hop.pointStart});
ret.push_back({hop.pointStart, hop.pointEnd});
}
} else {
const auto& l = getLine(hop, rAttrs, colors);
@ -1222,3 +1219,17 @@ uint32_t ShapeBuilder::getTextColor(uint32_t c) const {
if (a < 140) return 0x00FFFFFF;
return 0;
}
// _____________________________________________________________________________
double ShapeBuilder::emWeight(double mDist) const {
if (_motCfg.routingOpts.emPenMethod == "exp") {
return mDist * _motCfg.routingOpts.stationDistPenFactor;
}
if (_motCfg.routingOpts.emPenMethod == "norm") {
double s = mDist * _motCfg.routingOpts.stationDistPenFactor;
return 0.5 * s * s;
}
return mDist;
}

2
src/pfaedle/router/ShapeBuilder.h
View file

@ -119,6 +119,8 @@ class ShapeBuilder {
std::map<size_t, router::EdgeListHops> route(const TripTrie* trie,
const EdgeCandMap& ecm,
HopCache* hopCache) const;
double emWeight(double mDist) const;
void buildCandCache(const TripForests& clusters);
void buildIndex();

2
src/pfaedle/router/Weights.cpp
View file

@ -237,8 +237,6 @@ double DistDiffTransWeight::weight(uint32_t c, double d, double t0, double d0,
const RoutingOpts& rOpts) {
UNUSED(t0);
UNUSED(c);
// double mean = 250; // expectation value of 250 meters for buses
// double lambda = 1.0 / mean;
double w = fabs(d - d0);

51
src/shapevl/Collector.cpp
View file

@ -67,8 +67,15 @@ double Collector::add(const Trip* oldT, const Shape* oldS, const Trip* newT,
std::vector<double> newDists;
LINE newL = getWebMercLine(newS, &newDists);
auto oldSegs = segmentize(oldT, oldL, oldDists);
auto newSegs = segmentize(newT, newL, newDists);
std::vector<std::pair<double, double>> newLenDists;
std::vector<std::pair<double, double>> oldLenDists;
auto oldSegs = segmentize(oldT, oldL, oldDists, newLenDists);
auto newSegs = segmentize(newT, newL, newDists, oldLenDists);
for (const auto& p : oldLenDists) {
_distDiffs.push_back(fabs(p.first - p.second));
}
// new lines build from cleaned-up shapes
LINE oldLCut;
@ -77,8 +84,9 @@ double Collector::add(const Trip* oldT, const Shape* oldS, const Trip* newT,
for (auto oldL : oldSegs)
oldLCut.insert(oldLCut.end(), oldL.begin(), oldL.end());
for (auto newL : newSegs)
for (auto newL : newSegs) {
newLCut.insert(newLCut.end(), newL.begin(), newL.end());
}
// determine the scale factor between the distance in projected
// coordinates and the real-world distance in meters
@ -150,8 +158,9 @@ double Collector::add(const Trip* oldT, const Shape* oldS, const Trip* newT,
}
// _____________________________________________________________________________
std::vector<LINE> Collector::segmentize(const Trip* t, const LINE& shape,
const std::vector<double>& dists) {
std::vector<LINE> Collector::segmentize(
const Trip* t, const LINE& shape, const std::vector<double>& dists,
std::vector<std::pair<double, double>>& lenDist) {
// The straightforward way to segmentize the shape would be to just cut it at
// the exact measurements in stop_times.txt. We have tried that, but found
// that it produces misleading results for the following reason:
@ -182,7 +191,7 @@ std::vector<LINE> Collector::segmentize(const Trip* t, const LINE& shape,
if (t->getStopTimes().size() < 2) return ret;
POLYLINE pl(shape);
std::vector<std::pair<POINT, double> > cuts;
std::vector<std::pair<POINT, double>> cuts;
size_t i = 0;
for (auto st : t->getStopTimes()) {
@ -213,7 +222,17 @@ std::vector<LINE> Collector::segmentize(const Trip* t, const LINE& shape,
auto curLp = beforePl.projectOnAfter(cuts[i].first, lastLp.lastIndex);
ret.push_back(pl.getSegment(lastLp, curLp).getLine());
auto curL = pl.getSegment(lastLp, curLp).getLine();
double dist =
util::geo::haversine(t->getStopTimes()[i - 1].getStop()->getLat(),
t->getStopTimes()[i - 1].getStop()->getLng(),
t->getStopTimes()[i].getStop()->getLat(),
t->getStopTimes()[i].getStop()->getLng());
double len = util::geo::webMercLen(curL);
lenDist.push_back({dist, len});
ret.push_back(curL);
lastLp = curLp;
}
@ -365,13 +384,27 @@ void Collector::printStats(std::ostream* os) const {
}
// _____________________________________________________________________________
util::json::Dict Collector::getJSONStats() const {
util::json::Dict stats = {};
std::map<string, double> Collector::getStats() {
std::map<string, double> stats;
if (_distDiffs.size()) {
auto i = _distDiffs.begin() + _distDiffs.size() / 2;
std::nth_element(_distDiffs.begin(), i, _distDiffs.end());
stats["median-dist-diff"] = *i;
} else {
stats["median-dist-diff"] = -1;
}
stats["num-trips"] = _trips;
stats["num-trips-matched"] = _results.size();
stats["num-trips-wo-shapes"] = _noOrigShp;
stats["avg-fr"] = getAvgDist();
if (_results.size()) {
stats["max-avg-frech-dist"] = (--_results.end())->getDist();
} else {
stats["max-avg-frech-dist"] = -1;
}
stats["an-0"] =
(static_cast<double>(_an0) / static_cast<double>(_results.size())) * 100;
stats["an-5"] =

7
src/shapevl/Collector.h
View file

@ -59,7 +59,7 @@ class Collector {
void printShortStats(std::ostream* os) const;
// Get JSON stats
util::json::Dict getJSONStats() const;
std::map<string, double> getStats();
// Print a CSV for the results to os
void printCsv(std::ostream* os, const std::set<Result>& result) const;
@ -79,6 +79,8 @@ class Collector {
size_t _trips;
size_t _noOrigShp;
std::vector<double> _distDiffs;
double _fdSum;
size_t _unmatchedSegSum;
double _unmatchedSegLengthSum;
@ -97,7 +99,8 @@ class Collector {
const std::vector<LINE>& b);
static std::vector<LINE> segmentize(const Trip* t, const LINE& shape,
const std::vector<double>& dists);
const std::vector<double>& dists,
std::vector<std::pair<double, double>>& lenDist);
static std::vector<double> getBins(double mind, double maxd, size_t steps);
};

35
src/shapevl/ShapevlMain.cpp
View file

@ -27,6 +27,7 @@ void printHelp(int argc, char** argv) {
<< "\nAllowed arguments:\n -g <gtfs> Ground truth GTFS file\n";
std::cout << " -s Only output summary\n";
std::cout << " --json Output JSON\n";
std::cout << " --avg Take avg of all inputs (only for --json)\n";
std::cout << " -f <folder> Output full reports (per feed) to <folder>\n";
std::cout
<< " -m MOTs to match (GTFS MOT or string, default: all)\n";
@ -87,6 +88,7 @@ int main(int argc, char** argv) {
std::vector<std::ofstream> reportStreams;
bool summarize = false;
bool json = false;
bool avg = false;
for (int i = 1; i < argc; i++) {
std::string cur = argv[i];
@ -103,6 +105,8 @@ int main(int argc, char** argv) {
summarize = true;
} else if (cur == "--json") {
json = true;
} else if (cur == "--avg") {
avg = true;
} else if (cur == "-f") {
if (++i >= argc) {
LOG(ERROR) << "Missing argument for full reports (-f).";
@ -173,7 +177,11 @@ int main(int argc, char** argv) {
util::json::Dict stats = {};
for (size_t i = 0; i < evalColls.size(); i++) {
stats[evlFeedPaths[i]] = evalColls[i].getJSONStats();
util::json::Dict locStats = {};
for (const auto& kv : evalColls[i].getStats()) {
locStats[kv.first] = kv.second;
}
stats[evlFeedPaths[i]] = locStats;
}
util::json::Dict jsonStats;
@ -183,9 +191,28 @@ int main(int argc, char** argv) {
{"statistics", stats[evlFeedPaths[0]]
}};
} else {
jsonStats = {
{"statistics", stats
}};
if (avg) {
double count = evalColls.size();
std::vector<std::string> keys;
for (const auto& a : evalColls[0].getStats()) {
keys.push_back(a.first);
}
util::json::Dict avgStats;
for (const auto& k : keys) {
double sum = 0;
for (size_t i = 0; i < evalColls.size(); i++) {
sum += evalColls[i].getStats()[k];
}
avgStats[k] = sum / count;
}
jsonStats = {
{"statistics", avgStats
}};
} else {
jsonStats = {
{"statistics", stats
}};
}
}
util::json::Writer wr(&std::cout, 10, true);

1
src/util/http/Server.cpp
View file

@ -319,6 +319,7 @@ std::string HttpServer::compress(const std::string& str, std::string* enc) {
*enc = "gzip";
return ret;
#else
UNUSED(enc);
return str;
#endif
}