* speed up hop-to-hop calculations

* better and faster trip clustering: trip tries
* add --write-colors to extract line colors from OSM data
* refactor config parameter names, update default pfaedle.cfg
* add --stats for writing a stats.json file
* add --no-fast-hops, --no-a-star, --no-trie for debugging
* general refactoring
This commit is contained in:
Patrick Brosi 2022-01-03 22:27:59 +01:00
parent f1822868c5
commit 4c29892658
126 changed files with 14576 additions and 12196 deletions

View file

@ -31,10 +31,10 @@ bool BBoxIdx::contains(const Point<double>& p) const {
// _____________________________________________________________________________
BOX BBoxIdx::getFullWebMercBox() const {
return BOX(
util::geo::latLngToWebMerc<PFAEDLE_PRECISION>(
_root.box.getLowerLeft().getY(), _root.box.getLowerLeft().getX()),
util::geo::latLngToWebMerc<PFAEDLE_PRECISION>(
_root.box.getUpperRight().getY(), _root.box.getUpperRight().getX()));
util::geo::latLngToWebMerc<PFDL_PREC>(_root.box.getLowerLeft().getY(),
_root.box.getLowerLeft().getX()),
util::geo::latLngToWebMerc<PFDL_PREC>(_root.box.getUpperRight().getY(),
_root.box.getUpperRight().getX()));
}
// _____________________________________________________________________________

File diff suppressed because it is too large Load diff

View file

@ -30,20 +30,19 @@
namespace pfaedle {
namespace osm {
using ad::cppgtfs::gtfs::Stop;
using pfaedle::router::NodeSet;
using pfaedle::trgraph::Component;
using pfaedle::trgraph::Edge;
using pfaedle::trgraph::EdgeGrid;
using pfaedle::trgraph::NodeGrid;
using pfaedle::trgraph::Normalizer;
using pfaedle::trgraph::EdgePL;
using pfaedle::trgraph::Graph;
using pfaedle::trgraph::Node;
using pfaedle::trgraph::NodeGrid;
using pfaedle::trgraph::NodePL;
using pfaedle::trgraph::Edge;
using pfaedle::trgraph::EdgePL;
using pfaedle::trgraph::TransitEdgeLine;
using pfaedle::trgraph::Normalizer;
using pfaedle::trgraph::StatInfo;
using pfaedle::trgraph::StatGroup;
using pfaedle::trgraph::Component;
using pfaedle::router::NodeSet;
using ad::cppgtfs::gtfs::Stop;
using pfaedle::trgraph::TransitEdgeLine;
using util::Nullable;
struct NodeCand {
@ -58,9 +57,8 @@ struct SearchFunc {
};
struct EqSearch : public SearchFunc {
explicit EqSearch(bool orphanSnap) : orphanSnap(orphanSnap) {}
EqSearch() {}
double minSimi = 0.9;
bool orphanSnap;
bool operator()(const Node* cand, const StatInfo* si) const;
};
@ -87,8 +85,7 @@ class OsmBuilder {
// Read the OSM file at path, and write a graph to g. Only elements
// inside the bounding box will be read
void read(const std::string& path, const OsmReadOpts& opts, Graph* g,
const BBoxIdx& box, size_t gridSize, router::FeedStops* fs,
Restrictor* res);
const BBoxIdx& box, double gridSize, Restrictor* res);
// Based on the list of options, output an overpass XML query for getting
// the data needed for routing
@ -103,8 +100,8 @@ class OsmBuilder {
private:
pfxml::parser_state readBBoxNds(pfxml::file* xml, OsmIdSet* nodes,
OsmIdSet* noHupNodes, const OsmFilter& filter,
const BBoxIdx& bbox) const;
OsmIdSet* noHupNodes, const OsmFilter& filter,
const BBoxIdx& bbox) const;
void readRels(pfxml::file* f, RelLst* rels, RelMap* nodeRels, RelMap* wayRels,
const OsmFilter& filter, const AttrKeySet& keepAttrs,
@ -140,13 +137,14 @@ class OsmBuilder {
Restrictor* restor, const FlatRels& flatRels,
EdgTracks* etracks, const OsmReadOpts& opts);
void readEdges(pfxml::file* xml, const RelMap& wayRels, const OsmFilter& filter,
const OsmIdSet& bBoxNodes, const AttrKeySet& keepAttrs,
OsmIdList* ret, NIdMap* nodes, const FlatRels& flatRels);
void readEdges(pfxml::file* xml, const RelMap& wayRels,
const OsmFilter& filter, const OsmIdSet& bBoxNodes,
const AttrKeySet& keepAttrs, OsmIdList* ret, NIdMap* nodes,
const FlatRels& flatRels);
OsmWay nextWay(pfxml::file* xml, const RelMap& wayRels, const OsmFilter& filter,
const OsmIdSet& bBoxNodes, const AttrKeySet& keepAttrs,
const FlatRels& flatRels) const;
OsmWay nextWay(pfxml::file* xml, const RelMap& wayRels,
const OsmFilter& filter, const OsmIdSet& bBoxNodes,
const AttrKeySet& keepAttrs, const FlatRels& flatRels) const;
bool keepWay(const OsmWay& w, const RelMap& wayRels, const OsmFilter& filter,
const OsmIdSet& bBoxNodes, const FlatRels& fl) const;
@ -168,52 +166,46 @@ class OsmBuilder {
const AttrKeySet& keepAttrs) const;
protected:
Nullable<StatInfo> getStatInfo(Node* node, osmid nid, const POINT& pos,
const AttrMap& m, StAttrGroups* groups,
Nullable<StatInfo> getStatInfo(osmid nid, const AttrMap& m,
const RelMap& nodeRels, const RelLst& rels,
const OsmReadOpts& ops) const;
static void snapStats(const OsmReadOpts& opts, Graph* g, const BBoxIdx& bbox,
size_t gridSize, router::FeedStops* fs, Restrictor* res,
double gridSize, Restrictor* res,
const NodeSet& orphanStations);
static void writeGeoms(Graph* g);
static void deleteOrphNds(Graph* g);
static void writeGeoms(Graph* g, const OsmReadOpts& opts);
static void deleteOrphEdgs(Graph* g, const OsmReadOpts& opts);
static void deleteOrphNds(Graph* g, const OsmReadOpts& opts);
static double dist(const Node* a, const Node* b);
static double webMercDist(const Node* a, const Node* b);
static NodeGrid buildNodeIdx(Graph* g, size_t size, const BOX& webMercBox,
static NodeGrid buildNodeIdx(Graph* g, double size, const BOX& box,
bool which);
static EdgeGrid buildEdgeIdx(Graph* g, size_t size, const BOX& webMercBox);
static EdgeGrid buildEdgeIdx(Graph* g, double size, const BOX& box);
static void fixGaps(Graph* g, NodeGrid* ng);
static void collapseEdges(Graph* g);
static void writeODirEdgs(Graph* g, Restrictor* restor);
static void writeSelfEdgs(Graph* g);
static void writeOneWayPens(Graph* g, const OsmReadOpts& opts);
static void writeNoLinePens(Graph* g, const OsmReadOpts& opts);
static void writeEdgeTracks(const EdgTracks& tracks);
static void simplifyGeoms(Graph* g);
static uint32_t writeComps(Graph* g);
static uint32_t writeComps(Graph* g, const OsmReadOpts& opts);
static bool edgesSim(const Edge* a, const Edge* b);
static const EdgePL& mergeEdgePL(Edge* a, Edge* b);
static void getEdgCands(const POINT& s, EdgeCandPQ* ret, EdgeGrid* eg,
double d);
static std::set<Node*> getMatchingNds(const NodePL& s, NodeGrid* ng,
double d);
static Node* getMatchingNd(const NodePL& s, NodeGrid* ng, double d);
static NodeSet snapStation(Graph* g, NodePL* s, EdgeGrid* eg, NodeGrid* sng,
const OsmReadOpts& opts, Restrictor* restor,
bool surHeur, bool orphSnap, double maxD);
static void snapStation(Graph* g, NodePL* s, EdgeGrid* eg, NodeGrid* sng,
const OsmReadOpts& opts, Restrictor* restor,
double maxD);
// Checks if from the edge e, a station similar to si can be reach with less
// than maxD distance and less or equal to "maxFullTurns" full turns. If
// such a station exists, it is returned. If not, 0 is returned.
static Node* eqStatReach(const Edge* e, const StatInfo* si, const POINT& p,
double maxD, int maxFullTurns, double maxAng,
bool orph);
double maxD, int maxFullTurns, double maxAng);
static Node* depthSearch(const Edge* e, const StatInfo* si, const POINT& p,
double maxD, int maxFullTurns, double minAngle,
@ -223,8 +215,6 @@ class OsmBuilder {
double maxD, int maxFullTurns, double minAngle);
static bool keepFullTurn(const trgraph::Node* n, double ang);
static StatGroup* groupStats(const NodeSet& s);
static NodePL plFromGtfs(const Stop* s, const OsmReadOpts& ops);
std::vector<TransitEdgeLine*> getLines(const std::vector<size_t>& edgeRels,
@ -254,6 +244,10 @@ class OsmBuilder {
bool relKeep(osmid id, const RelMap& rels, const FlatRels& fl) const;
uint32_t parseHexColor(std::string) const;
static uint32_t costToInt(double c);
std::map<TransitEdgeLine, TransitEdgeLine*> _lines;
std::map<size_t, TransitEdgeLine*> _relLines;
};

View file

@ -26,6 +26,7 @@ OsmFilter::OsmFilter(const OsmReadOpts& o)
_posRestr(o.restrPosRestr),
_negRestr(o.restrNegRestr),
_noRestr(o.noRestrFilter),
_turnCycle(o.turnCycleFilter),
_levels(o.levelFilters) {}
// _____________________________________________________________________________
@ -72,6 +73,11 @@ uint64_t OsmFilter::blocker(const AttrMap& attrs) const {
return contained(attrs, _blocker, NODE);
}
// _____________________________________________________________________________
uint64_t OsmFilter::turnCycle(const AttrMap& attrs) const {
return contained(attrs, _turnCycle, NODE);
}
// _____________________________________________________________________________
uint64_t OsmFilter::contained(const AttrMap& attrs, const MultAttrMap& map,
Type t) {

View file

@ -27,6 +27,7 @@ class OsmFilter {
uint64_t onewayrev(const AttrMap& attrs) const;
uint64_t station(const AttrMap& attrs) const;
uint64_t blocker(const AttrMap& attrs) const;
uint64_t turnCycle(const AttrMap& attrs) const;
uint64_t negRestr(const AttrMap& attrs) const;
uint64_t posRestr(const AttrMap& attrs) const;
std::vector<std::string> getAttrKeys() const;
@ -46,7 +47,7 @@ class OsmFilter {
private:
MultAttrMap _keep, _drop, _nohup, _oneway, _onewayrev, _twoway, _station,
_blocker, _posRestr, _negRestr, _noRestr;
_blocker, _posRestr, _negRestr, _noRestr, _turnCycle;
const MultAttrMap* _levels;
};
} // namespace osm

View file

@ -15,6 +15,7 @@
#include <string>
#include "pfaedle/Def.h"
#include "pfaedle/osm/OsmIdSet.h"
#include "util/3rdparty/MurmurHash3.h"
using pfaedle::osm::OsmIdSet;
@ -28,26 +29,46 @@ OsmIdSet::OsmIdSet()
_last(0),
_smallest(-1),
_biggest(0),
_hasInv(false),
_obufpos(0),
_curBlock(-1),
_fsize(0) {
_bitset = new std::bitset<BLOOMF_BITS>();
_bitsetNotIn = new std::bitset<BLOOMF_BITS>();
_file = openTmpFile();
_buffer = new unsigned char[BUFFER_S];
_outBuffer = new unsigned char[OBUFFER_S];
_outBuffer = new unsigned char[BUFFER_S];
}
// _____________________________________________________________________________
OsmIdSet::~OsmIdSet() {
delete _bitset;
delete _bitsetNotIn;
delete[] _buffer;
if (!_closed) delete[] _outBuffer;
}
// _____________________________________________________________________________
void OsmIdSet::nadd(osmid id) {
if (_closed) throw std::exception();
_hasInv = true;
uint32_t h1, h2;
MurmurHash3_x86_32(&id, 8, 469954432, &h1);
h2 = jenkins(id);
for (int i = 0; i < 5; i++) {
uint32_t h = (h1 + i * h2) % BLOOMF_BITS;
(*_bitsetNotIn)[h] = 1;
}
}
// _____________________________________________________________________________
void OsmIdSet::add(osmid id) {
if (_closed) throw std::exception();
diskAdd(id);
if (_last > id) _sorted = false;
@ -55,7 +76,14 @@ void OsmIdSet::add(osmid id) {
if (id < _smallest) _smallest = id;
if (id > _biggest) _biggest = id;
for (int i = 0; i < 10; i++) (*_bitset)[hash(id, i)] = 1;
uint32_t h1, h2;
MurmurHash3_x86_32(&id, 8, 469954432, &h1);
h2 = jenkins(id);
for (int i = 0; i < 5; i++) {
uint32_t h = (h1 + i * h2) % BLOOMF_BITS;
(*_bitset)[h] = 1;
}
}
// _____________________________________________________________________________
@ -69,8 +97,8 @@ void OsmIdSet::diskAdd(osmid id) {
_blockEnds.push_back(id);
}
if (_obufpos >= OBUFFER_S) {
ssize_t w = cwrite(_file, _outBuffer, OBUFFER_S);
if (_obufpos >= BUFFER_S) {
ssize_t w = cwrite(_file, _outBuffer, BUFFER_S);
_fsize += w;
_obufpos = 0;
}
@ -86,7 +114,8 @@ size_t OsmIdSet::getBlock(osmid id) const {
bool OsmIdSet::diskHas(osmid id) const {
assert(_sorted);
if (std::find(_blockEnds.begin(), _blockEnds.end(), id) != _blockEnds.end()) {
auto a = std::lower_bound(_blockEnds.begin(), _blockEnds.end(), id);
if (a != _blockEnds.end() && *a == id) {
return true;
}
@ -125,12 +154,23 @@ bool OsmIdSet::has(osmid id) const {
LOOKUPS++;
if (!_closed) close();
// trivial cases
if (id < _smallest || id > _biggest) {
return false;
}
for (int i = 0; i < 10; i++) {
if ((*_bitset)[hash(id, i)] == 0) return false;
uint32_t h1, h2;
MurmurHash3_x86_32(&id, 8, 469954432, &h1);
h2 = jenkins(id);
for (int i = 0; i < 5; i++) {
uint32_t h = (h1 + i * h2) % BLOOMF_BITS;
if ((*_bitset)[h] == 0) {
return false;
}
if (_hasInv && (*_bitsetNotIn)[h] == 0) {
return true;
}
}
bool has = diskHas(id);
@ -249,8 +289,8 @@ size_t OsmIdSet::cread(int f, void* buf, size_t n) const {
// _____________________________________________________________________________
uint32_t OsmIdSet::knuth(uint32_t in) const {
const uint32_t prime = 2654435769;
return (in * prime) >> 2;
const uint32_t a = 2654435769;
return (in * a) >> 2;
}
// _____________________________________________________________________________
@ -264,11 +304,6 @@ uint32_t OsmIdSet::jenkins(uint32_t in) const {
return in >> 2;
}
// _____________________________________________________________________________
uint32_t OsmIdSet::hash(uint32_t in, int i) const {
return (knuth(in) + jenkins(in) * i) % BLOOMF_BITS;
}
// _____________________________________________________________________________
int OsmIdSet::openTmpFile() const {
const std::string& fname = getTmpFName("", "");

View file

@ -25,7 +25,7 @@ static const size_t BUFFER_S = 8 * 64 * 1024;
static const size_t SORT_BUFFER_S = 8 * 64 * 1024;
static const size_t OBUFFER_S = 8 * 1024 * 1024;
#define BLOOMF_BITS 400000000
#define BLOOMF_BITS 214748357
/*
* A disk-based set for OSM ids. Read-access for checking the presence is
@ -39,6 +39,9 @@ class OsmIdSet {
// Add an OSM id
void add(osmid id);
// Add an OSM id that is NOT contained
void nadd(osmid id);
// Check if an OSM id is contained
bool has(osmid id) const;
@ -57,6 +60,8 @@ class OsmIdSet {
osmid _smallest;
osmid _biggest;
bool _hasInv;
size_t _obufpos;
mutable size_t _curBlock;
mutable ssize_t _curBlockSize;
@ -64,13 +69,14 @@ class OsmIdSet {
// bloom filter
std::bitset<BLOOMF_BITS>* _bitset;
std::bitset<BLOOMF_BITS>* _bitsetNotIn;
mutable std::vector<osmid> _blockEnds;
mutable size_t _fsize;
uint32_t knuth(uint32_t in) const;
uint32_t jenkins(uint32_t in) const;
uint32_t hash(uint32_t in, int i) const;
void diskAdd(osmid id);
void close() const;
void sort() const;

View file

@ -5,14 +5,14 @@
#ifndef PFAEDLE_OSM_OSMREADOPTS_H_
#define PFAEDLE_OSM_OSMREADOPTS_H_
#include <map>
#include <queue>
#include <unordered_set>
#include <set>
#include <string>
#include <unordered_map>
#include <map>
#include <unordered_set>
#include <utility>
#include <vector>
#include <set>
#include "pfaedle/osm/Osm.h"
#include "pfaedle/trgraph/Graph.h"
#include "pfaedle/trgraph/Normalizer.h"
@ -77,11 +77,12 @@ struct RelLineRules {
AttrLst sNameRule;
AttrLst fromNameRule;
AttrLst toNameRule;
AttrLst colorRule;
};
inline bool operator==(const RelLineRules& a, const RelLineRules& b) {
return a.sNameRule == b.sNameRule && a.fromNameRule == b.fromNameRule &&
a.toNameRule == b.toNameRule;
a.toNameRule == b.toNameRule && a.colorRule == b.colorRule;
}
struct StationAttrRules {
@ -94,21 +95,6 @@ inline bool operator==(const StationAttrRules& a, const StationAttrRules& b) {
return a.nameRule == b.nameRule && a.platformRule == b.platformRule;
}
struct StatGroupNAttrRule {
DeepAttrRule attr;
double maxDist;
};
inline bool operator==(const StatGroupNAttrRule& a,
const StatGroupNAttrRule& b) {
return a.attr == b.attr && a.maxDist == b.maxDist;
}
typedef std::unordered_map<
std::string,
std::unordered_map<std::string, std::vector<trgraph::StatGroup*>>>
StAttrGroups;
struct OsmReadOpts {
OsmReadOpts() {}
@ -121,7 +107,7 @@ struct OsmReadOpts {
MultAttrMap twoWayFilter;
MultAttrMap stationFilter;
MultAttrMap stationBlockerFilter;
std::vector<StatGroupNAttrRule> statGroupNAttrRules;
MultAttrMap turnCycleFilter;
trgraph::Normalizer statNormzer;
trgraph::Normalizer lineNormzer;
@ -136,14 +122,23 @@ struct OsmReadOpts {
uint8_t maxSnapLevel;
double maxAngleSnapReach;
std::vector<double> maxSnapDistances;
double maxSnapFallbackHeurDistance;
double maxSnapDistance;
double maxStationCandDistance;
double maxBlockDistance;
double maxOsmStationDistance;
double maxSpeed;
double maxSpeedCorFac;
// TODO(patrick): this is not implemented yet
double levelSnapPunishFac[7] = {0, 0, 0, 0, 0, 0, 0};
std::vector<double> maxOsmStationDistances;
// given in km/h, but store in m/s
double levelDefSpeed[8] = {85 * 0.2777, 70 * 0.2777, 55 * 0.2777, 40 * 0.2777,
30 * 0.2777, 20 * 0.2777, 10 * 0.2777, 5 * 0.2777};
double oneWaySpeedPen;
double oneWayEntryCost;
double noLinesPunishFact;
double fullTurnAngle;
@ -154,9 +149,10 @@ struct OsmReadOpts {
};
inline bool operator==(const OsmReadOpts& a, const OsmReadOpts& b) {
if (a.maxSnapDistances.size() != b.maxSnapDistances.size()) return false;
for (size_t i = 0; i < a.maxSnapDistances.size(); i++) {
if (fabs(a.maxSnapDistances[i] - b.maxSnapDistances[i]) >= 0.1)
if (a.maxOsmStationDistances.size() != b.maxOsmStationDistances.size())
return false;
for (size_t i = 0; i < a.maxOsmStationDistances.size(); i++) {
if (fabs(a.maxOsmStationDistances[i] - b.maxOsmStationDistances[i]) >= 0.1)
return false;
}
@ -173,24 +169,29 @@ inline bool operator==(const OsmReadOpts& a, const OsmReadOpts& b) {
a.twoWayFilter == b.twoWayFilter &&
a.stationFilter == b.stationFilter &&
a.stationBlockerFilter == b.stationBlockerFilter &&
a.statGroupNAttrRules == b.statGroupNAttrRules &&
a.turnCycleFilter == b.turnCycleFilter &&
a.statNormzer == b.statNormzer && a.lineNormzer == b.lineNormzer &&
a.trackNormzer == b.trackNormzer && a.relLinerules == b.relLinerules &&
a.statAttrRules == b.statAttrRules &&
a.maxSnapLevel == b.maxSnapLevel &&
fabs(a.maxAngleSnapReach - b.maxAngleSnapReach) < 0.1 &&
fabs(a.maxOsmStationDistance - b.maxOsmStationDistance) < 0.1 &&
fabs(a.maxSnapFallbackHeurDistance - b.maxSnapFallbackHeurDistance) <
0.1 &&
fabs(a.maxSnapDistance - b.maxSnapDistance) < 0.1 &&
fabs(a.maxStationCandDistance - b.maxStationCandDistance) < 0.1 &&
fabs(a.maxBlockDistance - b.maxBlockDistance) < 0.1 &&
fabs(a.levelSnapPunishFac[0] - b.levelSnapPunishFac[0]) < 0.1 &&
fabs(a.levelSnapPunishFac[1] - b.levelSnapPunishFac[1]) < 0.1 &&
fabs(a.levelSnapPunishFac[2] - b.levelSnapPunishFac[2]) < 0.1 &&
fabs(a.levelSnapPunishFac[3] - b.levelSnapPunishFac[3]) < 0.1 &&
fabs(a.levelSnapPunishFac[4] - b.levelSnapPunishFac[4]) < 0.1 &&
fabs(a.levelSnapPunishFac[5] - b.levelSnapPunishFac[5]) < 0.1 &&
fabs(a.levelSnapPunishFac[6] - b.levelSnapPunishFac[6]) < 0.1 &&
fabs(a.levelDefSpeed[0] - b.levelDefSpeed[0]) < 0.1 &&
fabs(a.levelDefSpeed[1] - b.levelDefSpeed[1]) < 0.1 &&
fabs(a.levelDefSpeed[2] - b.levelDefSpeed[2]) < 0.1 &&
fabs(a.levelDefSpeed[3] - b.levelDefSpeed[3]) < 0.1 &&
fabs(a.levelDefSpeed[4] - b.levelDefSpeed[4]) < 0.1 &&
fabs(a.levelDefSpeed[5] - b.levelDefSpeed[5]) < 0.1 &&
fabs(a.levelDefSpeed[6] - b.levelDefSpeed[6]) < 0.1 &&
fabs(a.levelDefSpeed[7] - b.levelDefSpeed[7]) < 0.1 &&
fabs(a.oneWaySpeedPen - b.oneWaySpeedPen) < 0.1 &&
fabs(a.oneWayEntryCost - b.oneWayEntryCost) < 0.1 &&
fabs(a.noLinesPunishFact - b.noLinesPunishFact) < 0.1 &&
fabs(a.fullTurnAngle - b.fullTurnAngle) < 0.1 &&
fabs(a.maxSpeedCorFac - b.maxSpeedCorFac) < 0.1 &&
fabs(a.maxSpeed - b.maxSpeed) < 0.1 &&
a.restrPosRestr == b.restrPosRestr &&
a.restrNegRestr == b.restrNegRestr &&
a.noRestrFilter == b.noRestrFilter;