clean up and refactor half-baked development commits and squash them into a new version.

Changes:

* support for multiple GTFS feeds as input in filtering, read default global and local configuration files
* be a bit more memory conservative
* make caching optional
* dont delete orphan edge if it would transform a degree 3 node with a possible full turn into a degree 2 node eligible for contraction
* dedicated filters for funicular and gondola
* make max snap level option more intuitive
* allow filter rules for level 0
* additional fallback for station snapping
* dont try to route for MOT unequal to trip in -T mode, force-snap to orphaned OSM station if not snap was possible
* write bounds to filtered osm
* remove unused surrounding heuristic
* use bus lanes info
* be a bit more tolerant for bus oneway streets
* create missing directories
* error if no cfg is present, clean up evaluation Makefile

CMakeLists.txt

@@ -19,6 +19,7 @@ if(OPENMP_FOUND)
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
 endif()
 
+
 # set compiler flags, see http://stackoverflow.com/questions/7724569/debug-vs-release-in-cmake
 if(OPENMP_FOUND)
 	set(CMAKE_CXX_FLAGS            "-fopenmp -Ofast -fno-signed-zeros -fno-trapping-math -Wall -Wno-format-extra-args -Wextra -Wformat-nonliteral -Wformat-security -Wformat=2 -Wextra -Wno-implicit-fallthrough -pedantic")
@@ -77,7 +78,6 @@ add_custom_target(
 	)
 
 # handles install target
-
 install(
 	FILES pfaedle.cfg DESTINATION etc/${PROJECT_NAME} COMPONENT config PERMISSIONS WORLD_READ
 	)

README.md

@@ -40,7 +40,7 @@ make install
 ## Generating shapes for a GTFS feed
 
 ```
-pfaedle -c <CFG FILE> -x <OSM FILE> <GTFS INPUT FOLDER>
+pfaedle -x <OSM FILE> <GTFS INPUT FOLDER>
 ```
 
 A shape'd version of the input GTFS feed will be written to `./gtfs-out`.
@@ -51,22 +51,15 @@ input feed. To drop all existing shapes, use the `-D` flag.
 For example, you may generate (and replace existing, see -D parameter) shapes for the GTFS dataset for Freiburg like this:
 
 ```
-$ mkdir freiburg_gtfs && cd freiburg_gtfs
+$ wget https://fritz.freiburg.de/csv_Downloads/VAGFR.zip && unzip VAGFR.zip
-$ wget https://fritz.freiburg.de/csv_Downloads/VAGFR.zip
+$ wget http://download.geofabrik.de/europe/germany/baden-wuerttemberg/freiburg-regbez-latest.osm.bz2 && bunzip2 freiburg-regbez-latest.osm.bz2
-$ unzip VAGFR.zip
+$ pfaedle -D -x freiburg-regbez-latest.osm .
-$ wget http://download.geofabrik.de/europe/germany/baden-wuerttemberg/freiburg-regbez-latest.osm.bz2
-$ bunzip2 freiburg-regbez-latest.osm.bz2
-$ mkdir gtfs-out
-$ pfaedle -D -c pfaedle.cfg -x freiburg-regbez-latest.osm .
 ```
 
-A default configuration file `pfaedle.cfg` can be found in this repo.
-
-
 ## Generating shapes for a specific MOT
 
 To generate shapes only for a specific mot, use the `-m` option. Possible
-values are either `tram`, `bus`, `rail`, `subway`, `ferry`, `funicular`,
+values are either `tram`, `bus`, `coach`, `rail`, `subway`, `ferry`, `funicular`,
 `gondola`, `all` (default) or GTFS vehicle type codes (0, 1, 2, 3, 4, 5, 6, 7).
 
 Multiple values can be specified (comma separated).
@@ -75,7 +68,7 @@ Multiple values can be specified (comma separated).
 
 `pfaedle` comes with the ability to filter OpenStreetMap data. If you specify
 the `-X` flag, `pfaedle` will filter the input OSM file and output a new OSM
-file which contains *exactly* the data needed to calculate the shapes for the
+file which contains exactly the data needed to calculate the shapes for the
 input GTFS feed and the input configuration.
 
 This can be used to avoid parsing (for example) the entire world.osm on each
@@ -95,9 +88,7 @@ The following flags may be useful for debugging:
 
 # Configuration
 
-The main config file distributed with this repository is `pfaedle.cfg`. The
+A default configuration file `pfaedle.cfg` can be found in this repo and will be installed with `make install`. Custom configuration files can be specified with the `-c` flag. If no `-c` flag is set, `pfaedle` will parse and merge the following cfg files in the given order (if present): `<install prefix>/etc/pfaedle/pfaedle.cfg`, `$HOME/.config/pfaedle/pfaedle.cfg`, `<CWD>/pfaedle.cfg`. Values given in later files will overwrite earlier defined values.
-config file has some comments which hopefully explain the meaning behind the
-parameters.
 
 # Evaluation
 

eval/Makefile

@@ -7,111 +7,127 @@ lighteval: vitoria.lighteval stuttgart.lighteval paris.lighteval switzerland.lig
 eval: vitoria.eval stuttgart.eval paris.eval switzerland.eval
 
 clean:
-	rm -f *.eval
+	@rm -f *.eval
-	rm -rf gtfs
+	@rm -rf gtfs
-	rf -rf osm
+	@rm -rf osm
-	rm -rf evalout
+	@rm -rf evalout
 
 osmconvert:
-	wget -O - http://m.m.i24.cc/osmconvert.c | cc -x c - -lz -O3 -o osmconvert
+	@echo `date +"[%F %T.%3N]"` "EVAL : Fetching osmconvert..."
+	@curl http://m.m.i24.cc/osmconvert.c | cc -x c - -lz -O3 -o osmconvert
 
 %.lighteval: osm/%.osm gtfs/%/stops.txt gtfs/%/stop_times.txt gtfs/%/trips.txt gtfs/%/routes.txt eval.cfg
-	mkdir -p gtfs/$*/shaped
+	@echo `date +"[%F %T.%3N]"` "EVAL : Running light (without stats) evaluation for '"$*"'..."
-	rm -f gtfs/$*/shaped/*
+	@mkdir -p gtfs/$*/shaped
-	../build/pfaedle -x $< -i gtfs/$* -c eval.cfg -o gtfs/$*/shaped -D -m all 2>&1 | tee $@
+	@rm -f gtfs/$*/shaped/*
+	@../build/pfaedle -x $< -i gtfs/$* -c eval.cfg -o gtfs/$*/shaped -D -m all 2>&1 | tee $@
 
 %.eval: osm/%.osm gtfs/%/stops.txt gtfs/%/stop_times.txt gtfs/%/trips.txt gtfs/%/routes.txt eval.cfg eval-wo-osm-line-rels.cfg
-	mkdir -p gtfs/$*/shaped
+	@echo `date +"[%F %T.%3N]"` "EVAL : Running evaluation for '"$*"'..."
-	rm -f gtfs/$*/shaped/*
+	@mkdir -p gtfs/$*/shaped
-	mkdir -p evalout/
+	@rm -f gtfs/$*/shaped/*
-	mkdir -p evalout/$*/
+	@mkdir -p evalout/
-	mkdir -p evalout/$*/hmm+osm
+	@mkdir -p evalout/$*/
-	../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --eval-path evalout/$*/hmm+osm -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@
+	@mkdir -p evalout/$*/hmm+osm
-	find evalout/$*/hmm+osm/ -name "*.json" -print0 | xargs -0 rm
+	@../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --eval-path evalout/$*/hmm+osm -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@
+	@find evalout/$*/hmm+osm/ -name "*.json" -print0 | xargs -0 rm
 
-	mkdir -p evalout/$*/greedy
+	@mkdir -p evalout/$*/greedy
-	../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --method greedy --eval-path evalout/$*/greedy -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@
+	@../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --method greedy --eval-path evalout/$*/greedy -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@
-	find evalout/$*/greedy/ -name "*.json" -print0 | xargs -0 rm
+	@find evalout/$*/greedy/ -name "*.json" -print0 | xargs -0 rm
 
-	mkdir -p evalout/$*/greedy2
+	@mkdir -p evalout/$*/greedy2
-	../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --method greedy2 --eval-path evalout/$*/greedy2 -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@
+	@../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --method greedy2 --eval-path evalout/$*/greedy2 -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@
-	find evalout/$*/greedy2/ -name "*.json" -print0 | xargs -0 rm
+	@find evalout/$*/greedy2/ -name "*.json" -print0 | xargs -0 rm
 
-	mkdir -p evalout/$*/hmm
+	@mkdir -p evalout/$*/hmm
-	../build/pfaedle -x $< -i gtfs/$* -c eval-wo-osm-line-rels.cfg --eval-path evalout/$*/hmm -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@
+	@../build/pfaedle -x $< -i gtfs/$* -c eval-wo-osm-line-rels.cfg --eval-path evalout/$*/hmm -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@
-	find evalout/$*/hmm/ -name "*.json" -print0 | xargs -0 rm
+	@find evalout/$*/hmm/ -name "*.json" -print0 | xargs -0 rm
 
 osm/spain-latest.osm.pbf:
-	mkdir -p osm
+	@mkdir -p osm
-	wget http://download.geofabrik.de/europe/spain-latest.osm.pbf -O $@
+	@echo `date +"[%F %T.%3N]"` "EVAL : Downloading OSM data for Spain..."
+	@curl --progress-bar http://download.geofabrik.de/europe/spain-latest.osm.pbf > $@
 
 osm/spain-latest.osm: osm/spain-latest.osm.pbf osmconvert
 	@# pre-filter to vitoria gasteiz
-	osmconvert -b=-2.8661,42.7480,-2.4788,43.0237 $< > $@
+	@echo `date +"[%F %T.%3N]"` "EVAL : Pre-filtering OSM data to Vitoria-Gasteiz..."
+	@osmconvert -b=-2.8661,42.7480,-2.4788,43.0237 $< > $@
 
 osm/baden-wuerttemberg-latest.osm.pbf:
-	mkdir -p osm
+	@mkdir -p osm
-	wget http://download.geofabrik.de/europe/germany/baden-wuerttemberg-latest.osm.pbf -O $@
+	@echo `date +"[%F %T.%3N]"` "EVAL : Downloading OSM data for Baden-Württemberg..."
+	@curl --progress-bar http://download.geofabrik.de/europe/germany/baden-wuerttemberg-latest.osm.pbf > $@
 
 osm/baden-wuerttemberg-latest.osm: osm/baden-wuerttemberg-latest.osm.pbf osmconvert
-	osmconvert $< > $@
+	@echo `date +"[%F %T.%3N]"` "EVAL : Extracting OSM data..."
+	@osmconvert $< > $@
 
 osm/france-latest.osm.pbf:
-	mkdir -p osm
+	@mkdir -p osm
-	wget http://download.geofabrik.de/europe/france-latest.osm.pbf -O $@
+	@echo `date +"[%F %T.%3N]"` "EVAL : Downloading OSM data for France..."
+	@curl --progress-bar http://download.geofabrik.de/europe/france-latest.osm.pbf > $@
 
 osm/paris-latest.osm: osm/france-latest.osm.pbf osmconvert
 	@# pre-filter to greater ile de france
-	osmconvert -b=0.374,47.651,4.241,50.261 $< > $@
+	@echo `date +"[%F %T.%3N]"` "EVAL : Pre-filtering OSM data to Île-de-France..."
+	@osmconvert -b=0.374,47.651,4.241,50.261 $< > $@
 
 osm/europe-latest.osm.pbf:
-	mkdir -p osm
+	@mkdir -p osm
-	wget http://download.geofabrik.de/europe-latest.osm.pbf -O $@
+	@echo `date +"[%F %T.%3N]"` "EVAL : Downloading OSM data for Europe..."
+	@curl --progress-bar http://download.geofabrik.de/europe-latest.osm.pbf > $@
 
 osm/switzerland-latest.osm: osm/europe-latest.osm.pbf osmconvert
 	@# pre-filter to greater switzerland
-	osmconvert -b=3.757,44.245,15.579,52.670 $< > $@
+	@echo `date +"[%F %T.%3N]"` "EVAL : Pre-filtering OSM data to Switzerland..."
+	@osmconvert -b=3.757,44.245,15.579,52.670 $< > $@
 
 gtfs/vitoria/%.txt:
-	mkdir -p gtfs
+	@echo `date +"[%F %T.%3N]"` "EVAL : Downloading GTFS data for Vitoria-Gasteiz..."
-	mkdir -p gtfs/vitoria
+	@mkdir -p gtfs
-	wget https://transitfeeds.com/p/tuvisa-euskotran/239/latest/download -O gtfs/vitoria/gtfs.zip
+	@mkdir -p gtfs/vitoria
-	cd gtfs/vitoria && unzip -o gtfs.zip
+	@curl --progress-bar https://transitfeeds.com/p/tuvisa-euskotran/239/latest/download > gtfs/vitoria/gtfs.zip
-	rm gtfs/vitoria/gtfs.zip
+	@cd gtfs/vitoria && unzip -qq -o gtfs.zip
+	@rm gtfs/vitoria/gtfs.zip
 
 gtfs/stuttgart/%.txt:
-	mkdir -p gtfs
+	@echo `date +"[%F %T.%3N]"` "EVAL : Downloading GTFS data for Stuttgart..."
-	mkdir -p gtfs/stuttgart
+	@mkdir -p gtfs
-	echo "******************************************************************"
+	@mkdir -p gtfs/stuttgart
-	echo "* A password is required to access the VVS dataset. Send a mail  *"
+	@echo "******************************************************************"
-	echo "* to brosi@cs.informatik.uni-freiburg.de to receive the password. "
+	@echo "* A password is required to access the VVS dataset. Send a mail  *"
-	echo "******************************************************************"
+	@echo "* to brosi@cs.informatik.uni-freiburg.de to receive the password. "
-	wget http://www.vvs.de/download/opendata/VVS_GTFS.zip --ask-password --user vvsopendata01 -O gtfs/stuttgart/gtfs.zip
+	@echo "******************************************************************"
-	cd gtfs/stuttgart && unzip -o gtfs.zip
+	@curl --progress-bar http://www.vvs.de/download/opendata/VVS_GTFS.zip -su vvsopendata01 > gtfs/stuttgart/gtfs.zip
-	rm gtfs/stuttgart/gtfs.zip
+	@cd gtfs/stuttgart && unzip -qq -o gtfs.zip
+	@rm gtfs/stuttgart/gtfs.zip
 
 gtfs/paris/%.txt:
-	mkdir -p gtfs
+	@echo `date +"[%F %T.%3N]"` "EVAL : Downloading GTFS data for Paris..."
-	mkdir -p gtfs/paris
+	@mkdir -p gtfs
-	wget https://transitfeeds.com/p/stif/822/latest/download -O gtfs/paris/gtfs.zip
+	@mkdir -p gtfs/paris
-	cd gtfs/paris && unzip -o gtfs.zip
+	@curl --progress-bar https://transitfeeds.com/p/stif/822/latest/download > gtfs/paris/gtfs.zip
-	rm gtfs/paris/gtfs.zip
+	@cd gtfs/paris && unzip -qq -o gtfs.zip
+	@rm gtfs/paris/gtfs.zip
 
 gtfs/switzerland/%.txt:
-	mkdir -p gtfs
+	@echo `date +"[%F %T.%3N]"` "EVAL : Downloading GTFS data for Switzerland..."
-	mkdir -p gtfs/switzerland
+	@mkdir -p gtfs
-	wget http://gtfs.geops.ch/dl/gtfs_complete.zip -O gtfs/switzerland/gtfs.zip
+	@mkdir -p gtfs/switzerland
-	cd gtfs/switzerland && unzip -o gtfs.zip
+	@curl --progress-bar http://gtfs.geops.ch/dl/gtfs_complete.zip > gtfs/switzerland/gtfs.zip
-	rm gtfs/switzerland/gtfs.zip
+	@cd gtfs/switzerland && unzip -qq -o gtfs.zip
+	@rm gtfs/switzerland/gtfs.zip
 
-osm/vitoria.osm: osm/spain-latest.osm gtfs/vitoria/stops.txt eval.cfg
-	../build/pfaedle -x $< -i gtfs/vitoria/ -c eval.cfg -m all -X $@
 
-osm/stuttgart.osm: osm/baden-wuerttemberg-latest.osm gtfs/stuttgart/stops.txt eval.cfg
+osm/vitoria.osm: osm/spain-latest.osm gtfs/vitoria/stops.txt gtfs/vitoria/trips.txt gtfs/vitoria/routes.txt gtfs/vitoria/stop_times.txt eval.cfg
-	../build/pfaedle -x $< -i gtfs/stuttgart/ -c eval.cfg -m all -X $@
+	@../build/pfaedle -x $< -i gtfs/vitoria/ -c eval.cfg -m all -X $@
 
-osm/paris.osm: osm/paris-latest.osm gtfs/paris/stops.txt eval.cfg
+osm/stuttgart.osm: osm/baden-wuerttemberg-latest.osm gtfs/stuttgart/stops.txt gtfs/stuttgart/trips.txt gtfs/stuttgart/routes.txt gtfs/stuttgart/stop_times.txt eval.cfg
-	../build/pfaedle -x $< -i gtfs/paris/ -c eval.cfg -m all -X $@
+	@../build/pfaedle -x $< -i gtfs/stuttgart/ -c eval.cfg -m all -X $@
+
+osm/paris.osm: osm/paris-latest.osm gtfs/paris/stops.txt gtfs/paris/trips.txt gtfs/paris/routes.txt gtfs/paris/stop_times.txt eval.cfg
+	@../build/pfaedle -x $< -i gtfs/paris/ -c eval.cfg -m all -X $@
 
 osm/switzerland.osm: osm/switzerland-latest.osm gtfs/switzerland/stops.txt eval.cfg
-	../build/pfaedle -x $< -i gtfs/switzerland/ -c eval.cfg -m all -X $@
+	@../build/pfaedle -x $< -i gtfs/switzerland/ -c eval.cfg -m all -X $@

pfaedle.cfg

@@ -2,6 +2,216 @@
 # Chair of Algorithms and Datastructures
 # Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
 
+[tram, bus, coach, subway, rail, gondola, funicular, ferry]
+# Regular expressions and station comparision is
+# always case insensitive!
+station_normalize_chain:
+	, -> ' ';
+	- -> ' ';
+	— -> ' ';
+	_ -> ' ';
+	" -> '';
+	' -> '';
+	` -> '';
+	\( -> ' ';
+	\) -> ' ';
+	\[ -> ' ';
+	\] -> ' ';
+	/ -> ' ';
+	'\\' -> ' ';
+	< -> ' ';
+	> -> ' ';
+	& -> '+';
+	Ä -> Ae;
+	Ö -> Oe;
+	Ü -> Ue;
+	ä -> ae;
+	ö -> oe;
+	ü -> ue;
+	ß -> ss;
+	è -> e;
+	é -> e;
+	á -> a;
+	à -> a;
+	ó -> o;
+	ò -> o;
+	ô -> o;
+	ç -> c;
+	í -> i;
+	ú -> u;
+	ù -> u;
+	ë -> e;
+	å -> ae;
+	â -> a;
+	ê -> e;
+	ï -> i;
+	œ -> oe;
+	ø -> oe;
+	str\. -> strasse;
+	av\. -> avenue;
+
+	# always separate 'street', 'strasse'
+	'([a-zA-Z])strasse($| )' -> '\1 strasse\2';
+	'([a-zA-Z])street($| )' -> '\1 street\2';
+
+	# always use "street"
+	'(^| )strasse($| )' -> '\1street\2';
+
+	# always use "avenue"
+	'(^| )avenida($| )' -> '\1avenue\2';
+	'(^| )avenu($| )' -> '\1avenue\2';
+
+	# normalize every possible abbr. of german "Bahnhof", "Hauptbahnhof", "Busbahnhof"
+	'(^| )hauptbf\.($| )' -> '\1hauptbahnhof\2';
+	'(^| )hauptbf($| )' -> '\1hauptbahnhof\2';
+	'(^| )hauptbhf\.($| )' -> '\1hauptbahnhof\2';
+	'(^| )hauptbhf($| )' -> '\1hauptbahnhof\2';
+	'(^| )zentraler busbahnhof($| )$' -> \1busbahnhof\2;
+	'(^| )zentraler omnibusbahnhof($| )$' -> \1busbahnhof\2;
+	'(^| )omnibusbahnhof($| )' -> '\1busbahnhof\2';
+	'(^| )omnibusbhf($| )' -> '\1busbahnhof\2';
+	'(^| )busbf\.($| )' -> '\1busbahnhof\2';
+	'(^| )busbf($| )' -> '\1busbahnhof\2';
+	'(^| )bus bf\.($| )' -> '\1busbahnhof\2';
+	'(^| )bus bf($| )' -> '\1busbahnhof\2';
+	'(^| )busbhf\.($| )' -> '\1busbahnhof\2';
+	'(^| )busbhf($| )' -> '\1busbahnhof\2';
+	'(^| )bus bhf\.($| )' -> '\1busbahnhof\2';
+	'(^| )bus bhf($| )' -> '\1busbahnhof\2';
+	'(^| )zob($| )' -> '\1busbahnhof\2';
+	'(^| )hbf\.($| )' -> '\1hauptbahnhof\2';
+	'(^| )hbf($| )' -> '\1hauptbahnhof\2';
+	'(^| )hb\.($| )' -> '\1hauptbahnhof\2';
+	'(^| )hb($| )' -> '\1hauptbahnhof\2';
+	'(^| )bf\.($| )' -> '\1bahnhof\2';
+	'(^| )bf($| )' -> '\1bahnhof\2';
+	'(^| )bhf\.($| )' -> '\1bahnhof\2';
+	'(^| )bhf($| )' -> '\1bahnhof\2';
+	'(^| )bhfeingang($| )' -> '\1bahnhofeingang\2';
+	'(^| )gare de($| )' -> '\1gare\2';
+
+
+	# if a stations starts with  single station identifier
+	# always put it at the end (for example, "hauptbahnhof freiburg" becomes "freiburg hauptbahnhof")
+	'^hauptbahnhof (.+)$' -> \1 hauptbahnhof;
+	'^bahnhof (.+)$' -> \1 bahnhof;
+	'^busbahnhof (.+)$' -> \1 busbahnhof;
+	'^gare (.+)$' -> \1 gare;
+	'^station (.+)$' -> \1 station;
+
+	'(^| )busbahnhof($| )' -> '\1bbahnhof\2';
+
+	# normalize line types in station names
+	'(^| )u bahn\.($| )' -> '\1ubahn\2';
+	'(^| )metro\.($| )' -> '\1ubahn\2';
+	'(^| )subway\.($| )' -> '\1ubahn\2';
+	'(^| )underground\.($| )' -> '\1ubahn\2';
+	'(^| )ubahn($| )' -> '\1u\2';
+	'(^| )s bahn\.($| )' -> '\1sbahn\2';
+	'(^| )sbahn($| )' -> '\1s\2';
+	'(^| )tramway($| )' -> '\1tram\2';
+	'(^| )stadtbahn($| )' -> '\1tram\2';
+	'(^| )strassenbahn($| )' -> '\1tram\2';
+	'(^| )streetcar($| )' -> '\1tram\2';
+	'(^| )tram($| )' -> '\1t\2';
+
+	# delete track information from name
+	'(^| )kante [a-zA-Z0-9]{1,2}($| )' -> ' ';
+	'(^| )gleis [a-zA-Z0-9]{1,2}($| )' -> ' ';
+	'(^| )track [a-zA-Z0-9]{1,2}($| )' -> ' ';
+	'(^| )voie [a-zA-Z0-9]{1,2}($| )' -> ' ';
+
+	# abbrv
+	'(^| )und($| )' -> '\1+\2';
+	'(^| )and($| )' -> '\1+\2';
+	'(^| )et($| )' -> '\1+\2';
+
+	# noise
+	'\sde\s' -> ' ';
+	'\sda\s' -> ' ';
+	'\sdi\s' -> ' ';
+	'\sdel\s' -> ' ';
+	'\sdal\s' -> ' ';
+
+	# abbrv in most western languages
+	'(^| )saint ' -> '\1st. ';
+	'(^| )sankt ' -> '\1st. ';
+	'(^| )sanct ' -> '\1st. ';
+
+	\. -> ' ';
+
+	# whitespace
+	\s+ -> ' ';
+	^\s -> '';
+	\s$ -> '';
+
+line_normalize_chain:
+	, -> ' ';
+	- -> ' ';
+	_ -> ' ';
+	" -> '';
+	' -> '';
+	` -> '';
+	/ -> ' ';
+	< -> ' ';
+	> -> ' ';
+	& -> '+';
+	ä -> ae;
+	ö -> oe;
+	ü -> ue;
+	Ä -> Ae;
+	Ö -> Oe;
+	Ü -> Ue;
+	ß -> ss;
+	è -> e;
+	é -> e;
+	á -> a;
+	à -> a;
+	ó -> o;
+	ò -> o;
+	í -> i;
+	ú -> u;
+	ù -> u;
+	ë -> e;
+	å -> ae;
+	ç -> c;
+	â -> a;
+	ê -> e;
+	ï -> i;
+	œ -> oe;
+	ø -> oe;
+	^line -> '';
+	^linie -> '';
+	^metro -> '';
+	^tram -> '';
+	^strassenbahn -> '';
+	^bus -> '';
+
+	# delete everything in brackets
+	\(.+\) -> ' ';
+	\[.+\] -> ' ';
+
+	# whitespace
+	\s+ -> ' ';
+	^\s -> '';
+	\s$ -> '';
+
+	# line/number combs ALWAYS without whitespace (T 2 -> T2)
+	^([a-zA-Z]+) ([0-9]+)$ -> \1\2;
+
+track_normalize_chain:
+	'(^| )gleis($| )' -> '';
+	'(^| )gl\.($| )' -> '';
+	'(^| )platform($| )' -> '';
+	'(^| )track($| )' -> '';
+	'(^| )rail($| )' -> '';
+	# line/number combs ALWAYS without whitespace (1 A -> 1A)
+	^([a-zA-Z]+) ([0-9]+)$ -> \1\2;
+	^([0-9]+) ([a-zA-Z]+)$ -> \1\2;
+
+	# delete track numbers greater than 999
+	^[0-9]{4,}$ -> '';
+
 [rail]
 
 # OSM entities to keep on different levels, as k=v. Applies
@@ -12,8 +222,10 @@
 osm_filter_keep:
 	railway=rail
 	railway=light_rail
+	railway=tram
 	railway=narrow_gauge
 	route=rail
+	route=light_rail
 	route=train
 	public_transport=stop_area|rel_flat
 
@@ -21,10 +233,11 @@ osm_filter_lvl1:
 	usage=branch
 
 osm_filter_lvl2:
+	railway=tram
+	service=siding
 
 osm_filter_lvl3:
 	service=crossover
-	service=siding
 	# we cannot completely drop service=yard, because it is often used
 	# incorrectly for crossovers
 	service=yard
@@ -213,6 +426,9 @@ line_normalize_chain:
 	ä -> ae;
 	ö -> oe;
 	ü -> ue;
+	Ä -> Ae;
+	Ö -> Oe;
+	Ü -> Ue;
 	ß -> ss;
 	è -> e;
 	é -> e;
@@ -253,7 +469,21 @@ line_normalize_chain:
 	# if a character line number is present, delete the numeric part
 	^([a-zA-Z]+) [0-9]+$ -> \1;
 
-[bus]
+track_normalize_chain:
+	'(^| )gleis($| )' -> '';
+	'(^| )gl\.($| )' -> '';
+	'(^| )platform($| )' -> '';
+	'(^| )track($| )' -> '';
+	'(^| )rail($| )' -> '';
+	^([a-zA-Z]+) ([0-9]+)$ -> \1\2;
+	# number/char combs ALWAYS without char
+	^([0-9]+) ([a-zA-Z]+)$ -> \1;
+	^([0-9]+)([a-zA-Z]+)$ -> \1;
+
+	# delete track numbers greater than 999
+	^[0-9]{4,}$ -> '';
+
+[bus, coach]
 
 # OSM entities to keep on different levels, as k=v. Applies
 # to nodes, edges and relations.
@@ -287,6 +517,18 @@ osm_filter_keep:
 	psv=yes
 	psv=designated
 
+	bus:lanes=yes
+	bus:lanes=designated
+	bus:lanes=1
+
+	lanes:bus=1
+	lanes:bus=2
+	lanes:bus=3
+
+	lanes:psv=1
+	lanes:psv=2
+	lanes:psv=3
+
 	trolley_wire=yes
 	trolleywire=yes
 	trolleybus=yes
@@ -448,6 +690,14 @@ osm_filter_undirected:
 	busway:right=opposite_lane
 	psv=opposite_lane
 	psv=opposite
+	lanes:psv:backward=1
+	lanes:psv:backward=2
+	lanes:bus:backward=1
+	lanes:bus:backward=2
+	bus:lanes:backward=yes
+	bus:lanes:backward=designated
+	bus:lanes:backward=1
+
 
 # Nodes that are stations.
 # Only nodes that have been kept during the filtering above will be
@@ -488,9 +738,11 @@ osm_station_group_attrs:
 # original station position
 osm_max_snap_distance: 10, 50, 100
 
+osm_max_snap_fallback_distance: 300
+
 osm_max_snap_level: 5
 
-osm_max_osm_station_distance: 7.5
+osm_max_osm_station_distance: 8.0
 
 # sorted by priority, first found attr will be taken
 osm_station_name_attrs:
@@ -541,7 +793,65 @@ routing_one_way_edge_punish:	5000
 # information
 # routing_line_unmatched_punish_fac:  1.75
 
-[tram, subway, funicular]
+[coach]
+
+# OSM entities to keep on different levels, as k=v. Applies
+# to nodes, edges and relations.
+# Nodes included in kept ways are always kept.
+# Ways included in kept relations are always kept.
+
+osm_filter_lvl0:
+	highway=motorway
+	highway=motorway_link
+
+osm_filter_lvl1:
+	highway=trunk
+	highway=trunk_link
+
+osm_filter_lvl2:
+	highway=primary
+	highway=primary_link
+
+osm_filter_lvl3:
+	highway=secondary
+	highway=secondary_link
+
+osm_filter_lvl4:
+	highway=tertiary
+	highway=tertiary_link
+
+osm_filter_lvl5:
+	highway=unclassified
+	highway=residential
+	highway=road
+	highway=service
+
+osm_filter_lvl6:
+	highway=living_street
+	highway=pedestrian
+	psv=no
+
+osm_filter_lvl7:
+	bus=no
+	service=siding
+	access=permissive
+	access=private
+	access=no
+	service=parking_aisle
+	highway=footway
+
+routing_lvl0_fac:	1  # default level
+routing_lvl1_fac:	1.15
+routing_lvl2_fac:	1.5
+routing_lvl3_fac:	1.75
+routing_lvl4_fac:	2.25
+routing_lvl5_fac:	2.5
+routing_lvl6_fac:	3
+routing_lvl7_fac:	4
+
+osm_max_snap_level: 5
+
+[tram, subway]
 
 # OSM entities to keep on different levels, as k=v. Applies
 # to nodes, edges and relations.
@@ -550,6 +860,7 @@ routing_one_way_edge_punish:	5000
 
 osm_filter_keep:
 	route=tram
+	route=funicular
 	railway=subway
 	railway=light_rail
 	railway=tram
@@ -562,9 +873,326 @@ osm_filter_keep:
 	subway=yes
 	tram=yes
 
-osm_filter_lv2:
+osm_filter_lvl2:
 	service=siding
 
+osm_filter_lvl3:
+	railway=funicular
+	route=funicular
+
+osm_filter_lvl5:
+	service=crossover
+	service=yard
+
+# OSM entities to drop, as k=v. Applies to nodes, edges and
+# relations.
+# Nodes included in non-dropped ways are kept regardless of
+# a matching drop filter.
+# Ways included in non-dropped relations are kept regardless of
+# a matching drop filter.
+
+osm_filter_drop:
+	area=yes
+	public_transport=stop_area
+	type=multipolygon
+	railway=platform
+	public_transport=platform
+	service=alley
+
+# Nodes that should act as "no-hup" nodes. These are nodes
+# that are contained in multiple ways, but cannot be used
+# to switch from one way to another (for example, a
+# track crossing in rail networks)
+
+osm_filter_nohup:
+	railway:switch=no
+	railway=railway_crossing
+
+# Edges that should act as one-way nodes.
+
+osm_filter_oneway:
+	oneway=yes
+
+# Edges that may explicitely be used in
+# both directions. May be used to set exception
+# to "osm_filter_oneway"
+
+osm_filter_undirected:
+
+# Nodes that are stations.
+# Only nodes that have been kept during the filtering above will be
+# checked.
+osm_filter_station:
+	public_transport=stop_position
+	station=subway
+	station=tram
+	railway=stop
+	railway=halt
+	railway=station
+	railway=tram_stop
+	railway=subway_stop
+	tram_stop=*
+	stop=*
+
+# Relation fields that should be used for catching the lines that
+# occur on an edge. Only relations that have been kept during the
+# filtering above will be checked. The 'linename' will be normalized
+# according to the rules in line_normalization_chain.
+# The 'from_name' and 'to_name' will be normalized according to the
+# rules in station_normalization_chain.
+# The relations tags are given in the order of their relevance -
+# the first normalized tag-value that is not null/empty will be
+# taken.
+osm_line_relation_tags:
+	line_name=ref,name   # careful, no space after/before comma allowed!
+	from_name=from
+	to_name=to
+
+
+# attr name together with the
+# max distance in meters between any of the groups members and
+# a potential new member
+# first matching rule will be taken
+# only applies to nodes that match osm_filter_station!
+osm_station_group_attrs:
+	uic_ref=500
+	wikidata=500
+	name=100
+
+# max distance in meters between a snapped station position and the
+# original station position
+osm_max_snap_distance:	10, 50, 100
+
+osm_max_snap_level: 4
+
+
+# sorted by priority, first found attr will be taken
+osm_station_name_attrs:
+	name
+	uic_name
+
+# the track number tag in stop nodes, first one is taken
+osm_track_number_tags:	local_ref
+
+routing_lvl0_fac:	1  # default level
+routing_lvl1_fac:	1.5
+routing_lvl2_fac:	2
+routing_lvl3_fac:	2.5
+routing_lvl4_fac:	3.5
+routing_lvl5_fac:	5
+routing_lvl6_fac:	5
+routing_lvl7_fac:	5
+
+# Punishment (in meters) to add to the distance
+# function if a vehicle performans a full turn
+routing_full_turn_punish:			2000
+
+routing_station_distance_punish_fac: 3.14
+
+routing_non_osm_station_punish:	235
+
+# Max angle that should be counted as a full turn
+routing_full_turn_angle:			80
+
+# Max angle in a route from a station to an already reachable neighbar
+routing_snap_full_turn_angle:	80
+
+# Punishment (in meters) to add to the distance
+# function if a vehicle passes a station node without
+# stopping there
+routing_pass_thru_station_punish:   100
+
+# Punishment factor for every meter a vehicle
+# travels through a one-way edge
+routing_one_way_meter_punish_fac:   1
+
+# Punishment factor for every meter a vehicle
+# travels through an edge without any matching line
+# information
+routing_line_unmatched_punish_fac:  0.5
+
+[gondola]
+
+# OSM entities to keep on different levels, as k=v. Applies
+# to nodes, edges and relations.
+# Nodes included in kept ways are always kept.
+# Ways included in kept relations are always kept.
+
+osm_filter_keep:
+	aerialway=gondola
+	aerialway=cable_car
+	aerialway=chair_lift
+	aerialway=mixed_lift
+
+
+# OSM entities to drop, as k=v. Applies to nodes, edges and
+# relations.
+# Nodes included in non-dropped ways are kept regardless of
+# a matching drop filter.
+# Ways included in non-dropped relations are kept regardless of
+# a matching drop filter.
+
+osm_filter_drop:
+	area=yes
+	public_transport=stop_area
+	type=multipolygon
+	railway=platform
+	public_transport=platform
+	service=alley
+
+# Nodes that should act as "no-hup" nodes. These are nodes
+# that are contained in multiple ways, but cannot be used
+# to switch from one way to another (for example, a
+# track crossing in rail networks)
+
+osm_filter_nohup:
+
+# Edges that should act as one-way nodes.
+
+osm_filter_oneway:
+	oneway=yes
+
+# Edges that may explicitely be used in
+# both directions. May be used to set exception
+# to "osm_filter_oneway"
+
+osm_filter_undirected:
+
+# Nodes that are stations.
+# Only nodes that have been kept during the filtering above will be
+# checked.
+osm_filter_station:
+	aerialway=station
+	aerialway=stop
+	public_transport=stop_position
+	station=subway
+	station=tram
+	railway=stop
+	railway=halt
+	railway=station
+	railway=tram_stop
+	railway=subway_stop
+	tram_stop=*
+	stop=*
+
+# Relation fields that should be used for catching the lines that
+# occur on an edge. Only relations that have been kept during the
+# filtering above will be checked. The 'linename' will be normalized
+# according to the rules in line_normalization_chain.
+# The 'from_name' and 'to_name' will be normalized according to the
+# rules in station_normalization_chain.
+# The relations tags are given in the order of their relevance -
+# the first normalized tag-value that is not null/empty will be
+# taken.
+osm_line_relation_tags:
+	line_name=ref,name   # careful, no space after/before comma allowed!
+	from_name=from
+	to_name=to
+
+
+# attr name together with the
+# max distance in meters between any of the groups members and
+# a potential new member
+# first matching rule will be taken
+# only applies to nodes that match osm_filter_station!
+osm_station_group_attrs:
+	uic_ref=500
+	wikidata=500
+	name=100
+
+# max distance in meters between a snapped station position and the
+# original station position
+osm_max_snap_distance:	10, 50, 100
+
+osm_max_snap_level: 4
+
+
+# sorted by priority, first found attr will be taken
+osm_station_name_attrs:
+	name
+	uic_name
+
+# the track number tag in stop nodes, first one is taken
+osm_track_number_tags:	local_ref
+
+routing_lvl0_fac:	1  # default level
+routing_lvl1_fac:	1.5
+routing_lvl2_fac:	2
+routing_lvl3_fac:	2.5
+routing_lvl4_fac:	3.5
+routing_lvl5_fac:	5
+routing_lvl6_fac:	5
+routing_lvl7_fac:	5
+
+# Punishment (in meters) to add to the distance
+# function if a vehicle performans a full turn
+routing_full_turn_punish:			2000
+
+routing_station_distance_punish_fac: 3.14
+
+routing_non_osm_station_punish:	235
+
+# Max angle that should be counted as a full turn
+routing_full_turn_angle:			80
+
+# Max angle in a route from a station to an already reachable neighbar
+routing_snap_full_turn_angle:	80
+
+# Punishment (in meters) to add to the distance
+# function if a vehicle passes a station node without
+# stopping there
+routing_pass_thru_station_punish:   100
+
+# Punishment factor for every meter a vehicle
+# travels through a one-way edge
+routing_one_way_meter_punish_fac:   1
+
+# Punishment factor for every meter a vehicle
+# travels through an edge without any matching line
+# information
+routing_line_unmatched_punish_fac:  0.5
+
+[funicular]
+
+# OSM entities to keep on different levels, as k=v. Applies
+# to nodes, edges and relations.
+# Nodes included in kept ways are always kept.
+# Ways included in kept relations are always kept.
+
+osm_filter_keep:
+	route=funicular
+	railway=funicular
+	railway=narrow_gauge
+	route=tram
+	railway=subway
+	railway=light_rail
+	railway=tram
+	railway=station
+	railway=halt
+	railway=tram_stop
+	route=subway
+	route=light_rail
+	subway=yes
+	tram=yes
+
+osm_filter_lvl2:
+	service=siding
+
+osm_filter_lvl3:
+	route=tram
+	route=narrow_gauge
+	railway=subway
+	railway=narrow_gauge
+	railway=light_rail
+	railway=tram
+	railway=station
+	railway=halt
+	railway=tram_stop
+	route=subway
+	route=light_rail
+	subway=yes
+	tram=yes
+
 osm_filter_lvl5:
 	service=crossover
 	service=yard
@@ -797,206 +1425,3 @@ routing_one_way_meter_punish_fac:   1
 # information
 routing_line_unmatched_punish_fac:  0.5
 
-[tram, bus, subway, rail, gondola, funicular, ferry]
-# Regular expressions and station comparision is
-# always case insensitive!
-station_normalize_chain:
-	, -> ' ';
-	- -> ' ';
-	— -> ' ';
-	_ -> ' ';
-	" -> '';
-	' -> '';
-	` -> '';
-	\( -> ' ';
-	\) -> ' ';
-	\[ -> ' ';
-	\] -> ' ';
-	/ -> ' ';
-	'\\' -> ' ';
-	< -> ' ';
-	> -> ' ';
-	& -> '+';
-	ä -> ae;
-	ö -> oe;
-	ü -> ue;
-	ß -> ss;
-	è -> e;
-	é -> e;
-	á -> a;
-	à -> a;
-	ó -> o;
-	ò -> o;
-	ô -> o;
-	ç -> c;
-	í -> i;
-	ú -> u;
-	ù -> u;
-	ë -> e;
-	å -> ae;
-	â -> a;
-	ê -> e;
-	ï -> i;
-	œ -> oe;
-	ø -> oe;
-	str\. -> strasse;
-	av\. -> avenue;
-
-	# always separate 'street', 'strasse'
-	'([a-zA-Z])strasse($| )' -> '\1 strasse\2';
-	'([a-zA-Z])street($| )' -> '\1 street\2';
-
-	# always use "street"
-	'(^| )strasse($| )' -> '\1street\2';
-
-	# always use "avenue"
-	'(^| )avenida($| )' -> '\1avenue\2';
-	'(^| )avenu($| )' -> '\1avenue\2';
-
-	# normalize every possible abbr. of german "Bahnhof", "Hauptbahnhof", "Busbahnhof"
-	'(^| )hauptbf\.($| )' -> '\1hauptbahnhof\2';
-	'(^| )hauptbf($| )' -> '\1hauptbahnhof\2';
-	'(^| )hauptbhf\.($| )' -> '\1hauptbahnhof\2';
-	'(^| )hauptbhf($| )' -> '\1hauptbahnhof\2';
-	'(^| )zentraler busbahnhof($| )$' -> \1busbahnhof\2;
-	'(^| )zentraler omnibusbahnhof($| )$' -> \1busbahnhof\2;
-	'(^| )omnibusbahnhof($| )' -> '\1busbahnhof\2';
-	'(^| )omnibusbhf($| )' -> '\1busbahnhof\2';
-	'(^| )busbf\.($| )' -> '\1busbahnhof\2';
-	'(^| )busbf($| )' -> '\1busbahnhof\2';
-	'(^| )bus bf\.($| )' -> '\1busbahnhof\2';
-	'(^| )bus bf($| )' -> '\1busbahnhof\2';
-	'(^| )busbhf\.($| )' -> '\1busbahnhof\2';
-	'(^| )busbhf($| )' -> '\1busbahnhof\2';
-	'(^| )bus bhf\.($| )' -> '\1busbahnhof\2';
-	'(^| )bus bhf($| )' -> '\1busbahnhof\2';
-	'(^| )zob($| )' -> '\1busbahnhof\2';
-	'(^| )hbf\.($| )' -> '\1hauptbahnhof\2';
-	'(^| )hbf($| )' -> '\1hauptbahnhof\2';
-	'(^| )hb\.($| )' -> '\1hauptbahnhof\2';
-	'(^| )hb($| )' -> '\1hauptbahnhof\2';
-	'(^| )bf\.($| )' -> '\1bahnhof\2';
-	'(^| )bf($| )' -> '\1bahnhof\2';
-	'(^| )bhf\.($| )' -> '\1bahnhof\2';
-	'(^| )bhf($| )' -> '\1bahnhof\2';
-	'(^| )bhfeingang($| )' -> '\1bahnhofeingang\2';
-	'(^| )gare de($| )' -> '\1gare\2';
-
-
-	# if a stations starts with  single station identifier
-	# always put it at the end (for example, "hauptbahnhof freiburg" becomes "freiburg hauptbahnhof")
-	'^hauptbahnhof (.+)$' -> \1 hauptbahnhof;
-	'^bahnhof (.+)$' -> \1 bahnhof;
-	'^busbahnhof (.+)$' -> \1 busbahnhof;
-	'^gare (.+)$' -> \1 gare;
-	'^station (.+)$' -> \1 station;
-
-	'(^| )busbahnhof($| )' -> '\1bbahnhof\2';
-
-	# normalize line types in station names
-	'(^| )u bahn\.($| )' -> '\1ubahn\2';
-	'(^| )metro\.($| )' -> '\1ubahn\2';
-	'(^| )subway\.($| )' -> '\1ubahn\2';
-	'(^| )underground\.($| )' -> '\1ubahn\2';
-	'(^| )ubahn($| )' -> '\1u\2';
-	'(^| )s bahn\.($| )' -> '\1sbahn\2';
-	'(^| )sbahn($| )' -> '\1s\2';
-	'(^| )tramway($| )' -> '\1tram\2';
-	'(^| )stadtbahn($| )' -> '\1tram\2';
-	'(^| )strassenbahn($| )' -> '\1tram\2';
-	'(^| )streetcar($| )' -> '\1tram\2';
-	'(^| )tram($| )' -> '\1t\2';
-
-	# delete track information from name
-	'(^| )kante [a-zA-Z0-9]{1,2}($| )' -> ' ';
-	'(^| )gleis [a-zA-Z0-9]{1,2}($| )' -> ' ';
-	'(^| )track [a-zA-Z0-9]{1,2}($| )' -> ' ';
-	'(^| )voie [a-zA-Z0-9]{1,2}($| )' -> ' ';
-
-	# abbrv
-	'(^| )und($| )' -> '\1+\2';
-	'(^| )and($| )' -> '\1+\2';
-	'(^| )et($| )' -> '\1+\2';
-
-	# noise
-	'\sde\s' -> ' ';
-	'\sda\s' -> ' ';
-	'\sdi\s' -> ' ';
-	'\sdel\s' -> ' ';
-	'\sdal\s' -> ' ';
-
-	# abbrv in most western languages
-	'(^| )saint ' -> '\1st. ';
-	'(^| )sankt ' -> '\1st. ';
-	'(^| )sanct ' -> '\1st. ';
-
-	\. -> ' ';
-
-	# whitespace
-	\s+ -> ' ';
-	^\s -> '';
-	\s$ -> '';
-
-line_normalize_chain:
-	, -> ' ';
-	- -> ' ';
-	_ -> ' ';
-	" -> '';
-	' -> '';
-	` -> '';
-	/ -> ' ';
-	< -> ' ';
-	> -> ' ';
-	& -> '+';
-	ä -> ae;
-	ö -> oe;
-	ü -> ue;
-	ß -> ss;
-	è -> e;
-	é -> e;
-	á -> a;
-	à -> a;
-	ó -> o;
-	ò -> o;
-	í -> i;
-	ú -> u;
-	ù -> u;
-	ë -> e;
-	å -> ae;
-	ç -> c;
-	â -> a;
-	ê -> e;
-	ï -> i;
-	œ -> oe;
-	ø -> oe;
-	^line -> '';
-	^linie -> '';
-	^metro -> '';
-	^tram -> '';
-	^strassenbahn -> '';
-	^bus -> '';
-
-	# delete everything in brackets
-	\(.+\) -> ' ';
-	\[.+\] -> ' ';
-
-	# whitespace
-	\s+ -> ' ';
-	^\s -> '';
-	\s$ -> '';
-
-	# line/number combs ALWAYS without whitespace (T 2 -> T2)
-	^([a-zA-Z]+) ([0-9]+)$ -> \1\2;
-
-track_normalize_chain:
-	'(^| )gleis($| )' -> '';
-	'(^| )gl\.($| )' -> '';
-	'(^| )platform($| )' -> '';
-	'(^| )track($| )' -> '';
-	'(^| )rail($| )' -> '';
-	# line/number combs ALWAYS without whitespace (1 A -> 1A)
-	^([a-zA-Z]+) ([0-9]+)$ -> \1\2;
-	^([0-9]+) ([a-zA-Z]+)$ -> \1\2;
-
-	# delete track numbers greater than 999
-	^[0-9]{4,}$ -> '';

src/configparser

@@ -1 +1 @@
-Subproject commit 5a1f788dde8f334d40505268f71bcc473d1968d8
+Subproject commit 63fcb1d54eb4889b376b76cafe140317326b5c56

src/cppgtfs

@@ -1 +1 @@
-Subproject commit 727ddfecc952e7bd8e4b11ef34436454a50e7532
+Subproject commit 32b081e352fc7496a4e2b9a90cf46eecaf7c63fd

src/pfaedle/Def.h

@@ -25,4 +25,6 @@
 #define BOX util::geo::Box<PFAEDLE_PRECISION>
 #define POLYLINE util::geo::PolyLine<PFAEDLE_PRECISION>
 
+#define BOX_PADDING 2500
+
 #endif  // PFAEDLE_DEF_H_

src/pfaedle/PfaedleMain.cpp

@@ -2,19 +2,25 @@
 // Chair of Algorithms and Data Structures.
 // Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
 
+#include <limits.h>
+#include <pwd.h>
 #include <signal.h>
 #include <stdio.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
 #include <fstream>
 #include <map>
 #include <string>
 #include <vector>
 #include "ad/cppgtfs/Parser.h"
 #include "ad/cppgtfs/Writer.h"
-#include "ad/cppgtfs/gtfs/Feed.h"
 #include "pfaedle/config/ConfigReader.h"
 #include "pfaedle/config/MotConfig.h"
 #include "pfaedle/config/MotConfigReader.h"
 #include "pfaedle/eval/Collector.h"
+#include "pfaedle/gtfs/Feed.h"
+#include "pfaedle/gtfs/Writer.h"
 #include "pfaedle/netgraph/Graph.h"
 #include "pfaedle/osm/OsmIdSet.h"
 #include "pfaedle/router/ShapeBuilder.h"
@@ -30,13 +36,28 @@ using pfaedle::osm::OsmBuilder;
 using pfaedle::config::MotConfig;
 using pfaedle::config::Config;
 using pfaedle::router::ShapeBuilder;
+using configparser::ParseFileExc;
 using pfaedle::config::MotConfigReader;
 using pfaedle::config::ConfigReader;
 using pfaedle::eval::Collector;
 
+enum class RetCode {
+  SUCCESS = 0,
+  NO_INPUT_FEED = 1,
+  MULT_FEEDS_NOT_ALWD = 2,
+  TRIP_NOT_FOUND = 3,
+  GTFS_PARSE_ERR = 4,
+  NO_OSM_INPUT = 5,
+  MOT_CFG_PARSE_ERR = 6,
+  OSM_PARSE_ERR = 7,
+  GTFS_WRITE_ERR = 8,
+  NO_MOT_CFG = 9
+};
+
 std::string getMotStr(const MOTs& mots);
 std::string getFileNameMotStr(const MOTs& mots);
 MOTs getContMots(const MotConfig& motCfg, const MOTs& mots);
+std::vector<std::string> getCfgPaths(const Config& cfg);
 
 // _____________________________________________________________________________
 int main(int argc, char** argv) {
@@ -52,44 +73,90 @@ int main(int argc, char** argv) {
   ConfigReader cr;
   cr.read(&cfg, argc, argv);
 
-  ad::cppgtfs::gtfs::Feed gtfs;
+  std::vector<pfaedle::gtfs::Feed> gtfs(cfg.feedPaths.size());
+  // feed containing the shapeas in memory for evaluation
+  ad::cppgtfs::gtfs::Feed evalFeed;
 
-  motCfgReader.parse(cfg.configPaths);
+  std::vector<std::string> cfgPaths = getCfgPaths(cfg);
 
-  if (cfg.osmPath.empty()) {
+  try {
+    motCfgReader.parse(cfgPaths);
+  } catch (configparser::ParseExc ex) {
+    LOG(ERROR) << "Could not parse MOT configurations, reason was:";
+    std::cerr << ex.what() << std::endl;
+    exit(static_cast<int>(RetCode::MOT_CFG_PARSE_ERR));
+  }
+
+  if (cfg.osmPath.empty() && !cfg.writeOverpass) {
     std::cerr << "No OSM input file specified (-x), see --help." << std::endl;
-    exit(5);
+    exit(static_cast<int>(RetCode::NO_OSM_INPUT));
+  }
+
+  if (motCfgReader.getConfigs().size() == 0) {
+    LOG(ERROR) << "No MOT configurations specified and no implicit "
+                  "configurations found, see --help.";
+    exit(static_cast<int>(RetCode::NO_MOT_CFG));
   }
 
   if (cfg.feedPaths.size() == 1) {
+    if (cfg.inPlace) cfg.outputPath = cfg.feedPaths[0];
+    if (!cfg.writeOverpass)
       LOG(INFO) << "Reading " << cfg.feedPaths[0] << " ...";
+    try {
       ad::cppgtfs::Parser p;
-    p.parse(&gtfs, cfg.feedPaths[0]);
+      p.parse(&gtfs[0], cfg.feedPaths[0]);
-    LOG(INFO) << "Done.";
+      if (cfg.evaluate) {
+        // read the shapes and store them in memory
+        p.parseShapes(&evalFeed, cfg.feedPaths[0]);
+      }
+    } catch (ad::cppgtfs::ParserException ex) {
+      LOG(ERROR) << "Could not parse input GTFS feed, reason was:";
+      std::cerr << ex.what() << std::endl;
+      exit(static_cast<int>(RetCode::GTFS_PARSE_ERR));
+    }
+    if (!cfg.writeOverpass) LOG(INFO) << "Done.";
+  } else if (cfg.writeOsm.size() || cfg.writeOverpass) {
+    for (size_t i = 0; i < cfg.feedPaths.size(); i++) {
+      if (!cfg.writeOverpass)
+        LOG(INFO) << "Reading " << cfg.feedPaths[i] << " ...";
+      ad::cppgtfs::Parser p;
+      try {
+        p.parse(&gtfs[i], cfg.feedPaths[i]);
+      } catch (ad::cppgtfs::ParserException ex) {
+        LOG(ERROR) << "Could not parse input GTFS feed, reason was:";
+        std::cerr << ex.what() << std::endl;
+        exit(static_cast<int>(RetCode::GTFS_PARSE_ERR));
+      }
+      if (!cfg.writeOverpass) LOG(INFO) << "Done.";
+    }
   } else if (cfg.feedPaths.size() > 1) {
-    std::cerr << "Maximally one input feed allowed." << std::endl;
+    std::cerr << "Multiple feeds only allowed in filter mode." << std::endl;
-    exit(2);
+    exit(static_cast<int>(RetCode::MULT_FEEDS_NOT_ALWD));
   }
 
   LOG(DEBUG) << "Read " << motCfgReader.getConfigs().size()
              << " unique MOT configs.";
   MOTs cmdCfgMots = cfg.mots;
-  ad::cppgtfs::gtfs::Trip* singleTrip = 0;
+  pfaedle::gtfs::Trip* singleTrip = 0;
 
   if (cfg.shapeTripId.size()) {
-    singleTrip = gtfs.getTrips().get(cfg.shapeTripId);
+    if (!cfg.feedPaths.size()) {
+      std::cout << "No input feed specified, see --help" << std::endl;
+      exit(static_cast<int>(RetCode::NO_INPUT_FEED));
+    }
+    singleTrip = gtfs[0].getTrips().get(cfg.shapeTripId);
     if (!singleTrip) {
       LOG(ERROR) << "Trip #" << cfg.shapeTripId << " not found.";
-      exit(3);
+      exit(static_cast<int>(RetCode::TRIP_NOT_FOUND));
     }
   }
 
   if (cfg.writeOsm.size()) {
     LOG(INFO) << "Writing filtered XML to " << cfg.writeOsm << " ...";
-    BBoxIdx box(2500);
+    BBoxIdx box(BOX_PADDING);
-    if (cfg.feedPaths.size()) {
+    for (size_t i = 0; i < cfg.feedPaths.size(); i++) {
-      box = ShapeBuilder::getPaddedGtfsBox(&gtfs, 2500, cmdCfgMots,
+      ShapeBuilder::getGtfsBox(&gtfs[i], cmdCfgMots, cfg.shapeTripId, true,
-                                           cfg.shapeTripId, true);
+                               &box);
     }
     OsmBuilder osmBuilder;
     std::vector<pfaedle::osm::OsmReadOpts> opts;
@@ -99,15 +166,33 @@ int main(int argc, char** argv) {
         opts.push_back(o.osmBuildOpts);
       }
     }
+    try {
       osmBuilder.filterWrite(cfg.osmPath, cfg.writeOsm, opts, box);
-    exit(0);
+    } catch (xml::XmlFileException ex) {
+      LOG(ERROR) << "Could not parse OSM data, reason was:";
+      std::cerr << ex.what() << std::endl;
+      exit(static_cast<int>(RetCode::OSM_PARSE_ERR));
+    }
+    exit(static_cast<int>(RetCode::SUCCESS));
+  } else if (cfg.writeOverpass) {
+    BBoxIdx box(BOX_PADDING);
+    for (size_t i = 0; i < cfg.feedPaths.size(); i++) {
+      ShapeBuilder::getGtfsBox(&gtfs[i], cmdCfgMots, cfg.shapeTripId, true,
+                               &box);
+    }
+    OsmBuilder osmBuilder;
+    std::vector<pfaedle::osm::OsmReadOpts> opts;
+    for (const auto& o : motCfgReader.getConfigs()) {
+      if (std::find_first_of(o.mots.begin(), o.mots.end(), cmdCfgMots.begin(),
+                             cmdCfgMots.end()) != o.mots.end()) {
+        opts.push_back(o.osmBuildOpts);
+      }
+    }
+    osmBuilder.overpassQryWrite(&std::cout, opts, box);
+    exit(static_cast<int>(RetCode::SUCCESS));
   } else if (!cfg.feedPaths.size()) {
     std::cout << "No input feed specified, see --help" << std::endl;
-    exit(1);
+    exit(static_cast<int>(RetCode::NO_INPUT_FEED));
-  }
-
-  if (motCfgReader.getConfigs().size() == 0) {
-    LOG(WARN) << "No MOT configurations specified, see --help.";
   }
 
   std::vector<double> dfBins;
@@ -119,17 +204,22 @@ int main(int argc, char** argv) {
     std::string filePost;
     auto usedMots = getContMots(motCfg, cmdCfgMots);
     if (!usedMots.size()) continue;
+    if (singleTrip && !usedMots.count(singleTrip->getRoute()->getType()))
+      continue;
     if (motCfgReader.getConfigs().size() > 1)
       filePost = getFileNameMotStr(usedMots);
 
     std::string motStr = getMotStr(usedMots);
     LOG(INFO) << "Calculating shapes for mots " << motStr;
 
-    ShapeBuilder shapeBuilder(&gtfs, cmdCfgMots, motCfg, &ecoll, cfg);
+    try {
+      ShapeBuilder shapeBuilder(&gtfs[0], &evalFeed, cmdCfgMots, motCfg, &ecoll,
+                                cfg);
 
       if (cfg.writeGraph) {
         LOG(INFO) << "Outputting graph.json...";
         util::geo::output::GeoGraphJsonOutput out;
+        mkdir(cfg.dbgOutputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
         std::ofstream fstr(cfg.dbgOutputPath + "/graph.json");
         out.print(*shapeBuilder.getGraph(), fstr);
         fstr.close();
@@ -137,21 +227,17 @@ int main(int argc, char** argv) {
 
       if (singleTrip) {
         LOG(INFO) << "Outputting path.json...";
+        mkdir(cfg.dbgOutputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
         std::ofstream pstr(cfg.dbgOutputPath + "/path.json");
         util::geo::output::GeoJsonOutput o(pstr);
 
         auto l = shapeBuilder.shapeL(singleTrip);
 
-      if (singleTrip->getShape()) {
-        auto orig = Collector::getWebMercLine(singleTrip->getShape(), -1, -1);
-        o.print(orig, util::json::Dict{{"ver", "old"}});
-      }
-
         o.print(l, util::json::Dict{{"ver", "new"}});
         o.flush();
         pstr.close();
 
-      exit(0);
+        exit(static_cast<int>(RetCode::SUCCESS));
       }
 
       pfaedle::netgraph::Graph ng;
@@ -160,21 +246,34 @@ int main(int argc, char** argv) {
       if (cfg.buildTransitGraph) {
         util::geo::output::GeoGraphJsonOutput out;
         LOG(INFO) << "Outputting trgraph" + filePost + ".json...";
+        mkdir(cfg.dbgOutputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
         std::ofstream fstr(cfg.dbgOutputPath + "/trgraph" + filePost + ".json");
         out.print(ng, fstr);
         fstr.close();
       }
+    } catch (xml::XmlFileException ex) {
+      LOG(ERROR) << "Could not parse OSM data, reason was:";
+      std::cerr << ex.what() << std::endl;
+      exit(static_cast<int>(RetCode::OSM_PARSE_ERR));
+    }
   }
 
   if (cfg.evaluate) ecoll.printStats(&std::cout);
 
   if (cfg.feedPaths.size()) {
+    try {
+      mkdir(cfg.outputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
       LOG(INFO) << "Writing output GTFS to " << cfg.outputPath << " ...";
-    ad::cppgtfs::Writer w;
+      pfaedle::gtfs::Writer w;
-    w.write(&gtfs, cfg.outputPath);
+      w.write(&gtfs[0], cfg.outputPath);
+    } catch (ad::cppgtfs::WriterException ex) {
+      LOG(ERROR) << "Could not write final GTFS feed, reason was:";
+      std::cerr << ex.what() << std::endl;
+      exit(static_cast<int>(RetCode::GTFS_WRITE_ERR));
+    }
   }
 
-  return (0);
+  return static_cast<int>(RetCode::SUCCESS);
 }
 
 // _____________________________________________________________________________
@@ -183,7 +282,7 @@ std::string getMotStr(const MOTs& mots) {
   std::string motStr;
   for (const auto& mot : mots) {
     if (first) motStr += ", ";
-    motStr += "<" + Route::getTypeString(mot) + ">";
+    motStr += "<" + ad::cppgtfs::gtfs::flat::Route::getTypeString(mot) + ">";
     first = true;
   }
 
@@ -194,7 +293,7 @@ std::string getMotStr(const MOTs& mots) {
 std::string getFileNameMotStr(const MOTs& mots) {
   std::string motStr;
   for (const auto& mot : mots) {
-    motStr += "-" + Route::getTypeString(mot);
+    motStr += "-" + ad::cppgtfs::gtfs::flat::Route::getTypeString(mot);
   }
 
   return motStr;
@@ -211,3 +310,73 @@ MOTs getContMots(const MotConfig& motCfg, const MOTs& mots) {
 
   return ret;
 }
+
+// _____________________________________________________________________________
+std::vector<std::string> getCfgPaths(const Config& cfg) {
+  if (cfg.configPaths.size()) return cfg.configPaths;
+  std::vector<std::string> ret;
+
+  // parse implicit paths
+  const char* homedir = 0;
+  char* buf = 0;
+
+  if ((homedir = getenv("HOME")) == 0) {
+    homedir = "";
+    struct passwd pwd;
+    struct passwd* result;
+    size_t bufsize;
+    bufsize = sysconf(_SC_GETPW_R_SIZE_MAX);
+    if (bufsize == static_cast<size_t>(-1)) bufsize = 0x4000;
+    buf = static_cast<char*>(malloc(bufsize));
+    if (buf != 0) {
+      getpwuid_r(getuid(), &pwd, buf, bufsize, &result);
+      if (result != NULL) homedir = result->pw_dir;
+    }
+  }
+
+  // install prefix global configuration path, if available
+  {
+    auto path = std::string(INSTALL_PREFIX) +
+                std::string(XDG_CONFIG_DIRS_DEFAULT) + "/" + "pfaedle" + "/" +
+                CFG_FILE_NAME;
+    std::ifstream is(path);
+
+    LOG(DEBUG) << "Testing for config file at " << path;
+    if (is.good()) {
+      ret.push_back(path);
+      LOG(DEBUG) << "Found implicit config file " << path;
+    }
+  }
+
+  // local user configuration path, if available
+  {
+    auto path = std::string(homedir) + XDG_CONFIG_HOME_SUFFIX + "/" +
+                "pfaedle" + "/" + CFG_FILE_NAME;
+    std::ifstream is(path);
+
+    LOG(DEBUG) << "Testing for config file at " << path;
+    if (is.good()) {
+      ret.push_back(path);
+      LOG(DEBUG) << "Found implicit config file " << path;
+    }
+  }
+
+  if (buf) free(buf);
+
+  // CWD
+  {
+    char cwd[PATH_MAX];
+    if (getcwd(cwd, sizeof(cwd))) {
+      auto path = std::string(cwd) + "/" + CFG_FILE_NAME;
+      std::ifstream is(path);
+
+      LOG(DEBUG) << "Testing for config file at " << path;
+      if (is.good()) {
+        ret.push_back(path);
+        LOG(DEBUG) << "Found implicit config file " << path;
+      }
+    }
+  }
+
+  return ret;
+}

src/pfaedle/_config.h.in

@@ -7,4 +7,7 @@
 // version number from cmake version module
 #define VERSION_FULL "@VERSION_GIT_FULL@"
 
+// version number from cmake version module
+#define INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@"
+
 #endif  // SRC_PFAEDLE_CONFIG_H_N

src/pfaedle/config/ConfigReader.cpp

@@ -26,13 +26,13 @@ static const char* AUTHORS = "Patrick Brosi <brosi@informatik.uni-freiburg.de>";
 
 // _____________________________________________________________________________
 void ConfigReader::help(const char* bin) {
-  std::cout
+  std::cout << std::setfill(' ') << std::left << "pfaedle GTFS map matcher "
-      << std::setfill(' ') << std::left << "pfaedle GTFS map matcher\n"
+            << VERSION_FULL << "\n(built " << __DATE__ << " " << __TIME__
-      << VERSION_FULL << " (built " << __DATE__ << " " << __TIME__
             << " with geometry precision <" << PFAEDLE_PRECISION_STR << ">)\n\n"
             << "(C) " << YEAR << " " << COPY << "\n"
             << "Authors: " << AUTHORS << "\n\n"
-      << "Usage: " << bin << " -x <OSM FILE> -c <CFG FILE> <GTFS FEED>\n\n"
+            << "Usage: " << bin
+            << " -x <OSM FILE> <GTFS FEED>\n\n"
             << "Allowed options:\n\n"
             << "General:\n"
             << std::setw(35) << "  -v [ --version ]"
@@ -40,53 +40,82 @@ void ConfigReader::help(const char* bin) {
             << std::setw(35) << "  -h [ --help ]"
             << "show this help message\n"
             << std::setw(35) << "  -D [ --drop-shapes ]"
-      << "drop shapes already present in the feed and recalculate them\n"
+            << "drop shapes already present in the feed and\n"
+            << std::setw(35) << " "
+            << "  recalculate them\n"
             << "\nInput:\n"
             << std::setw(35) << "  -c [ --config ] arg"
             << "pfaedle config file\n"
             << std::setw(35) << "  -i [ --input ] arg"
-      << "gtfs feed(s), may also be given as positional parameter (see usage)\n"
+            << "gtfs feed(s), may also be given as positional\n"
+            << std::setw(35) << " "
+            << "  parameter (see usage)\n"
             << std::setw(35) << "  -x [ --osm-file ] arg"
             << "OSM xml input file\n"
             << std::setw(35) << "  -m [ --mots ] arg (=all)"
-      << "MOTs to calculate shapes for, comma separated, either as string "
+            << "MOTs to calculate shapes for, comma sep.,\n"
-         "{all,\n"
             << std::setw(35) << " "
-      << "tram | streetcar, subway | metro, rail | train, bus, ferry | boat | "
+            << "  either as string "
-         "\n"
+               "{all, tram | streetcar,\n"
             << std::setw(35) << " "
-      << "ship, cableclar, gondola, funicular} or as GTFS mot codes\n"
+            << "  subway | metro, rail | train, bus,\n"
+            << std::setw(35) << " "
+            << "  ferry | boat | ship, cablecar, gondola,\n"
+            << std::setw(35) << " "
+            << "  funicular, coach} or as GTFS mot codes\n"
             << "\nOutput:\n"
             << std::setw(35) << "  -o [ --output ] arg (=gtfs-out)"
             << "GTFS output path\n"
             << std::setw(35) << "  -X [ --osm-out ] arg"
-      << "if specified, a filtered OSM file will be written to <arg>\n"
+            << "if specified, a filtered OSM file will be\n"
+            << std::setw(35) << " "
+            << "  written to <arg>\n"
+            << std::setw(35) << "  --inplace"
+            << "overwrite input GTFS feed with output feed\n"
             << "\nDebug Output:\n"
             << std::setw(35) << "  -d [ --dbg-path ] arg (=geo)"
             << "output path for debug files\n"
             << std::setw(35) << "  --write-trgraph"
-      << "write transit graph as GeoJSON to <dbg-path>/trgraph.json\n"
+            << "write transit graph as GeoJSON to\n"
+            << std::setw(35) << " "
+            << "  <dbg-path>/trgraph.json\n"
             << std::setw(35) << "  --write-graph"
-      << "write routing graph as GeoJSON to <dbg-path>/graph.json\n"
+            << "write routing graph as GeoJSON to\n"
+            << std::setw(35) << " "
+            << "  <dbg-path>/graph.json\n"
             << std::setw(35) << "  --write-cgraph"
-      << "if -T is set, write combination graph as GeoJSON to "
+            << "if -T is set, write combination graph as\n"
+            << std::setw(35) << " "
+            << "  GeoJSON to "
                "<dbg-path>/combgraph.json\n"
             << std::setw(35) << "  --method arg (=global)"
-      << "matching method to use, either 'global' (based on HMM), 'greedy' or "
+            << "matching method to use, either 'global'\n"
+            << std::setw(35) << " "
+            << "  (based on HMM), 'greedy' or "
                "'greedy2'\n"
             << std::setw(35) << "  --eval"
-      << "evaluate existing shapes against matched shapes and print results\n"
+            << "evaluate existing shapes against matched\n"
+            << std::setw(35) << " "
+            << "  shapes and print results\n"
             << std::setw(35) << "  --eval-path arg (=.)"
             << "path for eval file output\n"
             << std::setw(35) << "  --eval-df-bins arg (= )"
-      << "bins to use for d_f histogram, comma separated (e.g. 10,20,30,40)\n"
+            << "bins to use for d_f histogram, comma sep.\n"
+            << std::setw(35) << " "
+            << "  (e.g. 10,20,30,40)\n"
             << "\nMisc:\n"
             << std::setw(35) << "  -T [ --trip-id ] arg"
-      << "Do routing only for trip <arg>, write result to\n"
+            << "Do routing only for trip <arg>, write result \n"
             << std::setw(35) << " "
-      << "<dbg-path>/path.json\n"
+            << "  to <dbg-path>/path.json\n"
+            << std::setw(35) << "  --overpass"
+            << "Output overpass query for matching OSM data\n"
             << std::setw(35) << "  --grid-size arg (=2000)"
-      << "Grid cell size\n";
+            << "Grid cell size\n"
+            << std::setw(35) << "  --use-route-cache"
+            << "(experimental) cache intermediate routing\n"
+            << std::setw(35) << " "
+            << "  results\n";
 }
 
 // _____________________________________________________________________________
@@ -101,6 +130,7 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
                          {"drop-shapes", required_argument, 0, 'D'},
                          {"mots", required_argument, NULL, 'm'},
                          {"grid-size", required_argument, 0, 'g'},
+                         {"overpass", no_argument, 0, 'a'},
                          {"osm-out", required_argument, 0, 'X'},
                          {"trip-id", required_argument, 0, 'T'},
                          {"write-graph", no_argument, 0, 1},
@@ -113,6 +143,8 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
                          {"dbg-path", required_argument, 0, 'd'},
                          {"version", no_argument, 0, 'v'},
                          {"help", no_argument, 0, 'h'},
+                         {"inplace", no_argument, 0, 9},
+                         {"use-route-cache", no_argument, 0, 8},
                          {0, 0, 0, 0}};
 
   char c;
@@ -140,6 +172,9 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
       case 7:
         cfg->evalDfBins = optarg;
         break;
+      case 8:
+        cfg->useCaching = true;
+        break;
       case 'o':
         cfg->outputPath = optarg;
         break;
@@ -170,6 +205,12 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
       case 'd':
         cfg->dbgOutputPath = optarg;
         break;
+      case 'a':
+        cfg->writeOverpass = true;
+        break;
+      case 9:
+        cfg->inPlace = true;
+        break;
       case 'v':
         std::cout << "pfaedle " << VERSION_FULL << " (built " << __DATE__ << " "
                   << __TIME__ << " with geometry precision <"
@@ -204,7 +245,8 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
 
   auto v = util::split(motStr, ',');
   for (const auto& motStr : v) {
-    const auto& mots = Route::getTypesFromString(util::trim(motStr));
+    const auto& mots =
+        ad::cppgtfs::gtfs::flat::Route::getTypesFromString(util::trim(motStr));
     cfg->mots.insert(mots.begin(), mots.end());
   }
 

src/pfaedle/config/MotConfigReader.cpp

@@ -7,6 +7,7 @@
 #include "pfaedle/config/MotConfigReader.h"
 #include "util/Misc.h"
 #include "util/String.h"
+#include "util/log/Log.h"
 
 using pfaedle::config::MotConfigReader;
 using pfaedle::config::MotConfig;
@@ -23,9 +24,13 @@ MotConfigReader::MotConfigReader() {}
 
 // _____________________________________________________________________________
 void MotConfigReader::parse(const std::vector<std::string>& paths) {
-  for (const auto& s : paths) {
   ConfigFileParser p;
+
+  // parse explicitely given paths
+  for (const auto& s : paths) {
+    LOG(DEBUG) << "Reading config file " << s;
     p.parse(s);
+  }
 
   for (const auto& sec : p.getSecs()) {
     MotConfig curCfg;
@@ -39,9 +44,8 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
       }
     }
 
-      for (uint8_t i = 0; i < 7; i++) {
+    for (uint8_t i = 0; i < 8; i++) {
-        std::string name =
+      std::string name = std::string("osm_filter_lvl") + std::to_string(i);
-            std::string("osm_filter_lvl") + std::to_string(i + 1);
       if (p.hasKey(secStr, name)) {
         for (const auto& kvs : p.getStrArr(sec.first, name, ' ')) {
           auto fRule = getFRule(kvs);
@@ -67,8 +71,7 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
     }
 
     if (p.hasKey(secStr, "osm_filter_nohup")) {
-        for (const auto& kvs :
+      for (const auto& kvs : p.getStrArr(sec.first, "osm_filter_nohup", ' ')) {
-             p.getStrArr(sec.first, "osm_filter_nohup", ' ')) {
         auto fRule = getFRule(kvs);
         curCfg.osmBuildOpts.noHupFilter[fRule.kv.first].insert(
             osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags)));
@@ -76,8 +79,7 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
     }
 
     if (p.hasKey(secStr, "osm_filter_oneway")) {
-        for (const auto& kvs :
+      for (const auto& kvs : p.getStrArr(sec.first, "osm_filter_oneway", ' ')) {
-             p.getStrArr(sec.first, "osm_filter_oneway", ' ')) {
         auto fRule = getFRule(kvs);
         curCfg.osmBuildOpts.oneWayFilter[fRule.kv.first].insert(
             osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags)));
@@ -215,16 +217,6 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
           2;
     }
 
-      if (p.hasKey(secStr, "osm_max_group_search_distance")) {
-        curCfg.osmBuildOpts.maxGroupSearchDistance =
-            p.getDouble(secStr, "osm_max_group_search_distance");
-      } else {
-        curCfg.osmBuildOpts.maxGroupSearchDistance =
-            *std::max_element(curCfg.osmBuildOpts.maxSnapDistances.begin(),
-                              curCfg.osmBuildOpts.maxSnapDistances.end()) *
-            4;
-      }
-
     if (p.hasKey(secStr, "osm_max_osm_station_distance")) {
       curCfg.osmBuildOpts.maxOsmStationDistance =
           p.getDouble(secStr, "osm_max_osm_station_distance");
@@ -260,15 +252,16 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
     if (p.hasKey(secStr, "routing_full_turn_angle")) {
       double ang = p.getDouble(secStr, "routing_full_turn_angle");
       curCfg.routingOpts.fullTurnAngle = ang;
+      curCfg.osmBuildOpts.fullTurnAngle = ang;
     } else {
       curCfg.routingOpts.fullTurnAngle = 5;
+      curCfg.osmBuildOpts.fullTurnAngle = 5;
     }
     if (p.hasKey(secStr, "routing_snap_full_turn_angle")) {
       double ang = p.getDouble(secStr, "routing_snap_full_turn_angle");
       curCfg.osmBuildOpts.maxAngleSnapReach = ang;
     } else {
-        curCfg.osmBuildOpts.maxAngleSnapReach =
+      curCfg.osmBuildOpts.maxAngleSnapReach = curCfg.routingOpts.fullTurnAngle;
-            curCfg.routingOpts.fullTurnAngle;
     }
     if (p.hasKey(secStr, "routing_pass_thru_station_punish")) {
       curCfg.routingOpts.passThruStationsPunish =
@@ -312,7 +305,8 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
         throw ParseExc(p.getVal(secStr, "station_normalize_chain").line,
                        p.getVal(secStr, "station_normalize_chain").pos,
                        "<valid regular expression>",
-                         std::string("<regex error: ") + e.what() + ">", s);
+                       std::string("<regex error: ") + e.what() + ">",
+                       p.getVal(secStr, "station_normalize_chain").file);
       }
     }
 
@@ -323,9 +317,10 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
             trgraph::Normalizer(getNormRules(arr));
       } catch (const std::exception& e) {
         throw ParseExc(p.getVal(secStr, "track_normalize_chain").line,
-                         p.getVal(secStr, "station_normalize_chain").pos,
+                       p.getVal(secStr, "track_normalize_chain").pos,
                        "<valid regular expression>",
-                         std::string("<regex error: ") + e.what() + ">", s);
+                       std::string("<regex error: ") + e.what() + ">",
+                       p.getVal(secStr, "track_normalize_chain").file);
       }
     }
 
@@ -338,7 +333,8 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
         throw ParseExc(p.getVal(secStr, "station_normalize_chain").line,
                        p.getVal(secStr, "station_normalize_chain").pos,
                        "<valid regular expression>",
-                         std::string("<regex error: ") + e.what() + ">", s);
+                       std::string("<regex error: ") + e.what() + ">",
+                       p.getVal(secStr, "station_normalize_chain").file);
       }
     }
 
@@ -346,7 +342,8 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
 
     for (auto& cfg : _cfgs) {
       if (cfg == curCfg) {
-          for (auto mot : Route::getTypesFromString(secStr)) {
+        for (auto mot :
+             ad::cppgtfs::gtfs::flat::Route::getTypesFromString(secStr)) {
           cfg.mots.insert(mot);
         }
         found = true;
@@ -355,12 +352,11 @@ void MotConfigReader::parse(const std::vector<std::string>& paths) {
     }
 
     if (!found) {
-        curCfg.mots = Route::getTypesFromString(secStr);
+      curCfg.mots = ad::cppgtfs::gtfs::flat::Route::getTypesFromString(secStr);
       _cfgs.push_back(curCfg);
     }
   }
 }
-}
 
 // _____________________________________________________________________________
 ReplRules MotConfigReader::getNormRules(

src/pfaedle/config/MotConfigReader.h

@@ -5,6 +5,30 @@
 #ifndef PFAEDLE_CONFIG_MOTCONFIGREADER_H_
 #define PFAEDLE_CONFIG_MOTCONFIGREADER_H_
 
+#include "pfaedle/_config.h"
+
+#ifndef HOME_VAR
+#define HOME_VAR "HOME"
+#endif
+#ifndef XDG_DATA_HOME_SUFFIX
+#define XDG_DATA_HOME_SUFFIX "/.local/share"
+#endif
+#ifndef XDG_CONFIG_HOME_SUFFIX
+#define XDG_CONFIG_HOME_SUFFIX "/.config"
+#endif
+#ifndef XDG_CACHE_HOME_SUFFIX
+#define XDG_CACHE_HOME_SUFFIX "/.cache"
+#endif
+#ifndef XDG_DATA_DIRS_DEFAULT
+#define XDG_DATA_DIRS_DEFAULT "/usr/local/share"
+#endif
+#ifndef XDG_CONFIG_DIRS_DEFAULT
+#define XDG_CONFIG_DIRS_DEFAULT "/etc"
+#endif
+#ifndef CFG_FILE_NAME
+#define CFG_FILE_NAME "pfaedle.cfg"
+#endif
+
 #include <unordered_map>
 #include <vector>
 #include <set>

src/pfaedle/config/PfaedleConfig.h

@@ -28,6 +28,9 @@ struct Config {
         writeCombGraph(false),
         evaluate(false),
         buildTransitGraph(false),
+        useCaching(false),
+        writeOverpass(false),
+        inPlace(false),
         gridSize(2000) {}
   std::string dbgOutputPath;
   std::string solveMethod;
@@ -46,6 +49,9 @@ struct Config {
   bool writeCombGraph;
   bool evaluate;
   bool buildTransitGraph;
+  bool useCaching;
+  bool writeOverpass;
+  bool inPlace;
   double gridSize;
 
   std::string toString() {
@@ -60,6 +66,8 @@ struct Config {
        << "write-graph: " << writeGraph << "\n"
        << "write-cgraph: " << writeCombGraph << "\n"
        << "grid-size: " << gridSize << "\n"
+       << "use-cache: " << useCaching << "\n"
+       << "write-overpass: " << writeOverpass << "\n"
        << "feed-paths: ";
 
     for (const auto& p : feedPaths) {

src/pfaedle/eval/Collector.cpp

@@ -18,14 +18,14 @@
 
 using util::geo::PolyLine;
 
-using ad::cppgtfs::gtfs::Trip;
+using pfaedle::gtfs::Trip;
 using ad::cppgtfs::gtfs::Shape;
 using pfaedle::eval::Collector;
 using pfaedle::eval::Result;
 using util::geo::output::GeoJsonOutput;
 
 // _____________________________________________________________________________
-double Collector::add(const Trip* t, const Shape* oldS, const Shape* newS,
+double Collector::add(const Trip* t, const Shape* oldS, const Shape& newS,
                       const std::vector<double>& newTripDists) {
   if (!oldS) {
     _noOrigShp++;
@@ -51,7 +51,7 @@ double Collector::add(const Trip* t, const Shape* oldS, const Shape* newS,
       (--t->getStopTimes().end())->getShapeDistanceTravelled(), &oldDists);
 
   std::vector<double> newDists;
-  LINE newL = getWebMercLine(newS, -1, -1, &newDists);
+  LINE newL = getWebMercLine(&newS, -1, -1, &newDists);
 
   std::ofstream fstr(_evalOutPath + "/trip-" + t->getId() + ".json");
   GeoJsonOutput gjout(fstr);
@@ -123,19 +123,19 @@ double Collector::add(const Trip* t, const Shape* oldS, const Shape* newS,
                                 6378137.0)) -
                    1.5707965);
 
-  if (_dCache.count(oldS) && _dCache.find(oldS)->second.count(newS)) {
+  if (_dCache.count(oldS) && _dCache.find(oldS)->second.count(newS.getId())) {
-    fd = _dCache[oldS][newS];
+    fd = _dCache[oldS][newS.getId()];
   } else {
     fd = util::geo::accFrechetDistC(oldLCut, newLCut, 5 / fac) * fac;
-    _dCache[oldS][newS] = fd;
+    _dCache[oldS][newS.getId()] = fd;
   }
 
-  if (_dACache.count(oldS) && _dACache.find(oldS)->second.count(newS)) {
+  if (_dACache.count(oldS) && _dACache.find(oldS)->second.count(newS.getId())) {
-    unmatchedSegments = _dACache[oldS][newS].first;
+    unmatchedSegments = _dACache[oldS][newS.getId()].first;
-    unmatchedSegmentsLength = _dACache[oldS][newS].second;
+    unmatchedSegmentsLength = _dACache[oldS][newS.getId()].second;
   } else {
     auto dA = getDa(oldSegs, newSegs);
-    _dACache[oldS][newS] = dA;
+    _dACache[oldS][newS.getId()] = dA;
     unmatchedSegments = dA.first;
     unmatchedSegmentsLength = dA.second;
   }
@@ -199,6 +199,8 @@ std::vector<LINE> Collector::segmentize(
   // get first half of geometry, and search for start point there!
   size_t before = std::upper_bound(dists.begin(), dists.end(), cuts[1].second) -
                   dists.begin();
+  if (before + 1 > shape.size()) before = shape.size() - 1;
+  assert(shape.begin() + before + 1 <= shape.end());
   POLYLINE l(LINE(shape.begin(), shape.begin() + before + 1));
   auto lastLp = l.projectOn(cuts.front().first);
 

src/pfaedle/eval/Collector.h

@@ -12,11 +12,12 @@
 #include <utility>
 #include <vector>
 #include "ad/cppgtfs/gtfs/Feed.h"
+#include "pfaedle/gtfs/Feed.h"
 #include "pfaedle/Def.h"
 #include "pfaedle/eval/Result.h"
 #include "util/geo/Geo.h"
 
-using ad::cppgtfs::gtfs::Trip;
+using pfaedle::gtfs::Trip;
 using ad::cppgtfs::gtfs::Shape;
 
 namespace pfaedle {
@@ -37,7 +38,7 @@ class Collector {
 
   // Add a shape found by our tool newS for a trip t with newly calculated
   // station dist values with the old shape oldS
-  double add(const Trip* t, const Shape* oldS, const Shape* newS,
+  double add(const Trip* t, const Shape* oldS, const Shape& newS,
              const std::vector<double>& newDists);
 
   // Return the set of all Result objects
@@ -65,8 +66,8 @@ class Collector {
   std::set<Result> _results;
   std::set<Result> _resultsAN;
   std::set<Result> _resultsAL;
-  std::map<const Shape*, std::map<const Shape*, double> > _dCache;
+  std::map<const Shape*, std::map<std::string, double> > _dCache;
-  std::map<const Shape*, std::map<const Shape*, std::pair<size_t, double> > >
+  std::map<const Shape*, std::map<std::string, std::pair<size_t, double> > >
       _dACache;
   size_t _noOrigShp;
 

src/pfaedle/eval/Result.h

@@ -5,9 +5,10 @@
 #ifndef PFAEDLE_EVAL_RESULT_H_
 #define PFAEDLE_EVAL_RESULT_H_
 
+#include "pfaedle/gtfs/Feed.h"
 #include "ad/cppgtfs/gtfs/Feed.h"
 
-using ad::cppgtfs::gtfs::Trip;
+using pfaedle::gtfs::Trip;
 using ad::cppgtfs::gtfs::Shape;
 
 namespace pfaedle {

src/pfaedle/gtfs/Feed.h

@@ -0,0 +1,37 @@
+// Copyright 2016, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#ifndef PFAEDLE_GTFS_FEED_H_
+#define PFAEDLE_GTFS_FEED_H_
+
+#include <string>
+#include "Route.h"
+#include "Service.h"
+#include "ShapeContainer.h"
+#include "StopTime.h"
+#include "ad/cppgtfs/gtfs/ContContainer.h"
+#include "ad/cppgtfs/gtfs/Feed.h"
+#include "ad/cppgtfs/gtfs/NullContainer.h"
+#include "ad/cppgtfs/gtfs/Stop.h"
+#include "ad/cppgtfs/gtfs/StopTime.h"
+#include "ad/cppgtfs/gtfs/Trip.h"
+
+namespace pfaedle {
+namespace gtfs {
+
+typedef ad::cppgtfs::gtfs::FeedB<
+    ad::cppgtfs::gtfs::Agency, Route, ad::cppgtfs::gtfs::Stop, Service,
+    StopTime, Shape, ad::cppgtfs::gtfs::Fare, ad::cppgtfs::gtfs::Container,
+    ad::cppgtfs::gtfs::ContContainer, ad::cppgtfs::gtfs::NullContainer,
+    ad::cppgtfs::gtfs::ContContainer, ad::cppgtfs::gtfs::ContContainer,
+    ShapeContainer, ad::cppgtfs::gtfs::NullContainer>
+    Feed;
+typedef ad::cppgtfs::gtfs::TripB<StopTime<ad::cppgtfs::gtfs::Stop>, Service,
+                                 Route, Shape>
+    Trip;
+
+}  // namespace gtfs
+}  // namespace pfaedle
+
+#endif  // PFAEDLE_GTFS_FEED_H_

src/pfaedle/gtfs/Route.h

@@ -0,0 +1,61 @@
+// Copyright 2016, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#ifndef PFAEDLE_GTFS_ROUTE_H_
+#define PFAEDLE_GTFS_ROUTE_H_
+
+#include <stdint.h>
+#include <algorithm>
+#include <iomanip>
+#include <set>
+#include <sstream>
+#include <string>
+#include "ad/cppgtfs/gtfs/Agency.h"
+#include "ad/cppgtfs/gtfs/Route.h"
+#include "util/Misc.h"
+
+using std::exception;
+using std::string;
+
+namespace pfaedle {
+namespace gtfs {
+
+class Route {
+ public:
+  typedef Route* Ref;
+  static std::string getId(Ref r) { return r->getId(); }
+
+  Route() {}
+
+  Route(const string& id, ad::cppgtfs::gtfs::Agency* agency,
+        const string& short_name, const string& long_name, const string& desc,
+        ad::cppgtfs::gtfs::flat::Route::TYPE type, const string& url,
+        uint32_t color, uint32_t text_color)
+      : _id(id), _short_name(short_name), _long_name(long_name), _type(type) {
+    UNUSED(agency);
+    UNUSED(desc);
+    UNUSED(url);
+    UNUSED(color);
+    UNUSED(text_color);
+  }
+
+  const std::string& getId() const { return _id; }
+
+  const std::string& getShortName() const { return _short_name; }
+
+  const std::string& getLongName() const { return _long_name; }
+
+  ad::cppgtfs::gtfs::flat::Route::TYPE getType() const { return _type; }
+
+ private:
+  string _id;
+  string _short_name;
+  string _long_name;
+  ad::cppgtfs::gtfs::flat::Route::TYPE _type;
+};
+
+}  // namespace gtfs
+}  // namespace pfaedle
+
+#endif  // PFAEDLE_GTFS_ROUTE_H_

src/pfaedle/gtfs/Service.h

@@ -0,0 +1,43 @@
+// Copyright 2016, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#ifndef PFAEDLE_GTFS_SERVICE_H_
+#define PFAEDLE_GTFS_SERVICE_H_
+
+#include <string>
+#include "ad/cppgtfs/gtfs/Service.h"
+#include "util/Misc.h"
+
+namespace pfaedle {
+namespace gtfs {
+
+class Service {
+ public:
+  typedef std::string Ref;
+  static std::string getId(Ref r) { return r; }
+
+  explicit Service(const string& id) : _id(id) {}
+  Service(const string& id, uint8_t serviceDays,
+          ad::cppgtfs::gtfs::ServiceDate start,
+          ad::cppgtfs::gtfs::ServiceDate end)
+      : _id(id) {
+    UNUSED(serviceDays);
+    UNUSED(start);
+    UNUSED(end);
+  }
+
+  const std::string& getId() const { return _id; }
+  void addException(const ad::cppgtfs::gtfs::ServiceDate& d,
+                    ad::cppgtfs::gtfs::Service::EXCEPTION_TYPE t) {
+    UNUSED(d);
+    UNUSED(t);
+  }
+
+ private:
+  std::string _id;
+};
+}  // namespace gtfs
+}  // namespace pfaedle
+
+#endif  // PFAEDLE_GTFS_SERVICE_H_

src/pfaedle/gtfs/ShapeContainer.h

@@ -0,0 +1,69 @@
+// Copyright 2018, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#ifndef PFAEDLE_GTFS_SHAPECONTAINER_H_
+#define PFAEDLE_GTFS_SHAPECONTAINER_H_
+
+#include <fstream>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <string>
+#include "ad/cppgtfs/gtfs/Shape.h"
+#include "ad/cppgtfs/gtfs/flat/Shape.h"
+#include "util/Misc.h"
+
+namespace pfaedle {
+namespace gtfs {
+
+struct Shape {
+  explicit Shape(const std::string& id) : id(id) {}
+  typedef std::string Ref;
+  static std::string getId(Ref r) { return r; }
+
+  template <typename T>
+  bool addPoint(T p) {
+    UNUSED(p);
+    return true;
+  }
+
+  const std::string& getId() const { return id; }
+
+  std::string id;
+};
+
+template <typename T>
+class ShapeContainer {
+ public:
+  ShapeContainer();
+  ~ShapeContainer();
+  T* add(const T& obj);
+  bool remove(const std::string& id);
+  const T* get(const std::string& id) const;
+  T* get(const std::string& id);
+  const std::string getRef(const std::string& id) const;
+  std::string getRef(const std::string& id);
+  size_t size() const;
+  void finalize() {}
+  bool has(const std::string& id) const;
+
+  std::string add(const ad::cppgtfs::gtfs::Shape& s);
+  void open();
+  bool nextStoragePt(ad::cppgtfs::gtfs::flat::ShapePoint* ret);
+
+ private:
+  std::set<std::string> _ids;
+  std::fstream _storage;
+  size_t _ptr;
+  size_t _max;
+  std::string _curId;
+  std::stringstream _writeBuffer;
+};
+
+#include "ShapeContainer.tpp"
+
+}  // namespace gtfs
+}  // namespace pfaedle
+
+#endif  // PFAEDLE_GTFS_SHAPECONTAINER_H_

src/pfaedle/gtfs/ShapeContainer.tpp

@@ -0,0 +1,154 @@
+// Copyright 2018, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#include <unistd.h>
+#include <string>
+
+// ____________________________________________________________________________
+template <typename T>
+ShapeContainer<T>::ShapeContainer() {
+  std::string f = ".pfaedle-tmp";
+
+  while (access(f.c_str(), F_OK) != -1) {
+    std::stringstream ss;
+    ss << ".pfaedle-tmp-";
+    ss << std::rand();
+    f = ss.str().c_str();
+  }
+
+  _storage.open(f, std::fstream::in | std::fstream::out | std::fstream::trunc);
+
+  // immediately unlink
+  unlink(f.c_str());
+
+  if (!_storage.good()) {
+    std::cerr << "Could not open temporary file " << f << std::endl;
+    exit(1);
+  }
+}
+
+// ____________________________________________________________________________
+template <typename T>
+ShapeContainer<T>::~ShapeContainer() {
+  _storage.close();
+}
+
+// ____________________________________________________________________________
+template <typename T>
+T* ShapeContainer<T>::add(const T& ent) {
+  _ids.insert(ent.getId());
+  return reinterpret_cast<T*>(1);
+}
+
+// ____________________________________________________________________________
+template <typename T>
+bool ShapeContainer<T>::remove(const std::string& id) {
+  _ids.erase(id);
+  return true;
+}
+
+// ____________________________________________________________________________
+template <typename T>
+T* ShapeContainer<T>::get(const std::string& id) {
+  if (!has(id)) return 0;
+  return reinterpret_cast<T*>(1);
+}
+
+// ____________________________________________________________________________
+template <typename T>
+const T* ShapeContainer<T>::get(const std::string& id) const {
+  if (!has(id)) return 0;
+  return reinterpret_cast<T*>(1);
+}
+
+// ____________________________________________________________________________
+template <typename T>
+bool ShapeContainer<T>::has(const std::string& id) const {
+  return _ids.count(id);
+}
+
+// ____________________________________________________________________________
+template <typename T>
+size_t ShapeContainer<T>::size() const {
+  return _ids.size();
+}
+
+// ____________________________________________________________________________
+template <typename T>
+std::string ShapeContainer<T>::add(const ad::cppgtfs::gtfs::Shape& s) {
+  if (has(s.getId())) return s.getId();
+  _ids.insert(s.getId());
+
+  _writeBuffer << s.getId() << '\t' << s.getPoints().size();
+  _writeBuffer << std::setprecision(11);
+  for (auto p : s.getPoints()) {
+    _writeBuffer << " " << p.lat << " " << p.lng << " " << p.travelDist;
+  }
+  // entries are newline separated
+  _writeBuffer << '\n';
+
+  if (_writeBuffer.tellp() > 1000 * 5000) {
+    _storage << _writeBuffer.rdbuf();
+    _writeBuffer.clear();
+  }
+
+  return s.getId();
+}
+
+// ____________________________________________________________________________
+template <typename T>
+void ShapeContainer<T>::open() {
+  _storage << _writeBuffer.rdbuf();
+  _writeBuffer.clear();
+
+  _ptr = 0;
+  _max = 0;
+  _storage.clear();
+  _storage.seekg(0, std::ios::beg);
+}
+
+// ____________________________________________________________________________
+template <typename T>
+bool ShapeContainer<T>::nextStoragePt(
+    ad::cppgtfs::gtfs::flat::ShapePoint* ret) {
+  while (_storage.good() && !_storage.fail()) {
+    if (!_ptr) {
+      _storage >> _curId;
+      _storage >> _max;
+    }
+
+    if (!_storage.good() || _storage.fail()) return false;
+
+    _storage >> ret->lat;
+    _storage >> ret->lng;
+    _storage >> ret->travelDist;
+    ret->seq = _ptr + 1;
+    ret->id = _curId;
+
+    if (_ptr + 1 == _max)
+      _ptr = 0;
+    else
+      _ptr++;
+
+    if (!_storage.good() || _storage.fail()) return false;
+
+    if (has(ret->id)) return true;
+  }
+
+  return false;
+}
+
+// ____________________________________________________________________________
+template <typename T>
+const std::string ShapeContainer<T>::getRef(const std::string& id) const {
+  if (!has(id)) return "";
+  return id;
+}
+
+// ____________________________________________________________________________
+template <typename T>
+std::string ShapeContainer<T>::getRef(const std::string& id) {
+  if (!has(id)) return "";
+  return id;
+}

src/pfaedle/gtfs/StopTime.h

@@ -0,0 +1,71 @@
+// Copyright 2016, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#ifndef PFAEDLE_GTFS_STOPTIME_H_
+#define PFAEDLE_GTFS_STOPTIME_H_
+
+#include <iomanip>
+#include <sstream>
+#include <string>
+#include <vector>
+#include "ad/cppgtfs/gtfs/Stop.h"
+#include "ad/cppgtfs/gtfs/StopTime.h"
+#include "util/Misc.h"
+
+using std::exception;
+using std::string;
+
+namespace pfaedle {
+namespace gtfs {
+
+template <typename StopT>
+class StopTime {
+ public:
+  StopTime(const ad::cppgtfs::gtfs::Time& at, const ad::cppgtfs::gtfs::Time& dt,
+           typename StopT::Ref s, uint32_t seq, const std::string& hs,
+           ad::cppgtfs::gtfs::flat::StopTime::PU_DO_TYPE put,
+           ad::cppgtfs::gtfs::flat::StopTime::PU_DO_TYPE dot, float distTrav,
+           bool isTp)
+      : _s(s), _sequence(seq), _dist(distTrav) {
+    UNUSED(at);
+    UNUSED(dt);
+    UNUSED(hs);
+    UNUSED(put);
+    UNUSED(dot);
+    UNUSED(distTrav);
+    UNUSED(isTp);
+  }
+
+  const typename StopT::Ref getStop() const { return _s; }
+  typename StopT::Ref getStop() { return _s; }
+  void setShapeDistanceTravelled(double d) { _dist = d; }
+
+  ad::cppgtfs::gtfs::Time getArrivalTime() const {
+    return ad::cppgtfs::gtfs::Time(0, 0, 0);
+  }
+  ad::cppgtfs::gtfs::Time getDepartureTime() const {
+    return ad::cppgtfs::gtfs::Time(0, 0, 0);
+  }
+
+  float getShapeDistanceTravelled() const { return _dist; }
+
+  uint16_t getSeq() const { return _sequence; }
+
+ private:
+  typename StopT::Ref _s;
+  uint32_t _sequence;
+  float _dist;
+};
+
+template <typename StopTimeT>
+struct StopTimeCompare {
+  bool operator()(const StopTimeT& lh, const StopTimeT& rh) const {
+    return lh.getSeq() < rh.getSeq();
+  }
+};
+
+}  // namespace gtfs
+}  // namespace pfaedle
+
+#endif  // PFAEDLE_GTFS_STOPTIME_H_

src/pfaedle/gtfs/Writer.cpp

@@ -0,0 +1,495 @@
+// Copyright 2018, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#include <cstdio>
+#include <fstream>
+#include <map>
+#include <string>
+#include <utility>
+#include "ad/cppgtfs/Parser.h"
+#include "ad/cppgtfs/Writer.h"
+#include "ad/cppgtfs/gtfs/flat/Agency.h"
+#include "ad/util/CsvWriter.h"
+#include "pfaedle/gtfs/Writer.h"
+
+using ad::util::CsvWriter;
+using ad::cppgtfs::Parser;
+using pfaedle::gtfs::Writer;
+
+// ____________________________________________________________________________
+bool Writer::write(gtfs::Feed* sourceFeed, const std::string& path) const {
+  std::ofstream fs;
+  std::ifstream is;
+  std::string gtfsPath(path);
+  std::string curFile;
+  std::string curFileTg;
+
+  curFile = gtfsPath + "/.agency.txt";
+  curFileTg = gtfsPath + "/agency.txt";
+  fs.open(curFile.c_str());
+  if (!fs.good()) cannotWrite(curFile, curFileTg);
+  writeAgency(sourceFeed, &fs);
+  fs.close();
+  std::rename(curFile.c_str(), curFileTg.c_str());
+
+  curFile = gtfsPath + "/.stops.txt";
+  curFileTg = gtfsPath + "/stops.txt";
+  fs.open(curFile.c_str());
+  if (!fs.good()) cannotWrite(curFile, curFileTg);
+  writeStops(sourceFeed, &fs);
+  fs.close();
+  std::rename(curFile.c_str(), curFileTg.c_str());
+
+  curFile = gtfsPath + "/.routes.txt";
+  curFileTg = gtfsPath + "/routes.txt";
+  fs.open(curFile.c_str());
+  if (!fs.good()) cannotWrite(curFile, curFileTg);
+  writeRoutes(sourceFeed, &fs);
+  fs.close();
+  std::rename(curFile.c_str(), curFileTg.c_str());
+
+  is.open((sourceFeed->getPath() + "/calendar.txt").c_str());
+  if (is.good()) {
+    is.close();
+    curFile = gtfsPath + "/.calendar.txt";
+    curFileTg = gtfsPath + "/calendar.txt";
+    fs.open(curFile.c_str());
+    if (!fs.good()) cannotWrite(curFile, curFileTg);
+    writeCalendar(sourceFeed, &fs);
+    fs.close();
+    std::rename(curFile.c_str(), curFileTg.c_str());
+  }
+
+  is.open((sourceFeed->getPath() + "/calendar_dates.txt").c_str());
+  if (is.good()) {
+    is.close();
+    curFile = gtfsPath + "/.calendar_dates.txt";
+    curFileTg = gtfsPath + "/calendar_dates.txt";
+    fs.open(curFile.c_str());
+    if (!fs.good()) cannotWrite(curFile, curFileTg);
+    writeCalendarDates(sourceFeed, &fs);
+    fs.close();
+    std::rename(curFile.c_str(), curFileTg.c_str());
+  }
+
+  is.open((sourceFeed->getPath() + "/transfers.txt").c_str());
+  if (is.good()) {
+    is.close();
+    curFile = gtfsPath + "/.transfers.txt";
+    curFileTg = gtfsPath + "/transfers.txt";
+    fs.open(curFile.c_str());
+    if (!fs.good()) cannotWrite(curFile, curFileTg);
+    writeTransfers(sourceFeed, &fs);
+    fs.close();
+    std::rename(curFile.c_str(), curFileTg.c_str());
+  }
+
+  is.open((sourceFeed->getPath() + "/fare_attributes.txt").c_str());
+  if (is.good()) {
+    is.close();
+    curFile = gtfsPath + "/.fare_attributes.txt";
+    curFileTg = gtfsPath + "/fare_attributes.txt";
+    fs.open(curFile.c_str());
+    if (!fs.good()) cannotWrite(curFile, curFileTg);
+    writeFares(sourceFeed, &fs);
+    fs.close();
+    std::rename(curFile.c_str(), curFileTg.c_str());
+  }
+
+  is.open((sourceFeed->getPath() + "/fare_rules.txt").c_str());
+  if (is.good()) {
+    is.close();
+    curFile = gtfsPath + "/.fare_rules.txt";
+    curFileTg = gtfsPath + "/fare_rules.txt";
+    fs.open(curFile.c_str());
+    if (!fs.good()) cannotWrite(curFile, curFileTg);
+    writeFareRules(sourceFeed, &fs);
+    fs.close();
+    std::rename(curFile.c_str(), curFileTg.c_str());
+  }
+
+  is.close();
+  curFile = gtfsPath + "/.shapes.txt";
+  curFileTg = gtfsPath + "/shapes.txt";
+  fs.open(curFile.c_str());
+  if (!fs.good()) cannotWrite(curFile, curFileTg);
+  writeShapes(sourceFeed, &fs);
+  fs.close();
+  std::rename(curFile.c_str(), curFileTg.c_str());
+
+  is.close();
+  curFile = gtfsPath + "/.trips.txt";
+  curFileTg = gtfsPath + "/trips.txt";
+  fs.open(curFile.c_str());
+  if (!fs.good()) cannotWrite(curFile, curFileTg);
+  bool hasFreqs = writeTrips(sourceFeed, &fs);
+  fs.close();
+  std::rename(curFile.c_str(), curFileTg.c_str());
+
+  is.open((sourceFeed->getPath() + "/frequencies.txt").c_str());
+  if (hasFreqs && is.good()) {
+    is.close();
+    curFile = gtfsPath + "/.frequencies.txt";
+    curFileTg = gtfsPath + "/frequencies.txt";
+    fs.open(curFile.c_str());
+    if (!fs.good()) cannotWrite(curFile, curFileTg);
+    writeFrequencies(sourceFeed, &fs);
+    fs.close();
+    std::rename(curFile.c_str(), curFileTg.c_str());
+  }
+
+  is.close();
+  curFile = gtfsPath + "/.stop_times.txt";
+  curFileTg = gtfsPath + "/stop_times.txt";
+  fs.open(curFile.c_str());
+  if (!fs.good()) cannotWrite(curFile, curFileTg);
+  writeStopTimes(sourceFeed, &fs);
+  fs.close();
+  std::rename(curFile.c_str(), curFileTg.c_str());
+
+  if (!sourceFeed->getPublisherUrl().empty() &&
+      !sourceFeed->getPublisherName().empty()) {
+    curFile = gtfsPath + "/.feed_info.txt";
+    curFileTg = gtfsPath + "/feed_info.txt";
+    fs.open(curFile.c_str());
+    if (!fs.good()) cannotWrite(curFile, curFileTg);
+    writeFeedInfo(sourceFeed, &fs);
+    fs.close();
+    std::rename(curFile.c_str(), curFileTg.c_str());
+  }
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeFeedInfo(gtfs::Feed* f, std::ostream* os) const {
+  auto csvw = ad::cppgtfs::Writer::getFeedInfoCsvw(os);
+  csvw.flushLine();
+  csvw.writeString(f->getPublisherName());
+  csvw.writeString(f->getPublisherUrl());
+  csvw.writeString(f->getLang());
+  if (!f->getStartDate().empty())
+    csvw.writeInt(f->getStartDate().getYYYYMMDD());
+  else
+    csvw.skip();
+  if (!f->getEndDate().empty())
+    csvw.writeInt(f->getEndDate().getYYYYMMDD());
+  else
+    csvw.skip();
+  csvw.writeString(f->getVersion());
+  csvw.flushLine();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeAgency(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/agency.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getAgencyCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::Agency fa;
+  auto flds = Parser::getAgencyFlds(&csvp);
+
+  while (p.nextAgency(&csvp, &fa, flds)) {
+    w.writeAgency(fa, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeStops(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/stops.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getStopsCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::Stop s;
+  auto flds = Parser::getStopFlds(&csvp);
+
+  while (p.nextStop(&csvp, &s, flds)) {
+    w.writeStop(s, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeRoutes(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/routes.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getRoutesCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::Route s;
+  auto flds = Parser::getRouteFlds(&csvp);
+
+  while (p.nextRoute(&csvp, &s, flds)) {
+    w.writeRoute(s, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeCalendar(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/calendar.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getCalendarCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::Calendar c;
+  auto flds = Parser::getCalendarFlds(&csvp);
+
+  while (p.nextCalendar(&csvp, &c, flds)) {
+    w.writeCalendar(c, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeCalendarDates(gtfs::Feed* sourceFeed,
+                                std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/calendar_dates.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getCalendarDatesCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::CalendarDate c;
+  auto flds = Parser::getCalendarDateFlds(&csvp);
+
+  while (p.nextCalendarDate(&csvp, &c, flds)) {
+    w.writeCalendarDate(c, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeFrequencies(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/frequencies.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getFrequencyCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::Frequency f;
+  auto flds = Parser::getFrequencyFlds(&csvp);
+
+  while (p.nextFrequency(&csvp, &f, flds)) {
+    w.writeFrequency(f, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeTransfers(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/transfers.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getTransfersCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::Transfer t;
+  auto flds = Parser::getTransfersFlds(&csvp);
+
+  while (p.nextTransfer(&csvp, &t, flds)) {
+    w.writeTransfer(t, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeFares(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/fare_attributes.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getFaresCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::Fare f;
+  auto flds = Parser::getFareFlds(&csvp);
+
+  while (p.nextFare(&csvp, &f, flds)) {
+    w.writeFare(f, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeFareRules(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/fare_rules.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getFareRulesCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::FareRule f;
+  auto flds = Parser::getFareRuleFlds(&csvp);
+
+  while (p.nextFareRule(&csvp, &f, flds)) {
+    w.writeFareRule(f, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeShapes(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/shapes.txt").c_str());
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getShapesCsvw(os);
+  csvw.flushLine();
+  ad::cppgtfs::gtfs::flat::ShapePoint sp;
+  ad::cppgtfs::Writer w;
+
+  if (fs.good()) {
+    CsvParser csvp(&fs);
+    Parser p;
+
+    auto flds = Parser::getShapeFlds(&csvp);
+
+    std::string curShapeId;
+    std::string curSkipShapeId;
+
+    while (p.nextShapePoint(&csvp, &sp, flds)) {
+      if (sp.id == curSkipShapeId) continue;
+      if (sp.id != curShapeId) {
+        if (sourceFeed->getShapes().has(sp.id)) {
+          curShapeId = sp.id;
+        } else {
+          curSkipShapeId = sp.id;
+          continue;
+        }
+      }
+
+      w.writeShapePoint(sp, &csvw);
+    }
+  }
+
+  sourceFeed->getShapes().open();
+  while (sourceFeed->getShapes().nextStoragePt(&sp)) {
+    w.writeShapePoint(sp, &csvw);
+  }
+
+  fs.close();
+
+  return true;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeTrips(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  ad::cppgtfs::Writer w;
+  bool hasFreqs = false;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getTripsCsvw(os);
+  csvw.flushLine();
+
+  for (auto t : sourceFeed->getTrips()) {
+    if (t.getFrequencies().size()) hasFreqs = true;
+    w.writeTrip(t.getFlat(), &csvw);
+  }
+
+  return hasFreqs;
+}
+
+// ____________________________________________________________________________
+bool Writer::writeStopTimes(gtfs::Feed* sourceFeed, std::ostream* os) const {
+  std::ifstream fs;
+  fs.open((sourceFeed->getPath() + "/stop_times.txt").c_str());
+
+  CsvParser csvp(&fs);
+  Parser p;
+  ad::cppgtfs::Writer w;
+
+  CsvWriter csvw = ad::cppgtfs::Writer::getStopTimesCsvw(os);
+  csvw.flushLine();
+
+  ad::cppgtfs::gtfs::flat::StopTime st;
+  auto flds = Parser::getStopTimeFlds(&csvp);
+
+  std::string curTripId;
+  Trip* cur = 0;
+
+  while (p.nextStopTime(&csvp, &st, flds)) {
+    // we may have changed to distance field
+    if (curTripId != st.trip) {
+      cur = sourceFeed->getTrips().get(st.trip);
+      curTripId = st.trip;
+    }
+    for (const auto& stN : cur->getStopTimes()) {
+      if (stN.getSeq() == st.sequence)
+        st.shapeDistTravelled = stN.getShapeDistanceTravelled();
+    }
+
+    w.writeStopTime(st, &csvw);
+  }
+  fs.close();
+
+  return true;
+}
+
+// ___________________________________________________________________________
+void Writer::cannotWrite(const std::string& file, const std::string& file2) {
+  std::stringstream ss;
+  ss << "(temporary file for " << file2 << ") Could not write to file";
+  throw ad::cppgtfs::WriterException(ss.str(), file);
+}

src/pfaedle/gtfs/Writer.h

@@ -0,0 +1,41 @@
+// Copyright 2016, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#ifndef PFAEDLE_GTFS_WRITER_H_
+#define PFAEDLE_GTFS_WRITER_H_
+
+#include <string>
+#include "Feed.h"
+
+namespace pfaedle {
+namespace gtfs {
+
+class Writer {
+ public:
+  Writer() {}
+
+  bool write(Feed* sourceFeed, const std::string& path) const;
+
+ private:
+  bool writeFeedInfo(Feed* f, std::ostream* os) const;
+  bool writeAgency(Feed* f, std::ostream* os) const;
+  bool writeStops(Feed* f, std::ostream* os) const;
+  bool writeRoutes(Feed* f, std::ostream* os) const;
+  bool writeCalendar(Feed* f, std::ostream* os) const;
+  bool writeCalendarDates(Feed* f, std::ostream* os) const;
+  bool writeFrequencies(Feed* f, std::ostream* os) const;
+  bool writeTransfers(Feed* f, std::ostream* os) const;
+  bool writeFares(Feed* f, std::ostream* os) const;
+  bool writeFareRules(Feed* f, std::ostream* os) const;
+  bool writeShapes(Feed* f, std::ostream* os) const;
+  bool writeTrips(Feed* f, std::ostream* os) const;
+  bool writeStopTimes(Feed* f, std::ostream* os) const;
+
+  static void cannotWrite(const std::string& file, const std::string& file2);
+};
+
+}  // namespace gtfs
+}  // namespace pfaedle
+
+#endif  // PFAEDLE_GTFS_WRITER_H_

src/pfaedle/netgraph/EdgePL.h

@@ -9,11 +9,12 @@
 #include <string>
 #include <vector>
 #include "ad/cppgtfs/gtfs/Feed.h"
+#include "pfaedle/gtfs/Feed.h"
 #include "util/String.h"
 #include "util/geo/GeoGraph.h"
 
 using util::geograph::GeoEdgePL;
-using ad::cppgtfs::gtfs::Trip;
+using pfaedle::gtfs::Trip;
 
 namespace pfaedle {
 namespace netgraph {
@@ -22,7 +23,7 @@ namespace netgraph {
  * A payload class for edges on a network graph - that is a graph
  * that exactly represents a physical public transit network
  */
-class EdgePL : public GeoEdgePL<PFAEDLE_PRECISION> {
+class EdgePL {
  public:
   EdgePL() {}
   EdgePL(const LINE& l, const std::set<const Trip*>& trips)
@@ -36,10 +37,10 @@ class EdgePL : public GeoEdgePL<PFAEDLE_PRECISION> {
   util::json::Dict getAttrs() const {
     util::json::Dict obj;
     obj["num_trips"] = static_cast<int>(_trips.size());
-    obj["route_short_names"] = util::json::Array(
+    obj["route_short_names"] =
-        _routeShortNames.begin(), _routeShortNames.end());
+        util::json::Array(_routeShortNames.begin(), _routeShortNames.end());
-    obj["trip_short_names"] = util::json::Array(_tripShortNames.begin(),
+    obj["trip_short_names"] =
-                                                       _tripShortNames.end());
+        util::json::Array(_tripShortNames.begin(), _tripShortNames.end());
     return obj;
   }
 

src/pfaedle/netgraph/NodePL.h

@@ -12,7 +12,6 @@
 
 using util::geograph::GeoNodePL;
 
-
 namespace pfaedle {
 namespace netgraph {
 
@@ -20,15 +19,13 @@ namespace netgraph {
  * A payload class for edges on a network graph - that is a graph
  * that exactly represents a physical public transit network
  */
-class NodePL : public GeoNodePL<PFAEDLE_PRECISION> {
+class NodePL {
  public:
   NodePL() {}
   NodePL(const POINT& geom) { _geom = geom; }  // NOLINT
 
   const POINT* getGeom() const { return &_geom; }
-  util::json::Dict getAttrs() const {
+  util::json::Dict getAttrs() const { return util::json::Dict(); }
-    return util::json::Dict();
-  }
 
  private:
   POINT _geom;

src/pfaedle/osm/BBoxIdx.cpp

@@ -37,6 +37,31 @@ BOX BBoxIdx::getFullWebMercBox() const {
           _root.box.getUpperRight().getY(), _root.box.getUpperRight().getX()));
 }
 
+// _____________________________________________________________________________
+BOX BBoxIdx::getFullBox() const { return _root.box; }
+
+// _____________________________________________________________________________
+std::vector<util::geo::Box<double>> BBoxIdx::getLeafs() const {
+  std::vector<util::geo::Box<double>> ret;
+  getLeafsRec(_root, &ret);
+  return ret;
+}
+
+// _____________________________________________________________________________
+void BBoxIdx::getLeafsRec(const BBoxIdxNd& nd,
+                          std::vector<util::geo::Box<double>>* ret) const {
+  if (!nd.childs.size()) {
+    ret->push_back(nd.box);
+    return;
+  }
+
+  for (const auto& child : nd.childs) {
+    getLeafsRec(child, ret);
+  }
+
+  return;
+}
+
 // _____________________________________________________________________________
 bool BBoxIdx::treeHas(const Point<double>& p, const BBoxIdxNd& nd) const {
   if (!nd.childs.size()) return util::geo::contains(p, nd.box);

src/pfaedle/osm/BBoxIdx.h

@@ -38,9 +38,15 @@ class BBoxIdx {
   // Return the full total bounding box of this index
   BOX getFullWebMercBox() const;
 
+  // Return the full total bounding box of this index
+  BOX getFullBox() const;
+
   // Return the size of this index
   size_t size() const;
 
+  // return the leaf bounding boxes of this idx
+  std::vector<Box<double>> getLeafs() const;
+
  private:
   double _padding;
   size_t _size;
@@ -50,6 +56,9 @@ class BBoxIdx {
   void addToTree(const Box<double>& box, BBoxIdxNd* nd, size_t lvl);
   bool treeHas(const Point<double>& p, const BBoxIdxNd& nd) const;
 
+  void getLeafsRec(const BBoxIdxNd& nd,
+                   std::vector<util::geo::Box<double>>* ret) const;
+
   static const size_t MAX_LVL = 5;
   static constexpr double MIN_COM_AREA = 0.0;
 };

src/pfaedle/osm/OsmBuilder.cpp

@@ -43,8 +43,20 @@ using pfaedle::osm::OsmRel;
 using pfaedle::osm::OsmNode;
 using pfaedle::osm::EdgeGrid;
 using pfaedle::osm::NodeGrid;
+using pfaedle::osm::EqSearch;
+using pfaedle::osm::BlockSearch;
 using ad::cppgtfs::gtfs::Stop;
 
+// _____________________________________________________________________________
+bool EqSearch::operator()(const Node* cand, const StatInfo* si) const {
+  if (orphanSnap && cand->pl().getSI() &&
+      (!cand->pl().getSI()->getGroup() ||
+       cand->pl().getSI()->getGroup()->getStops().size() == 0)) {
+    return true;
+  }
+  return cand->pl().getSI() && cand->pl().getSI()->simi(si) > minSimi;
+}
+
 // _____________________________________________________________________________
 OsmBuilder::OsmBuilder() {}
 
@@ -140,7 +152,8 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
         POINT geom = *s->pl().getGeom();
         NodePL pl = s->pl();
         pl.getSI()->setIsFromOsm(false);
-        const auto& r = snapStation(g, &pl, &eg, &sng, opts, res, false, d);
+        const auto& r =
+            snapStation(g, &pl, &eg, &sng, opts, res, false, false, d);
         groupStats(r);
         for (auto n : r) {
           // if the snapped station is very near to the original OSM
@@ -153,32 +166,70 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
       }
     }
 
+    std::vector<const Stop*> notSnapped;
+
     for (size_t i = 0; i < opts.maxSnapDistances.size(); i++) {
       double d = opts.maxSnapDistances[i];
       for (auto& s : *fs) {
         auto pl = plFromGtfs(s.first, opts);
 
-        StatGroup* group =
+        StatGroup* group = groupStats(
-            groupStats(snapStation(g, &pl, &eg, &sng, opts, res,
+            snapStation(g, &pl, &eg, &sng, opts, res,
-                                   i == opts.maxSnapDistances.size() - 1, d));
+                        i == opts.maxSnapDistances.size() - 1, false, d));
 
         if (group) {
           group->addStop(s.first);
           (*fs)[s.first] = *group->getNodes().begin();
+        } else if (i == opts.maxSnapDistances.size() - 1) {
+          LOG(VDEBUG) << "Could not snap station "
+                      << "(" << pl.getSI()->getName() << ")"
+                      << " (" << s.first->getLat() << "," << s.first->getLng()
+                      << ") in normal run, trying again later in orphan mode.";
+          notSnapped.push_back(s.first);
+        }
+      }
+    }
+
+    if (notSnapped.size())
+      LOG(VDEBUG) << notSnapped.size() << " stations could not be snapped in "
+                                          "normal run, trying again in orphan "
+                                          "mode.";
+
+    // try again, but aggressively snap to orphan OSM stations which have
+    // not been assigned to any GTFS stop yet
+    for (size_t i = 0; i < opts.maxSnapDistances.size(); i++) {
+      double d = opts.maxSnapDistances[i];
+      for (auto& s : notSnapped) {
+        auto pl = plFromGtfs(s, opts);
+
+        StatGroup* group = groupStats(
+            snapStation(g, &pl, &eg, &sng, opts, res,
+                        i == opts.maxSnapDistances.size() - 1, true, d));
+
+        if (group) {
+          group->addStop(s);
+          // add the added station name as an alt name to ensure future
+          // similarity
+          for (auto n : group->getNodes()) {
+            if (n->pl().getSI())
+              n->pl().getSI()->addAltName(pl.getSI()->getName());
+          }
+          (*fs)[s] = *group->getNodes().begin();
         } else if (i ==
                    opts.maxSnapDistances.size() - 1) {  // only fail on last
+          // finally give up
+
           // add a group with only this stop in it
           StatGroup* dummyGroup = new StatGroup();
           Node* dummyNode = g->addNd(pl);
 
           dummyNode->pl().getSI()->setGroup(dummyGroup);
           dummyGroup->addNode(dummyNode);
-          dummyGroup->addStop(s.first);
+          dummyGroup->addStop(s);
-          (*fs)[s.first] = dummyNode;
+          (*fs)[s] = dummyNode;
           LOG(WARN) << "Could not snap station "
                     << "(" << pl.getSI()->getName() << ")"
-                    << " (" << s.first->getLat() << "," << s.first->getLng()
+                    << " (" << s->getLat() << "," << s->getLng() << ")";
-                    << ")";
         }
       }
     }
@@ -188,7 +239,7 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
   deleteOrphNds(g);
 
   LOG(VDEBUG) << "Deleting orphan edges...";
-  deleteOrphEdgs(g);
+  deleteOrphEdgs(g, opts);
 
   LOG(VDEBUG) << "Collapsing edges...";
   collapseEdges(g);
@@ -197,7 +248,7 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
   deleteOrphNds(g);
 
   LOG(VDEBUG) << "Deleting orphan edges...";
-  deleteOrphEdgs(g);
+  deleteOrphEdgs(g, opts);
 
   LOG(VDEBUG) << "Writing graph components...";
   // the restrictor is needed here to prevent connections in the graph
@@ -223,6 +274,89 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
              << " edges and " << comps << " connected component(s)";
 }
 
+// _____________________________________________________________________________
+void OsmBuilder::overpassQryWrite(std::ostream* out,
+                                  const std::vector<OsmReadOpts>& opts,
+                                  const BBoxIdx& latLngBox) const {
+  OsmIdSet bboxNodes, noHupNodes;
+  MultAttrMap emptyF;
+
+  RelLst rels;
+  OsmIdList ways;
+  RelMap nodeRels, wayRels;
+
+  // TODO(patrick): not needed here!
+  Restrictions rests;
+
+  NIdMap nodes;
+
+  // always empty
+  NIdMultMap multNodes;
+  util::xml::XmlWriter wr(out, true, 4);
+
+  *out << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+  wr.openComment();
+  wr.writeText(" - written by pfaedle -");
+  wr.closeTag();
+  wr.openTag("osm-script",
+             {{"timeout", "99999"}, {"element-limit", "1073741824"}});
+
+  OsmFilter filter;
+
+  for (const OsmReadOpts& o : opts) {
+    filter = filter.merge(OsmFilter(o.keepFilter, o.dropFilter));
+  }
+
+  wr.openTag("union");
+  size_t c = 0;
+  for (auto box : latLngBox.getLeafs()) {
+    if (box.getLowerLeft().getX() > box.getUpperRight().getX()) continue;
+    c++;
+    wr.openComment();
+    wr.writeText(std::string("Bounding box #") + std::to_string(c) + " (" +
+                 std::to_string(box.getLowerLeft().getY()) + ", " +
+                 std::to_string(box.getLowerLeft().getX()) + ", " +
+                 std::to_string(box.getUpperRight().getY()) + ", " +
+                 std::to_string(box.getUpperRight().getX()) + ")");
+    wr.closeTag();
+    for (auto t : std::vector<std::string>{"way", "node", "relation"}) {
+      for (auto r : filter.getKeepRules()) {
+        for (auto val : r.second) {
+          if (t == "way" && (val.second & OsmFilter::WAY)) continue;
+          if (t == "relation" && (val.second & OsmFilter::REL)) continue;
+          if (t == "node" && (val.second & OsmFilter::NODE)) continue;
+
+          wr.openTag("query", {{"type", t}});
+          if (val.first == "*")
+            wr.openTag("has-kv", {{"k", r.first}});
+          else
+            wr.openTag("has-kv", {{"k", r.first}, {"v", val.first}});
+          wr.closeTag();
+          wr.openTag("bbox-query",
+                     {{"s", std::to_string(box.getLowerLeft().getY())},
+                      {"w", std::to_string(box.getLowerLeft().getX())},
+                      {"n", std::to_string(box.getUpperRight().getY())},
+                      {"e", std::to_string(box.getUpperRight().getX())}});
+          wr.closeTag();
+          wr.closeTag();
+        }
+      }
+    }
+  }
+
+  wr.closeTag();
+
+  wr.openTag("union");
+  wr.openTag("item");
+  wr.closeTag();
+  wr.openTag("recurse", {{"type", "down"}});
+  wr.closeTag();
+  wr.closeTag();
+  wr.openTag("print");
+
+  wr.closeTags();
+}
+
 // _____________________________________________________________________________
 void OsmBuilder::filterWrite(const std::string& in, const std::string& out,
                              const std::vector<OsmReadOpts>& opts,
@@ -250,8 +384,15 @@ void OsmBuilder::filterWrite(const std::string& in, const std::string& out,
 
   outstr << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
   wr.openTag("osm");
-
+  wr.openTag(
-  // TODO(patrick): write bounding box tag
+      "bounds",
+      {{"minlat", std::to_string(latLngBox.getFullBox().getLowerLeft().getY())},
+       {"minlon", std::to_string(latLngBox.getFullBox().getLowerLeft().getX())},
+       {"maxlat",
+        std::to_string(latLngBox.getFullBox().getUpperRight().getY())},
+       {"maxlon",
+        std::to_string(latLngBox.getFullBox().getUpperRight().getX())}});
+  wr.closeTag();
 
   OsmFilter filter;
   AttrKeySet attrKeys[3] = {};
@@ -1071,8 +1212,7 @@ EdgeGrid OsmBuilder::buildEdgeIdx(Graph* g, size_t size,
 }
 
 // _____________________________________________________________________________
-NodeGrid OsmBuilder::buildNodeIdx(Graph* g, size_t size,
+NodeGrid OsmBuilder::buildNodeIdx(Graph* g, size_t size, const BOX& webMercBox,
-                                  const BOX& webMercBox,
                                   bool which) const {
   NodeGrid ret(size, size, webMercBox, false);
   for (auto* n : *g->getNds()) {
@@ -1158,9 +1298,10 @@ bool OsmBuilder::isBlocked(const Edge* e, const StatInfo* si, const POINT& p,
 
 // _____________________________________________________________________________
 Node* OsmBuilder::eqStatReach(const Edge* e, const StatInfo* si, const POINT& p,
-                              double maxD, int maxFullTurns,
+                              double maxD, int maxFullTurns, double minAngle,
-                              double minAngle) const {
+                              bool orphanSnap) const {
-  return depthSearch(e, si, p, maxD, maxFullTurns, minAngle, EqSearch());
+  return depthSearch(e, si, p, maxD, maxFullTurns, minAngle,
+                     EqSearch(orphanSnap));
 }
 
 // _____________________________________________________________________________
@@ -1187,8 +1328,7 @@ std::set<Node*> OsmBuilder::getMatchingNds(const NodePL& s, NodeGrid* ng,
   std::set<Node*> ret;
   double distor = webMercDistFactor(*s.getGeom());
   std::set<Node*> neighs;
-  BOX box =
+  BOX box = util::geo::pad(util::geo::getBoundingBox(*s.getGeom()), d / distor);
-      util::geo::pad(util::geo::getBoundingBox(*s.getGeom()), d / distor);
   ng->get(box, &neighs);
 
   for (auto* n : neighs) {
@@ -1205,8 +1345,7 @@ std::set<Node*> OsmBuilder::getMatchingNds(const NodePL& s, NodeGrid* ng,
 Node* OsmBuilder::getMatchingNd(const NodePL& s, NodeGrid* ng, double d) const {
   double distor = webMercDistFactor(*s.getGeom());
   std::set<Node*> neighs;
-  BOX box =
+  BOX box = util::geo::pad(util::geo::getBoundingBox(*s.getGeom()), d / distor);
-      util::geo::pad(util::geo::getBoundingBox(*s.getGeom()), d / distor);
   ng->get(box, &neighs);
 
   Node* ret = 0;
@@ -1229,7 +1368,7 @@ Node* OsmBuilder::getMatchingNd(const NodePL& s, NodeGrid* ng, double d) const {
 std::set<Node*> OsmBuilder::snapStation(Graph* g, NodePL* s, EdgeGrid* eg,
                                         NodeGrid* sng, const OsmReadOpts& opts,
                                         Restrictor* restor, bool surrHeur,
-                                        double d) const {
+                                        bool orphSnap, double d) const {
   assert(s->getSI());
   std::set<Node*> ret;
 
@@ -1239,10 +1378,14 @@ std::set<Node*> OsmBuilder::snapStation(Graph* g, NodePL* s, EdgeGrid* eg,
 
   if (pq.empty() && surrHeur) {
     // no station found in the first round, try again with the nearest
-    // surrounding
+    // surrounding station with matching name
-    // station with matching name
     const Node* best = getMatchingNd(*s, sng, opts.maxSnapFallbackHeurDistance);
-    if (best) getEdgCands(*best->pl().getGeom(), &pq, eg, d);
+    if (best) {
+      getEdgCands(*best->pl().getGeom(), &pq, eg, d);
+    } else {
+      // if still no luck, get edge cands in fallback snap distance
+      getEdgCands(*s->getGeom(), &pq, eg, opts.maxSnapFallbackHeurDistance);
+    }
   }
 
   while (!pq.empty()) {
@@ -1254,7 +1397,7 @@ std::set<Node*> OsmBuilder::snapStation(Graph* g, NodePL* s, EdgeGrid* eg,
 
     Node* eq = 0;
     if (!(eq = eqStatReach(e, s->getSI(), geom, 2 * d, 0,
-                           opts.maxAngleSnapReach))) {
+                           opts.maxAngleSnapReach, orphSnap))) {
       if (e->pl().lvl() > opts.maxSnapLevel) continue;
       if (isBlocked(e, s->getSI(), geom, opts.maxBlockDistance, 0,
                     opts.maxAngleSnapReach)) {
@@ -1309,11 +1452,6 @@ std::set<Node*> OsmBuilder::snapStation(Graph* g, NodePL* s, EdgeGrid* eg,
     }
   }
 
-  // get surrounding nodes
-  // TODO(patrick): own distance configuration for this!
-  const auto& sur = getMatchingNds(*s, sng, opts.maxGroupSearchDistance);
-  ret.insert(sur.begin(), sur.end());
-
   return ret;
 }
 
@@ -1336,7 +1474,10 @@ StatGroup* OsmBuilder::groupStats(const NodeSet& s) const {
     }
   }
 
-  if (!used) delete ret;
+  if (!used) {
+    delete ret;
+    return 0;
+  }
 
   return ret;
 }
@@ -1507,7 +1648,7 @@ void OsmBuilder::getKeptAttrKeys(const OsmReadOpts& opts,
 }
 
 // _____________________________________________________________________________
-void OsmBuilder::deleteOrphEdgs(Graph* g) const {
+void OsmBuilder::deleteOrphEdgs(Graph* g, const OsmReadOpts& opts) const {
   size_t ROUNDS = 3;
   for (size_t c = 0; c < ROUNDS; c++) {
     for (auto i = g->getNds()->begin(); i != g->getNds()->end();) {
@@ -1515,6 +1656,15 @@ void OsmBuilder::deleteOrphEdgs(Graph* g) const {
         ++i;
         continue;
       }
+
+      // check if the removal of this edge would transform a steep angle
+      // full turn at an intersection into a node 2 eligible for contraction
+      // if so, dont delete
+      if (keepFullTurn(*i, opts.fullTurnAngle)) {
+        ++i;
+        continue;
+      }
+
       i = g->delNd(*i);
       continue;
       i++;
@@ -1706,3 +1856,43 @@ void OsmBuilder::writeSelfEdgs(Graph* g) const {
     }
   }
 }
+
+// _____________________________________________________________________________
+bool OsmBuilder::keepFullTurn(const trgraph::Node* n, double ang) const {
+  if (n->getInDeg() + n->getOutDeg() != 1) return false;
+
+  const trgraph::Edge* e = 0;
+  if (n->getOutDeg())
+    e = n->getAdjListOut().front();
+  else
+    e = n->getAdjListIn().front();
+
+  auto other = e->getOtherNd(n);
+
+  if (other->getInDeg() + other->getOutDeg() == 3) {
+    const trgraph::Edge* a = 0;
+    const trgraph::Edge* b = 0;
+    for (auto f : other->getAdjListIn()) {
+      if (f != e && !a)
+        a = f;
+      else if (f != e && !b)
+        b = f;
+    }
+
+    for (auto f : other->getAdjListOut()) {
+      if (f != e && !a)
+        a = f;
+      else if (f != e && !b)
+        b = f;
+    }
+
+    auto ap = a->pl().backHop();
+    auto bp = b->pl().backHop();
+    if (a->getTo() != other) ap = a->pl().frontHop();
+    if (b->getTo() != other) bp = b->pl().frontHop();
+
+    return router::angSmaller(ap, *other->pl().getGeom(), bp, ang);
+  }
+
+  return false;
+}

src/pfaedle/osm/OsmBuilder.h

@@ -59,10 +59,10 @@ struct SearchFunc {
 };
 
 struct EqSearch : public SearchFunc {
+  explicit EqSearch(bool orphanSnap) : orphanSnap(orphanSnap) {}
   double minSimi = 0.9;
-  bool operator()(const Node* cand, const StatInfo* si) const {
+  bool orphanSnap;
-    return cand->pl().getSI() && cand->pl().getSI()->simi(si) > minSimi;
+  bool operator()(const Node* cand, const StatInfo* si) const;
-  }
 };
 
 struct BlockSearch : public SearchFunc {
@@ -91,6 +91,11 @@ class OsmBuilder {
             const BBoxIdx& box, size_t gridSize, router::FeedStops* fs,
             Restrictor* res);
 
+  // Based on the list of options, output an overpass XML query for getting
+  // the data needed for routing
+  void overpassQryWrite(std::ostream* out, const std::vector<OsmReadOpts>& opts,
+                        const BBoxIdx& latLngBox) const;
+
   // Based on the list of options, read an OSM file from in and output an
   // OSM file to out which contains exactly the entities that are needed
   // from the file at in
@@ -170,7 +175,7 @@ class OsmBuilder {
 
   void writeGeoms(Graph* g) const;
   void deleteOrphNds(Graph* g) const;
-  void deleteOrphEdgs(Graph* g) const;
+  void deleteOrphEdgs(Graph* g, const OsmReadOpts& opts) const;
   double dist(const Node* a, const Node* b) const;
   double webMercDist(const Node* a, const Node* b) const;
   double webMercDistFactor(const POINT& a) const;
@@ -198,13 +203,14 @@ class OsmBuilder {
 
   NodeSet snapStation(Graph* g, NodePL* s, EdgeGrid* eg, NodeGrid* sng,
                       const OsmReadOpts& opts, Restrictor* restor, bool surHeur,
-                      double maxD) const;
+                      bool orphSnap, double maxD) const;
 
   // Checks if from the edge e, a station similar to si can be reach with less
   // than maxD distance and less or equal to "maxFullTurns" full turns. If
   // such a station exists, it is returned. If not, 0 is returned.
   Node* eqStatReach(const Edge* e, const StatInfo* si, const POINT& p,
-                    double maxD, int maxFullTurns, double maxAng) const;
+                    double maxD, int maxFullTurns, double maxAng,
+                    bool orph) const;
 
   Node* depthSearch(const Edge* e, const StatInfo* si, const POINT& p,
                     double maxD, int maxFullTurns, double minAngle,
@@ -243,6 +249,8 @@ class OsmBuilder {
 
   bool relKeep(osmid id, const RelMap& rels, const FlatRels& fl) const;
 
+  bool keepFullTurn(const trgraph::Node* n, double ang) const;
+
   std::map<TransitEdgeLine, TransitEdgeLine*> _lines;
   std::map<size_t, TransitEdgeLine*> _relLines;
 };

src/pfaedle/osm/OsmFilter.cpp

@@ -102,12 +102,12 @@ uint64_t OsmFilter::contained(const AttrMap& attrs, const Attr& attr) {
 // _____________________________________________________________________________
 uint8_t OsmFilter::level(const AttrMap& attrs) const {
   // the best matching level is always returned
-  for (int16_t i = 0; i < 7; i++) {
+  for (int16_t i = 0; i < 8; i++) {
     for (const auto& kv : attrs) {
       const auto& lkv = (_levels + i)->find(kv.first);
       if (lkv != (_levels + i)->end()) {
         for (const auto& val : lkv->second) {
-          if (valMatches(kv.second, val.first)) return i + 1;
+          if (valMatches(kv.second, val.first)) return i;
         }
       }
     }
@@ -169,7 +169,7 @@ std::vector<std::string> OsmFilter::getAttrKeys() const {
   for (const auto& kv : _noRestr) {
     ret.push_back(kv.first);
   }
-  for (uint8_t i = 0; i < 7; i++) {
+  for (uint8_t i = 0; i < 8; i++) {
     for (const auto& kv : *(_levels + i)) {
       ret.push_back(kv.first);
     }
@@ -191,27 +191,6 @@ OsmFilter OsmFilter::merge(const OsmFilter& other) const {
     keep[kv.first].insert(kv.second.begin(), kv.second.end());
   }
 
-  // TODO(patrick): multi-level combination for filters. otherwise
-  // filter drop filters meant as a refinement for keep filters
-  // interfere with other keeps
-
-  // const auto* d = &_drop;
-
-  // for (size_t i = 0; i < 2; i++) {
-    // for (const auto& kv : *d) {
-      // if (keep.find(kv.first) != keep.end()) {
-        // for (const auto& val : kv.second) {
-          // if (keep[kv.first].find(val.first) == keep[kv.first].end()) {
-            // drop[kv.first].insert(val);
-          // }
-        // }
-      // } else {
-        // drop[kv.first].insert(kv.second.begin(), kv.second.end());
-      // }
-    // }
-    // d = &other._drop;
-  // }
-
   return OsmFilter(keep, drop);
 }
 
@@ -258,3 +237,13 @@ uint64_t OsmFilter::posRestr(const AttrMap& attrs) const {
   if (contained(attrs, _noRestr, ALL)) return false;
   return (contained(attrs, _posRestr, ALL));
 }
+
+// _____________________________________________________________________________
+const pfaedle::osm::MultAttrMap& OsmFilter::getKeepRules() const {
+  return _keep;
+}
+
+// _____________________________________________________________________________
+const pfaedle::osm::MultAttrMap& OsmFilter::getDropRules() const {
+  return _drop;
+}

src/pfaedle/osm/OsmFilter.h

@@ -33,6 +33,9 @@ class OsmFilter {
 
   OsmFilter merge(const OsmFilter& other) const;
 
+  const MultAttrMap& getKeepRules() const;
+  const MultAttrMap& getDropRules() const;
+
   std::string toString() const;
 
   static bool valMatches(const std::string& a, const std::string& b, bool m);

src/pfaedle/osm/OsmReadOpts.h

@@ -113,7 +113,7 @@ struct OsmReadOpts {
 
   MultAttrMap noHupFilter;
   MultAttrMap keepFilter;
-  MultAttrMap levelFilters[7];
+  MultAttrMap levelFilters[8];
   MultAttrMap dropFilter;
   MultAttrMap oneWayFilter;
   MultAttrMap oneWayFilterRev;
@@ -136,7 +136,6 @@ struct OsmReadOpts {
   double maxAngleSnapReach;
   std::vector<double> maxSnapDistances;
   double maxSnapFallbackHeurDistance;
-  double maxGroupSearchDistance;
   double maxBlockDistance;
 
   double maxOsmStationDistance;
@@ -144,6 +143,8 @@ struct OsmReadOpts {
   // TODO(patrick): this is not implemented yet
   double levelSnapPunishFac[7] = {0, 0, 0, 0, 0, 0, 0};
 
+  double fullTurnAngle;
+
   // restriction system
   MultAttrMap restrPosRestr;
   MultAttrMap restrNegRestr;
@@ -179,7 +180,6 @@ inline bool operator==(const OsmReadOpts& a, const OsmReadOpts& b) {
          fabs(a.maxOsmStationDistance - b.maxOsmStationDistance) < 0.1 &&
          fabs(a.maxSnapFallbackHeurDistance - b.maxSnapFallbackHeurDistance) <
              0.1 &&
-         fabs(a.maxGroupSearchDistance - b.maxGroupSearchDistance) < 0.1 &&
          fabs(a.maxBlockDistance - b.maxBlockDistance) < 0.1 &&
          fabs(a.levelSnapPunishFac[0] - b.levelSnapPunishFac[0]) < 0.1 &&
          fabs(a.levelSnapPunishFac[1] - b.levelSnapPunishFac[1]) < 0.1 &&
@@ -188,6 +188,7 @@ inline bool operator==(const OsmReadOpts& a, const OsmReadOpts& b) {
          fabs(a.levelSnapPunishFac[4] - b.levelSnapPunishFac[4]) < 0.1 &&
          fabs(a.levelSnapPunishFac[5] - b.levelSnapPunishFac[5]) < 0.1 &&
          fabs(a.levelSnapPunishFac[6] - b.levelSnapPunishFac[6]) < 0.1 &&
+         fabs(a.fullTurnAngle - b.fullTurnAngle) < 0.1 &&
          a.restrPosRestr == b.restrPosRestr &&
          a.restrNegRestr == b.restrNegRestr &&
          a.noRestrFilter == b.noRestrFilter;

src/pfaedle/router/Comp.h

@@ -60,12 +60,29 @@ inline double lineSimi(const std::string& a, const std::string& b) {
 
   if (a.empty() || b.empty()) return 0;
 
-  // if one of the lines is completely contained in the other, return 1
+  if (a.size() > b.size() + 1) {
-  if (a.find(b) != std::string::npos) {
+    // check if a begins with b
+    if (a.compare(0, b.size() + 1, b + " ") == 0) {
       return 1;
-  } else if (b.find(a) != std::string::npos) {
+    }
+
+    // check if a ends with b
+    if (a.compare(a.size() - (b.size() + 1), b.size() + 1, " " + b) == 0) {
       return 1;
     }
+  }
+
+  if (b.size() > a.size() + 1) {
+    // check if b begins with a
+    if (b.compare(0, a.size() + 1, a + " ") == 0) {
+      return 1;
+    }
+
+    // check if b ends with a
+    if (b.compare(b.size() - (a.size() + 1), a.size() + 1, " " + a) == 0) {
+      return 1;
+    }
+  }
 
   return 0;
 }

src/pfaedle/router/EdgePL.cpp

@@ -82,21 +82,6 @@ util::json::Dict EdgePL::getAttrs() const {
   obj["cost"] = std::to_string(_cost.getValue());
   obj["from_edge"] = util::toString(_startE);
   obj["to_edge"] = util::toString(_endE);
-  obj["cost_m_lvl1"] = std::to_string(_cost.meterDistLvl1);
-  obj["cost_m_lvl0"] = std::to_string(_cost.meterDist);
-  obj["cost_m_lvl1"] = std::to_string(_cost.meterDistLvl1);
-  obj["cost_m_lvl2"] = std::to_string(_cost.meterDistLvl2);
-  obj["cost_m_lvl3"] = std::to_string(_cost.meterDistLvl3);
-  obj["cost_m_lvl4"] = std::to_string(_cost.meterDistLvl4);
-  obj["cost_m_lvl5"] = std::to_string(_cost.meterDistLvl5);
-  obj["cost_m_lvl6"] = std::to_string(_cost.meterDistLvl6);
-  obj["cost_m_lvl7"] = std::to_string(_cost.meterDistLvl7);
-  obj["cost_fullturn"] = std::to_string(_cost.fullTurns);
-  obj["cost_st_passthru"] = std::to_string(_cost.passThruStations);
-  obj["cost_m_oneway"] = std::to_string(_cost.oneWayMeters);
-  obj["cost_m_lineunmatch"] = std::to_string(_cost.lineUnmatchedMeters);
-  obj["cost_reach_node_pen"] = std::to_string(_cost.reachPen);
-  obj["cost_oneway_event"] = std::to_string(_cost.oneWayEdges);
   obj["dummy"] = _edges.size() ? "no" : "yes";
 
   return obj;

src/pfaedle/router/EdgePL.h

@@ -17,7 +17,7 @@ using util::geograph::GeoEdgePL;
 namespace pfaedle {
 namespace router {
 
-class EdgePL : public GeoEdgePL<PFAEDLE_PRECISION> {
+class EdgePL {
  public:
   EdgePL() : _cost(), _start(0), _end(0), _startE(0), _endE(0) {}
   const LINE* getGeom() const;

src/pfaedle/router/Misc.h

@@ -7,8 +7,8 @@
 
 #include <set>
 #include <string>
-#include <vector>
 #include <unordered_map>
+#include <vector>
 #include "ad/cppgtfs/gtfs/Feed.h"
 #include "ad/cppgtfs/gtfs/Route.h"
 #include "pfaedle/trgraph/Graph.h"
@@ -67,90 +67,35 @@ inline bool operator==(const RoutingOpts& a, const RoutingOpts& b) {
 }
 
 struct EdgeCost {
-  EdgeCost()
+  EdgeCost() : _cost(0) {}
-      : meterDist(0),
+  explicit EdgeCost(double cost) : _cost(cost) {}
-        meterDistLvl1(0),
-        meterDistLvl2(0),
-        meterDistLvl3(0),
-        meterDistLvl4(0),
-        meterDistLvl5(0),
-        meterDistLvl6(0),
-        meterDistLvl7(0),
-        fullTurns(0),
-        passThruStations(0),
-        oneWayMeters(0),
-        oneWayEdges(0),
-        lineUnmatchedMeters(0),
-        reachPen(0),
-        o(0) {}
   EdgeCost(double mDist, double mDistLvl1, double mDistLvl2, double mDistLvl3,
            double mDistLvl4, double mDistLvl5, double mDistLvl6,
            double mDistLvl7, uint32_t fullTurns, int32_t passThru,
            double oneWayMeters, size_t oneWayEdges, double lineUnmatchedMeters,
-           double reachPen, const RoutingOpts* o)
+           double reachPen, const RoutingOpts* o) {
-      : meterDist(mDist),
+    if (!o) {
-        meterDistLvl1(mDistLvl1),
+      _cost = mDist + reachPen;
-        meterDistLvl2(mDistLvl2),
+    } else {
-        meterDistLvl3(mDistLvl3),
+      _cost = mDist * o->levelPunish[0] + mDistLvl1 * o->levelPunish[1] +
-        meterDistLvl4(mDistLvl4),
+              mDistLvl2 * o->levelPunish[2] + mDistLvl3 * o->levelPunish[3] +
-        meterDistLvl5(mDistLvl5),
+              mDistLvl4 * o->levelPunish[4] + mDistLvl5 * o->levelPunish[5] +
-        meterDistLvl6(mDistLvl6),
+              mDistLvl6 * o->levelPunish[6] + mDistLvl7 * o->levelPunish[7] +
-        meterDistLvl7(mDistLvl7),
-        fullTurns(fullTurns),
-        passThruStations(passThru),
-        oneWayMeters(oneWayMeters),
-        oneWayEdges(oneWayEdges),
-        lineUnmatchedMeters(lineUnmatchedMeters),
-        reachPen(reachPen),
-        o(o) {}
-  double meterDist;
-  double meterDistLvl1;
-  double meterDistLvl2;
-  double meterDistLvl3;
-  double meterDistLvl4;
-  double meterDistLvl5;
-  double meterDistLvl6;
-  double meterDistLvl7;
-  uint32_t fullTurns;
-  int32_t passThruStations;
-  double oneWayMeters;
-  size_t oneWayEdges;
-  double lineUnmatchedMeters;
-  double reachPen;
-  const RoutingOpts* o;
-
-  double getValue() const {
-    if (!o) return meterDist + reachPen;
-    return meterDist * o->levelPunish[0] + meterDistLvl1 * o->levelPunish[1] +
-           meterDistLvl2 * o->levelPunish[2] +
-           meterDistLvl3 * o->levelPunish[3] +
-           meterDistLvl4 * o->levelPunish[4] +
-           meterDistLvl5 * o->levelPunish[5] +
-           meterDistLvl6 * o->levelPunish[6] +
-           meterDistLvl7 * o->levelPunish[7] +
               oneWayMeters * o->oneWayPunishFac +
               oneWayEdges * o->oneWayEdgePunish +
               lineUnmatchedMeters * o->lineUnmatchedPunishFact +
               fullTurns * o->fullTurnPunishFac +
-           passThruStations * o->passThruStationsPunish + reachPen;
+              passThru * o->passThruStationsPunish + reachPen;
+    }
   }
 
-  double getTotalMeters() const {
+  float _cost;
-    return meterDist + meterDistLvl1 + meterDistLvl2 + meterDistLvl3 +
+
-           meterDistLvl4 + meterDistLvl5 + meterDistLvl6 + meterDistLvl7;
+  double getValue() const { return _cost; }
-  }
 };
 
 inline EdgeCost operator+(const EdgeCost& a, const EdgeCost& b) {
-  return EdgeCost(
+  return EdgeCost(a.getValue() + b.getValue());
-      a.meterDist + b.meterDist, a.meterDistLvl1 + b.meterDistLvl1,
-      a.meterDistLvl2 + b.meterDistLvl2, a.meterDistLvl3 + b.meterDistLvl3,
-      a.meterDistLvl4 + b.meterDistLvl4, a.meterDistLvl5 + b.meterDistLvl5,
-      a.meterDistLvl6 + b.meterDistLvl6, a.meterDistLvl7 + b.meterDistLvl7,
-      a.fullTurns + b.fullTurns, a.passThruStations + b.passThruStations,
-      a.oneWayMeters + b.oneWayMeters, a.oneWayEdges + b.oneWayEdges,
-      a.lineUnmatchedMeters + b.lineUnmatchedMeters, a.reachPen + b.reachPen,
-      a.o ? a.o : b.o);
 }
 
 inline bool operator<=(const EdgeCost& a, const EdgeCost& b) {

src/pfaedle/router/NodePL.h

@@ -18,7 +18,7 @@ using util::geograph::GeoNodePL;
 namespace pfaedle {
 namespace router {
 
-class NodePL : public GeoNodePL<PFAEDLE_PRECISION> {
+class NodePL {
  public:
   NodePL() : _n(0) {}
   NodePL(const pfaedle::trgraph::Node* n) : _n(n) {}  // NOLINT

src/pfaedle/router/Router.cpp

@@ -191,7 +191,8 @@ double CombCostFunc::operator()(const router::Edge* from, const router::Node* n,
 }
 
 // _____________________________________________________________________________
-Router::Router(size_t numThreads) : _cache(numThreads) {
+Router::Router(size_t numThreads, bool caching)
+    : _cache(numThreads), _caching(caching) {
   for (size_t i = 0; i < numThreads; i++) {
     _cache[i] = new Cache();
   }
@@ -219,6 +220,9 @@ bool Router::compConned(const NodeCandGroup& a, const NodeCandGroup& b) const {
 HopBand Router::getHopBand(const NodeCandGroup& a, const NodeCandGroup& b,
                            const RoutingAttrs& rAttrs, const RoutingOpts& rOpts,
                            const osm::Restrictor& rest) const {
+  assert(a.size());
+  assert(b.size());
+
   double pend = 0;
   for (size_t i = 0; i < a.size(); i++) {
     for (size_t j = 0; j < b.size(); j++) {
@@ -231,6 +235,7 @@ HopBand Router::getHopBand(const NodeCandGroup& a, const NodeCandGroup& b,
   LOG(VDEBUG) << "Pending max hop distance is " << pend << " meters";
 
   const trgraph::StatGroup* tgGrpTo = 0;
+
   if (b.begin()->nd->pl().getSI())
     tgGrpTo = b.begin()->nd->pl().getSI()->getGroup();
 
@@ -556,6 +561,7 @@ void Router::nestedCache(const EdgeList* el,
                          const std::set<trgraph::Edge*>& froms,
                          const CostFunc& cost,
                          const RoutingAttrs& rAttrs) const {
+  if (!_caching) return;
   if (el->size() == 0) return;
   // iterate over result edges backwards
   EdgeList curEdges;
@@ -586,7 +592,7 @@ std::set<pfaedle::trgraph::Edge*> Router::getCachedHops(
     const RoutingAttrs& rAttrs) const {
   std::set<trgraph::Edge*> ret;
   for (auto to : tos) {
-    if ((*_cache[omp_get_thread_num()])[rAttrs][from].count(to)) {
+    if (_caching && (*_cache[omp_get_thread_num()])[rAttrs][from].count(to)) {
       const auto& cv = (*_cache[omp_get_thread_num()])[rAttrs][from][to];
       (*rCosts)[to] = cv.first;
       *edgesRet.at(to) = cv.second;
@@ -601,6 +607,7 @@ std::set<pfaedle::trgraph::Edge*> Router::getCachedHops(
 // _____________________________________________________________________________
 void Router::cache(trgraph::Edge* from, trgraph::Edge* to, const EdgeCost& c,
                    EdgeList* edges, const RoutingAttrs& rAttrs) const {
+  if (!_caching) return;
   if (from == to) return;
   (*_cache[omp_get_thread_num()])[rAttrs][from][to] =
       std::pair<EdgeCost, EdgeList>(c, *edges);

src/pfaedle/router/Router.h

@@ -137,7 +137,7 @@ struct CombCostFunc
 class Router {
  public:
   // Init this router with caches for numThreads threads
-  explicit Router(size_t numThreads);
+  explicit Router(size_t numThreads, bool caching);
   ~Router();
 
   // Find the most likely path through the graph for a node candidate route.
@@ -163,6 +163,7 @@ class Router {
 
  private:
   mutable std::vector<Cache*> _cache;
+  bool _caching;
   HopBand getHopBand(const NodeCandGroup& a, const NodeCandGroup& b,
                      const RoutingAttrs& rAttrs, const RoutingOpts& rOpts,
                      const osm::Restrictor& rest) const;

src/pfaedle/router/ShapeBuilder.cpp

@@ -9,6 +9,7 @@
 #define omp_get_num_procs() 1
 #endif
 
+#include <exception>
 #include <map>
 #include <mutex>
 #include <unordered_map>
@@ -16,6 +17,8 @@
 #include "ad/cppgtfs/gtfs/Feed.h"
 #include "pfaedle/Def.h"
 #include "pfaedle/eval/Collector.h"
+#include "pfaedle/gtfs/Feed.h"
+#include "pfaedle/gtfs/StopTime.h"
 #include "pfaedle/osm/OsmBuilder.h"
 #include "pfaedle/router/ShapeBuilder.h"
 #include "pfaedle/trgraph/StatGroup.h"
@@ -43,38 +46,36 @@ using pfaedle::router::EdgeListHops;
 using pfaedle::router::Clusters;
 using pfaedle::osm::BBoxIdx;
 using ad::cppgtfs::gtfs::Stop;
-using ad::cppgtfs::gtfs::Trip;
+using pfaedle::gtfs::Trip;
-using ad::cppgtfs::gtfs::Feed;
+using pfaedle::gtfs::Feed;
-using ad::cppgtfs::gtfs::StopTime;
+using pfaedle::gtfs::StopTime;
 using ad::cppgtfs::gtfs::ShapePoint;
 
 // _____________________________________________________________________________
-ShapeBuilder::ShapeBuilder(Feed* feed, MOTs mots,
+ShapeBuilder::ShapeBuilder(Feed* feed, ad::cppgtfs::gtfs::Feed* evalFeed,
-                           const config::MotConfig& motCfg,
+                           MOTs mots, const config::MotConfig& motCfg,
                            eval::Collector* ecoll, const config::Config& cfg)
     : _feed(feed),
+      _evalFeed(evalFeed),
       _mots(mots),
       _motCfg(motCfg),
       _ecoll(ecoll),
       _cfg(cfg),
-      _crouter(omp_get_num_procs()),
+      _crouter(omp_get_num_procs(), cfg.useCaching),
       _curShpCnt(0) {
   _numThreads = _crouter.getCacheNumber();
   writeMotStops();
 
-  // TODO(patrick): maybe do this on demand to avoid graph filtering / reading
-  // for input where no routing is necessary (already shape'd)
   buildGraph();
 }
 
 // _____________________________________________________________________________
 void ShapeBuilder::writeMotStops() {
   for (auto t : _feed->getTrips()) {
-    if (!_cfg.shapeTripId.empty() && t.second->getId() != _cfg.shapeTripId)
+    if (!_cfg.shapeTripId.empty() && t.getId() != _cfg.shapeTripId) continue;
-      continue;
+    if (_mots.count(t.getRoute()->getType()) &&
-    if (_mots.count(t.second->getRoute()->getType()) &&
+        _motCfg.mots.count(t.getRoute()->getType())) {
-        _motCfg.mots.count(t.second->getRoute()->getType())) {
+      for (auto st : t.getStopTimes()) {
-      for (auto st : t.second->getStopTimes()) {
         _stops[st.getStop()] = 0;
       }
     }
@@ -95,6 +96,7 @@ const NodeCandGroup& ShapeBuilder::getNodeCands(const Stop* s) const {
 // _____________________________________________________________________________
 LINE ShapeBuilder::shapeL(const router::NodeCandRoute& ncr,
                           const router::RoutingAttrs& rAttrs) {
+  try {
     const router::EdgeListHops& res = route(ncr, rAttrs);
 
     LINE l;
@@ -107,7 +109,8 @@ LINE ShapeBuilder::shapeL(const router::NodeCandRoute& ncr,
       for (auto i = hop.edges.rbegin(); i != hop.edges.rend(); i++) {
         const auto* e = *i;
         if ((e->getFrom() == last) ^ e->pl().isRev()) {
-        l.insert(l.end(), e->pl().getGeom()->begin(), e->pl().getGeom()->end());
+          l.insert(l.end(), e->pl().getGeom()->begin(),
+                   e->pl().getGeom()->end());
         } else {
           l.insert(l.end(), e->pl().getGeom()->rbegin(),
                    e->pl().getGeom()->rend());
@@ -117,6 +120,10 @@ LINE ShapeBuilder::shapeL(const router::NodeCandRoute& ncr,
     }
 
     return l;
+  } catch (const std::runtime_error& e) {
+    LOG(ERROR) << e.what();
+    return LINE();
+  }
 }
 
 // _____________________________________________________________________________
@@ -193,9 +200,9 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) {
   Clusters clusters = clusterTrips(_feed, _mots);
   LOG(DEBUG) << "Clustered trips into " << clusters.size() << " clusters.";
 
-  std::map<ad::cppgtfs::gtfs::Shape*, size_t> shpUsage;
+  std::map<std::string, size_t> shpUsage;
   for (auto t : _feed->getTrips()) {
-    if (t.second->getShape()) shpUsage[t.second->getShape()]++;
+    if (!t.getShape().empty()) shpUsage[t.getShape()]++;
   }
 
   // to avoid unfair load balance on threads
@@ -223,7 +230,7 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) {
                   << "%, " << (EDijkstra::ITERS - oiters) << " iters, "
                   /**
                     TODO: this is actually misleading. We are counting the
-                    Dijkstra iterations, but the measuring them against
+                    Dijkstra iterations, but measuring them against
                     the total running time (including all overhead + HMM solve)
                   << tput "
                   << (static_cast<double>(EDijkstra::ITERS - oiters)) /
@@ -249,7 +256,7 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) {
     }
 
     std::vector<double> distances;
-    ad::cppgtfs::gtfs::Shape* shp =
+    const ad::cppgtfs::gtfs::Shape& shp =
         getGtfsShape(cshp, clusters[i][0], &distances);
 
     LOG(DEBUG) << "Took " << EDijkstra::ITERS - iters << " iterations.";
@@ -259,14 +266,14 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) {
 
     for (auto t : clusters[i]) {
       if (_cfg.evaluate) {
-        _ecoll->add(t, t->getShape(), shp, distances);
+        _ecoll->add(t, _evalFeed->getShapes().get(t->getShape()), shp,
+                    distances);
       }
 
-      if (t->getShape() && shpUsage[t->getShape()] > 0) {
+      if (!t->getShape().empty() && shpUsage[t->getShape()] > 0) {
         shpUsage[t->getShape()]--;
         if (shpUsage[t->getShape()] == 0) {
-          _feed->getShapes().remove(t->getShape()->getId());
+          _feed->getShapes().remove(t->getShape());
-          delete t->getShape();
         }
       }
       setShape(t, shp, distances);
@@ -295,27 +302,25 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) {
 }
 
 // _____________________________________________________________________________
-void ShapeBuilder::setShape(Trip* t, ad::cppgtfs::gtfs::Shape* s,
+void ShapeBuilder::setShape(Trip* t, const ad::cppgtfs::gtfs::Shape& s,
                             const std::vector<double>& distances) {
   assert(distances.size() == t->getStopTimes().size());
   // set distances
   size_t i = 0;
-  for (const StopTime& st : t->getStopTimes()) {
+  for (const auto& st : t->getStopTimes()) {
-    const_cast<StopTime&>(st).setShapeDistanceTravelled(distances[i]);
+    const_cast<StopTime<Stop>&>(st).setShapeDistanceTravelled(distances[i]);
     i++;
   }
 
-  t->setShape(s);
-
   std::lock_guard<std::mutex> guard(_shpMutex);
-  _feed->getShapes().add(s);
+  // TODO(patrick):
+  t->setShape(_feed->getShapes().add(s));
 }
 
 // _____________________________________________________________________________
-ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape(
+ad::cppgtfs::gtfs::Shape ShapeBuilder::getGtfsShape(
     const Shape& shp, Trip* t, std::vector<double>* hopDists) {
-  ad::cppgtfs::gtfs::Shape* ret =
+  ad::cppgtfs::gtfs::Shape ret(getFreeShapeId(t));
-      new ad::cppgtfs::gtfs::Shape(getFreeShapeId(t));
 
   assert(shp.hops.size() == t->getStopTimes().size() - 1);
 
@@ -338,7 +343,7 @@ ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape(
       last = *hop.start->pl().getGeom();
 
       if (dist - lastDist > 0.01) {
-        ret->addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq));
+        ret.addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq));
         seq++;
         lastDist = dist;
       }
@@ -349,11 +354,12 @@ ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape(
       if (dist - lastDist > 0.01) {
         ll = webMercToLatLng<PFAEDLE_PRECISION>(
             hop.end->pl().getGeom()->getX(), hop.end->pl().getGeom()->getY());
-        ret->addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq));
+        ret.addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq));
         seq++;
         lastDist = dist;
       }
     }
+
     for (auto i = hop.edges.rbegin(); i != hop.edges.rend(); i++) {
       const auto* e = *i;
       if ((e->getFrom() == l) ^ e->pl().isRev()) {
@@ -367,7 +373,7 @@ ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape(
           if (dist - lastDist > 0.01) {
             POINT ll =
                 webMercToLatLng<PFAEDLE_PRECISION>(cur.getX(), cur.getY());
-            ret->addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq));
+            ret.addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq));
             seq++;
             lastDist = dist;
           }
@@ -383,7 +389,7 @@ ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape(
           if (dist - lastDist > 0.01) {
             POINT ll =
                 webMercToLatLng<PFAEDLE_PRECISION>(cur.getX(), cur.getY());
-            ret->addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq));
+            ret.addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq));
             seq++;
             lastDist = dist;
           }
@@ -447,33 +453,30 @@ const RoutingAttrs& ShapeBuilder::getRAttrs(const Trip* trip) const {
 }
 
 // _____________________________________________________________________________
-BBoxIdx ShapeBuilder::getPaddedGtfsBox(const Feed* feed, double pad,
+void ShapeBuilder::getGtfsBox(const Feed* feed, const MOTs& mots,
-                                       const MOTs& mots, const std::string& tid,
+                              const std::string& tid, bool dropShapes,
-                                       bool dropShapes) {
+                              osm::BBoxIdx* box) {
-  osm::BBoxIdx box(pad);
   for (const auto& t : feed->getTrips()) {
-    if (!tid.empty() && t.second->getId() != tid) continue;
+    if (!tid.empty() && t.getId() != tid) continue;
-    if (tid.empty() && t.second->getShape() && !dropShapes) continue;
+    if (tid.empty() && !t.getShape().empty() && !dropShapes) continue;
-    if (t.second->getStopTimes().size() < 2) continue;
+    if (t.getStopTimes().size() < 2) continue;
-    if (mots.count(t.second->getRoute()->getType())) {
+    if (mots.count(t.getRoute()->getType())) {
       DBox cur;
-      for (const auto& st : t.second->getStopTimes()) {
+      for (const auto& st : t.getStopTimes()) {
         cur = extendBox(DPoint(st.getStop()->getLng(), st.getStop()->getLat()),
                         cur);
       }
-      box.add(cur);
+      box->add(cur);
     }
   }
-
-  return box;
 }
 
 // _____________________________________________________________________________
 void ShapeBuilder::buildGraph() {
   osm::OsmBuilder osmBuilder;
 
-  osm::BBoxIdx box =
+  osm::BBoxIdx box(BOX_PADDING);
-      getPaddedGtfsBox(_feed, 2500, _mots, _cfg.shapeTripId, _cfg.dropShapes);
+  getGtfsBox(_feed, _mots, _cfg.shapeTripId, _cfg.dropShapes, &box);
 
   osmBuilder.read(_cfg.osmPath, _motCfg.osmBuildOpts, &_g, box, _cfg.gridSize,
                   getFeedStops(), &_restr);
@@ -497,6 +500,11 @@ NodeCandRoute ShapeBuilder::getNCR(Trip* trip) const {
 
   for (const auto& st : trip->getStopTimes()) {
     ncr[i] = getNodeCands(st.getStop());
+    if (ncr[i].size() == 0) {
+      throw std::runtime_error("No node candidate found for station '" +
+                               st.getStop()->getName() + "' on trip '" +
+                               trip->getId() + "'");
+    }
     i++;
   }
   return ncr;
@@ -535,29 +543,29 @@ Clusters ShapeBuilder::clusterTrips(Feed* f, MOTs mots) {
   size_t j = 0;
 
   Clusters ret;
-  for (const auto& trip : f->getTrips()) {
+  for (auto& trip : f->getTrips()) {
-    if (trip.second->getShape() && !_cfg.dropShapes) continue;
+    if (!trip.getShape().empty() && !_cfg.dropShapes) continue;
-    if (trip.second->getStopTimes().size() < 2) continue;
+    if (trip.getStopTimes().size() < 2) continue;
-    if (!mots.count(trip.second->getRoute()->getType()) ||
+    if (!mots.count(trip.getRoute()->getType()) ||
-        !_motCfg.mots.count(trip.second->getRoute()->getType()))
+        !_motCfg.mots.count(trip.getRoute()->getType()))
       continue;
     bool found = false;
-    auto spair = StopPair(trip.second->getStopTimes().begin()->getStop(),
+    auto spair = StopPair(trip.getStopTimes().begin()->getStop(),
-                          trip.second->getStopTimes().rbegin()->getStop());
+                          trip.getStopTimes().rbegin()->getStop());
     const auto& c = clusterIdx[spair];
 
     for (size_t i = 0; i < c.size(); i++) {
       j++;
-      if (routingEqual(ret[c[i]][0], trip.second)) {
+      if (routingEqual(ret[c[i]][0], &trip)) {
-        ret[c[i]].push_back(trip.second);
+        ret[c[i]].push_back(&trip);
         found = true;
         break;
       }
     }
     if (!found) {
-      ret.push_back({trip.second});
+      ret.push_back(Cluster{&trip});
       // explicit call to write render attrs to cache
-      getRAttrs(trip.second);
+      getRAttrs(&trip);
       clusterIdx[spair].push_back(ret.size() - 1);
     }
   }

src/pfaedle/router/ShapeBuilder.h

@@ -16,6 +16,7 @@
 #include "pfaedle/config/MotConfig.h"
 #include "pfaedle/config/PfaedleConfig.h"
 #include "pfaedle/eval/Collector.h"
+#include "pfaedle/gtfs/Feed.h"
 #include "pfaedle/netgraph/Graph.h"
 #include "pfaedle/osm/Restrictor.h"
 #include "pfaedle/router/Misc.h"
@@ -27,8 +28,8 @@ namespace pfaedle {
 namespace router {
 
 using ad::cppgtfs::gtfs::Stop;
-using ad::cppgtfs::gtfs::Trip;
+using pfaedle::gtfs::Trip;
-using ad::cppgtfs::gtfs::Feed;
+using pfaedle::gtfs::Feed;
 
 struct Shape {
   router::EdgeListHops hops;
@@ -48,8 +49,9 @@ typedef std::unordered_map<const trgraph::Edge*, std::set<const Trip*>>
  */
 class ShapeBuilder {
  public:
-  ShapeBuilder(Feed* feed, MOTs mots, const config::MotConfig& motCfg,
+  ShapeBuilder(Feed* feed, ad::cppgtfs::gtfs::Feed* evalFeed, MOTs mots,
-               eval::Collector* ecoll, const config::Config& cfg);
+               const config::MotConfig& motCfg, eval::Collector* ecoll,
+               const config::Config& cfg);
 
   void shape(pfaedle::netgraph::Graph* ng);
 
@@ -66,12 +68,13 @@ class ShapeBuilder {
 
   const trgraph::Graph* getGraph() const;
 
-  static osm::BBoxIdx getPaddedGtfsBox(const Feed* feed, double pad,
+  static void getGtfsBox(const Feed* feed, const MOTs& mots,
-                                       const MOTs& mots, const std::string& tid,
+                         const std::string& tid, bool dropShapes,
-                                       bool dropShapes);
+                         osm::BBoxIdx* box);
 
  private:
   Feed* _feed;
+  ad::cppgtfs::gtfs::Feed* _evalFeed;
   MOTs _mots;
   config::MotConfig _motCfg;
   eval::Collector* _ecoll;
@@ -101,10 +104,10 @@ class ShapeBuilder {
 
   std::string getFreeShapeId(Trip* t);
 
-  ad::cppgtfs::gtfs::Shape* getGtfsShape(const Shape& shp, Trip* t,
+  ad::cppgtfs::gtfs::Shape getGtfsShape(const Shape& shp, Trip* t,
                                          std::vector<double>* hopDists);
 
-  void setShape(Trip* t, ad::cppgtfs::gtfs::Shape* s,
+  void setShape(Trip* t, const ad::cppgtfs::gtfs::Shape& s,
                 const std::vector<double>& dists);
 
   router::NodeCandRoute getNCR(Trip* trip) const;

src/pfaedle/trgraph/EdgePL.cpp

@@ -39,7 +39,7 @@ EdgePL::EdgePL(const EdgePL& pl, bool geoflat)
   }
   _flines[_l]++;
 
-  for (auto l : _lines) addLine(l);
+  for (auto l : pl._lines) addLine(l);
 }
 
 // _____________________________________________________________________________
@@ -82,7 +82,9 @@ double EdgePL::getLength() const { return _length; }
 
 // _____________________________________________________________________________
 void EdgePL::addLine(const TransitEdgeLine* l) {
-  if (_lines.insert(l).second) {
+  if (std::find(_lines.begin(), _lines.end(), l) == _lines.end()) {
+    _lines.reserve(_lines.size() + 1);
+    _lines.push_back(l);
     if (_tlines.count(l))
       _tlines[l]++;
     else
@@ -96,7 +98,7 @@ void EdgePL::addLines(const std::vector<TransitEdgeLine*>& l) {
 }
 
 // _____________________________________________________________________________
-const std::set<const TransitEdgeLine*>& EdgePL::getLines() const {
+const std::vector<const TransitEdgeLine*>& EdgePL::getLines() const {
   return _lines;
 }
 

src/pfaedle/trgraph/EdgePL.h

@@ -45,7 +45,7 @@ inline bool operator<(const TransitEdgeLine& a, const TransitEdgeLine& b) {
 /*
  * An edge payload class for the transit graph.
  */
-class EdgePL : public GeoEdgePL<PFAEDLE_PRECISION> {
+class EdgePL {
  public:
   EdgePL();
   ~EdgePL();
@@ -101,7 +101,7 @@ class EdgePL : public GeoEdgePL<PFAEDLE_PRECISION> {
   void addLines(const std::vector<TransitEdgeLine*>& l);
 
   // Return the TransitEdgeLines stored for this payload
-  const std::set<const TransitEdgeLine*>& getLines() const;
+  const std::vector<const TransitEdgeLine*>& getLines() const;
 
   // Returns the last hop of the payload - this is the (n-2)th point in
   // the payload geometry of length n > 1
@@ -123,7 +123,7 @@ class EdgePL : public GeoEdgePL<PFAEDLE_PRECISION> {
 
   LINE* _l;
 
-  std::set<const TransitEdgeLine*> _lines;
+  std::vector<const TransitEdgeLine*> _lines;
 
   static void unRefTLine(const TransitEdgeLine* l);
 

src/pfaedle/trgraph/NodePL.h

@@ -26,7 +26,7 @@ struct Component {
 /*
  * A node payload class for the transit graph.
  */
-class NodePL : public GeoNodePL<PFAEDLE_PRECISION> {
+class NodePL {
  public:
   NodePL();
   NodePL(const NodePL& pl);   // NOLINT

src/util/CMakeLists.txt

@@ -2,4 +2,11 @@ file(GLOB_RECURSE util_SRC *.cpp)
 list(REMOVE_ITEM util_SRC TestMain.cpp)
 add_library(util ${util_SRC})
 
+find_package( ZLIB )
+if (ZLIB_FOUND)
+	include_directories( ${ZLIB_INCLUDE_DIRS} )
+	target_link_libraries( util ${ZLIB_LIBRARIES} )
+	add_definitions( -DZLIB_FOUND=${ZLIB_FOUND} )
+endif( ZLIB_FOUND )
+
 add_subdirectory(tests)

src/util/Nullable.h

@@ -26,7 +26,7 @@ class Nullable {
    : val(other.val), null(other.isNull()) {}
 
   Nullable& operator=(const Nullable& other) {
-    val = other.get();
+    if (!other.isNull()) val = other.get();
     null = other.isNull();
     return *this;
   }

src/util/String.h

@@ -142,6 +142,39 @@ inline size_t editDist(const std::string& s1, const std::string& s2) {
   return prev[len2];
 }
 
+// _____________________________________________________________________________
+inline size_t prefixEditDist(const std::string& prefix, const std::string& s,
+                             size_t deltaMax) {
+  // https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#C++
+  size_t len1 = prefix.size();
+  size_t len2 = std::min(s.size(), prefix.size() + deltaMax + 1);
+  std::vector<size_t> d((len1 + 1) * (len2 + 1));
+
+  d[0] = 0;
+  for (size_t i = 1; i <= len1; ++i) d[i * (len2 + 1)] = i;
+  for (size_t i = 1; i <= len2; ++i) d[ i] = i;
+
+  for (size_t i = 1; i <= len1; i++) {
+    for (size_t j = 1; j <= len2; j++) {
+      d[i * (len2 + 1) + j] = std::min(std::min(d[(i - 1) * (len2 + 1) + j] + 1, d[i * (len2 + 1) + j - 1] + 1),
+                         d[(i - 1) * (len2 + 1) + j - 1] + (prefix[i - 1] == s[j - 1] ? 0 : 1));
+    }
+  }
+
+  // take min of last row
+  size_t deltaMin = std::max(std::max(deltaMax + 1, prefix.size()), s.size());
+  for (size_t i = 0; i <= len2; i++) {
+    if (d[len1 * (len2 + 1) + i] < deltaMin) deltaMin = d[len1 * (len2 + 1) + i];
+  }
+
+  return deltaMin;
+}
+
+// _____________________________________________________________________________
+inline size_t prefixEditDist(const std::string& prefix, const std::string& s) {
+  return prefixEditDist(prefix, s, s.size());
+}
+
 // _____________________________________________________________________________
 template <class Iter>
 inline std::string implode(Iter begin, const Iter& end, const char* del) {

src/util/geo/Geo.h

@@ -47,7 +47,7 @@ typedef Polygon<double> DPolygon;
 typedef Polygon<float> FPolygon;
 typedef Polygon<int> IPolygon;
 
-const static double EPSILON = 0.00000000001;
+const static double EPSILON = 0.00001;
 const static double RAD = 0.017453292519943295;  // PI/180
 
 // _____________________________________________________________________________
@@ -236,7 +236,7 @@ inline RotatedBox<T> shrink(const RotatedBox<T>& b, double d) {
 }
 
 // _____________________________________________________________________________
-inline bool doubleEq(double a, double b) { return fabs(a - b) < 0.000001; }
+inline bool doubleEq(double a, double b) { return fabs(a - b) < EPSILON; }
 
 // _____________________________________________________________________________
 template <typename T>
@@ -1153,7 +1153,7 @@ inline size_t convexHullImpl(const MultiPoint<T>& a, size_t p1, size_t p2,
   for (const auto& p : a) {
     double tmpDist = distToSegment((*h)[p1], (*h)[p2], p);
     double cp = crossProd(p, LineSegment<T>((*h)[p1], (*h)[p2]));
-    if (((cp > 0 && !d) || (cp < 0 && d)) && tmpDist > maxDist) {
+    if (((cp > 0 && !d) || (cp < 0 && d)) && tmpDist >= maxDist + EPSILON) {
       pa = p;
       found = true;
       maxDist = tmpDist;
@@ -1462,7 +1462,8 @@ inline Point<T> latLngToWebMerc(double lat, double lng) {
 // _____________________________________________________________________________
 template <typename T>
 inline Point<T> webMercToLatLng(double x, double y) {
-  double lat = 114.591559026 * (atan(exp(y / 6378137.0)) - 0.78539825);
+  double lat =
+      (1.5707963267948966 - (2.0 * atan(exp(-y / 6378137.0)))) * (180.0 / M_PI);
   double lon = x / 111319.4907932735677;
   return Point<T>(lon, lat);
 }

src/util/geo/Point.h

@@ -5,6 +5,8 @@
 #ifndef UTIL_GEO_POINT_H_
 #define UTIL_GEO_POINT_H_
 
+#include <vector>
+
 namespace util {
 namespace geo {
 

src/util/graph/Algorithm.h

@@ -0,0 +1,33 @@
+// Copyright 2017, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#ifndef UTIL_GRAPH_ALGORITHM_H_
+#define UTIL_GRAPH_ALGORITHM_H_
+
+#include <stack>
+#include "util/graph/Edge.h"
+#include "util/graph/UndirGraph.h"
+#include "util/graph/Node.h"
+
+namespace util {
+namespace graph {
+
+using util::graph::Graph;
+using util::graph::Node;
+using util::graph::Edge;
+
+// collection of general graph algorithms
+class Algorithm {
+ public:
+  template <typename N, typename E>
+  static std::vector<std::set<Node<N, E>*> > connectedComponents(
+      const UndirGraph<N, E>& g);
+};
+
+#include "util/graph/Algorithm.tpp"
+
+}
+}
+
+#endif  // UTIL_GRAPH_ALGORITHM_H_

src/util/graph/Algorithm.tpp

@@ -0,0 +1,32 @@
+// Copyright 2017, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+// _____________________________________________________________________________
+template <typename N, typename E>
+std::vector<std::set<Node<N, E>*> > Algorithm::connectedComponents(
+    const UndirGraph<N, E>& g) {
+  std::vector<std::set<Node<N, E>*>> ret;
+  std::set<Node<N, E>*> visited;
+
+  for (auto* n : g.getNds()) {
+    if (!visited.count(n)) {
+      ret.resize(ret.size() + 1);
+      std::stack<Node<N, E>*> q;
+      q.push(n);
+      while (!q.empty()) {
+        Node<N, E>* cur = q.top();
+        q.pop();
+
+        ret.back().insert(cur);
+        visited.insert(cur);
+
+        for (auto* e : cur->getAdjList()) {
+          if (!visited.count(e->getOtherNd(cur))) q.push(e->getOtherNd(cur));
+        }
+      }
+    }
+  }
+
+  return ret;
+}

src/util/graph/DirNode.h

@@ -6,6 +6,7 @@
 #define UTIL_GRAPH_DIRNODE_H_
 
 #include <vector>
+#include <algorithm>
 #include "util/graph/Node.h"
 
 namespace util {

src/util/graph/Node.h

@@ -33,12 +33,15 @@ class Node {
   virtual void addEdge(Edge<N, E>* e) = 0;
   virtual void removeEdge(Edge<N, E>* e) = 0;
 
-  virtual ~Node() {};
+  virtual ~Node() = 0;
 
   virtual N& pl() = 0;
   virtual const N& pl() const = 0;
 };
 
+template <typename N, typename E>
+inline Node<N, E>::~Node() {}
+
 }}
 
 #endif  // UTIL_GRAPH_NODE_H_

src/util/graph/UndirNode.h

@@ -6,6 +6,7 @@
 #define UTIL_GRAPH_UNDIRNODE_H_
 
 #include <vector>
+#include <algorithm>
 #include "util/graph/Node.h"
 
 namespace util {

src/util/http/Server.cpp

@@ -0,0 +1,347 @@
+// Copyright 2018, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#ifndef ZLIB_CONST
+#define ZLIB_CONST
+#endif
+
+#include <fcntl.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <algorithm>
+#include <memory>
+#include <sstream>
+#include <thread>
+#include <unordered_map>
+#ifdef ZLIB_FOUND
+#include <zlib.h>
+#endif
+#include <vector>
+#include "Server.h"
+#include "util/String.h"
+
+using util::http::Socket;
+using util::http::Queue;
+using util::http::Req;
+using util::http::HttpErr;
+using util::http::HttpServer;
+using util::http::HeaderState;
+
+// _____________________________________________________________________________
+Socket::Socket(int port) {
+  int y = 1;
+  _sock = socket(PF_INET, SOCK_STREAM, 0);
+  if (_sock < 0)
+    throw std::runtime_error(std::string("Could not create socket (") +
+                             std::strerror(errno) + ")");
+
+  struct sockaddr_in addr;
+
+  addr.sin_family = AF_INET;
+  addr.sin_port = htons(port);
+  addr.sin_addr.s_addr = INADDR_ANY;
+  memset(&(addr.sin_zero), '\0', 8);
+
+  setsockopt(_sock, SOL_SOCKET, SO_REUSEADDR, &y, sizeof(y));
+  // https://news.ycombinator.com/item?id=10608356
+  setsockopt(_sock, IPPROTO_TCP, TCP_QUICKACK, &y, sizeof(y));
+
+  if (bind(_sock, reinterpret_cast<sockaddr*>(&addr), sizeof(addr)) < 0) {
+    throw std::runtime_error(std::string("Could not bind to port ") +
+                             std::to_string(port) + " (" +
+                             std::strerror(errno) + ")");
+  }
+}
+
+// _____________________________________________________________________________
+Socket::~Socket() { close(_sock); }
+
+// _____________________________________________________________________________
+int Socket::wait() {
+  if (listen(_sock, BLOG) < 0)
+    throw std::runtime_error(std::string("Cannot listen to socket (") +
+                             std::strerror(errno) + ")");
+  sockaddr_in cli_addr;
+  socklen_t clilen = sizeof(cli_addr);
+  int sock = accept(_sock, reinterpret_cast<sockaddr*>(&cli_addr), &clilen);
+  return sock;
+}
+
+// _____________________________________________________________________________
+void HttpServer::send(int sock, Answer* aw) {
+  std::string enc = "identity";
+  if (aw->gzip) aw->pl = compress(aw->pl, &enc);
+
+  aw->params["Content-Encoding"] = enc;
+  aw->params["Content-Length"] = std::to_string(aw->pl.size());
+
+  std::stringstream ss;
+  ss << "HTTP/1.1 " << aw->status << "\r\n";
+  for (const auto& kv : aw->params)
+    ss << kv.first << ": " << kv.second << "\r\n";
+  ss << "\r\n" << aw->pl;
+  std::string buff = ss.str();
+
+  size_t writes = 0;
+  // https://news.ycombinator.com/item?id=10608356
+  int y = 1;
+  setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, &y, sizeof(y));
+
+  while (writes != buff.size()) {
+    int64_t out = write(sock, buff.c_str() + writes, buff.size() - writes);
+    if (out < 0) {
+      if (errno == EWOULDBLOCK || errno == EAGAIN || errno == EINTR) continue;
+      throw std::runtime_error("Failed to write to socket");
+    }
+    writes += out;
+  }
+}
+
+// _____________________________________________________________________________
+void HttpServer::handle() {
+  int connection = -1;
+  while ((connection = _jobs.get()) != -1) {
+    Answer answ;
+
+    try {
+      Req req = getReq(connection);
+      answ = _handler->handle(req, connection);
+      answ.gzip = gzipSupport(req);
+    } catch (HttpErr err) {
+      answ = Answer{err.what(), err.what(), false, {}};
+    } catch (...) {
+      // catch everything to make sure the server continues running
+      answ = Answer{
+          "500 Internal Server Error", "500 Internal Server Error", false, {}};
+    }
+
+    send(connection, &answ);
+    close(connection);
+  }
+}
+
+// _____________________________________________________________________________
+bool HttpServer::gzipSupport(const Req& req) {
+  bool accepts = false;
+  // decide according to
+  // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
+  for (const auto& kv : req.params) {
+    if (kv.first == "Accept-Encoding") {
+      for (const auto& encoding : split(kv.second, ',')) {
+        std::vector<std::string> parts = split(encoding, ';');
+        for (size_t i = 0; i < parts.size(); i++) {
+          parts[i] = trim(parts[i]);
+        }
+        if (parts[0] == "*" && ((parts.size() == 1) || parts[1] != "q=0"))
+          accepts = true;
+        if (parts[0] == "gzip") accepts = true;
+        if (parts.size() > 1 && parts[1] == "q=0") accepts = false;
+      }
+    }
+  }
+  return accepts;
+}
+
+// _____________________________________________________________________________
+Req HttpServer::getReq(int connection) {
+  char buf[BSIZE + 1];
+  size_t rcvd = 0;
+  int64_t curRcvd = 0;
+  HeaderState state = NONE;
+  Req ret{"", "", "", "", {}};
+  char *tmp, *tmp2;
+  char* brk = 0;
+
+  while ((curRcvd = read(connection, buf + rcvd, BSIZE - rcvd))) {
+    if (curRcvd < 0) {
+      if (errno == EAGAIN || errno == EINTR) continue;
+      throw HttpErr("500 Internal Server Error");
+    }
+
+    // parse request
+    for (int i = 0; i < curRcvd; i++) {
+      if (brk) break;
+      char* c = buf + rcvd + i;
+      switch (state) {
+        case NONE:
+          state = I_COM;
+          tmp = c;
+          continue;
+        case I_VER:
+          if (*c == '\n') {
+            *c = 0;
+            ret.ver = trim(tmp);
+            state = A_KEY;
+          }
+          continue;
+        case I_URL:
+          if (*c == ' ') {
+            *c = 0, ret.url = trim(tmp);
+            tmp = c + 1;
+            state = I_VER;
+          } else if (*c == '\n') {
+            *c = 0, ret.url = trim(tmp);
+            state = A_KEY;
+          }
+          continue;
+        case I_COM:
+          if (*c == ' ') {
+            *c = 0, ret.cmd = trim(tmp);
+            tmp = c + 1;
+            state = I_URL;
+          } else if (*c == '\n') {
+            *c = 0, ret.cmd = trim(tmp);
+            state = A_KEY;
+          }
+          continue;
+        case A_KEY:
+          if (*c == '\r') *c = ' ';
+          if (*c == '\n')
+            brk = c + 1;
+          else if (*c != ' ') {
+            state = I_KEY;
+            tmp = c;
+          }
+          continue;
+        case I_KEY:
+          if (*c == ':') {
+            *c = 0;
+            state = A_VAL;
+          }
+          continue;
+        case A_VAL:
+          if (*c != ' ') {
+            state = I_VAL;
+            tmp2 = c;
+          }
+          continue;
+        case I_VAL:
+          if (*c == '\r') *c = ' ';
+          if (*c == '\n') {
+            *c = 0;
+            ret.params[tmp] = trim(tmp2);
+            state = A_KEY;
+          }
+          continue;
+      }
+    }
+
+    rcvd += curRcvd;
+
+    // buffer is full
+    if (rcvd == BSIZE) throw HttpErr("431 Request Header Fields Too Large");
+    if (brk) break;
+  }
+
+  // POST payload
+  if (ret.cmd == "POST") {
+    size_t size = 0;
+    if (ret.params.count("Content-Length"))
+      size = atoi(ret.params["Content-Length"].c_str());
+    if (size) {
+      char* postBuf = new char[size + 1];
+      postBuf[size] = 0;
+      size_t rem = 0;
+
+      // copy existing to new buffer
+      if ((int)rcvd > brk - buf) {
+        rem = std::min(size, rcvd - (brk - buf));
+        memcpy(postBuf, brk, rem);
+      }
+
+      rcvd = 0;
+
+      if (rem < size) {
+        while ((curRcvd = read(connection, postBuf + rcvd + rem, size - rem))) {
+          if (curRcvd == -1 && (errno == EAGAIN || errno == EINTR)) continue;
+          if (curRcvd == -1) {
+            postBuf[rcvd + 1] = 0;
+            break;
+          }
+          rcvd += curRcvd;
+          if (rcvd == size - rem) break;
+        }
+      }
+
+      ret.payload = postBuf;
+      delete[] postBuf;
+    }
+  }
+
+  return ret;
+}
+
+// _____________________________________________________________________________
+std::string HttpServer::compress(const std::string& str, std::string* enc) {
+#ifdef ZLIB_FOUND
+  // do not compress small payloads
+  if (str.size() < 500) return str;
+
+  std::string ret;
+
+  // based on http://www.zlib.net/zlib_how.html
+  z_stream defStr;
+  defStr.zalloc = Z_NULL;
+  defStr.zfree = Z_NULL;
+  defStr.opaque = Z_NULL;
+  defStr.avail_in = 0;
+  defStr.next_in = Z_NULL;
+
+  // fail silently with no compression at all
+  if (deflateInit2(&defStr, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8,
+                   Z_DEFAULT_STRATEGY) != Z_OK)
+    return str;
+
+  defStr.next_in = reinterpret_cast<z_const Bytef*>(str.c_str());
+  defStr.avail_in = static_cast<unsigned int>(str.size());
+
+  size_t cSize = 0;
+  do {
+    if (ret.size() < (cSize + BSIZE_C)) ret.resize(cSize + BSIZE_C);
+    defStr.avail_out = BSIZE_C;
+    defStr.next_out = reinterpret_cast<Bytef*>(&ret[0] + cSize);
+    deflate(&defStr, Z_FINISH);
+    cSize += BSIZE_C - defStr.avail_out;
+  } while (defStr.avail_out == 0);
+
+  deflateEnd(&defStr);
+  ret.resize(cSize);
+
+  if (ret.size() > str.size()) return str;
+  *enc = "gzip";
+  return ret;
+#else
+  return str;
+#endif
+}
+
+// _____________________________________________________________________________
+void HttpServer::run() {
+  Socket socket(_port);
+
+  std::vector<std::thread> thrds(_threads);
+  for (auto& thr : thrds) thr = std::thread(&HttpServer::handle, this);
+
+  while (1) _jobs.add(socket.wait());
+}
+
+// _____________________________________________________________________________
+void Queue::add(int c) {
+  if (c < 0) return;
+  {
+    std::unique_lock<std::mutex> lock(_mut);
+    _jobs.push(c);
+  }
+  _hasNew.notify_one();
+}
+
+// _____________________________________________________________________________
+int Queue::get() {
+  std::unique_lock<std::mutex> lock(_mut);
+  while (_jobs.empty()) _hasNew.wait(lock);
+  int next = _jobs.front();
+  _jobs.pop();
+  return next;
+}

src/util/http/Server.h

@@ -0,0 +1,134 @@
+// Copyright 2018, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
+
+#include <condition_variable>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <queue>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include <unistd.h>
+#include <cerrno>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+#ifndef UTIL_HTTP_SERVER_H_
+#define UTIL_HTTP_SERVER_H_
+
+namespace util {
+namespace http {
+
+// socket backlog size
+const static size_t BLOG = 128;
+// socket read buffer size
+const static size_t BSIZE = 4 * 1024;
+// zlib compression buffer size
+const size_t BSIZE_C = 128 * 1024;
+
+// states for HTTP header parser
+enum HeaderState { NONE, I_COM, I_URL, I_VER, A_KEY, I_KEY, A_VAL, I_VAL };
+
+/*
+ * HTTP Error
+ */
+class HttpErr : public std::exception {
+ public:
+  HttpErr(std::string msg) : _msg(msg) {}
+  ~HttpErr() throw() {}
+
+  virtual const char* what() const throw() { return _msg.c_str(); }
+
+ private:
+  std::string _msg;
+  uint16_t _code;
+};
+
+/*
+ * HTTP Request
+ */
+struct Req {
+  std::string cmd, url, ver, payload;
+  std::unordered_map<std::string, std::string> params;
+};
+
+/*
+ * HTTP Answer
+ */
+struct Answer {
+  std::string status, pl;
+  bool gzip;
+  std::unordered_map<std::string, std::string> params;
+};
+
+/*
+ * Virtual handler provider class
+ */
+class Handler {
+ public:
+  virtual Answer handle(const Req& request, int connection) const = 0;
+};
+
+/*
+ * Queue of connections to handle
+ */
+class Queue {
+ public:
+  void add(int c);
+  int get();
+
+ private:
+  std::mutex _mut;
+  std::queue<int> _jobs;
+  std::condition_variable _hasNew;
+};
+
+/*
+ * Socket wrapper
+ */
+class Socket {
+ public:
+  Socket(int port);
+  ~Socket();
+  int wait();
+
+ private:
+  int _sock;
+};
+
+/*
+ * Simple HTTP server, must provide a pointer to a class instance implementing
+ * virtual class Handler.
+ */
+class HttpServer {
+ public:
+  HttpServer(int port, const Handler* h) : HttpServer(port, h, 0) {}
+  HttpServer(int port, const Handler* h, size_t threads)
+      : _port(port), _handler(h), _threads(threads) {
+    if (!_threads) _threads = 8 * std::thread::hardware_concurrency();
+  }
+  void run();
+
+ private:
+  int _port;
+  Queue _jobs;
+  const Handler* _handler;
+  size_t _threads;
+
+  void handle();
+
+  static void send(int sock, Answer* aw);
+  static Req getReq(int connection);
+  static std::string compress(const std::string& str, std::string* enc);
+  static bool gzipSupport(const Req& req);
+};
+}  // http
+}  // util
+
+#endif  // UTIL_HTTP_SERVER_H_

src/util/tests/TestMain.cpp

@@ -8,6 +8,7 @@
 #include "util/String.h"
 #include "util/geo/Geo.h"
 #include "util/json/Writer.h"
+#include "util/graph/Algorithm.h"
 #include "util/graph/DirGraph.h"
 #include "util/graph/UndirGraph.h"
 #include "util/graph/Dijkstra.h"
@@ -403,6 +404,46 @@ CASE("editdist") {
   EXPECT(util::editDist("hello", "hello") == (size_t)0);
 }},
 
+// ___________________________________________________________________________
+{
+CASE("prefixeditdist") {
+  EXPECT(util::prefixEditDist("hello", "hello", 0) == (size_t)0);
+  EXPECT(util::prefixEditDist("hello", "hello", 100) == (size_t)0);
+  EXPECT(util::prefixEditDist("hello", "hello") == (size_t)0);
+  EXPECT(util::prefixEditDist("hel", "hello") == (size_t)0);
+  EXPECT(util::prefixEditDist("hel", "hello", 0) == (size_t)0);
+  EXPECT(util::prefixEditDist("hel", "hello", 1) == (size_t)0);
+  EXPECT(util::prefixEditDist("hel", "hello", 2) == (size_t)0);
+  EXPECT(util::prefixEditDist("hal", "hello", 2) == (size_t)1);
+  EXPECT(util::prefixEditDist("hal", "hello", 1) == (size_t)1);
+  EXPECT(util::prefixEditDist("hal", "hello", 0) > (size_t)0);
+  EXPECT(util::prefixEditDist("fel", "hello", 0) > (size_t)0);
+  EXPECT(util::prefixEditDist("fel", "hello", 1) == (size_t)1);
+  EXPECT(util::prefixEditDist("fel", "hello", 2) == (size_t)1);
+  EXPECT(util::prefixEditDist("fal", "hello", 2) == (size_t)2);
+  EXPECT(util::prefixEditDist("fal", "hello", 1) > (size_t)1);
+  EXPECT(util::prefixEditDist("fal", "hello", 0) > (size_t)0);
+  EXPECT(util::prefixEditDist("far", "hello", 0) > (size_t)0);
+  EXPECT(util::prefixEditDist("far", "hello", 1) > (size_t)1);
+  EXPECT(util::prefixEditDist("far", "hello", 2) > (size_t)2);
+  EXPECT(util::prefixEditDist("far", "hello", 3) == (size_t)3);
+  EXPECT(util::prefixEditDist("far", "hello", 4) == (size_t)3);
+  EXPECT(util::prefixEditDist("far", "hello") == (size_t)3);
+  EXPECT(util::prefixEditDist("hefar", "hello") == (size_t)3);
+  EXPECT(util::prefixEditDist("hefaree", "hello") == (size_t)5);
+  EXPECT(util::prefixEditDist("helloo", "hello") == (size_t)1);
+  EXPECT(util::prefixEditDist("helloo", "hello", 0) > (size_t)0);
+  EXPECT(util::prefixEditDist("helloo", "hello", 1) == (size_t)1);
+  EXPECT(util::prefixEditDist("helloo", "hello", 2) == (size_t)1);
+  EXPECT(util::prefixEditDist("", "hello", 2) == (size_t)0);
+  EXPECT(util::prefixEditDist("e", "hello", 2) == (size_t)1);
+  EXPECT(util::prefixEditDist("el", "hello", 2) == (size_t)1);
+  EXPECT(util::prefixEditDist("ello", "hello", 2) == (size_t)1);
+  EXPECT(util::prefixEditDist("hell", "hello", 2) == (size_t)0);
+  EXPECT(util::prefixEditDist("hell", "", 2) > (size_t)2);
+  EXPECT(util::prefixEditDist("hell", "") == (size_t)4);
+}},
+
 // ___________________________________________________________________________
 {
 CASE("toString") {
@@ -447,6 +488,51 @@ CASE("replace") {
   EXPECT(b == "loree aaaau aaaau loree");
 }},
 
+// ___________________________________________________________________________
+{
+CASE("Connected components undirected") {
+  UndirGraph<std::string, int> g;
+
+  auto a = g.addNd("A");
+  auto b = g.addNd("B");
+  auto c = g.addNd("C");
+  auto d = g.addNd("D");
+  auto e = g.addNd("E");
+
+  g.addEdg(a, c, 1);
+  g.addEdg(a, b, 5);
+  g.addEdg(d, c, 1);
+  g.addEdg(d, b, 3);
+  g.addEdg(e, d, 1);
+  g.addEdg(e, b, 1);
+
+  auto comps = util::graph::Algorithm::connectedComponents(g);
+
+  EXPECT(comps.size() == static_cast<size_t>(1));
+  EXPECT(comps[0].count(a));
+  EXPECT(comps[0].count(b));
+  EXPECT(comps[0].count(c));
+  EXPECT(comps[0].count(d));
+  EXPECT(comps[0].count(e));
+
+  auto f = g.addNd("F");
+  comps = util::graph::Algorithm::connectedComponents(g);
+  EXPECT(comps.size() == static_cast<size_t>(2));
+
+  auto gn = g.addNd("G");
+  comps = util::graph::Algorithm::connectedComponents(g);
+  EXPECT(comps.size() == static_cast<size_t>(3));
+
+  g.addEdg(f, gn, 1);
+  comps = util::graph::Algorithm::connectedComponents(g);
+  EXPECT(comps.size() == static_cast<size_t>(2));
+
+  g.addEdg(f, a, 1);
+  comps = util::graph::Algorithm::connectedComponents(g);
+  EXPECT(comps.size() == static_cast<size_t>(1));
+
+}},
+
 // ___________________________________________________________________________
 {
 CASE("Edge-based Dijkstra directed, 1 to all") {

src/xml

@@ -1 +1 @@
-Subproject commit 00e19bfbdc300eb064fbceef2efaed2ccedfda88
+Subproject commit 5081d32879c30456f6cb515342a3096c5a0d7de6