diff --git a/CMakeLists.txt b/CMakeLists.txt index 9cb116e..190e2ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,11 +14,12 @@ set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build") find_package(OpenMP) -if(OPENMP_FOUND) +if (OPENMP_FOUND) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") endif() + # set compiler flags, see http://stackoverflow.com/questions/7724569/debug-vs-release-in-cmake if(OPENMP_FOUND) set(CMAKE_CXX_FLAGS "-fopenmp -Ofast -fno-signed-zeros -fno-trapping-math -Wall -Wno-format-extra-args -Wextra -Wformat-nonliteral -Wformat-security -Wformat=2 -Wextra -Wno-implicit-fallthrough -pedantic") @@ -71,18 +72,17 @@ add_test("utilTest" utilTest) # custom eval target add_custom_target( - eval - COMMAND make - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}//eval -) + eval + COMMAND make + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}//eval + ) # handles install target - install( FILES pfaedle.cfg DESTINATION etc/${PROJECT_NAME} COMPONENT config PERMISSIONS WORLD_READ -) + ) install( FILES build/pfaedle DESTINATION bin PERMISSIONS WORLD_EXECUTE COMPONENT binaries -) + ) diff --git a/README.md b/README.md index 90bb21a..9a55462 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ make install ## Generating shapes for a GTFS feed ``` -pfaedle -c -x +pfaedle -x ``` A shape'd version of the input GTFS feed will be written to `./gtfs-out`. @@ -51,22 +51,15 @@ input feed. To drop all existing shapes, use the `-D` flag. For example, you may generate (and replace existing, see -D parameter) shapes for the GTFS dataset for Freiburg like this: ``` -$ mkdir freiburg_gtfs && cd freiburg_gtfs -$ wget https://fritz.freiburg.de/csv_Downloads/VAGFR.zip -$ unzip VAGFR.zip -$ wget http://download.geofabrik.de/europe/germany/baden-wuerttemberg/freiburg-regbez-latest.osm.bz2 -$ bunzip2 freiburg-regbez-latest.osm.bz2 -$ mkdir gtfs-out -$ pfaedle -D -c pfaedle.cfg -x freiburg-regbez-latest.osm . +$ wget https://fritz.freiburg.de/csv_Downloads/VAGFR.zip && unzip VAGFR.zip +$ wget http://download.geofabrik.de/europe/germany/baden-wuerttemberg/freiburg-regbez-latest.osm.bz2 && bunzip2 freiburg-regbez-latest.osm.bz2 +$ pfaedle -D -x freiburg-regbez-latest.osm . ``` -A default configuration file `pfaedle.cfg` can be found in this repo. - - ## Generating shapes for a specific MOT To generate shapes only for a specific mot, use the `-m` option. Possible -values are either `tram`, `bus`, `rail`, `subway`, `ferry`, `funicular`, +values are either `tram`, `bus`, `coach`, `rail`, `subway`, `ferry`, `funicular`, `gondola`, `all` (default) or GTFS vehicle type codes (0, 1, 2, 3, 4, 5, 6, 7). Multiple values can be specified (comma separated). @@ -75,7 +68,7 @@ Multiple values can be specified (comma separated). `pfaedle` comes with the ability to filter OpenStreetMap data. If you specify the `-X` flag, `pfaedle` will filter the input OSM file and output a new OSM -file which contains *exactly* the data needed to calculate the shapes for the +file which contains exactly the data needed to calculate the shapes for the input GTFS feed and the input configuration. This can be used to avoid parsing (for example) the entire world.osm on each @@ -95,9 +88,7 @@ The following flags may be useful for debugging: # Configuration -The main config file distributed with this repository is `pfaedle.cfg`. The -config file has some comments which hopefully explain the meaning behind the -parameters. +A default configuration file `pfaedle.cfg` can be found in this repo and will be installed with `make install`. Custom configuration files can be specified with the `-c` flag. If no `-c` flag is set, `pfaedle` will parse and merge the following cfg files in the given order (if present): `/etc/pfaedle/pfaedle.cfg`, `$HOME/.config/pfaedle/pfaedle.cfg`, `/pfaedle.cfg`. Values given in later files will overwrite earlier defined values. # Evaluation diff --git a/eval/Makefile b/eval/Makefile index 83b3b6b..1cae7a8 100644 --- a/eval/Makefile +++ b/eval/Makefile @@ -7,111 +7,127 @@ lighteval: vitoria.lighteval stuttgart.lighteval paris.lighteval switzerland.lig eval: vitoria.eval stuttgart.eval paris.eval switzerland.eval clean: - rm -f *.eval - rm -rf gtfs - rf -rf osm - rm -rf evalout + @rm -f *.eval + @rm -rf gtfs + @rm -rf osm + @rm -rf evalout osmconvert: - wget -O - http://m.m.i24.cc/osmconvert.c | cc -x c - -lz -O3 -o osmconvert + @echo `date +"[%F %T.%3N]"` "EVAL : Fetching osmconvert..." + @curl http://m.m.i24.cc/osmconvert.c | cc -x c - -lz -O3 -o osmconvert %.lighteval: osm/%.osm gtfs/%/stops.txt gtfs/%/stop_times.txt gtfs/%/trips.txt gtfs/%/routes.txt eval.cfg - mkdir -p gtfs/$*/shaped - rm -f gtfs/$*/shaped/* - ../build/pfaedle -x $< -i gtfs/$* -c eval.cfg -o gtfs/$*/shaped -D -m all 2>&1 | tee $@ + @echo `date +"[%F %T.%3N]"` "EVAL : Running light (without stats) evaluation for '"$*"'..." + @mkdir -p gtfs/$*/shaped + @rm -f gtfs/$*/shaped/* + @../build/pfaedle -x $< -i gtfs/$* -c eval.cfg -o gtfs/$*/shaped -D -m all 2>&1 | tee $@ %.eval: osm/%.osm gtfs/%/stops.txt gtfs/%/stop_times.txt gtfs/%/trips.txt gtfs/%/routes.txt eval.cfg eval-wo-osm-line-rels.cfg - mkdir -p gtfs/$*/shaped - rm -f gtfs/$*/shaped/* - mkdir -p evalout/ - mkdir -p evalout/$*/ - mkdir -p evalout/$*/hmm+osm - ../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --eval-path evalout/$*/hmm+osm -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@ - find evalout/$*/hmm+osm/ -name "*.json" -print0 | xargs -0 rm + @echo `date +"[%F %T.%3N]"` "EVAL : Running evaluation for '"$*"'..." + @mkdir -p gtfs/$*/shaped + @rm -f gtfs/$*/shaped/* + @mkdir -p evalout/ + @mkdir -p evalout/$*/ + @mkdir -p evalout/$*/hmm+osm + @../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --eval-path evalout/$*/hmm+osm -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@ + @find evalout/$*/hmm+osm/ -name "*.json" -print0 | xargs -0 rm - mkdir -p evalout/$*/greedy - ../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --method greedy --eval-path evalout/$*/greedy -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@ - find evalout/$*/greedy/ -name "*.json" -print0 | xargs -0 rm + @mkdir -p evalout/$*/greedy + @../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --method greedy --eval-path evalout/$*/greedy -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@ + @find evalout/$*/greedy/ -name "*.json" -print0 | xargs -0 rm - mkdir -p evalout/$*/greedy2 - ../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --method greedy2 --eval-path evalout/$*/greedy2 -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@ - find evalout/$*/greedy2/ -name "*.json" -print0 | xargs -0 rm + @mkdir -p evalout/$*/greedy2 + @../build/pfaedle -x $< -i gtfs/$* -c eval.cfg --method greedy2 --eval-path evalout/$*/greedy2 -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@ + @find evalout/$*/greedy2/ -name "*.json" -print0 | xargs -0 rm - mkdir -p evalout/$*/hmm - ../build/pfaedle -x $< -i gtfs/$* -c eval-wo-osm-line-rels.cfg --eval-path evalout/$*/hmm -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@ - find evalout/$*/hmm/ -name "*.json" -print0 | xargs -0 rm + @mkdir -p evalout/$*/hmm + @../build/pfaedle -x $< -i gtfs/$* -c eval-wo-osm-line-rels.cfg --eval-path evalout/$*/hmm -o gtfs/$*/shaped -D -m all --eval --eval-df-bins $(EVAL_DF_BINS) 2>&1 | tee $@ + @find evalout/$*/hmm/ -name "*.json" -print0 | xargs -0 rm osm/spain-latest.osm.pbf: - mkdir -p osm - wget http://download.geofabrik.de/europe/spain-latest.osm.pbf -O $@ + @mkdir -p osm + @echo `date +"[%F %T.%3N]"` "EVAL : Downloading OSM data for Spain..." + @curl --progress-bar http://download.geofabrik.de/europe/spain-latest.osm.pbf > $@ osm/spain-latest.osm: osm/spain-latest.osm.pbf osmconvert @# pre-filter to vitoria gasteiz - osmconvert -b=-2.8661,42.7480,-2.4788,43.0237 $< > $@ + @echo `date +"[%F %T.%3N]"` "EVAL : Pre-filtering OSM data to Vitoria-Gasteiz..." + @osmconvert -b=-2.8661,42.7480,-2.4788,43.0237 $< > $@ osm/baden-wuerttemberg-latest.osm.pbf: - mkdir -p osm - wget http://download.geofabrik.de/europe/germany/baden-wuerttemberg-latest.osm.pbf -O $@ + @mkdir -p osm + @echo `date +"[%F %T.%3N]"` "EVAL : Downloading OSM data for Baden-Württemberg..." + @curl --progress-bar http://download.geofabrik.de/europe/germany/baden-wuerttemberg-latest.osm.pbf > $@ osm/baden-wuerttemberg-latest.osm: osm/baden-wuerttemberg-latest.osm.pbf osmconvert - osmconvert $< > $@ + @echo `date +"[%F %T.%3N]"` "EVAL : Extracting OSM data..." + @osmconvert $< > $@ osm/france-latest.osm.pbf: - mkdir -p osm - wget http://download.geofabrik.de/europe/france-latest.osm.pbf -O $@ + @mkdir -p osm + @echo `date +"[%F %T.%3N]"` "EVAL : Downloading OSM data for France..." + @curl --progress-bar http://download.geofabrik.de/europe/france-latest.osm.pbf > $@ osm/paris-latest.osm: osm/france-latest.osm.pbf osmconvert @# pre-filter to greater ile de france - osmconvert -b=0.374,47.651,4.241,50.261 $< > $@ + @echo `date +"[%F %T.%3N]"` "EVAL : Pre-filtering OSM data to Île-de-France..." + @osmconvert -b=0.374,47.651,4.241,50.261 $< > $@ osm/europe-latest.osm.pbf: - mkdir -p osm - wget http://download.geofabrik.de/europe-latest.osm.pbf -O $@ + @mkdir -p osm + @echo `date +"[%F %T.%3N]"` "EVAL : Downloading OSM data for Europe..." + @curl --progress-bar http://download.geofabrik.de/europe-latest.osm.pbf > $@ osm/switzerland-latest.osm: osm/europe-latest.osm.pbf osmconvert @# pre-filter to greater switzerland - osmconvert -b=3.757,44.245,15.579,52.670 $< > $@ + @echo `date +"[%F %T.%3N]"` "EVAL : Pre-filtering OSM data to Switzerland..." + @osmconvert -b=3.757,44.245,15.579,52.670 $< > $@ gtfs/vitoria/%.txt: - mkdir -p gtfs - mkdir -p gtfs/vitoria - wget https://transitfeeds.com/p/tuvisa-euskotran/239/latest/download -O gtfs/vitoria/gtfs.zip - cd gtfs/vitoria && unzip -o gtfs.zip - rm gtfs/vitoria/gtfs.zip + @echo `date +"[%F %T.%3N]"` "EVAL : Downloading GTFS data for Vitoria-Gasteiz..." + @mkdir -p gtfs + @mkdir -p gtfs/vitoria + @curl --progress-bar https://transitfeeds.com/p/tuvisa-euskotran/239/latest/download > gtfs/vitoria/gtfs.zip + @cd gtfs/vitoria && unzip -qq -o gtfs.zip + @rm gtfs/vitoria/gtfs.zip gtfs/stuttgart/%.txt: - mkdir -p gtfs - mkdir -p gtfs/stuttgart - echo "******************************************************************" - echo "* A password is required to access the VVS dataset. Send a mail *" - echo "* to brosi@cs.informatik.uni-freiburg.de to receive the password. " - echo "******************************************************************" - wget http://www.vvs.de/download/opendata/VVS_GTFS.zip --ask-password --user vvsopendata01 -O gtfs/stuttgart/gtfs.zip - cd gtfs/stuttgart && unzip -o gtfs.zip - rm gtfs/stuttgart/gtfs.zip + @echo `date +"[%F %T.%3N]"` "EVAL : Downloading GTFS data for Stuttgart..." + @mkdir -p gtfs + @mkdir -p gtfs/stuttgart + @echo "******************************************************************" + @echo "* A password is required to access the VVS dataset. Send a mail *" + @echo "* to brosi@cs.informatik.uni-freiburg.de to receive the password. " + @echo "******************************************************************" + @curl --progress-bar http://www.vvs.de/download/opendata/VVS_GTFS.zip -su vvsopendata01 > gtfs/stuttgart/gtfs.zip + @cd gtfs/stuttgart && unzip -qq -o gtfs.zip + @rm gtfs/stuttgart/gtfs.zip gtfs/paris/%.txt: - mkdir -p gtfs - mkdir -p gtfs/paris - wget https://transitfeeds.com/p/stif/822/latest/download -O gtfs/paris/gtfs.zip - cd gtfs/paris && unzip -o gtfs.zip - rm gtfs/paris/gtfs.zip + @echo `date +"[%F %T.%3N]"` "EVAL : Downloading GTFS data for Paris..." + @mkdir -p gtfs + @mkdir -p gtfs/paris + @curl --progress-bar https://transitfeeds.com/p/stif/822/latest/download > gtfs/paris/gtfs.zip + @cd gtfs/paris && unzip -qq -o gtfs.zip + @rm gtfs/paris/gtfs.zip gtfs/switzerland/%.txt: - mkdir -p gtfs - mkdir -p gtfs/switzerland - wget http://gtfs.geops.ch/dl/gtfs_complete.zip -O gtfs/switzerland/gtfs.zip - cd gtfs/switzerland && unzip -o gtfs.zip - rm gtfs/switzerland/gtfs.zip + @echo `date +"[%F %T.%3N]"` "EVAL : Downloading GTFS data for Switzerland..." + @mkdir -p gtfs + @mkdir -p gtfs/switzerland + @curl --progress-bar http://gtfs.geops.ch/dl/gtfs_complete.zip > gtfs/switzerland/gtfs.zip + @cd gtfs/switzerland && unzip -qq -o gtfs.zip + @rm gtfs/switzerland/gtfs.zip -osm/vitoria.osm: osm/spain-latest.osm gtfs/vitoria/stops.txt eval.cfg - ../build/pfaedle -x $< -i gtfs/vitoria/ -c eval.cfg -m all -X $@ -osm/stuttgart.osm: osm/baden-wuerttemberg-latest.osm gtfs/stuttgart/stops.txt eval.cfg - ../build/pfaedle -x $< -i gtfs/stuttgart/ -c eval.cfg -m all -X $@ +osm/vitoria.osm: osm/spain-latest.osm gtfs/vitoria/stops.txt gtfs/vitoria/trips.txt gtfs/vitoria/routes.txt gtfs/vitoria/stop_times.txt eval.cfg + @../build/pfaedle -x $< -i gtfs/vitoria/ -c eval.cfg -m all -X $@ -osm/paris.osm: osm/paris-latest.osm gtfs/paris/stops.txt eval.cfg - ../build/pfaedle -x $< -i gtfs/paris/ -c eval.cfg -m all -X $@ +osm/stuttgart.osm: osm/baden-wuerttemberg-latest.osm gtfs/stuttgart/stops.txt gtfs/stuttgart/trips.txt gtfs/stuttgart/routes.txt gtfs/stuttgart/stop_times.txt eval.cfg + @../build/pfaedle -x $< -i gtfs/stuttgart/ -c eval.cfg -m all -X $@ + +osm/paris.osm: osm/paris-latest.osm gtfs/paris/stops.txt gtfs/paris/trips.txt gtfs/paris/routes.txt gtfs/paris/stop_times.txt eval.cfg + @../build/pfaedle -x $< -i gtfs/paris/ -c eval.cfg -m all -X $@ osm/switzerland.osm: osm/switzerland-latest.osm gtfs/switzerland/stops.txt eval.cfg - ../build/pfaedle -x $< -i gtfs/switzerland/ -c eval.cfg -m all -X $@ + @../build/pfaedle -x $< -i gtfs/switzerland/ -c eval.cfg -m all -X $@ diff --git a/geo/pfaedle.qgs b/geo/pfaedle.qgs index 8636d80..edfabfd 100644 --- a/geo/pfaedle.qgs +++ b/geo/pfaedle.qgs @@ -1,5 +1,5 @@ - + @@ -13,14 +13,14 @@ - + - + - + @@ -36,41 +36,54 @@ - + - + + + + + + + - degrees + meters - 867288.79171589459292591 - 6099511.4607889149338007 - 890845.27921608532778919 - 6122540.06078738905489445 + 995121.91129447647836059 + 6259104.80718581937253475 + 1006900.15504457184579223 + 6270619.10718505643308163 0 - 0 + 1 - +proj=longlat +datum=WGS84 +no_defs - 3452 - 4326 - EPSG:4326 - WGS 84 - longlat + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + 3857 + 3857 + EPSG:3857 + WGS 84 / Pseudo Mercator + merc WGS84 - true + false 0 - + + + + + + + + @@ -81,6 +94,8 @@ path20180217155708341 trgraph_trgraph_LineString20180508200527144 trgraph_trgraph_Point20180508200527256 + OSM_Transportation20181215024818603 + OpenStreetMap_de20181215024846026 @@ -91,15 +106,15 @@ - - + + - + @@ -115,10 +130,10 @@ - - + + @@ -127,9 +142,19 @@ + + + + + + - + 875390.4375 6113024.5 @@ -144,14 +169,14 @@ OGRGeoJSON LineString - +proj=longlat +datum=WGS84 +no_defs - 3452 - 4326 - EPSG:4326 - WGS 84 - longlat + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + 3857 + 3857 + EPSG:3857 + WGS 84 / Pseudo Mercator + merc WGS84 - true + false ogr @@ -174,18 +199,12 @@ - - - - - - @@ -1170,6 +1189,7 @@ + @@ -1186,6 +1206,7 @@ + @@ -1195,8 +1216,11 @@ + + + @@ -1209,6 +1233,7 @@ + @@ -1218,6 +1243,7 @@ + @@ -1230,10 +1256,14 @@ + + + - + + @@ -1241,6 +1271,7 @@ + @@ -1257,11 +1288,13 @@ + + @@ -1274,6 +1307,7 @@ + @@ -1283,11 +1317,13 @@ + + @@ -1296,21 +1332,27 @@ + + + + + + 0 0 @@ -1337,41 +1379,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + . - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + + . 0 . - + 0 generatedlayout @@ -1384,10 +1489,8 @@ - - @@ -1413,14 +1516,14 @@ OGRGeoJSON LineString - +proj=longlat +datum=WGS84 +no_defs - 3452 - 4326 - EPSG:4326 - WGS 84 - longlat + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + 3857 + 3857 + EPSG:3857 + WGS 84 / Pseudo Mercator + merc WGS84 - true + false ogr @@ -1443,9 +1546,6 @@ - - - @@ -1473,48 +1573,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1667,7 +1725,7 @@ - + @@ -1821,36 +1879,21 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + @@ -1861,7 +1904,6 @@ - + OGRGeoJSON_Point20180206114956218 ./combgraph.json @@ -2532,14 +2542,14 @@ def my_form_open(dialog, layer, feature): OGRGeoJSON Point - +proj=longlat +datum=WGS84 +no_defs - 3452 - 4326 - EPSG:4326 - WGS 84 - longlat + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + 3857 + 3857 + EPSG:3857 + WGS 84 / Pseudo Mercator + merc WGS84 - true + false ogr @@ -2562,9 +2572,6 @@ def my_form_open(dialog, layer, feature): - - - @@ -2592,48 +2599,6 @@ def my_form_open(dialog, layer, feature): - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2679,7 +2644,149 @@ def my_form_open(dialog, layer, feature): - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 0 0 @@ -2705,53 +2812,102 @@ def my_form_open(dialog, layer, feature): + + + + + + + + + + + + + + + + + + + + + + + + + + + + . - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - + + . 0 . - + 0 generatedlayout @@ -2764,7 +2920,6 @@ def my_form_open(dialog, layer, feature): - @@ -2774,20 +2929,6 @@ def my_form_open(dialog, layer, feature): - - - - - - - - - - - - - - @@ -2796,6 +2937,98 @@ def my_form_open(dialog, layer, feature): + + + -20037508.34278924390673637 + -20037508.34278925508260727 + 20037508.34278924390673637 + 20037508.34278924390673637 + + OSM_Transportation20181215024818603 + type=xyz&zmin=0&zmax=19&url=http://tile.thunderforest.com/transport/{z}/{x}/{y}.png + + + + Maps © Thunderforest, Data © OpenStreetMap contributors + OSM Transportation + + + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + 3857 + 3857 + EPSG:3857 + WGS 84 / Pseudo Mercator + merc + WGS84 + false + + + + + + wms + + + + + + + + + + + + + + + 0 + + + + -20037508.34278924390673637 + -20037508.34278925508260727 + 20037508.34278924390673637 + 20037508.34278924390673637 + + OpenStreetMap_de20181215024846026 + type=xyz&zmin=0&zmax=18&url=http://a.tile.openstreetmap.de/tiles/osmde/{z}/{x}/{y}.png + + + + OpenStreetMap contributors, under ODbL + OpenStreetMap.de + + + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + 3857 + 3857 + EPSG:3857 + WGS 84 / Pseudo Mercator + merc + WGS84 + false + + + + + + wms + + + + + + + + + + + + + + + 0 + 735429.75 @@ -2811,14 +3044,14 @@ def my_form_open(dialog, layer, feature): path - +proj=longlat +datum=WGS84 +no_defs - 3452 - 4326 - EPSG:4326 - WGS 84 - longlat + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + 3857 + 3857 + EPSG:3857 + WGS 84 / Pseudo Mercator + merc WGS84 - true + false ogr @@ -2828,7 +3061,11 @@ def my_form_open(dialog, layer, feature): - + + + + + @@ -2891,6 +3128,7 @@ def my_form_open(dialog, layer, feature): + @@ -2907,6 +3145,7 @@ def my_form_open(dialog, layer, feature): + @@ -2916,8 +3155,11 @@ def my_form_open(dialog, layer, feature): + + + @@ -2930,6 +3172,7 @@ def my_form_open(dialog, layer, feature): + @@ -2939,6 +3182,7 @@ def my_form_open(dialog, layer, feature): + @@ -2951,10 +3195,14 @@ def my_form_open(dialog, layer, feature): + + + + @@ -2962,6 +3210,7 @@ def my_form_open(dialog, layer, feature): + @@ -2978,11 +3227,13 @@ def my_form_open(dialog, layer, feature): + + @@ -2995,6 +3246,7 @@ def my_form_open(dialog, layer, feature): + @@ -3004,11 +3256,13 @@ def my_form_open(dialog, layer, feature): + + @@ -3017,26 +3271,32 @@ def my_form_open(dialog, layer, feature): + + + + + + 0 0 30 - + ver + + + + + + + + + + + + + + + + + + + + + + + + + + + + . - + + + - + - + + . 0 . - + 0 generatedlayout @@ -3078,7 +3387,9 @@ def my_form_open(dialog, layer, feature): - + + + @@ -3090,14 +3401,14 @@ def my_form_open(dialog, layer, feature): trgraph trgraph LineString - +proj=longlat +datum=WGS84 +no_defs - 3452 - 4326 - EPSG:4326 - WGS 84 - longlat + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + 3857 + 3857 + EPSG:3857 + WGS 84 / Pseudo Mercator + merc WGS84 - true + false ogr @@ -3107,29 +3418,7 @@ def my_form_open(dialog, layer, feature): - - - - - - - - - - - - - - - - - - - - - - - + @@ -4212,7 +4501,7 @@ def my_form_open(dialog, layer, feature): - + @@ -4336,6 +4625,300 @@ def my_form_open(dialog, layer, feature): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + . + + + + + + + + + . + + 0 + . + + 0 + generatedlayout + + + + + + + + + + trgraph_trgraph_Point20180508200527256 + ./trgraph.json|layerid=0|geometrytype=Point + + + + trgraph trgraph Point + + + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + 3857 + 3857 + EPSG:3857 + WGS 84 / Pseudo Mercator + merc + WGS84 + false + + + ogr + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + 0 + 0 + id + + + @@ -4363,36 +4946,21 @@ def my_form_open(dialog, layer, feature): - - - - - - - - - - - + + . + - - + . 0 - + . - - - - - - - - - - - - - trgraph_trgraph_Point20180508200527256 - ./trgraph.json|layerid=0|geometrytype=Point - - - - trgraph trgraph Point - - - +proj=longlat +datum=WGS84 +no_defs - 3452 - 4326 - EPSG:4326 - WGS 84 - longlat - WGS84 - true - - - ogr - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0 - 0 - 0 - id - - - - - - - - - - - - - - - - - - - - - - 0 - - - 0 - generatedlayout - - - - - - - - - - - - - - + + + + + false @@ -4593,14 +5014,16 @@ def my_form_open(dialog, layer, feature): D - false + + false - NONE + WGS84 8 + false @@ -4623,6 +5046,8 @@ def my_form_open(dialog, layer, feature): OGRGeoJSON_Point20180203134333739 OGRGeoJSON_Point20180206114956218 path20180217155708341 + trgraph_trgraph_LineString20180508200527144 + trgraph_trgraph_Point20180508200527256 enabled @@ -4630,6 +5055,8 @@ def my_form_open(dialog, layer, feature): enabled enabled enabled + disabled + disabled current_layer @@ -4639,6 +5066,8 @@ def my_form_open(dialog, layer, feature): 2 2 2 + 2 + 2 to_vertex @@ -4646,6 +5075,8 @@ def my_form_open(dialog, layer, feature): to_vertex to_vertex to_vertex + to_vertex_and_segment + to_vertex_and_segment off 0 @@ -4655,6 +5086,8 @@ def my_form_open(dialog, layer, feature): 0.000000 0.000000 0.000000 + 0.000000 + 0.000000 @@ -4663,15 +5096,16 @@ def my_form_open(dialog, layer, feature): - + None false - +proj=longlat +datum=WGS84 +no_defs - EPSG:4326 - 3452 + +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs + EPSG:3857 + 3857 + 1 @@ -4681,11 +5115,11 @@ def my_form_open(dialog, layer, feature): 255 - + conditions unknown 90 meters - + m2 diff --git a/pfaedle.cfg b/pfaedle.cfg index ab4189c..346f463 100644 --- a/pfaedle.cfg +++ b/pfaedle.cfg @@ -2,6 +2,216 @@ # Chair of Algorithms and Datastructures # Authors: Patrick Brosi +[tram, bus, coach, subway, rail, gondola, funicular, ferry] +# Regular expressions and station comparision is +# always case insensitive! +station_normalize_chain: + , -> ' '; + - -> ' '; + — -> ' '; + _ -> ' '; + " -> ''; + ' -> ''; + ` -> ''; + \( -> ' '; + \) -> ' '; + \[ -> ' '; + \] -> ' '; + / -> ' '; + '\\' -> ' '; + < -> ' '; + > -> ' '; + & -> '+'; + Ä -> Ae; + Ö -> Oe; + Ü -> Ue; + ä -> ae; + ö -> oe; + ü -> ue; + ß -> ss; + è -> e; + é -> e; + á -> a; + à -> a; + ó -> o; + ò -> o; + ô -> o; + ç -> c; + í -> i; + ú -> u; + ù -> u; + ë -> e; + å -> ae; + â -> a; + ê -> e; + ï -> i; + œ -> oe; + ø -> oe; + str\. -> strasse; + av\. -> avenue; + + # always separate 'street', 'strasse' + '([a-zA-Z])strasse($| )' -> '\1 strasse\2'; + '([a-zA-Z])street($| )' -> '\1 street\2'; + + # always use "street" + '(^| )strasse($| )' -> '\1street\2'; + + # always use "avenue" + '(^| )avenida($| )' -> '\1avenue\2'; + '(^| )avenu($| )' -> '\1avenue\2'; + + # normalize every possible abbr. of german "Bahnhof", "Hauptbahnhof", "Busbahnhof" + '(^| )hauptbf\.($| )' -> '\1hauptbahnhof\2'; + '(^| )hauptbf($| )' -> '\1hauptbahnhof\2'; + '(^| )hauptbhf\.($| )' -> '\1hauptbahnhof\2'; + '(^| )hauptbhf($| )' -> '\1hauptbahnhof\2'; + '(^| )zentraler busbahnhof($| )$' -> \1busbahnhof\2; + '(^| )zentraler omnibusbahnhof($| )$' -> \1busbahnhof\2; + '(^| )omnibusbahnhof($| )' -> '\1busbahnhof\2'; + '(^| )omnibusbhf($| )' -> '\1busbahnhof\2'; + '(^| )busbf\.($| )' -> '\1busbahnhof\2'; + '(^| )busbf($| )' -> '\1busbahnhof\2'; + '(^| )bus bf\.($| )' -> '\1busbahnhof\2'; + '(^| )bus bf($| )' -> '\1busbahnhof\2'; + '(^| )busbhf\.($| )' -> '\1busbahnhof\2'; + '(^| )busbhf($| )' -> '\1busbahnhof\2'; + '(^| )bus bhf\.($| )' -> '\1busbahnhof\2'; + '(^| )bus bhf($| )' -> '\1busbahnhof\2'; + '(^| )zob($| )' -> '\1busbahnhof\2'; + '(^| )hbf\.($| )' -> '\1hauptbahnhof\2'; + '(^| )hbf($| )' -> '\1hauptbahnhof\2'; + '(^| )hb\.($| )' -> '\1hauptbahnhof\2'; + '(^| )hb($| )' -> '\1hauptbahnhof\2'; + '(^| )bf\.($| )' -> '\1bahnhof\2'; + '(^| )bf($| )' -> '\1bahnhof\2'; + '(^| )bhf\.($| )' -> '\1bahnhof\2'; + '(^| )bhf($| )' -> '\1bahnhof\2'; + '(^| )bhfeingang($| )' -> '\1bahnhofeingang\2'; + '(^| )gare de($| )' -> '\1gare\2'; + + + # if a stations starts with single station identifier + # always put it at the end (for example, "hauptbahnhof freiburg" becomes "freiburg hauptbahnhof") + '^hauptbahnhof (.+)$' -> \1 hauptbahnhof; + '^bahnhof (.+)$' -> \1 bahnhof; + '^busbahnhof (.+)$' -> \1 busbahnhof; + '^gare (.+)$' -> \1 gare; + '^station (.+)$' -> \1 station; + + '(^| )busbahnhof($| )' -> '\1bbahnhof\2'; + + # normalize line types in station names + '(^| )u bahn\.($| )' -> '\1ubahn\2'; + '(^| )metro\.($| )' -> '\1ubahn\2'; + '(^| )subway\.($| )' -> '\1ubahn\2'; + '(^| )underground\.($| )' -> '\1ubahn\2'; + '(^| )ubahn($| )' -> '\1u\2'; + '(^| )s bahn\.($| )' -> '\1sbahn\2'; + '(^| )sbahn($| )' -> '\1s\2'; + '(^| )tramway($| )' -> '\1tram\2'; + '(^| )stadtbahn($| )' -> '\1tram\2'; + '(^| )strassenbahn($| )' -> '\1tram\2'; + '(^| )streetcar($| )' -> '\1tram\2'; + '(^| )tram($| )' -> '\1t\2'; + + # delete track information from name + '(^| )kante [a-zA-Z0-9]{1,2}($| )' -> ' '; + '(^| )gleis [a-zA-Z0-9]{1,2}($| )' -> ' '; + '(^| )track [a-zA-Z0-9]{1,2}($| )' -> ' '; + '(^| )voie [a-zA-Z0-9]{1,2}($| )' -> ' '; + + # abbrv + '(^| )und($| )' -> '\1+\2'; + '(^| )and($| )' -> '\1+\2'; + '(^| )et($| )' -> '\1+\2'; + + # noise + '\sde\s' -> ' '; + '\sda\s' -> ' '; + '\sdi\s' -> ' '; + '\sdel\s' -> ' '; + '\sdal\s' -> ' '; + + # abbrv in most western languages + '(^| )saint ' -> '\1st. '; + '(^| )sankt ' -> '\1st. '; + '(^| )sanct ' -> '\1st. '; + + \. -> ' '; + + # whitespace + \s+ -> ' '; + ^\s -> ''; + \s$ -> ''; + +line_normalize_chain: + , -> ' '; + - -> ' '; + _ -> ' '; + " -> ''; + ' -> ''; + ` -> ''; + / -> ' '; + < -> ' '; + > -> ' '; + & -> '+'; + ä -> ae; + ö -> oe; + ü -> ue; + Ä -> Ae; + Ö -> Oe; + Ü -> Ue; + ß -> ss; + è -> e; + é -> e; + á -> a; + à -> a; + ó -> o; + ò -> o; + í -> i; + ú -> u; + ù -> u; + ë -> e; + å -> ae; + ç -> c; + â -> a; + ê -> e; + ï -> i; + œ -> oe; + ø -> oe; + ^line -> ''; + ^linie -> ''; + ^metro -> ''; + ^tram -> ''; + ^strassenbahn -> ''; + ^bus -> ''; + + # delete everything in brackets + \(.+\) -> ' '; + \[.+\] -> ' '; + + # whitespace + \s+ -> ' '; + ^\s -> ''; + \s$ -> ''; + + # line/number combs ALWAYS without whitespace (T 2 -> T2) + ^([a-zA-Z]+) ([0-9]+)$ -> \1\2; + +track_normalize_chain: + '(^| )gleis($| )' -> ''; + '(^| )gl\.($| )' -> ''; + '(^| )platform($| )' -> ''; + '(^| )track($| )' -> ''; + '(^| )rail($| )' -> ''; + # line/number combs ALWAYS without whitespace (1 A -> 1A) + ^([a-zA-Z]+) ([0-9]+)$ -> \1\2; + ^([0-9]+) ([a-zA-Z]+)$ -> \1\2; + + # delete track numbers greater than 999 + ^[0-9]{4,}$ -> ''; + [rail] # OSM entities to keep on different levels, as k=v. Applies @@ -12,8 +222,10 @@ osm_filter_keep: railway=rail railway=light_rail + railway=tram railway=narrow_gauge route=rail + route=light_rail route=train public_transport=stop_area|rel_flat @@ -21,10 +233,11 @@ osm_filter_lvl1: usage=branch osm_filter_lvl2: + railway=tram + service=siding osm_filter_lvl3: service=crossover - service=siding # we cannot completely drop service=yard, because it is often used # incorrectly for crossovers service=yard @@ -213,6 +426,9 @@ line_normalize_chain: ä -> ae; ö -> oe; ü -> ue; + Ä -> Ae; + Ö -> Oe; + Ü -> Ue; ß -> ss; è -> e; é -> e; @@ -253,7 +469,21 @@ line_normalize_chain: # if a character line number is present, delete the numeric part ^([a-zA-Z]+) [0-9]+$ -> \1; -[bus] +track_normalize_chain: + '(^| )gleis($| )' -> ''; + '(^| )gl\.($| )' -> ''; + '(^| )platform($| )' -> ''; + '(^| )track($| )' -> ''; + '(^| )rail($| )' -> ''; + ^([a-zA-Z]+) ([0-9]+)$ -> \1\2; + # number/char combs ALWAYS without char + ^([0-9]+) ([a-zA-Z]+)$ -> \1; + ^([0-9]+)([a-zA-Z]+)$ -> \1; + + # delete track numbers greater than 999 + ^[0-9]{4,}$ -> ''; + +[bus, coach] # OSM entities to keep on different levels, as k=v. Applies # to nodes, edges and relations. @@ -287,6 +517,18 @@ osm_filter_keep: psv=yes psv=designated + bus:lanes=yes + bus:lanes=designated + bus:lanes=1 + + lanes:bus=1 + lanes:bus=2 + lanes:bus=3 + + lanes:psv=1 + lanes:psv=2 + lanes:psv=3 + trolley_wire=yes trolleywire=yes trolleybus=yes @@ -448,6 +690,14 @@ osm_filter_undirected: busway:right=opposite_lane psv=opposite_lane psv=opposite + lanes:psv:backward=1 + lanes:psv:backward=2 + lanes:bus:backward=1 + lanes:bus:backward=2 + bus:lanes:backward=yes + bus:lanes:backward=designated + bus:lanes:backward=1 + # Nodes that are stations. # Only nodes that have been kept during the filtering above will be @@ -486,11 +736,13 @@ osm_station_group_attrs: # max distance in meters between a snapped station position and the # original station position -osm_max_snap_distance: 10 , 50, 100 +osm_max_snap_distance: 10, 50, 100 + +osm_max_snap_fallback_distance: 300 osm_max_snap_level: 5 -osm_max_osm_station_distance: 7.5 +osm_max_osm_station_distance: 8.0 # sorted by priority, first found attr will be taken osm_station_name_attrs: @@ -541,7 +793,65 @@ routing_one_way_edge_punish: 5000 # information # routing_line_unmatched_punish_fac: 1.75 -[tram, subway, funicular] +[coach] + +# OSM entities to keep on different levels, as k=v. Applies +# to nodes, edges and relations. +# Nodes included in kept ways are always kept. +# Ways included in kept relations are always kept. + +osm_filter_lvl0: + highway=motorway + highway=motorway_link + +osm_filter_lvl1: + highway=trunk + highway=trunk_link + +osm_filter_lvl2: + highway=primary + highway=primary_link + +osm_filter_lvl3: + highway=secondary + highway=secondary_link + +osm_filter_lvl4: + highway=tertiary + highway=tertiary_link + +osm_filter_lvl5: + highway=unclassified + highway=residential + highway=road + highway=service + +osm_filter_lvl6: + highway=living_street + highway=pedestrian + psv=no + +osm_filter_lvl7: + bus=no + service=siding + access=permissive + access=private + access=no + service=parking_aisle + highway=footway + +routing_lvl0_fac: 1 # default level +routing_lvl1_fac: 1.15 +routing_lvl2_fac: 1.5 +routing_lvl3_fac: 1.75 +routing_lvl4_fac: 2.25 +routing_lvl5_fac: 2.5 +routing_lvl6_fac: 3 +routing_lvl7_fac: 4 + +osm_max_snap_level: 5 + +[tram, subway] # OSM entities to keep on different levels, as k=v. Applies # to nodes, edges and relations. @@ -550,6 +860,7 @@ routing_one_way_edge_punish: 5000 osm_filter_keep: route=tram + route=funicular railway=subway railway=light_rail railway=tram @@ -562,9 +873,326 @@ osm_filter_keep: subway=yes tram=yes -osm_filter_lv2: +osm_filter_lvl2: service=siding +osm_filter_lvl3: + railway=funicular + route=funicular + +osm_filter_lvl5: + service=crossover + service=yard + +# OSM entities to drop, as k=v. Applies to nodes, edges and +# relations. +# Nodes included in non-dropped ways are kept regardless of +# a matching drop filter. +# Ways included in non-dropped relations are kept regardless of +# a matching drop filter. + +osm_filter_drop: + area=yes + public_transport=stop_area + type=multipolygon + railway=platform + public_transport=platform + service=alley + +# Nodes that should act as "no-hup" nodes. These are nodes +# that are contained in multiple ways, but cannot be used +# to switch from one way to another (for example, a +# track crossing in rail networks) + +osm_filter_nohup: + railway:switch=no + railway=railway_crossing + +# Edges that should act as one-way nodes. + +osm_filter_oneway: + oneway=yes + +# Edges that may explicitely be used in +# both directions. May be used to set exception +# to "osm_filter_oneway" + +osm_filter_undirected: + +# Nodes that are stations. +# Only nodes that have been kept during the filtering above will be +# checked. +osm_filter_station: + public_transport=stop_position + station=subway + station=tram + railway=stop + railway=halt + railway=station + railway=tram_stop + railway=subway_stop + tram_stop=* + stop=* + +# Relation fields that should be used for catching the lines that +# occur on an edge. Only relations that have been kept during the +# filtering above will be checked. The 'linename' will be normalized +# according to the rules in line_normalization_chain. +# The 'from_name' and 'to_name' will be normalized according to the +# rules in station_normalization_chain. +# The relations tags are given in the order of their relevance - +# the first normalized tag-value that is not null/empty will be +# taken. +osm_line_relation_tags: + line_name=ref,name # careful, no space after/before comma allowed! + from_name=from + to_name=to + + +# attr name together with the +# max distance in meters between any of the groups members and +# a potential new member +# first matching rule will be taken +# only applies to nodes that match osm_filter_station! +osm_station_group_attrs: + uic_ref=500 + wikidata=500 + name=100 + +# max distance in meters between a snapped station position and the +# original station position +osm_max_snap_distance: 10, 50, 100 + +osm_max_snap_level: 4 + + +# sorted by priority, first found attr will be taken +osm_station_name_attrs: + name + uic_name + +# the track number tag in stop nodes, first one is taken +osm_track_number_tags: local_ref + +routing_lvl0_fac: 1 # default level +routing_lvl1_fac: 1.5 +routing_lvl2_fac: 2 +routing_lvl3_fac: 2.5 +routing_lvl4_fac: 3.5 +routing_lvl5_fac: 5 +routing_lvl6_fac: 5 +routing_lvl7_fac: 5 + +# Punishment (in meters) to add to the distance +# function if a vehicle performans a full turn +routing_full_turn_punish: 2000 + +routing_station_distance_punish_fac: 3.14 + +routing_non_osm_station_punish: 235 + +# Max angle that should be counted as a full turn +routing_full_turn_angle: 80 + +# Max angle in a route from a station to an already reachable neighbar +routing_snap_full_turn_angle: 80 + +# Punishment (in meters) to add to the distance +# function if a vehicle passes a station node without +# stopping there +routing_pass_thru_station_punish: 100 + +# Punishment factor for every meter a vehicle +# travels through a one-way edge +routing_one_way_meter_punish_fac: 1 + +# Punishment factor for every meter a vehicle +# travels through an edge without any matching line +# information +routing_line_unmatched_punish_fac: 0.5 + +[gondola] + +# OSM entities to keep on different levels, as k=v. Applies +# to nodes, edges and relations. +# Nodes included in kept ways are always kept. +# Ways included in kept relations are always kept. + +osm_filter_keep: + aerialway=gondola + aerialway=cable_car + aerialway=chair_lift + aerialway=mixed_lift + + +# OSM entities to drop, as k=v. Applies to nodes, edges and +# relations. +# Nodes included in non-dropped ways are kept regardless of +# a matching drop filter. +# Ways included in non-dropped relations are kept regardless of +# a matching drop filter. + +osm_filter_drop: + area=yes + public_transport=stop_area + type=multipolygon + railway=platform + public_transport=platform + service=alley + +# Nodes that should act as "no-hup" nodes. These are nodes +# that are contained in multiple ways, but cannot be used +# to switch from one way to another (for example, a +# track crossing in rail networks) + +osm_filter_nohup: + +# Edges that should act as one-way nodes. + +osm_filter_oneway: + oneway=yes + +# Edges that may explicitely be used in +# both directions. May be used to set exception +# to "osm_filter_oneway" + +osm_filter_undirected: + +# Nodes that are stations. +# Only nodes that have been kept during the filtering above will be +# checked. +osm_filter_station: + aerialway=station + aerialway=stop + public_transport=stop_position + station=subway + station=tram + railway=stop + railway=halt + railway=station + railway=tram_stop + railway=subway_stop + tram_stop=* + stop=* + +# Relation fields that should be used for catching the lines that +# occur on an edge. Only relations that have been kept during the +# filtering above will be checked. The 'linename' will be normalized +# according to the rules in line_normalization_chain. +# The 'from_name' and 'to_name' will be normalized according to the +# rules in station_normalization_chain. +# The relations tags are given in the order of their relevance - +# the first normalized tag-value that is not null/empty will be +# taken. +osm_line_relation_tags: + line_name=ref,name # careful, no space after/before comma allowed! + from_name=from + to_name=to + + +# attr name together with the +# max distance in meters between any of the groups members and +# a potential new member +# first matching rule will be taken +# only applies to nodes that match osm_filter_station! +osm_station_group_attrs: + uic_ref=500 + wikidata=500 + name=100 + +# max distance in meters between a snapped station position and the +# original station position +osm_max_snap_distance: 10, 50, 100 + +osm_max_snap_level: 4 + + +# sorted by priority, first found attr will be taken +osm_station_name_attrs: + name + uic_name + +# the track number tag in stop nodes, first one is taken +osm_track_number_tags: local_ref + +routing_lvl0_fac: 1 # default level +routing_lvl1_fac: 1.5 +routing_lvl2_fac: 2 +routing_lvl3_fac: 2.5 +routing_lvl4_fac: 3.5 +routing_lvl5_fac: 5 +routing_lvl6_fac: 5 +routing_lvl7_fac: 5 + +# Punishment (in meters) to add to the distance +# function if a vehicle performans a full turn +routing_full_turn_punish: 2000 + +routing_station_distance_punish_fac: 3.14 + +routing_non_osm_station_punish: 235 + +# Max angle that should be counted as a full turn +routing_full_turn_angle: 80 + +# Max angle in a route from a station to an already reachable neighbar +routing_snap_full_turn_angle: 80 + +# Punishment (in meters) to add to the distance +# function if a vehicle passes a station node without +# stopping there +routing_pass_thru_station_punish: 100 + +# Punishment factor for every meter a vehicle +# travels through a one-way edge +routing_one_way_meter_punish_fac: 1 + +# Punishment factor for every meter a vehicle +# travels through an edge without any matching line +# information +routing_line_unmatched_punish_fac: 0.5 + +[funicular] + +# OSM entities to keep on different levels, as k=v. Applies +# to nodes, edges and relations. +# Nodes included in kept ways are always kept. +# Ways included in kept relations are always kept. + +osm_filter_keep: + route=funicular + railway=funicular + railway=narrow_gauge + route=tram + railway=subway + railway=light_rail + railway=tram + railway=station + railway=halt + railway=tram_stop + route=subway + route=light_rail + subway=yes + tram=yes + +osm_filter_lvl2: + service=siding + +osm_filter_lvl3: + route=tram + route=narrow_gauge + railway=subway + railway=narrow_gauge + railway=light_rail + railway=tram + railway=station + railway=halt + railway=tram_stop + route=subway + route=light_rail + subway=yes + tram=yes + osm_filter_lvl5: service=crossover service=yard @@ -797,206 +1425,3 @@ routing_one_way_meter_punish_fac: 1 # information routing_line_unmatched_punish_fac: 0.5 -[tram, bus, subway, rail, gondola, funicular, ferry] -# Regular expressions and station comparision is -# always case insensitive! -station_normalize_chain: - , -> ' '; - - -> ' '; - — -> ' '; - _ -> ' '; - " -> ''; - ' -> ''; - ` -> ''; - \( -> ' '; - \) -> ' '; - \[ -> ' '; - \] -> ' '; - / -> ' '; - '\\' -> ' '; - < -> ' '; - > -> ' '; - & -> '+'; - ä -> ae; - ö -> oe; - ü -> ue; - ß -> ss; - è -> e; - é -> e; - á -> a; - à -> a; - ó -> o; - ò -> o; - ô -> o; - ç -> c; - í -> i; - ú -> u; - ù -> u; - ë -> e; - å -> ae; - â -> a; - ê -> e; - ï -> i; - œ -> oe; - ø -> oe; - str\. -> strasse; - av\. -> avenue; - - # always separate 'street', 'strasse' - '([a-zA-Z])strasse($| )' -> '\1 strasse\2'; - '([a-zA-Z])street($| )' -> '\1 street\2'; - - # always use "street" - '(^| )strasse($| )' -> '\1street\2'; - - # always use "avenue" - '(^| )avenida($| )' -> '\1avenue\2'; - '(^| )avenu($| )' -> '\1avenue\2'; - - # normalize every possible abbr. of german "Bahnhof", "Hauptbahnhof", "Busbahnhof" - '(^| )hauptbf\.($| )' -> '\1hauptbahnhof\2'; - '(^| )hauptbf($| )' -> '\1hauptbahnhof\2'; - '(^| )hauptbhf\.($| )' -> '\1hauptbahnhof\2'; - '(^| )hauptbhf($| )' -> '\1hauptbahnhof\2'; - '(^| )zentraler busbahnhof($| )$' -> \1busbahnhof\2; - '(^| )zentraler omnibusbahnhof($| )$' -> \1busbahnhof\2; - '(^| )omnibusbahnhof($| )' -> '\1busbahnhof\2'; - '(^| )omnibusbhf($| )' -> '\1busbahnhof\2'; - '(^| )busbf\.($| )' -> '\1busbahnhof\2'; - '(^| )busbf($| )' -> '\1busbahnhof\2'; - '(^| )bus bf\.($| )' -> '\1busbahnhof\2'; - '(^| )bus bf($| )' -> '\1busbahnhof\2'; - '(^| )busbhf\.($| )' -> '\1busbahnhof\2'; - '(^| )busbhf($| )' -> '\1busbahnhof\2'; - '(^| )bus bhf\.($| )' -> '\1busbahnhof\2'; - '(^| )bus bhf($| )' -> '\1busbahnhof\2'; - '(^| )zob($| )' -> '\1busbahnhof\2'; - '(^| )hbf\.($| )' -> '\1hauptbahnhof\2'; - '(^| )hbf($| )' -> '\1hauptbahnhof\2'; - '(^| )hb\.($| )' -> '\1hauptbahnhof\2'; - '(^| )hb($| )' -> '\1hauptbahnhof\2'; - '(^| )bf\.($| )' -> '\1bahnhof\2'; - '(^| )bf($| )' -> '\1bahnhof\2'; - '(^| )bhf\.($| )' -> '\1bahnhof\2'; - '(^| )bhf($| )' -> '\1bahnhof\2'; - '(^| )bhfeingang($| )' -> '\1bahnhofeingang\2'; - '(^| )gare de($| )' -> '\1gare\2'; - - - # if a stations starts with single station identifier - # always put it at the end (for example, "hauptbahnhof freiburg" becomes "freiburg hauptbahnhof") - '^hauptbahnhof (.+)$' -> \1 hauptbahnhof; - '^bahnhof (.+)$' -> \1 bahnhof; - '^busbahnhof (.+)$' -> \1 busbahnhof; - '^gare (.+)$' -> \1 gare; - '^station (.+)$' -> \1 station; - - '(^| )busbahnhof($| )' -> '\1bbahnhof\2'; - - # normalize line types in station names - '(^| )u bahn\.($| )' -> '\1ubahn\2'; - '(^| )metro\.($| )' -> '\1ubahn\2'; - '(^| )subway\.($| )' -> '\1ubahn\2'; - '(^| )underground\.($| )' -> '\1ubahn\2'; - '(^| )ubahn($| )' -> '\1u\2'; - '(^| )s bahn\.($| )' -> '\1sbahn\2'; - '(^| )sbahn($| )' -> '\1s\2'; - '(^| )tramway($| )' -> '\1tram\2'; - '(^| )stadtbahn($| )' -> '\1tram\2'; - '(^| )strassenbahn($| )' -> '\1tram\2'; - '(^| )streetcar($| )' -> '\1tram\2'; - '(^| )tram($| )' -> '\1t\2'; - - # delete track information from name - '(^| )kante [a-zA-Z0-9]{1,2}($| )' -> ' '; - '(^| )gleis [a-zA-Z0-9]{1,2}($| )' -> ' '; - '(^| )track [a-zA-Z0-9]{1,2}($| )' -> ' '; - '(^| )voie [a-zA-Z0-9]{1,2}($| )' -> ' '; - - # abbrv - '(^| )und($| )' -> '\1+\2'; - '(^| )and($| )' -> '\1+\2'; - '(^| )et($| )' -> '\1+\2'; - - # noise - '\sde\s' -> ' '; - '\sda\s' -> ' '; - '\sdi\s' -> ' '; - '\sdel\s' -> ' '; - '\sdal\s' -> ' '; - - # abbrv in most western languages - '(^| )saint ' -> '\1st. '; - '(^| )sankt ' -> '\1st. '; - '(^| )sanct ' -> '\1st. '; - - \. -> ' '; - - # whitespace - \s+ -> ' '; - ^\s -> ''; - \s$ -> ''; - -line_normalize_chain: - , -> ' '; - - -> ' '; - _ -> ' '; - " -> ''; - ' -> ''; - ` -> ''; - / -> ' '; - < -> ' '; - > -> ' '; - & -> '+'; - ä -> ae; - ö -> oe; - ü -> ue; - ß -> ss; - è -> e; - é -> e; - á -> a; - à -> a; - ó -> o; - ò -> o; - í -> i; - ú -> u; - ù -> u; - ë -> e; - å -> ae; - ç -> c; - â -> a; - ê -> e; - ï -> i; - œ -> oe; - ø -> oe; - ^line -> ''; - ^linie -> ''; - ^metro -> ''; - ^tram -> ''; - ^strassenbahn -> ''; - ^bus -> ''; - - # delete everything in brackets - \(.+\) -> ' '; - \[.+\] -> ' '; - - # whitespace - \s+ -> ' '; - ^\s -> ''; - \s$ -> ''; - - # line/number combs ALWAYS without whitespace (T 2 -> T2) - ^([a-zA-Z]+) ([0-9]+)$ -> \1\2; - -track_normalize_chain: - '(^| )gleis($| )' -> ''; - '(^| )gl\.($| )' -> ''; - '(^| )platform($| )' -> ''; - '(^| )track($| )' -> ''; - '(^| )rail($| )' -> ''; - # line/number combs ALWAYS without whitespace (1 A -> 1A) - ^([a-zA-Z]+) ([0-9]+)$ -> \1\2; - ^([0-9]+) ([a-zA-Z]+)$ -> \1\2; - - # delete track numbers greater than 999 - ^[0-9]{4,}$ -> ''; diff --git a/src/configparser b/src/configparser index 5a1f788..63fcb1d 160000 --- a/src/configparser +++ b/src/configparser @@ -1 +1 @@ -Subproject commit 5a1f788dde8f334d40505268f71bcc473d1968d8 +Subproject commit 63fcb1d54eb4889b376b76cafe140317326b5c56 diff --git a/src/cppgtfs b/src/cppgtfs index 727ddfe..32b081e 160000 --- a/src/cppgtfs +++ b/src/cppgtfs @@ -1 +1 @@ -Subproject commit 727ddfecc952e7bd8e4b11ef34436454a50e7532 +Subproject commit 32b081e352fc7496a4e2b9a90cf46eecaf7c63fd diff --git a/src/pfaedle/Def.h b/src/pfaedle/Def.h index 620e9a4..02e8002 100644 --- a/src/pfaedle/Def.h +++ b/src/pfaedle/Def.h @@ -25,4 +25,6 @@ #define BOX util::geo::Box #define POLYLINE util::geo::PolyLine +#define BOX_PADDING 2500 + #endif // PFAEDLE_DEF_H_ diff --git a/src/pfaedle/PfaedleMain.cpp b/src/pfaedle/PfaedleMain.cpp index 1eb3baf..36ffb61 100644 --- a/src/pfaedle/PfaedleMain.cpp +++ b/src/pfaedle/PfaedleMain.cpp @@ -2,19 +2,25 @@ // Chair of Algorithms and Data Structures. // Authors: Patrick Brosi +#include +#include #include #include +#include +#include +#include #include #include #include #include #include "ad/cppgtfs/Parser.h" #include "ad/cppgtfs/Writer.h" -#include "ad/cppgtfs/gtfs/Feed.h" #include "pfaedle/config/ConfigReader.h" #include "pfaedle/config/MotConfig.h" #include "pfaedle/config/MotConfigReader.h" #include "pfaedle/eval/Collector.h" +#include "pfaedle/gtfs/Feed.h" +#include "pfaedle/gtfs/Writer.h" #include "pfaedle/netgraph/Graph.h" #include "pfaedle/osm/OsmIdSet.h" #include "pfaedle/router/ShapeBuilder.h" @@ -30,13 +36,28 @@ using pfaedle::osm::OsmBuilder; using pfaedle::config::MotConfig; using pfaedle::config::Config; using pfaedle::router::ShapeBuilder; +using configparser::ParseFileExc; using pfaedle::config::MotConfigReader; using pfaedle::config::ConfigReader; using pfaedle::eval::Collector; +enum class RetCode { + SUCCESS = 0, + NO_INPUT_FEED = 1, + MULT_FEEDS_NOT_ALWD = 2, + TRIP_NOT_FOUND = 3, + GTFS_PARSE_ERR = 4, + NO_OSM_INPUT = 5, + MOT_CFG_PARSE_ERR = 6, + OSM_PARSE_ERR = 7, + GTFS_WRITE_ERR = 8, + NO_MOT_CFG = 9 +}; + std::string getMotStr(const MOTs& mots); std::string getFileNameMotStr(const MOTs& mots); MOTs getContMots(const MotConfig& motCfg, const MOTs& mots); +std::vector getCfgPaths(const Config& cfg); // _____________________________________________________________________________ int main(int argc, char** argv) { @@ -52,44 +73,90 @@ int main(int argc, char** argv) { ConfigReader cr; cr.read(&cfg, argc, argv); - ad::cppgtfs::gtfs::Feed gtfs; + std::vector gtfs(cfg.feedPaths.size()); + // feed containing the shapeas in memory for evaluation + ad::cppgtfs::gtfs::Feed evalFeed; - motCfgReader.parse(cfg.configPaths); + std::vector cfgPaths = getCfgPaths(cfg); - if (cfg.osmPath.empty()) { + try { + motCfgReader.parse(cfgPaths); + } catch (configparser::ParseExc ex) { + LOG(ERROR) << "Could not parse MOT configurations, reason was:"; + std::cerr << ex.what() << std::endl; + exit(static_cast(RetCode::MOT_CFG_PARSE_ERR)); + } + + if (cfg.osmPath.empty() && !cfg.writeOverpass) { std::cerr << "No OSM input file specified (-x), see --help." << std::endl; - exit(5); + exit(static_cast(RetCode::NO_OSM_INPUT)); + } + + if (motCfgReader.getConfigs().size() == 0) { + LOG(ERROR) << "No MOT configurations specified and no implicit " + "configurations found, see --help."; + exit(static_cast(RetCode::NO_MOT_CFG)); } if (cfg.feedPaths.size() == 1) { - LOG(INFO) << "Reading " << cfg.feedPaths[0] << " ..."; - ad::cppgtfs::Parser p; - p.parse(>fs, cfg.feedPaths[0]); - LOG(INFO) << "Done."; + if (cfg.inPlace) cfg.outputPath = cfg.feedPaths[0]; + if (!cfg.writeOverpass) + LOG(INFO) << "Reading " << cfg.feedPaths[0] << " ..."; + try { + ad::cppgtfs::Parser p; + p.parse(>fs[0], cfg.feedPaths[0]); + if (cfg.evaluate) { + // read the shapes and store them in memory + p.parseShapes(&evalFeed, cfg.feedPaths[0]); + } + } catch (ad::cppgtfs::ParserException ex) { + LOG(ERROR) << "Could not parse input GTFS feed, reason was:"; + std::cerr << ex.what() << std::endl; + exit(static_cast(RetCode::GTFS_PARSE_ERR)); + } + if (!cfg.writeOverpass) LOG(INFO) << "Done."; + } else if (cfg.writeOsm.size() || cfg.writeOverpass) { + for (size_t i = 0; i < cfg.feedPaths.size(); i++) { + if (!cfg.writeOverpass) + LOG(INFO) << "Reading " << cfg.feedPaths[i] << " ..."; + ad::cppgtfs::Parser p; + try { + p.parse(>fs[i], cfg.feedPaths[i]); + } catch (ad::cppgtfs::ParserException ex) { + LOG(ERROR) << "Could not parse input GTFS feed, reason was:"; + std::cerr << ex.what() << std::endl; + exit(static_cast(RetCode::GTFS_PARSE_ERR)); + } + if (!cfg.writeOverpass) LOG(INFO) << "Done."; + } } else if (cfg.feedPaths.size() > 1) { - std::cerr << "Maximally one input feed allowed." << std::endl; - exit(2); + std::cerr << "Multiple feeds only allowed in filter mode." << std::endl; + exit(static_cast(RetCode::MULT_FEEDS_NOT_ALWD)); } LOG(DEBUG) << "Read " << motCfgReader.getConfigs().size() << " unique MOT configs."; MOTs cmdCfgMots = cfg.mots; - ad::cppgtfs::gtfs::Trip* singleTrip = 0; + pfaedle::gtfs::Trip* singleTrip = 0; if (cfg.shapeTripId.size()) { - singleTrip = gtfs.getTrips().get(cfg.shapeTripId); + if (!cfg.feedPaths.size()) { + std::cout << "No input feed specified, see --help" << std::endl; + exit(static_cast(RetCode::NO_INPUT_FEED)); + } + singleTrip = gtfs[0].getTrips().get(cfg.shapeTripId); if (!singleTrip) { LOG(ERROR) << "Trip #" << cfg.shapeTripId << " not found."; - exit(3); + exit(static_cast(RetCode::TRIP_NOT_FOUND)); } } if (cfg.writeOsm.size()) { LOG(INFO) << "Writing filtered XML to " << cfg.writeOsm << " ..."; - BBoxIdx box(2500); - if (cfg.feedPaths.size()) { - box = ShapeBuilder::getPaddedGtfsBox(>fs, 2500, cmdCfgMots, - cfg.shapeTripId, true); + BBoxIdx box(BOX_PADDING); + for (size_t i = 0; i < cfg.feedPaths.size(); i++) { + ShapeBuilder::getGtfsBox(>fs[i], cmdCfgMots, cfg.shapeTripId, true, + &box); } OsmBuilder osmBuilder; std::vector opts; @@ -99,15 +166,33 @@ int main(int argc, char** argv) { opts.push_back(o.osmBuildOpts); } } - osmBuilder.filterWrite(cfg.osmPath, cfg.writeOsm, opts, box); - exit(0); + try { + osmBuilder.filterWrite(cfg.osmPath, cfg.writeOsm, opts, box); + } catch (xml::XmlFileException ex) { + LOG(ERROR) << "Could not parse OSM data, reason was:"; + std::cerr << ex.what() << std::endl; + exit(static_cast(RetCode::OSM_PARSE_ERR)); + } + exit(static_cast(RetCode::SUCCESS)); + } else if (cfg.writeOverpass) { + BBoxIdx box(BOX_PADDING); + for (size_t i = 0; i < cfg.feedPaths.size(); i++) { + ShapeBuilder::getGtfsBox(>fs[i], cmdCfgMots, cfg.shapeTripId, true, + &box); + } + OsmBuilder osmBuilder; + std::vector opts; + for (const auto& o : motCfgReader.getConfigs()) { + if (std::find_first_of(o.mots.begin(), o.mots.end(), cmdCfgMots.begin(), + cmdCfgMots.end()) != o.mots.end()) { + opts.push_back(o.osmBuildOpts); + } + } + osmBuilder.overpassQryWrite(&std::cout, opts, box); + exit(static_cast(RetCode::SUCCESS)); } else if (!cfg.feedPaths.size()) { std::cout << "No input feed specified, see --help" << std::endl; - exit(1); - } - - if (motCfgReader.getConfigs().size() == 0) { - LOG(WARN) << "No MOT configurations specified, see --help."; + exit(static_cast(RetCode::NO_INPUT_FEED)); } std::vector dfBins; @@ -119,62 +204,76 @@ int main(int argc, char** argv) { std::string filePost; auto usedMots = getContMots(motCfg, cmdCfgMots); if (!usedMots.size()) continue; + if (singleTrip && !usedMots.count(singleTrip->getRoute()->getType())) + continue; if (motCfgReader.getConfigs().size() > 1) filePost = getFileNameMotStr(usedMots); std::string motStr = getMotStr(usedMots); LOG(INFO) << "Calculating shapes for mots " << motStr; - ShapeBuilder shapeBuilder(>fs, cmdCfgMots, motCfg, &ecoll, cfg); + try { + ShapeBuilder shapeBuilder(>fs[0], &evalFeed, cmdCfgMots, motCfg, &ecoll, + cfg); - if (cfg.writeGraph) { - LOG(INFO) << "Outputting graph.json..."; - util::geo::output::GeoGraphJsonOutput out; - std::ofstream fstr(cfg.dbgOutputPath + "/graph.json"); - out.print(*shapeBuilder.getGraph(), fstr); - fstr.close(); - } - - if (singleTrip) { - LOG(INFO) << "Outputting path.json..."; - std::ofstream pstr(cfg.dbgOutputPath + "/path.json"); - util::geo::output::GeoJsonOutput o(pstr); - - auto l = shapeBuilder.shapeL(singleTrip); - - if (singleTrip->getShape()) { - auto orig = Collector::getWebMercLine(singleTrip->getShape(), -1, -1); - o.print(orig, util::json::Dict{{"ver", "old"}}); + if (cfg.writeGraph) { + LOG(INFO) << "Outputting graph.json..."; + util::geo::output::GeoGraphJsonOutput out; + mkdir(cfg.dbgOutputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + std::ofstream fstr(cfg.dbgOutputPath + "/graph.json"); + out.print(*shapeBuilder.getGraph(), fstr); + fstr.close(); } - o.print(l, util::json::Dict{{"ver", "new"}}); - o.flush(); - pstr.close(); + if (singleTrip) { + LOG(INFO) << "Outputting path.json..."; + mkdir(cfg.dbgOutputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + std::ofstream pstr(cfg.dbgOutputPath + "/path.json"); + util::geo::output::GeoJsonOutput o(pstr); - exit(0); - } + auto l = shapeBuilder.shapeL(singleTrip); - pfaedle::netgraph::Graph ng; - shapeBuilder.shape(&ng); + o.print(l, util::json::Dict{{"ver", "new"}}); + o.flush(); + pstr.close(); - if (cfg.buildTransitGraph) { - util::geo::output::GeoGraphJsonOutput out; - LOG(INFO) << "Outputting trgraph" + filePost + ".json..."; - std::ofstream fstr(cfg.dbgOutputPath + "/trgraph" + filePost + ".json"); - out.print(ng, fstr); - fstr.close(); + exit(static_cast(RetCode::SUCCESS)); + } + + pfaedle::netgraph::Graph ng; + shapeBuilder.shape(&ng); + + if (cfg.buildTransitGraph) { + util::geo::output::GeoGraphJsonOutput out; + LOG(INFO) << "Outputting trgraph" + filePost + ".json..."; + mkdir(cfg.dbgOutputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + std::ofstream fstr(cfg.dbgOutputPath + "/trgraph" + filePost + ".json"); + out.print(ng, fstr); + fstr.close(); + } + } catch (xml::XmlFileException ex) { + LOG(ERROR) << "Could not parse OSM data, reason was:"; + std::cerr << ex.what() << std::endl; + exit(static_cast(RetCode::OSM_PARSE_ERR)); } } if (cfg.evaluate) ecoll.printStats(&std::cout); if (cfg.feedPaths.size()) { - LOG(INFO) << "Writing output GTFS to " << cfg.outputPath << " ..."; - ad::cppgtfs::Writer w; - w.write(>fs, cfg.outputPath); + try { + mkdir(cfg.outputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + LOG(INFO) << "Writing output GTFS to " << cfg.outputPath << " ..."; + pfaedle::gtfs::Writer w; + w.write(>fs[0], cfg.outputPath); + } catch (ad::cppgtfs::WriterException ex) { + LOG(ERROR) << "Could not write final GTFS feed, reason was:"; + std::cerr << ex.what() << std::endl; + exit(static_cast(RetCode::GTFS_WRITE_ERR)); + } } - return (0); + return static_cast(RetCode::SUCCESS); } // _____________________________________________________________________________ @@ -183,7 +282,7 @@ std::string getMotStr(const MOTs& mots) { std::string motStr; for (const auto& mot : mots) { if (first) motStr += ", "; - motStr += "<" + Route::getTypeString(mot) + ">"; + motStr += "<" + ad::cppgtfs::gtfs::flat::Route::getTypeString(mot) + ">"; first = true; } @@ -194,7 +293,7 @@ std::string getMotStr(const MOTs& mots) { std::string getFileNameMotStr(const MOTs& mots) { std::string motStr; for (const auto& mot : mots) { - motStr += "-" + Route::getTypeString(mot); + motStr += "-" + ad::cppgtfs::gtfs::flat::Route::getTypeString(mot); } return motStr; @@ -211,3 +310,73 @@ MOTs getContMots(const MotConfig& motCfg, const MOTs& mots) { return ret; } + +// _____________________________________________________________________________ +std::vector getCfgPaths(const Config& cfg) { + if (cfg.configPaths.size()) return cfg.configPaths; + std::vector ret; + + // parse implicit paths + const char* homedir = 0; + char* buf = 0; + + if ((homedir = getenv("HOME")) == 0) { + homedir = ""; + struct passwd pwd; + struct passwd* result; + size_t bufsize; + bufsize = sysconf(_SC_GETPW_R_SIZE_MAX); + if (bufsize == static_cast(-1)) bufsize = 0x4000; + buf = static_cast(malloc(bufsize)); + if (buf != 0) { + getpwuid_r(getuid(), &pwd, buf, bufsize, &result); + if (result != NULL) homedir = result->pw_dir; + } + } + + // install prefix global configuration path, if available + { + auto path = std::string(INSTALL_PREFIX) + + std::string(XDG_CONFIG_DIRS_DEFAULT) + "/" + "pfaedle" + "/" + + CFG_FILE_NAME; + std::ifstream is(path); + + LOG(DEBUG) << "Testing for config file at " << path; + if (is.good()) { + ret.push_back(path); + LOG(DEBUG) << "Found implicit config file " << path; + } + } + + // local user configuration path, if available + { + auto path = std::string(homedir) + XDG_CONFIG_HOME_SUFFIX + "/" + + "pfaedle" + "/" + CFG_FILE_NAME; + std::ifstream is(path); + + LOG(DEBUG) << "Testing for config file at " << path; + if (is.good()) { + ret.push_back(path); + LOG(DEBUG) << "Found implicit config file " << path; + } + } + + if (buf) free(buf); + + // CWD + { + char cwd[PATH_MAX]; + if (getcwd(cwd, sizeof(cwd))) { + auto path = std::string(cwd) + "/" + CFG_FILE_NAME; + std::ifstream is(path); + + LOG(DEBUG) << "Testing for config file at " << path; + if (is.good()) { + ret.push_back(path); + LOG(DEBUG) << "Found implicit config file " << path; + } + } + } + + return ret; +} diff --git a/src/pfaedle/_config.h.in b/src/pfaedle/_config.h.in index 78f0fb9..2fe88eb 100644 --- a/src/pfaedle/_config.h.in +++ b/src/pfaedle/_config.h.in @@ -7,4 +7,7 @@ // version number from cmake version module #define VERSION_FULL "@VERSION_GIT_FULL@" +// version number from cmake version module +#define INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@" + #endif // SRC_PFAEDLE_CONFIG_H_N diff --git a/src/pfaedle/config/ConfigReader.cpp b/src/pfaedle/config/ConfigReader.cpp index 98528cd..d38c347 100644 --- a/src/pfaedle/config/ConfigReader.cpp +++ b/src/pfaedle/config/ConfigReader.cpp @@ -26,67 +26,96 @@ static const char* AUTHORS = "Patrick Brosi "; // _____________________________________________________________________________ void ConfigReader::help(const char* bin) { - std::cout - << std::setfill(' ') << std::left << "pfaedle GTFS map matcher\n" - << VERSION_FULL << " (built " << __DATE__ << " " << __TIME__ - << " with geometry precision <" << PFAEDLE_PRECISION_STR << ">)\n\n" - << "(C) " << YEAR << " " << COPY << "\n" - << "Authors: " << AUTHORS << "\n\n" - << "Usage: " << bin << " -x -c \n\n" - << "Allowed options:\n\n" - << "General:\n" - << std::setw(35) << " -v [ --version ]" - << "print version\n" - << std::setw(35) << " -h [ --help ]" - << "show this help message\n" - << std::setw(35) << " -D [ --drop-shapes ]" - << "drop shapes already present in the feed and recalculate them\n" - << "\nInput:\n" - << std::setw(35) << " -c [ --config ] arg" - << "pfaedle config file\n" - << std::setw(35) << " -i [ --input ] arg" - << "gtfs feed(s), may also be given as positional parameter (see usage)\n" - << std::setw(35) << " -x [ --osm-file ] arg" - << "OSM xml input file\n" - << std::setw(35) << " -m [ --mots ] arg (=all)" - << "MOTs to calculate shapes for, comma separated, either as string " - "{all,\n" - << std::setw(35) << " " - << "tram | streetcar, subway | metro, rail | train, bus, ferry | boat | " - "\n" - << std::setw(35) << " " - << "ship, cableclar, gondola, funicular} or as GTFS mot codes\n" - << "\nOutput:\n" - << std::setw(35) << " -o [ --output ] arg (=gtfs-out)" - << "GTFS output path\n" - << std::setw(35) << " -X [ --osm-out ] arg" - << "if specified, a filtered OSM file will be written to \n" - << "\nDebug Output:\n" - << std::setw(35) << " -d [ --dbg-path ] arg (=geo)" - << "output path for debug files\n" - << std::setw(35) << " --write-trgraph" - << "write transit graph as GeoJSON to /trgraph.json\n" - << std::setw(35) << " --write-graph" - << "write routing graph as GeoJSON to /graph.json\n" - << std::setw(35) << " --write-cgraph" - << "if -T is set, write combination graph as GeoJSON to " - "/combgraph.json\n" - << std::setw(35) << " --method arg (=global)" - << "matching method to use, either 'global' (based on HMM), 'greedy' or " - "'greedy2'\n" - << std::setw(35) << " --eval" - << "evaluate existing shapes against matched shapes and print results\n" - << std::setw(35) << " --eval-path arg (=.)" - << "path for eval file output\n" - << std::setw(35) << " --eval-df-bins arg (= )" - << "bins to use for d_f histogram, comma separated (e.g. 10,20,30,40)\n" - << "\nMisc:\n" - << std::setw(35) << " -T [ --trip-id ] arg" - << "Do routing only for trip , write result to\n" - << std::setw(35) << " " - << "/path.json\n" - << std::setw(35) << " --grid-size arg (=2000)" - << "Grid cell size\n"; + std::cout << std::setfill(' ') << std::left << "pfaedle GTFS map matcher " + << VERSION_FULL << "\n(built " << __DATE__ << " " << __TIME__ + << " with geometry precision <" << PFAEDLE_PRECISION_STR << ">)\n\n" + << "(C) " << YEAR << " " << COPY << "\n" + << "Authors: " << AUTHORS << "\n\n" + << "Usage: " << bin + << " -x \n\n" + << "Allowed options:\n\n" + << "General:\n" + << std::setw(35) << " -v [ --version ]" + << "print version\n" + << std::setw(35) << " -h [ --help ]" + << "show this help message\n" + << std::setw(35) << " -D [ --drop-shapes ]" + << "drop shapes already present in the feed and\n" + << std::setw(35) << " " + << " recalculate them\n" + << "\nInput:\n" + << std::setw(35) << " -c [ --config ] arg" + << "pfaedle config file\n" + << std::setw(35) << " -i [ --input ] arg" + << "gtfs feed(s), may also be given as positional\n" + << std::setw(35) << " " + << " parameter (see usage)\n" + << std::setw(35) << " -x [ --osm-file ] arg" + << "OSM xml input file\n" + << std::setw(35) << " -m [ --mots ] arg (=all)" + << "MOTs to calculate shapes for, comma sep.,\n" + << std::setw(35) << " " + << " either as string " + "{all, tram | streetcar,\n" + << std::setw(35) << " " + << " subway | metro, rail | train, bus,\n" + << std::setw(35) << " " + << " ferry | boat | ship, cablecar, gondola,\n" + << std::setw(35) << " " + << " funicular, coach} or as GTFS mot codes\n" + << "\nOutput:\n" + << std::setw(35) << " -o [ --output ] arg (=gtfs-out)" + << "GTFS output path\n" + << std::setw(35) << " -X [ --osm-out ] arg" + << "if specified, a filtered OSM file will be\n" + << std::setw(35) << " " + << " written to \n" + << std::setw(35) << " --inplace" + << "overwrite input GTFS feed with output feed\n" + << "\nDebug Output:\n" + << std::setw(35) << " -d [ --dbg-path ] arg (=geo)" + << "output path for debug files\n" + << std::setw(35) << " --write-trgraph" + << "write transit graph as GeoJSON to\n" + << std::setw(35) << " " + << " /trgraph.json\n" + << std::setw(35) << " --write-graph" + << "write routing graph as GeoJSON to\n" + << std::setw(35) << " " + << " /graph.json\n" + << std::setw(35) << " --write-cgraph" + << "if -T is set, write combination graph as\n" + << std::setw(35) << " " + << " GeoJSON to " + "/combgraph.json\n" + << std::setw(35) << " --method arg (=global)" + << "matching method to use, either 'global'\n" + << std::setw(35) << " " + << " (based on HMM), 'greedy' or " + "'greedy2'\n" + << std::setw(35) << " --eval" + << "evaluate existing shapes against matched\n" + << std::setw(35) << " " + << " shapes and print results\n" + << std::setw(35) << " --eval-path arg (=.)" + << "path for eval file output\n" + << std::setw(35) << " --eval-df-bins arg (= )" + << "bins to use for d_f histogram, comma sep.\n" + << std::setw(35) << " " + << " (e.g. 10,20,30,40)\n" + << "\nMisc:\n" + << std::setw(35) << " -T [ --trip-id ] arg" + << "Do routing only for trip , write result \n" + << std::setw(35) << " " + << " to /path.json\n" + << std::setw(35) << " --overpass" + << "Output overpass query for matching OSM data\n" + << std::setw(35) << " --grid-size arg (=2000)" + << "Grid cell size\n" + << std::setw(35) << " --use-route-cache" + << "(experimental) cache intermediate routing\n" + << std::setw(35) << " " + << " results\n"; } // _____________________________________________________________________________ @@ -101,6 +130,7 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) { {"drop-shapes", required_argument, 0, 'D'}, {"mots", required_argument, NULL, 'm'}, {"grid-size", required_argument, 0, 'g'}, + {"overpass", no_argument, 0, 'a'}, {"osm-out", required_argument, 0, 'X'}, {"trip-id", required_argument, 0, 'T'}, {"write-graph", no_argument, 0, 1}, @@ -113,6 +143,8 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) { {"dbg-path", required_argument, 0, 'd'}, {"version", no_argument, 0, 'v'}, {"help", no_argument, 0, 'h'}, + {"inplace", no_argument, 0, 9}, + {"use-route-cache", no_argument, 0, 8}, {0, 0, 0, 0}}; char c; @@ -140,6 +172,9 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) { case 7: cfg->evalDfBins = optarg; break; + case 8: + cfg->useCaching = true; + break; case 'o': cfg->outputPath = optarg; break; @@ -170,6 +205,12 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) { case 'd': cfg->dbgOutputPath = optarg; break; + case 'a': + cfg->writeOverpass = true; + break; + case 9: + cfg->inPlace = true; + break; case 'v': std::cout << "pfaedle " << VERSION_FULL << " (built " << __DATE__ << " " << __TIME__ << " with geometry precision <" @@ -204,7 +245,8 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) { auto v = util::split(motStr, ','); for (const auto& motStr : v) { - const auto& mots = Route::getTypesFromString(util::trim(motStr)); + const auto& mots = + ad::cppgtfs::gtfs::flat::Route::getTypesFromString(util::trim(motStr)); cfg->mots.insert(mots.begin(), mots.end()); } diff --git a/src/pfaedle/config/MotConfigReader.cpp b/src/pfaedle/config/MotConfigReader.cpp index dc5ac2e..c31866a 100644 --- a/src/pfaedle/config/MotConfigReader.cpp +++ b/src/pfaedle/config/MotConfigReader.cpp @@ -7,6 +7,7 @@ #include "pfaedle/config/MotConfigReader.h" #include "util/Misc.h" #include "util/String.h" +#include "util/log/Log.h" using pfaedle::config::MotConfigReader; using pfaedle::config::MotConfig; @@ -23,341 +24,336 @@ MotConfigReader::MotConfigReader() {} // _____________________________________________________________________________ void MotConfigReader::parse(const std::vector& paths) { + ConfigFileParser p; + + // parse explicitely given paths for (const auto& s : paths) { - ConfigFileParser p; + LOG(DEBUG) << "Reading config file " << s; p.parse(s); + } - for (const auto& sec : p.getSecs()) { - MotConfig curCfg; - std::string secStr = sec.first; + for (const auto& sec : p.getSecs()) { + MotConfig curCfg; + std::string secStr = sec.first; - if (p.hasKey(secStr, "osm_filter_keep")) { - for (const auto& kvs : p.getStrArr(sec.first, "osm_filter_keep", ' ')) { + if (p.hasKey(secStr, "osm_filter_keep")) { + for (const auto& kvs : p.getStrArr(sec.first, "osm_filter_keep", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.keepFilter[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + } + } + + for (uint8_t i = 0; i < 8; i++) { + std::string name = std::string("osm_filter_lvl") + std::to_string(i); + if (p.hasKey(secStr, name)) { + for (const auto& kvs : p.getStrArr(sec.first, name, ' ')) { auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.keepFilter[fRule.kv.first].insert( + curCfg.osmBuildOpts.levelFilters[i][fRule.kv.first].insert( osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); } } + } - for (uint8_t i = 0; i < 7; i++) { - std::string name = - std::string("osm_filter_lvl") + std::to_string(i + 1); - if (p.hasKey(secStr, name)) { - for (const auto& kvs : p.getStrArr(sec.first, name, ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.levelFilters[i][fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } - } + if (p.hasKey(secStr, "osm_filter_drop")) { + for (const auto& kvs : p.getStrArr(sec.first, "osm_filter_drop", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.dropFilter[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); } + } - if (p.hasKey(secStr, "osm_filter_drop")) { - for (const auto& kvs : p.getStrArr(sec.first, "osm_filter_drop", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.dropFilter[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } + if (p.hasKey(secStr, "osm_max_snap_level")) { + curCfg.osmBuildOpts.maxSnapLevel = + p.getInt(sec.first, "osm_max_snap_level"); + } else { + curCfg.osmBuildOpts.maxSnapLevel = 7; + } + + if (p.hasKey(secStr, "osm_filter_nohup")) { + for (const auto& kvs : p.getStrArr(sec.first, "osm_filter_nohup", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.noHupFilter[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); } + } - if (p.hasKey(secStr, "osm_max_snap_level")) { - curCfg.osmBuildOpts.maxSnapLevel = - p.getInt(sec.first, "osm_max_snap_level"); + if (p.hasKey(secStr, "osm_filter_oneway")) { + for (const auto& kvs : p.getStrArr(sec.first, "osm_filter_oneway", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.oneWayFilter[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + } + } + + if (p.hasKey(secStr, "osm_filter_oneway_reverse")) { + for (const auto& kvs : + p.getStrArr(sec.first, "osm_filter_oneway_reverse", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.oneWayFilterRev[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + } + } + + if (p.hasKey(secStr, "osm_filter_undirected")) { + for (const auto& kvs : + p.getStrArr(sec.first, "osm_filter_undirected", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.twoWayFilter[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + } + } + + if (p.hasKey(secStr, "osm_filter_station")) { + for (const auto& kvs : + p.getStrArr(sec.first, "osm_filter_station", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.stationFilter[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + } + } + + if (p.hasKey(secStr, "osm_filter_station_blocker")) { + for (const auto& kvs : + p.getStrArr(sec.first, "osm_filter_station_blocker", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.stationBlockerFilter[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + } + } + + if (p.hasKey(secStr, "osm_node_positive_restriction")) { + for (const auto& kvs : + p.getStrArr(sec.first, "osm_node_positive_restriction", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.restrPosRestr[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + } + } + + if (p.hasKey(secStr, "osm_node_negative_restriction")) { + for (const auto& kvs : + p.getStrArr(sec.first, "osm_node_negative_restriction", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.restrNegRestr[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + } + } + + if (p.hasKey(secStr, "osm_filter_no_restriction")) { + for (const auto& kvs : + p.getStrArr(sec.first, "osm_filter_no_restriction", ' ')) { + auto fRule = getFRule(kvs); + curCfg.osmBuildOpts.noRestrFilter[fRule.kv.first].insert( + osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + } + } + + if (p.hasKey(secStr, "osm_station_name_attrs")) { + for (const std::string& r : + p.getStrArr(sec.first, "osm_station_name_attrs", ' ')) { + curCfg.osmBuildOpts.statAttrRules.nameRule.push_back( + getDeepAttrRule(r)); + } + } + + if (p.hasKey(secStr, "osm_track_number_tags")) { + for (const std::string& r : + p.getStrArr(sec.first, "osm_track_number_tags", ' ')) { + curCfg.osmBuildOpts.statAttrRules.platformRule.push_back( + getDeepAttrRule(r)); + } + } + + if (p.hasKey(secStr, "osm_edge_track_number_tags")) { + for (const std::string& r : + p.getStrArr(sec.first, "osm_edge_track_number_tags", ' ')) { + curCfg.osmBuildOpts.edgePlatformRules.push_back(getDeepAttrRule(r)); + } + } + + if (p.hasKey(secStr, "osm_station_group_attrs")) { + auto arr = p.getStrArr(secStr, "osm_station_group_attrs", ' '); + + for (const auto& ruleStr : arr) { + auto deep = getDeepAttrRule(ruleStr); + // TODO(patrick): getKv is misused here as a a=b parser + auto attrD = getKv(deep.attr); + deep.attr = attrD.first; + double dist = atof(attrD.second.c_str()); + curCfg.osmBuildOpts.statGroupNAttrRules.push_back({deep, dist}); + } + } + + if (p.hasKey(secStr, "osm_line_relation_tags")) { + auto arr = p.getStrArr(secStr, "osm_line_relation_tags", ' '); + + for (const auto& ruleStr : arr) { + auto rule = getKv(ruleStr); + auto tags = util::split(rule.second, ','); + if (rule.first == "from_name") + curCfg.osmBuildOpts.relLinerules.fromNameRule = tags; + else if (rule.first == "to_name") + curCfg.osmBuildOpts.relLinerules.toNameRule = tags; + else if (rule.first == "line_name") + curCfg.osmBuildOpts.relLinerules.sNameRule = tags; + } + } + + if (p.hasKey(secStr, "osm_max_snap_distance")) { + curCfg.osmBuildOpts.maxSnapDistances = + p.getDoubleArr(secStr, "osm_max_snap_distance", ','); + } else { + curCfg.osmBuildOpts.maxSnapDistances.push_back(50); + } + + if (p.hasKey(secStr, "osm_max_snap_fallback_distance")) { + curCfg.osmBuildOpts.maxSnapFallbackHeurDistance = + p.getDouble(secStr, "osm_max_snap_fallback_distance"); + } else { + curCfg.osmBuildOpts.maxSnapFallbackHeurDistance = + *std::max_element(curCfg.osmBuildOpts.maxSnapDistances.begin(), + curCfg.osmBuildOpts.maxSnapDistances.end()) * + 2; + } + + if (p.hasKey(secStr, "osm_max_osm_station_distance")) { + curCfg.osmBuildOpts.maxOsmStationDistance = + p.getDouble(secStr, "osm_max_osm_station_distance"); + } else { + curCfg.osmBuildOpts.maxOsmStationDistance = 5; + } + + if (p.hasKey(secStr, "osm_max_node_block_distance")) { + curCfg.osmBuildOpts.maxBlockDistance = + p.getDouble(secStr, "osm_max_node_block_distance"); + } else { + curCfg.osmBuildOpts.maxBlockDistance = + *std::max_element(curCfg.osmBuildOpts.maxSnapDistances.begin(), + curCfg.osmBuildOpts.maxSnapDistances.end()) / + 8; + } + + for (uint8_t i = 0; i < 8; i++) { + std::string name = + std::string("routing_lvl") + std::to_string(i) + "_fac"; + if (p.hasKey(secStr, name)) { + double v = p.getDouble(sec.first, name); + curCfg.routingOpts.levelPunish[i] = v; } else { - curCfg.osmBuildOpts.maxSnapLevel = 7; + curCfg.routingOpts.levelPunish[i] = 1; } + } - if (p.hasKey(secStr, "osm_filter_nohup")) { - for (const auto& kvs : - p.getStrArr(sec.first, "osm_filter_nohup", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.noHupFilter[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); + if (p.hasKey(secStr, "routing_full_turn_punish")) { + curCfg.routingOpts.fullTurnPunishFac = + p.getDouble(secStr, "routing_full_turn_punish"); + } + if (p.hasKey(secStr, "routing_full_turn_angle")) { + double ang = p.getDouble(secStr, "routing_full_turn_angle"); + curCfg.routingOpts.fullTurnAngle = ang; + curCfg.osmBuildOpts.fullTurnAngle = ang; + } else { + curCfg.routingOpts.fullTurnAngle = 5; + curCfg.osmBuildOpts.fullTurnAngle = 5; + } + if (p.hasKey(secStr, "routing_snap_full_turn_angle")) { + double ang = p.getDouble(secStr, "routing_snap_full_turn_angle"); + curCfg.osmBuildOpts.maxAngleSnapReach = ang; + } else { + curCfg.osmBuildOpts.maxAngleSnapReach = curCfg.routingOpts.fullTurnAngle; + } + if (p.hasKey(secStr, "routing_pass_thru_station_punish")) { + curCfg.routingOpts.passThruStationsPunish = + p.getDouble(secStr, "routing_pass_thru_station_punish"); + } + if (p.hasKey(secStr, "routing_one_way_meter_punish_fac")) { + curCfg.routingOpts.oneWayPunishFac = + p.getDouble(secStr, "routing_one_way_meter_punish_fac"); + } + if (p.hasKey(secStr, "routing_one_way_edge_punish")) { + curCfg.routingOpts.oneWayEdgePunish = + p.getDouble(secStr, "routing_one_way_edge_punish"); + } + if (p.hasKey(secStr, "routing_line_unmatched_punish_fac")) { + curCfg.routingOpts.lineUnmatchedPunishFact = + p.getDouble(secStr, "routing_line_unmatched_punish_fac"); + } + if (p.hasKey(secStr, "routing_platform_unmatched_punish")) { + curCfg.routingOpts.platformUnmatchedPen = + p.getDouble(secStr, "routing_platform_unmatched_punish"); + } + if (p.hasKey(secStr, "routing_non_osm_station_punish")) { + curCfg.routingOpts.nonOsmPen = + p.getDouble(secStr, "routing_non_osm_station_punish"); + } else { + curCfg.routingOpts.nonOsmPen = 0; + } + if (p.hasKey(secStr, "routing_station_distance_punish_fac")) { + curCfg.routingOpts.stationDistPenFactor = + p.getDouble(secStr, "routing_station_distance_punish_fac"); + } else { + curCfg.routingOpts.stationDistPenFactor = 1; + } + + if (p.hasKey(secStr, "station_normalize_chain")) { + try { + auto arr = p.getStrArr(secStr, "station_normalize_chain", ';'); + curCfg.osmBuildOpts.statNormzer = + trgraph::Normalizer(getNormRules(arr)); + } catch (const std::exception& e) { + throw ParseExc(p.getVal(secStr, "station_normalize_chain").line, + p.getVal(secStr, "station_normalize_chain").pos, + "", + std::string("", + p.getVal(secStr, "station_normalize_chain").file); + } + } + + if (p.hasKey(secStr, "track_normalize_chain")) { + try { + auto arr = p.getStrArr(secStr, "track_normalize_chain", ';'); + curCfg.osmBuildOpts.trackNormzer = + trgraph::Normalizer(getNormRules(arr)); + } catch (const std::exception& e) { + throw ParseExc(p.getVal(secStr, "track_normalize_chain").line, + p.getVal(secStr, "track_normalize_chain").pos, + "", + std::string("", + p.getVal(secStr, "track_normalize_chain").file); + } + } + + if (p.hasKey(secStr, "line_normalize_chain")) { + try { + auto arr = p.getStrArr(secStr, "line_normalize_chain", ';'); + curCfg.osmBuildOpts.lineNormzer = + trgraph::Normalizer(getNormRules(arr)); + } catch (const std::exception& e) { + throw ParseExc(p.getVal(secStr, "station_normalize_chain").line, + p.getVal(secStr, "station_normalize_chain").pos, + "", + std::string("", + p.getVal(secStr, "station_normalize_chain").file); + } + } + + bool found = false; + + for (auto& cfg : _cfgs) { + if (cfg == curCfg) { + for (auto mot : + ad::cppgtfs::gtfs::flat::Route::getTypesFromString(secStr)) { + cfg.mots.insert(mot); } + found = true; + break; } + } - if (p.hasKey(secStr, "osm_filter_oneway")) { - for (const auto& kvs : - p.getStrArr(sec.first, "osm_filter_oneway", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.oneWayFilter[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } - } - - if (p.hasKey(secStr, "osm_filter_oneway_reverse")) { - for (const auto& kvs : - p.getStrArr(sec.first, "osm_filter_oneway_reverse", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.oneWayFilterRev[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } - } - - if (p.hasKey(secStr, "osm_filter_undirected")) { - for (const auto& kvs : - p.getStrArr(sec.first, "osm_filter_undirected", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.twoWayFilter[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } - } - - if (p.hasKey(secStr, "osm_filter_station")) { - for (const auto& kvs : - p.getStrArr(sec.first, "osm_filter_station", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.stationFilter[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } - } - - if (p.hasKey(secStr, "osm_filter_station_blocker")) { - for (const auto& kvs : - p.getStrArr(sec.first, "osm_filter_station_blocker", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.stationBlockerFilter[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } - } - - if (p.hasKey(secStr, "osm_node_positive_restriction")) { - for (const auto& kvs : - p.getStrArr(sec.first, "osm_node_positive_restriction", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.restrPosRestr[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } - } - - if (p.hasKey(secStr, "osm_node_negative_restriction")) { - for (const auto& kvs : - p.getStrArr(sec.first, "osm_node_negative_restriction", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.restrNegRestr[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } - } - - if (p.hasKey(secStr, "osm_filter_no_restriction")) { - for (const auto& kvs : - p.getStrArr(sec.first, "osm_filter_no_restriction", ' ')) { - auto fRule = getFRule(kvs); - curCfg.osmBuildOpts.noRestrFilter[fRule.kv.first].insert( - osm::AttrFlagPair(fRule.kv.second, getFlags(fRule.flags))); - } - } - - if (p.hasKey(secStr, "osm_station_name_attrs")) { - for (const std::string& r : - p.getStrArr(sec.first, "osm_station_name_attrs", ' ')) { - curCfg.osmBuildOpts.statAttrRules.nameRule.push_back( - getDeepAttrRule(r)); - } - } - - if (p.hasKey(secStr, "osm_track_number_tags")) { - for (const std::string& r : - p.getStrArr(sec.first, "osm_track_number_tags", ' ')) { - curCfg.osmBuildOpts.statAttrRules.platformRule.push_back( - getDeepAttrRule(r)); - } - } - - if (p.hasKey(secStr, "osm_edge_track_number_tags")) { - for (const std::string& r : - p.getStrArr(sec.first, "osm_edge_track_number_tags", ' ')) { - curCfg.osmBuildOpts.edgePlatformRules.push_back(getDeepAttrRule(r)); - } - } - - if (p.hasKey(secStr, "osm_station_group_attrs")) { - auto arr = p.getStrArr(secStr, "osm_station_group_attrs", ' '); - - for (const auto& ruleStr : arr) { - auto deep = getDeepAttrRule(ruleStr); - // TODO(patrick): getKv is misused here as a a=b parser - auto attrD = getKv(deep.attr); - deep.attr = attrD.first; - double dist = atof(attrD.second.c_str()); - curCfg.osmBuildOpts.statGroupNAttrRules.push_back({deep, dist}); - } - } - - if (p.hasKey(secStr, "osm_line_relation_tags")) { - auto arr = p.getStrArr(secStr, "osm_line_relation_tags", ' '); - - for (const auto& ruleStr : arr) { - auto rule = getKv(ruleStr); - auto tags = util::split(rule.second, ','); - if (rule.first == "from_name") - curCfg.osmBuildOpts.relLinerules.fromNameRule = tags; - else if (rule.first == "to_name") - curCfg.osmBuildOpts.relLinerules.toNameRule = tags; - else if (rule.first == "line_name") - curCfg.osmBuildOpts.relLinerules.sNameRule = tags; - } - } - - if (p.hasKey(secStr, "osm_max_snap_distance")) { - curCfg.osmBuildOpts.maxSnapDistances = - p.getDoubleArr(secStr, "osm_max_snap_distance", ','); - } else { - curCfg.osmBuildOpts.maxSnapDistances.push_back(50); - } - - if (p.hasKey(secStr, "osm_max_snap_fallback_distance")) { - curCfg.osmBuildOpts.maxSnapFallbackHeurDistance = - p.getDouble(secStr, "osm_max_snap_fallback_distance"); - } else { - curCfg.osmBuildOpts.maxSnapFallbackHeurDistance = - *std::max_element(curCfg.osmBuildOpts.maxSnapDistances.begin(), - curCfg.osmBuildOpts.maxSnapDistances.end()) * - 2; - } - - if (p.hasKey(secStr, "osm_max_group_search_distance")) { - curCfg.osmBuildOpts.maxGroupSearchDistance = - p.getDouble(secStr, "osm_max_group_search_distance"); - } else { - curCfg.osmBuildOpts.maxGroupSearchDistance = - *std::max_element(curCfg.osmBuildOpts.maxSnapDistances.begin(), - curCfg.osmBuildOpts.maxSnapDistances.end()) * - 4; - } - - if (p.hasKey(secStr, "osm_max_osm_station_distance")) { - curCfg.osmBuildOpts.maxOsmStationDistance = - p.getDouble(secStr, "osm_max_osm_station_distance"); - } else { - curCfg.osmBuildOpts.maxOsmStationDistance = 5; - } - - if (p.hasKey(secStr, "osm_max_node_block_distance")) { - curCfg.osmBuildOpts.maxBlockDistance = - p.getDouble(secStr, "osm_max_node_block_distance"); - } else { - curCfg.osmBuildOpts.maxBlockDistance = - *std::max_element(curCfg.osmBuildOpts.maxSnapDistances.begin(), - curCfg.osmBuildOpts.maxSnapDistances.end()) / - 8; - } - - for (uint8_t i = 0; i < 8; i++) { - std::string name = - std::string("routing_lvl") + std::to_string(i) + "_fac"; - if (p.hasKey(secStr, name)) { - double v = p.getDouble(sec.first, name); - curCfg.routingOpts.levelPunish[i] = v; - } else { - curCfg.routingOpts.levelPunish[i] = 1; - } - } - - if (p.hasKey(secStr, "routing_full_turn_punish")) { - curCfg.routingOpts.fullTurnPunishFac = - p.getDouble(secStr, "routing_full_turn_punish"); - } - if (p.hasKey(secStr, "routing_full_turn_angle")) { - double ang = p.getDouble(secStr, "routing_full_turn_angle"); - curCfg.routingOpts.fullTurnAngle = ang; - } else { - curCfg.routingOpts.fullTurnAngle = 5; - } - if (p.hasKey(secStr, "routing_snap_full_turn_angle")) { - double ang = p.getDouble(secStr, "routing_snap_full_turn_angle"); - curCfg.osmBuildOpts.maxAngleSnapReach = ang; - } else { - curCfg.osmBuildOpts.maxAngleSnapReach = - curCfg.routingOpts.fullTurnAngle; - } - if (p.hasKey(secStr, "routing_pass_thru_station_punish")) { - curCfg.routingOpts.passThruStationsPunish = - p.getDouble(secStr, "routing_pass_thru_station_punish"); - } - if (p.hasKey(secStr, "routing_one_way_meter_punish_fac")) { - curCfg.routingOpts.oneWayPunishFac = - p.getDouble(secStr, "routing_one_way_meter_punish_fac"); - } - if (p.hasKey(secStr, "routing_one_way_edge_punish")) { - curCfg.routingOpts.oneWayEdgePunish = - p.getDouble(secStr, "routing_one_way_edge_punish"); - } - if (p.hasKey(secStr, "routing_line_unmatched_punish_fac")) { - curCfg.routingOpts.lineUnmatchedPunishFact = - p.getDouble(secStr, "routing_line_unmatched_punish_fac"); - } - if (p.hasKey(secStr, "routing_platform_unmatched_punish")) { - curCfg.routingOpts.platformUnmatchedPen = - p.getDouble(secStr, "routing_platform_unmatched_punish"); - } - if (p.hasKey(secStr, "routing_non_osm_station_punish")) { - curCfg.routingOpts.nonOsmPen = - p.getDouble(secStr, "routing_non_osm_station_punish"); - } else { - curCfg.routingOpts.nonOsmPen = 0; - } - if (p.hasKey(secStr, "routing_station_distance_punish_fac")) { - curCfg.routingOpts.stationDistPenFactor = - p.getDouble(secStr, "routing_station_distance_punish_fac"); - } else { - curCfg.routingOpts.stationDistPenFactor = 1; - } - - if (p.hasKey(secStr, "station_normalize_chain")) { - try { - auto arr = p.getStrArr(secStr, "station_normalize_chain", ';'); - curCfg.osmBuildOpts.statNormzer = - trgraph::Normalizer(getNormRules(arr)); - } catch (const std::exception& e) { - throw ParseExc(p.getVal(secStr, "station_normalize_chain").line, - p.getVal(secStr, "station_normalize_chain").pos, - "", - std::string("", s); - } - } - - if (p.hasKey(secStr, "track_normalize_chain")) { - try { - auto arr = p.getStrArr(secStr, "track_normalize_chain", ';'); - curCfg.osmBuildOpts.trackNormzer = - trgraph::Normalizer(getNormRules(arr)); - } catch (const std::exception& e) { - throw ParseExc(p.getVal(secStr, "track_normalize_chain").line, - p.getVal(secStr, "station_normalize_chain").pos, - "", - std::string("", s); - } - } - - if (p.hasKey(secStr, "line_normalize_chain")) { - try { - auto arr = p.getStrArr(secStr, "line_normalize_chain", ';'); - curCfg.osmBuildOpts.lineNormzer = - trgraph::Normalizer(getNormRules(arr)); - } catch (const std::exception& e) { - throw ParseExc(p.getVal(secStr, "station_normalize_chain").line, - p.getVal(secStr, "station_normalize_chain").pos, - "", - std::string("", s); - } - } - - bool found = false; - - for (auto& cfg : _cfgs) { - if (cfg == curCfg) { - for (auto mot : Route::getTypesFromString(secStr)) { - cfg.mots.insert(mot); - } - found = true; - break; - } - } - - if (!found) { - curCfg.mots = Route::getTypesFromString(secStr); - _cfgs.push_back(curCfg); - } + if (!found) { + curCfg.mots = ad::cppgtfs::gtfs::flat::Route::getTypesFromString(secStr); + _cfgs.push_back(curCfg); } } } diff --git a/src/pfaedle/config/MotConfigReader.h b/src/pfaedle/config/MotConfigReader.h index 912a148..2e7fee2 100644 --- a/src/pfaedle/config/MotConfigReader.h +++ b/src/pfaedle/config/MotConfigReader.h @@ -5,6 +5,30 @@ #ifndef PFAEDLE_CONFIG_MOTCONFIGREADER_H_ #define PFAEDLE_CONFIG_MOTCONFIGREADER_H_ +#include "pfaedle/_config.h" + +#ifndef HOME_VAR +#define HOME_VAR "HOME" +#endif +#ifndef XDG_DATA_HOME_SUFFIX +#define XDG_DATA_HOME_SUFFIX "/.local/share" +#endif +#ifndef XDG_CONFIG_HOME_SUFFIX +#define XDG_CONFIG_HOME_SUFFIX "/.config" +#endif +#ifndef XDG_CACHE_HOME_SUFFIX +#define XDG_CACHE_HOME_SUFFIX "/.cache" +#endif +#ifndef XDG_DATA_DIRS_DEFAULT +#define XDG_DATA_DIRS_DEFAULT "/usr/local/share" +#endif +#ifndef XDG_CONFIG_DIRS_DEFAULT +#define XDG_CONFIG_DIRS_DEFAULT "/etc" +#endif +#ifndef CFG_FILE_NAME +#define CFG_FILE_NAME "pfaedle.cfg" +#endif + #include #include #include diff --git a/src/pfaedle/config/PfaedleConfig.h b/src/pfaedle/config/PfaedleConfig.h index ba9a2a0..f209c0c 100644 --- a/src/pfaedle/config/PfaedleConfig.h +++ b/src/pfaedle/config/PfaedleConfig.h @@ -28,6 +28,9 @@ struct Config { writeCombGraph(false), evaluate(false), buildTransitGraph(false), + useCaching(false), + writeOverpass(false), + inPlace(false), gridSize(2000) {} std::string dbgOutputPath; std::string solveMethod; @@ -46,6 +49,9 @@ struct Config { bool writeCombGraph; bool evaluate; bool buildTransitGraph; + bool useCaching; + bool writeOverpass; + bool inPlace; double gridSize; std::string toString() { @@ -60,6 +66,8 @@ struct Config { << "write-graph: " << writeGraph << "\n" << "write-cgraph: " << writeCombGraph << "\n" << "grid-size: " << gridSize << "\n" + << "use-cache: " << useCaching << "\n" + << "write-overpass: " << writeOverpass << "\n" << "feed-paths: "; for (const auto& p : feedPaths) { diff --git a/src/pfaedle/eval/Collector.cpp b/src/pfaedle/eval/Collector.cpp index 14bbb34..07c1ae8 100644 --- a/src/pfaedle/eval/Collector.cpp +++ b/src/pfaedle/eval/Collector.cpp @@ -18,14 +18,14 @@ using util::geo::PolyLine; -using ad::cppgtfs::gtfs::Trip; +using pfaedle::gtfs::Trip; using ad::cppgtfs::gtfs::Shape; using pfaedle::eval::Collector; using pfaedle::eval::Result; using util::geo::output::GeoJsonOutput; // _____________________________________________________________________________ -double Collector::add(const Trip* t, const Shape* oldS, const Shape* newS, +double Collector::add(const Trip* t, const Shape* oldS, const Shape& newS, const std::vector& newTripDists) { if (!oldS) { _noOrigShp++; @@ -51,7 +51,7 @@ double Collector::add(const Trip* t, const Shape* oldS, const Shape* newS, (--t->getStopTimes().end())->getShapeDistanceTravelled(), &oldDists); std::vector newDists; - LINE newL = getWebMercLine(newS, -1, -1, &newDists); + LINE newL = getWebMercLine(&newS, -1, -1, &newDists); std::ofstream fstr(_evalOutPath + "/trip-" + t->getId() + ".json"); GeoJsonOutput gjout(fstr); @@ -123,19 +123,19 @@ double Collector::add(const Trip* t, const Shape* oldS, const Shape* newS, 6378137.0)) - 1.5707965); - if (_dCache.count(oldS) && _dCache.find(oldS)->second.count(newS)) { - fd = _dCache[oldS][newS]; + if (_dCache.count(oldS) && _dCache.find(oldS)->second.count(newS.getId())) { + fd = _dCache[oldS][newS.getId()]; } else { fd = util::geo::accFrechetDistC(oldLCut, newLCut, 5 / fac) * fac; - _dCache[oldS][newS] = fd; + _dCache[oldS][newS.getId()] = fd; } - if (_dACache.count(oldS) && _dACache.find(oldS)->second.count(newS)) { - unmatchedSegments = _dACache[oldS][newS].first; - unmatchedSegmentsLength = _dACache[oldS][newS].second; + if (_dACache.count(oldS) && _dACache.find(oldS)->second.count(newS.getId())) { + unmatchedSegments = _dACache[oldS][newS.getId()].first; + unmatchedSegmentsLength = _dACache[oldS][newS.getId()].second; } else { auto dA = getDa(oldSegs, newSegs); - _dACache[oldS][newS] = dA; + _dACache[oldS][newS.getId()] = dA; unmatchedSegments = dA.first; unmatchedSegmentsLength = dA.second; } @@ -199,6 +199,8 @@ std::vector Collector::segmentize( // get first half of geometry, and search for start point there! size_t before = std::upper_bound(dists.begin(), dists.end(), cuts[1].second) - dists.begin(); + if (before + 1 > shape.size()) before = shape.size() - 1; + assert(shape.begin() + before + 1 <= shape.end()); POLYLINE l(LINE(shape.begin(), shape.begin() + before + 1)); auto lastLp = l.projectOn(cuts.front().first); diff --git a/src/pfaedle/eval/Collector.h b/src/pfaedle/eval/Collector.h index 0bbc208..853314e 100644 --- a/src/pfaedle/eval/Collector.h +++ b/src/pfaedle/eval/Collector.h @@ -12,11 +12,12 @@ #include #include #include "ad/cppgtfs/gtfs/Feed.h" +#include "pfaedle/gtfs/Feed.h" #include "pfaedle/Def.h" #include "pfaedle/eval/Result.h" #include "util/geo/Geo.h" -using ad::cppgtfs::gtfs::Trip; +using pfaedle::gtfs::Trip; using ad::cppgtfs::gtfs::Shape; namespace pfaedle { @@ -37,7 +38,7 @@ class Collector { // Add a shape found by our tool newS for a trip t with newly calculated // station dist values with the old shape oldS - double add(const Trip* t, const Shape* oldS, const Shape* newS, + double add(const Trip* t, const Shape* oldS, const Shape& newS, const std::vector& newDists); // Return the set of all Result objects @@ -65,8 +66,8 @@ class Collector { std::set _results; std::set _resultsAN; std::set _resultsAL; - std::map > _dCache; - std::map > > + std::map > _dCache; + std::map > > _dACache; size_t _noOrigShp; diff --git a/src/pfaedle/eval/Result.h b/src/pfaedle/eval/Result.h index 0bfa667..b039234 100644 --- a/src/pfaedle/eval/Result.h +++ b/src/pfaedle/eval/Result.h @@ -5,9 +5,10 @@ #ifndef PFAEDLE_EVAL_RESULT_H_ #define PFAEDLE_EVAL_RESULT_H_ +#include "pfaedle/gtfs/Feed.h" #include "ad/cppgtfs/gtfs/Feed.h" -using ad::cppgtfs::gtfs::Trip; +using pfaedle::gtfs::Trip; using ad::cppgtfs::gtfs::Shape; namespace pfaedle { diff --git a/src/pfaedle/gtfs/Feed.h b/src/pfaedle/gtfs/Feed.h new file mode 100644 index 0000000..8e5cc7b --- /dev/null +++ b/src/pfaedle/gtfs/Feed.h @@ -0,0 +1,37 @@ +// Copyright 2016, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#ifndef PFAEDLE_GTFS_FEED_H_ +#define PFAEDLE_GTFS_FEED_H_ + +#include +#include "Route.h" +#include "Service.h" +#include "ShapeContainer.h" +#include "StopTime.h" +#include "ad/cppgtfs/gtfs/ContContainer.h" +#include "ad/cppgtfs/gtfs/Feed.h" +#include "ad/cppgtfs/gtfs/NullContainer.h" +#include "ad/cppgtfs/gtfs/Stop.h" +#include "ad/cppgtfs/gtfs/StopTime.h" +#include "ad/cppgtfs/gtfs/Trip.h" + +namespace pfaedle { +namespace gtfs { + +typedef ad::cppgtfs::gtfs::FeedB< + ad::cppgtfs::gtfs::Agency, Route, ad::cppgtfs::gtfs::Stop, Service, + StopTime, Shape, ad::cppgtfs::gtfs::Fare, ad::cppgtfs::gtfs::Container, + ad::cppgtfs::gtfs::ContContainer, ad::cppgtfs::gtfs::NullContainer, + ad::cppgtfs::gtfs::ContContainer, ad::cppgtfs::gtfs::ContContainer, + ShapeContainer, ad::cppgtfs::gtfs::NullContainer> + Feed; +typedef ad::cppgtfs::gtfs::TripB, Service, + Route, Shape> + Trip; + +} // namespace gtfs +} // namespace pfaedle + +#endif // PFAEDLE_GTFS_FEED_H_ diff --git a/src/pfaedle/gtfs/Route.h b/src/pfaedle/gtfs/Route.h new file mode 100644 index 0000000..15ba7f0 --- /dev/null +++ b/src/pfaedle/gtfs/Route.h @@ -0,0 +1,61 @@ +// Copyright 2016, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#ifndef PFAEDLE_GTFS_ROUTE_H_ +#define PFAEDLE_GTFS_ROUTE_H_ + +#include +#include +#include +#include +#include +#include +#include "ad/cppgtfs/gtfs/Agency.h" +#include "ad/cppgtfs/gtfs/Route.h" +#include "util/Misc.h" + +using std::exception; +using std::string; + +namespace pfaedle { +namespace gtfs { + +class Route { + public: + typedef Route* Ref; + static std::string getId(Ref r) { return r->getId(); } + + Route() {} + + Route(const string& id, ad::cppgtfs::gtfs::Agency* agency, + const string& short_name, const string& long_name, const string& desc, + ad::cppgtfs::gtfs::flat::Route::TYPE type, const string& url, + uint32_t color, uint32_t text_color) + : _id(id), _short_name(short_name), _long_name(long_name), _type(type) { + UNUSED(agency); + UNUSED(desc); + UNUSED(url); + UNUSED(color); + UNUSED(text_color); + } + + const std::string& getId() const { return _id; } + + const std::string& getShortName() const { return _short_name; } + + const std::string& getLongName() const { return _long_name; } + + ad::cppgtfs::gtfs::flat::Route::TYPE getType() const { return _type; } + + private: + string _id; + string _short_name; + string _long_name; + ad::cppgtfs::gtfs::flat::Route::TYPE _type; +}; + +} // namespace gtfs +} // namespace pfaedle + +#endif // PFAEDLE_GTFS_ROUTE_H_ diff --git a/src/pfaedle/gtfs/Service.h b/src/pfaedle/gtfs/Service.h new file mode 100644 index 0000000..245ce31 --- /dev/null +++ b/src/pfaedle/gtfs/Service.h @@ -0,0 +1,43 @@ +// Copyright 2016, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#ifndef PFAEDLE_GTFS_SERVICE_H_ +#define PFAEDLE_GTFS_SERVICE_H_ + +#include +#include "ad/cppgtfs/gtfs/Service.h" +#include "util/Misc.h" + +namespace pfaedle { +namespace gtfs { + +class Service { + public: + typedef std::string Ref; + static std::string getId(Ref r) { return r; } + + explicit Service(const string& id) : _id(id) {} + Service(const string& id, uint8_t serviceDays, + ad::cppgtfs::gtfs::ServiceDate start, + ad::cppgtfs::gtfs::ServiceDate end) + : _id(id) { + UNUSED(serviceDays); + UNUSED(start); + UNUSED(end); + } + + const std::string& getId() const { return _id; } + void addException(const ad::cppgtfs::gtfs::ServiceDate& d, + ad::cppgtfs::gtfs::Service::EXCEPTION_TYPE t) { + UNUSED(d); + UNUSED(t); + } + + private: + std::string _id; +}; +} // namespace gtfs +} // namespace pfaedle + +#endif // PFAEDLE_GTFS_SERVICE_H_ diff --git a/src/pfaedle/gtfs/ShapeContainer.h b/src/pfaedle/gtfs/ShapeContainer.h new file mode 100644 index 0000000..6a675b8 --- /dev/null +++ b/src/pfaedle/gtfs/ShapeContainer.h @@ -0,0 +1,69 @@ +// Copyright 2018, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#ifndef PFAEDLE_GTFS_SHAPECONTAINER_H_ +#define PFAEDLE_GTFS_SHAPECONTAINER_H_ + +#include +#include +#include +#include +#include +#include "ad/cppgtfs/gtfs/Shape.h" +#include "ad/cppgtfs/gtfs/flat/Shape.h" +#include "util/Misc.h" + +namespace pfaedle { +namespace gtfs { + +struct Shape { + explicit Shape(const std::string& id) : id(id) {} + typedef std::string Ref; + static std::string getId(Ref r) { return r; } + + template + bool addPoint(T p) { + UNUSED(p); + return true; + } + + const std::string& getId() const { return id; } + + std::string id; +}; + +template +class ShapeContainer { + public: + ShapeContainer(); + ~ShapeContainer(); + T* add(const T& obj); + bool remove(const std::string& id); + const T* get(const std::string& id) const; + T* get(const std::string& id); + const std::string getRef(const std::string& id) const; + std::string getRef(const std::string& id); + size_t size() const; + void finalize() {} + bool has(const std::string& id) const; + + std::string add(const ad::cppgtfs::gtfs::Shape& s); + void open(); + bool nextStoragePt(ad::cppgtfs::gtfs::flat::ShapePoint* ret); + + private: + std::set _ids; + std::fstream _storage; + size_t _ptr; + size_t _max; + std::string _curId; + std::stringstream _writeBuffer; +}; + +#include "ShapeContainer.tpp" + +} // namespace gtfs +} // namespace pfaedle + +#endif // PFAEDLE_GTFS_SHAPECONTAINER_H_ diff --git a/src/pfaedle/gtfs/ShapeContainer.tpp b/src/pfaedle/gtfs/ShapeContainer.tpp new file mode 100644 index 0000000..081a47b --- /dev/null +++ b/src/pfaedle/gtfs/ShapeContainer.tpp @@ -0,0 +1,154 @@ +// Copyright 2018, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#include +#include + +// ____________________________________________________________________________ +template +ShapeContainer::ShapeContainer() { + std::string f = ".pfaedle-tmp"; + + while (access(f.c_str(), F_OK) != -1) { + std::stringstream ss; + ss << ".pfaedle-tmp-"; + ss << std::rand(); + f = ss.str().c_str(); + } + + _storage.open(f, std::fstream::in | std::fstream::out | std::fstream::trunc); + + // immediately unlink + unlink(f.c_str()); + + if (!_storage.good()) { + std::cerr << "Could not open temporary file " << f << std::endl; + exit(1); + } +} + +// ____________________________________________________________________________ +template +ShapeContainer::~ShapeContainer() { + _storage.close(); +} + +// ____________________________________________________________________________ +template +T* ShapeContainer::add(const T& ent) { + _ids.insert(ent.getId()); + return reinterpret_cast(1); +} + +// ____________________________________________________________________________ +template +bool ShapeContainer::remove(const std::string& id) { + _ids.erase(id); + return true; +} + +// ____________________________________________________________________________ +template +T* ShapeContainer::get(const std::string& id) { + if (!has(id)) return 0; + return reinterpret_cast(1); +} + +// ____________________________________________________________________________ +template +const T* ShapeContainer::get(const std::string& id) const { + if (!has(id)) return 0; + return reinterpret_cast(1); +} + +// ____________________________________________________________________________ +template +bool ShapeContainer::has(const std::string& id) const { + return _ids.count(id); +} + +// ____________________________________________________________________________ +template +size_t ShapeContainer::size() const { + return _ids.size(); +} + +// ____________________________________________________________________________ +template +std::string ShapeContainer::add(const ad::cppgtfs::gtfs::Shape& s) { + if (has(s.getId())) return s.getId(); + _ids.insert(s.getId()); + + _writeBuffer << s.getId() << '\t' << s.getPoints().size(); + _writeBuffer << std::setprecision(11); + for (auto p : s.getPoints()) { + _writeBuffer << " " << p.lat << " " << p.lng << " " << p.travelDist; + } + // entries are newline separated + _writeBuffer << '\n'; + + if (_writeBuffer.tellp() > 1000 * 5000) { + _storage << _writeBuffer.rdbuf(); + _writeBuffer.clear(); + } + + return s.getId(); +} + +// ____________________________________________________________________________ +template +void ShapeContainer::open() { + _storage << _writeBuffer.rdbuf(); + _writeBuffer.clear(); + + _ptr = 0; + _max = 0; + _storage.clear(); + _storage.seekg(0, std::ios::beg); +} + +// ____________________________________________________________________________ +template +bool ShapeContainer::nextStoragePt( + ad::cppgtfs::gtfs::flat::ShapePoint* ret) { + while (_storage.good() && !_storage.fail()) { + if (!_ptr) { + _storage >> _curId; + _storage >> _max; + } + + if (!_storage.good() || _storage.fail()) return false; + + _storage >> ret->lat; + _storage >> ret->lng; + _storage >> ret->travelDist; + ret->seq = _ptr + 1; + ret->id = _curId; + + if (_ptr + 1 == _max) + _ptr = 0; + else + _ptr++; + + if (!_storage.good() || _storage.fail()) return false; + + if (has(ret->id)) return true; + } + + return false; +} + +// ____________________________________________________________________________ +template +const std::string ShapeContainer::getRef(const std::string& id) const { + if (!has(id)) return ""; + return id; +} + +// ____________________________________________________________________________ +template +std::string ShapeContainer::getRef(const std::string& id) { + if (!has(id)) return ""; + return id; +} diff --git a/src/pfaedle/gtfs/StopTime.h b/src/pfaedle/gtfs/StopTime.h new file mode 100644 index 0000000..fe18a52 --- /dev/null +++ b/src/pfaedle/gtfs/StopTime.h @@ -0,0 +1,71 @@ +// Copyright 2016, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#ifndef PFAEDLE_GTFS_STOPTIME_H_ +#define PFAEDLE_GTFS_STOPTIME_H_ + +#include +#include +#include +#include +#include "ad/cppgtfs/gtfs/Stop.h" +#include "ad/cppgtfs/gtfs/StopTime.h" +#include "util/Misc.h" + +using std::exception; +using std::string; + +namespace pfaedle { +namespace gtfs { + +template +class StopTime { + public: + StopTime(const ad::cppgtfs::gtfs::Time& at, const ad::cppgtfs::gtfs::Time& dt, + typename StopT::Ref s, uint32_t seq, const std::string& hs, + ad::cppgtfs::gtfs::flat::StopTime::PU_DO_TYPE put, + ad::cppgtfs::gtfs::flat::StopTime::PU_DO_TYPE dot, float distTrav, + bool isTp) + : _s(s), _sequence(seq), _dist(distTrav) { + UNUSED(at); + UNUSED(dt); + UNUSED(hs); + UNUSED(put); + UNUSED(dot); + UNUSED(distTrav); + UNUSED(isTp); + } + + const typename StopT::Ref getStop() const { return _s; } + typename StopT::Ref getStop() { return _s; } + void setShapeDistanceTravelled(double d) { _dist = d; } + + ad::cppgtfs::gtfs::Time getArrivalTime() const { + return ad::cppgtfs::gtfs::Time(0, 0, 0); + } + ad::cppgtfs::gtfs::Time getDepartureTime() const { + return ad::cppgtfs::gtfs::Time(0, 0, 0); + } + + float getShapeDistanceTravelled() const { return _dist; } + + uint16_t getSeq() const { return _sequence; } + + private: + typename StopT::Ref _s; + uint32_t _sequence; + float _dist; +}; + +template +struct StopTimeCompare { + bool operator()(const StopTimeT& lh, const StopTimeT& rh) const { + return lh.getSeq() < rh.getSeq(); + } +}; + +} // namespace gtfs +} // namespace pfaedle + +#endif // PFAEDLE_GTFS_STOPTIME_H_ diff --git a/src/pfaedle/gtfs/Writer.cpp b/src/pfaedle/gtfs/Writer.cpp new file mode 100644 index 0000000..de5a856 --- /dev/null +++ b/src/pfaedle/gtfs/Writer.cpp @@ -0,0 +1,495 @@ +// Copyright 2018, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#include +#include +#include +#include +#include +#include "ad/cppgtfs/Parser.h" +#include "ad/cppgtfs/Writer.h" +#include "ad/cppgtfs/gtfs/flat/Agency.h" +#include "ad/util/CsvWriter.h" +#include "pfaedle/gtfs/Writer.h" + +using ad::util::CsvWriter; +using ad::cppgtfs::Parser; +using pfaedle::gtfs::Writer; + +// ____________________________________________________________________________ +bool Writer::write(gtfs::Feed* sourceFeed, const std::string& path) const { + std::ofstream fs; + std::ifstream is; + std::string gtfsPath(path); + std::string curFile; + std::string curFileTg; + + curFile = gtfsPath + "/.agency.txt"; + curFileTg = gtfsPath + "/agency.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeAgency(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + + curFile = gtfsPath + "/.stops.txt"; + curFileTg = gtfsPath + "/stops.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeStops(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + + curFile = gtfsPath + "/.routes.txt"; + curFileTg = gtfsPath + "/routes.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeRoutes(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + + is.open((sourceFeed->getPath() + "/calendar.txt").c_str()); + if (is.good()) { + is.close(); + curFile = gtfsPath + "/.calendar.txt"; + curFileTg = gtfsPath + "/calendar.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeCalendar(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + } + + is.open((sourceFeed->getPath() + "/calendar_dates.txt").c_str()); + if (is.good()) { + is.close(); + curFile = gtfsPath + "/.calendar_dates.txt"; + curFileTg = gtfsPath + "/calendar_dates.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeCalendarDates(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + } + + is.open((sourceFeed->getPath() + "/transfers.txt").c_str()); + if (is.good()) { + is.close(); + curFile = gtfsPath + "/.transfers.txt"; + curFileTg = gtfsPath + "/transfers.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeTransfers(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + } + + is.open((sourceFeed->getPath() + "/fare_attributes.txt").c_str()); + if (is.good()) { + is.close(); + curFile = gtfsPath + "/.fare_attributes.txt"; + curFileTg = gtfsPath + "/fare_attributes.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeFares(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + } + + is.open((sourceFeed->getPath() + "/fare_rules.txt").c_str()); + if (is.good()) { + is.close(); + curFile = gtfsPath + "/.fare_rules.txt"; + curFileTg = gtfsPath + "/fare_rules.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeFareRules(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + } + + is.close(); + curFile = gtfsPath + "/.shapes.txt"; + curFileTg = gtfsPath + "/shapes.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeShapes(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + + is.close(); + curFile = gtfsPath + "/.trips.txt"; + curFileTg = gtfsPath + "/trips.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + bool hasFreqs = writeTrips(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + + is.open((sourceFeed->getPath() + "/frequencies.txt").c_str()); + if (hasFreqs && is.good()) { + is.close(); + curFile = gtfsPath + "/.frequencies.txt"; + curFileTg = gtfsPath + "/frequencies.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeFrequencies(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + } + + is.close(); + curFile = gtfsPath + "/.stop_times.txt"; + curFileTg = gtfsPath + "/stop_times.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeStopTimes(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + + if (!sourceFeed->getPublisherUrl().empty() && + !sourceFeed->getPublisherName().empty()) { + curFile = gtfsPath + "/.feed_info.txt"; + curFileTg = gtfsPath + "/feed_info.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeFeedInfo(sourceFeed, &fs); + fs.close(); + std::rename(curFile.c_str(), curFileTg.c_str()); + } + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeFeedInfo(gtfs::Feed* f, std::ostream* os) const { + auto csvw = ad::cppgtfs::Writer::getFeedInfoCsvw(os); + csvw.flushLine(); + csvw.writeString(f->getPublisherName()); + csvw.writeString(f->getPublisherUrl()); + csvw.writeString(f->getLang()); + if (!f->getStartDate().empty()) + csvw.writeInt(f->getStartDate().getYYYYMMDD()); + else + csvw.skip(); + if (!f->getEndDate().empty()) + csvw.writeInt(f->getEndDate().getYYYYMMDD()); + else + csvw.skip(); + csvw.writeString(f->getVersion()); + csvw.flushLine(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeAgency(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/agency.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getAgencyCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::Agency fa; + auto flds = Parser::getAgencyFlds(&csvp); + + while (p.nextAgency(&csvp, &fa, flds)) { + w.writeAgency(fa, &csvw); + } + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeStops(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/stops.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getStopsCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::Stop s; + auto flds = Parser::getStopFlds(&csvp); + + while (p.nextStop(&csvp, &s, flds)) { + w.writeStop(s, &csvw); + } + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeRoutes(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/routes.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getRoutesCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::Route s; + auto flds = Parser::getRouteFlds(&csvp); + + while (p.nextRoute(&csvp, &s, flds)) { + w.writeRoute(s, &csvw); + } + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeCalendar(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/calendar.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getCalendarCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::Calendar c; + auto flds = Parser::getCalendarFlds(&csvp); + + while (p.nextCalendar(&csvp, &c, flds)) { + w.writeCalendar(c, &csvw); + } + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeCalendarDates(gtfs::Feed* sourceFeed, + std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/calendar_dates.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getCalendarDatesCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::CalendarDate c; + auto flds = Parser::getCalendarDateFlds(&csvp); + + while (p.nextCalendarDate(&csvp, &c, flds)) { + w.writeCalendarDate(c, &csvw); + } + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeFrequencies(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/frequencies.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getFrequencyCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::Frequency f; + auto flds = Parser::getFrequencyFlds(&csvp); + + while (p.nextFrequency(&csvp, &f, flds)) { + w.writeFrequency(f, &csvw); + } + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeTransfers(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/transfers.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getTransfersCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::Transfer t; + auto flds = Parser::getTransfersFlds(&csvp); + + while (p.nextTransfer(&csvp, &t, flds)) { + w.writeTransfer(t, &csvw); + } + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeFares(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/fare_attributes.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getFaresCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::Fare f; + auto flds = Parser::getFareFlds(&csvp); + + while (p.nextFare(&csvp, &f, flds)) { + w.writeFare(f, &csvw); + } + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeFareRules(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/fare_rules.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getFareRulesCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::FareRule f; + auto flds = Parser::getFareRuleFlds(&csvp); + + while (p.nextFareRule(&csvp, &f, flds)) { + w.writeFareRule(f, &csvw); + } + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeShapes(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/shapes.txt").c_str()); + + CsvWriter csvw = ad::cppgtfs::Writer::getShapesCsvw(os); + csvw.flushLine(); + ad::cppgtfs::gtfs::flat::ShapePoint sp; + ad::cppgtfs::Writer w; + + if (fs.good()) { + CsvParser csvp(&fs); + Parser p; + + auto flds = Parser::getShapeFlds(&csvp); + + std::string curShapeId; + std::string curSkipShapeId; + + while (p.nextShapePoint(&csvp, &sp, flds)) { + if (sp.id == curSkipShapeId) continue; + if (sp.id != curShapeId) { + if (sourceFeed->getShapes().has(sp.id)) { + curShapeId = sp.id; + } else { + curSkipShapeId = sp.id; + continue; + } + } + + w.writeShapePoint(sp, &csvw); + } + } + + sourceFeed->getShapes().open(); + while (sourceFeed->getShapes().nextStoragePt(&sp)) { + w.writeShapePoint(sp, &csvw); + } + + fs.close(); + + return true; +} + +// ____________________________________________________________________________ +bool Writer::writeTrips(gtfs::Feed* sourceFeed, std::ostream* os) const { + ad::cppgtfs::Writer w; + bool hasFreqs = false; + + CsvWriter csvw = ad::cppgtfs::Writer::getTripsCsvw(os); + csvw.flushLine(); + + for (auto t : sourceFeed->getTrips()) { + if (t.getFrequencies().size()) hasFreqs = true; + w.writeTrip(t.getFlat(), &csvw); + } + + return hasFreqs; +} + +// ____________________________________________________________________________ +bool Writer::writeStopTimes(gtfs::Feed* sourceFeed, std::ostream* os) const { + std::ifstream fs; + fs.open((sourceFeed->getPath() + "/stop_times.txt").c_str()); + + CsvParser csvp(&fs); + Parser p; + ad::cppgtfs::Writer w; + + CsvWriter csvw = ad::cppgtfs::Writer::getStopTimesCsvw(os); + csvw.flushLine(); + + ad::cppgtfs::gtfs::flat::StopTime st; + auto flds = Parser::getStopTimeFlds(&csvp); + + std::string curTripId; + Trip* cur = 0; + + while (p.nextStopTime(&csvp, &st, flds)) { + // we may have changed to distance field + if (curTripId != st.trip) { + cur = sourceFeed->getTrips().get(st.trip); + curTripId = st.trip; + } + for (const auto& stN : cur->getStopTimes()) { + if (stN.getSeq() == st.sequence) + st.shapeDistTravelled = stN.getShapeDistanceTravelled(); + } + + w.writeStopTime(st, &csvw); + } + fs.close(); + + return true; +} + +// ___________________________________________________________________________ +void Writer::cannotWrite(const std::string& file, const std::string& file2) { + std::stringstream ss; + ss << "(temporary file for " << file2 << ") Could not write to file"; + throw ad::cppgtfs::WriterException(ss.str(), file); +} diff --git a/src/pfaedle/gtfs/Writer.h b/src/pfaedle/gtfs/Writer.h new file mode 100644 index 0000000..27072c0 --- /dev/null +++ b/src/pfaedle/gtfs/Writer.h @@ -0,0 +1,41 @@ +// Copyright 2016, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#ifndef PFAEDLE_GTFS_WRITER_H_ +#define PFAEDLE_GTFS_WRITER_H_ + +#include +#include "Feed.h" + +namespace pfaedle { +namespace gtfs { + +class Writer { + public: + Writer() {} + + bool write(Feed* sourceFeed, const std::string& path) const; + + private: + bool writeFeedInfo(Feed* f, std::ostream* os) const; + bool writeAgency(Feed* f, std::ostream* os) const; + bool writeStops(Feed* f, std::ostream* os) const; + bool writeRoutes(Feed* f, std::ostream* os) const; + bool writeCalendar(Feed* f, std::ostream* os) const; + bool writeCalendarDates(Feed* f, std::ostream* os) const; + bool writeFrequencies(Feed* f, std::ostream* os) const; + bool writeTransfers(Feed* f, std::ostream* os) const; + bool writeFares(Feed* f, std::ostream* os) const; + bool writeFareRules(Feed* f, std::ostream* os) const; + bool writeShapes(Feed* f, std::ostream* os) const; + bool writeTrips(Feed* f, std::ostream* os) const; + bool writeStopTimes(Feed* f, std::ostream* os) const; + + static void cannotWrite(const std::string& file, const std::string& file2); +}; + +} // namespace gtfs +} // namespace pfaedle + +#endif // PFAEDLE_GTFS_WRITER_H_ diff --git a/src/pfaedle/netgraph/EdgePL.h b/src/pfaedle/netgraph/EdgePL.h index eb6bd37..58b767f 100644 --- a/src/pfaedle/netgraph/EdgePL.h +++ b/src/pfaedle/netgraph/EdgePL.h @@ -9,11 +9,12 @@ #include #include #include "ad/cppgtfs/gtfs/Feed.h" +#include "pfaedle/gtfs/Feed.h" #include "util/String.h" #include "util/geo/GeoGraph.h" using util::geograph::GeoEdgePL; -using ad::cppgtfs::gtfs::Trip; +using pfaedle::gtfs::Trip; namespace pfaedle { namespace netgraph { @@ -22,7 +23,7 @@ namespace netgraph { * A payload class for edges on a network graph - that is a graph * that exactly represents a physical public transit network */ -class EdgePL : public GeoEdgePL { +class EdgePL { public: EdgePL() {} EdgePL(const LINE& l, const std::set& trips) @@ -36,10 +37,10 @@ class EdgePL : public GeoEdgePL { util::json::Dict getAttrs() const { util::json::Dict obj; obj["num_trips"] = static_cast(_trips.size()); - obj["route_short_names"] = util::json::Array( - _routeShortNames.begin(), _routeShortNames.end()); - obj["trip_short_names"] = util::json::Array(_tripShortNames.begin(), - _tripShortNames.end()); + obj["route_short_names"] = + util::json::Array(_routeShortNames.begin(), _routeShortNames.end()); + obj["trip_short_names"] = + util::json::Array(_tripShortNames.begin(), _tripShortNames.end()); return obj; } diff --git a/src/pfaedle/netgraph/NodePL.h b/src/pfaedle/netgraph/NodePL.h index 3ef7f19..fd33a1f 100644 --- a/src/pfaedle/netgraph/NodePL.h +++ b/src/pfaedle/netgraph/NodePL.h @@ -12,7 +12,6 @@ using util::geograph::GeoNodePL; - namespace pfaedle { namespace netgraph { @@ -20,15 +19,13 @@ namespace netgraph { * A payload class for edges on a network graph - that is a graph * that exactly represents a physical public transit network */ -class NodePL : public GeoNodePL { +class NodePL { public: NodePL() {} NodePL(const POINT& geom) { _geom = geom; } // NOLINT const POINT* getGeom() const { return &_geom; } - util::json::Dict getAttrs() const { - return util::json::Dict(); - } + util::json::Dict getAttrs() const { return util::json::Dict(); } private: POINT _geom; diff --git a/src/pfaedle/osm/BBoxIdx.cpp b/src/pfaedle/osm/BBoxIdx.cpp index b427e21..c25d378 100644 --- a/src/pfaedle/osm/BBoxIdx.cpp +++ b/src/pfaedle/osm/BBoxIdx.cpp @@ -37,6 +37,31 @@ BOX BBoxIdx::getFullWebMercBox() const { _root.box.getUpperRight().getY(), _root.box.getUpperRight().getX())); } +// _____________________________________________________________________________ +BOX BBoxIdx::getFullBox() const { return _root.box; } + +// _____________________________________________________________________________ +std::vector> BBoxIdx::getLeafs() const { + std::vector> ret; + getLeafsRec(_root, &ret); + return ret; +} + +// _____________________________________________________________________________ +void BBoxIdx::getLeafsRec(const BBoxIdxNd& nd, + std::vector>* ret) const { + if (!nd.childs.size()) { + ret->push_back(nd.box); + return; + } + + for (const auto& child : nd.childs) { + getLeafsRec(child, ret); + } + + return; +} + // _____________________________________________________________________________ bool BBoxIdx::treeHas(const Point& p, const BBoxIdxNd& nd) const { if (!nd.childs.size()) return util::geo::contains(p, nd.box); diff --git a/src/pfaedle/osm/BBoxIdx.h b/src/pfaedle/osm/BBoxIdx.h index f443ff8..25be2ae 100644 --- a/src/pfaedle/osm/BBoxIdx.h +++ b/src/pfaedle/osm/BBoxIdx.h @@ -38,9 +38,15 @@ class BBoxIdx { // Return the full total bounding box of this index BOX getFullWebMercBox() const; + // Return the full total bounding box of this index + BOX getFullBox() const; + // Return the size of this index size_t size() const; + // return the leaf bounding boxes of this idx + std::vector> getLeafs() const; + private: double _padding; size_t _size; @@ -50,6 +56,9 @@ class BBoxIdx { void addToTree(const Box& box, BBoxIdxNd* nd, size_t lvl); bool treeHas(const Point& p, const BBoxIdxNd& nd) const; + void getLeafsRec(const BBoxIdxNd& nd, + std::vector>* ret) const; + static const size_t MAX_LVL = 5; static constexpr double MIN_COM_AREA = 0.0; }; diff --git a/src/pfaedle/osm/OsmBuilder.cpp b/src/pfaedle/osm/OsmBuilder.cpp index 460f1ec..efa5f4c 100644 --- a/src/pfaedle/osm/OsmBuilder.cpp +++ b/src/pfaedle/osm/OsmBuilder.cpp @@ -43,8 +43,20 @@ using pfaedle::osm::OsmRel; using pfaedle::osm::OsmNode; using pfaedle::osm::EdgeGrid; using pfaedle::osm::NodeGrid; +using pfaedle::osm::EqSearch; +using pfaedle::osm::BlockSearch; using ad::cppgtfs::gtfs::Stop; +// _____________________________________________________________________________ +bool EqSearch::operator()(const Node* cand, const StatInfo* si) const { + if (orphanSnap && cand->pl().getSI() && + (!cand->pl().getSI()->getGroup() || + cand->pl().getSI()->getGroup()->getStops().size() == 0)) { + return true; + } + return cand->pl().getSI() && cand->pl().getSI()->simi(si) > minSimi; +} + // _____________________________________________________________________________ OsmBuilder::OsmBuilder() {} @@ -140,7 +152,8 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts, POINT geom = *s->pl().getGeom(); NodePL pl = s->pl(); pl.getSI()->setIsFromOsm(false); - const auto& r = snapStation(g, &pl, &eg, &sng, opts, res, false, d); + const auto& r = + snapStation(g, &pl, &eg, &sng, opts, res, false, false, d); groupStats(r); for (auto n : r) { // if the snapped station is very near to the original OSM @@ -153,32 +166,70 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts, } } + std::vector notSnapped; + for (size_t i = 0; i < opts.maxSnapDistances.size(); i++) { double d = opts.maxSnapDistances[i]; for (auto& s : *fs) { auto pl = plFromGtfs(s.first, opts); - StatGroup* group = - groupStats(snapStation(g, &pl, &eg, &sng, opts, res, - i == opts.maxSnapDistances.size() - 1, d)); + StatGroup* group = groupStats( + snapStation(g, &pl, &eg, &sng, opts, res, + i == opts.maxSnapDistances.size() - 1, false, d)); if (group) { group->addStop(s.first); (*fs)[s.first] = *group->getNodes().begin(); + } else if (i == opts.maxSnapDistances.size() - 1) { + LOG(VDEBUG) << "Could not snap station " + << "(" << pl.getSI()->getName() << ")" + << " (" << s.first->getLat() << "," << s.first->getLng() + << ") in normal run, trying again later in orphan mode."; + notSnapped.push_back(s.first); + } + } + } + + if (notSnapped.size()) + LOG(VDEBUG) << notSnapped.size() << " stations could not be snapped in " + "normal run, trying again in orphan " + "mode."; + + // try again, but aggressively snap to orphan OSM stations which have + // not been assigned to any GTFS stop yet + for (size_t i = 0; i < opts.maxSnapDistances.size(); i++) { + double d = opts.maxSnapDistances[i]; + for (auto& s : notSnapped) { + auto pl = plFromGtfs(s, opts); + + StatGroup* group = groupStats( + snapStation(g, &pl, &eg, &sng, opts, res, + i == opts.maxSnapDistances.size() - 1, true, d)); + + if (group) { + group->addStop(s); + // add the added station name as an alt name to ensure future + // similarity + for (auto n : group->getNodes()) { + if (n->pl().getSI()) + n->pl().getSI()->addAltName(pl.getSI()->getName()); + } + (*fs)[s] = *group->getNodes().begin(); } else if (i == opts.maxSnapDistances.size() - 1) { // only fail on last + // finally give up + // add a group with only this stop in it StatGroup* dummyGroup = new StatGroup(); Node* dummyNode = g->addNd(pl); dummyNode->pl().getSI()->setGroup(dummyGroup); dummyGroup->addNode(dummyNode); - dummyGroup->addStop(s.first); - (*fs)[s.first] = dummyNode; + dummyGroup->addStop(s); + (*fs)[s] = dummyNode; LOG(WARN) << "Could not snap station " << "(" << pl.getSI()->getName() << ")" - << " (" << s.first->getLat() << "," << s.first->getLng() - << ")"; + << " (" << s->getLat() << "," << s->getLng() << ")"; } } } @@ -188,7 +239,7 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts, deleteOrphNds(g); LOG(VDEBUG) << "Deleting orphan edges..."; - deleteOrphEdgs(g); + deleteOrphEdgs(g, opts); LOG(VDEBUG) << "Collapsing edges..."; collapseEdges(g); @@ -197,7 +248,7 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts, deleteOrphNds(g); LOG(VDEBUG) << "Deleting orphan edges..."; - deleteOrphEdgs(g); + deleteOrphEdgs(g, opts); LOG(VDEBUG) << "Writing graph components..."; // the restrictor is needed here to prevent connections in the graph @@ -223,6 +274,89 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts, << " edges and " << comps << " connected component(s)"; } +// _____________________________________________________________________________ +void OsmBuilder::overpassQryWrite(std::ostream* out, + const std::vector& opts, + const BBoxIdx& latLngBox) const { + OsmIdSet bboxNodes, noHupNodes; + MultAttrMap emptyF; + + RelLst rels; + OsmIdList ways; + RelMap nodeRels, wayRels; + + // TODO(patrick): not needed here! + Restrictions rests; + + NIdMap nodes; + + // always empty + NIdMultMap multNodes; + util::xml::XmlWriter wr(out, true, 4); + + *out << "\n"; + wr.openComment(); + wr.writeText(" - written by pfaedle -"); + wr.closeTag(); + wr.openTag("osm-script", + {{"timeout", "99999"}, {"element-limit", "1073741824"}}); + + OsmFilter filter; + + for (const OsmReadOpts& o : opts) { + filter = filter.merge(OsmFilter(o.keepFilter, o.dropFilter)); + } + + wr.openTag("union"); + size_t c = 0; + for (auto box : latLngBox.getLeafs()) { + if (box.getLowerLeft().getX() > box.getUpperRight().getX()) continue; + c++; + wr.openComment(); + wr.writeText(std::string("Bounding box #") + std::to_string(c) + " (" + + std::to_string(box.getLowerLeft().getY()) + ", " + + std::to_string(box.getLowerLeft().getX()) + ", " + + std::to_string(box.getUpperRight().getY()) + ", " + + std::to_string(box.getUpperRight().getX()) + ")"); + wr.closeTag(); + for (auto t : std::vector{"way", "node", "relation"}) { + for (auto r : filter.getKeepRules()) { + for (auto val : r.second) { + if (t == "way" && (val.second & OsmFilter::WAY)) continue; + if (t == "relation" && (val.second & OsmFilter::REL)) continue; + if (t == "node" && (val.second & OsmFilter::NODE)) continue; + + wr.openTag("query", {{"type", t}}); + if (val.first == "*") + wr.openTag("has-kv", {{"k", r.first}}); + else + wr.openTag("has-kv", {{"k", r.first}, {"v", val.first}}); + wr.closeTag(); + wr.openTag("bbox-query", + {{"s", std::to_string(box.getLowerLeft().getY())}, + {"w", std::to_string(box.getLowerLeft().getX())}, + {"n", std::to_string(box.getUpperRight().getY())}, + {"e", std::to_string(box.getUpperRight().getX())}}); + wr.closeTag(); + wr.closeTag(); + } + } + } + } + + wr.closeTag(); + + wr.openTag("union"); + wr.openTag("item"); + wr.closeTag(); + wr.openTag("recurse", {{"type", "down"}}); + wr.closeTag(); + wr.closeTag(); + wr.openTag("print"); + + wr.closeTags(); +} + // _____________________________________________________________________________ void OsmBuilder::filterWrite(const std::string& in, const std::string& out, const std::vector& opts, @@ -250,8 +384,15 @@ void OsmBuilder::filterWrite(const std::string& in, const std::string& out, outstr << "\n"; wr.openTag("osm"); - - // TODO(patrick): write bounding box tag + wr.openTag( + "bounds", + {{"minlat", std::to_string(latLngBox.getFullBox().getLowerLeft().getY())}, + {"minlon", std::to_string(latLngBox.getFullBox().getLowerLeft().getX())}, + {"maxlat", + std::to_string(latLngBox.getFullBox().getUpperRight().getY())}, + {"maxlon", + std::to_string(latLngBox.getFullBox().getUpperRight().getX())}}); + wr.closeTag(); OsmFilter filter; AttrKeySet attrKeys[3] = {}; @@ -1071,8 +1212,7 @@ EdgeGrid OsmBuilder::buildEdgeIdx(Graph* g, size_t size, } // _____________________________________________________________________________ -NodeGrid OsmBuilder::buildNodeIdx(Graph* g, size_t size, - const BOX& webMercBox, +NodeGrid OsmBuilder::buildNodeIdx(Graph* g, size_t size, const BOX& webMercBox, bool which) const { NodeGrid ret(size, size, webMercBox, false); for (auto* n : *g->getNds()) { @@ -1158,9 +1298,10 @@ bool OsmBuilder::isBlocked(const Edge* e, const StatInfo* si, const POINT& p, // _____________________________________________________________________________ Node* OsmBuilder::eqStatReach(const Edge* e, const StatInfo* si, const POINT& p, - double maxD, int maxFullTurns, - double minAngle) const { - return depthSearch(e, si, p, maxD, maxFullTurns, minAngle, EqSearch()); + double maxD, int maxFullTurns, double minAngle, + bool orphanSnap) const { + return depthSearch(e, si, p, maxD, maxFullTurns, minAngle, + EqSearch(orphanSnap)); } // _____________________________________________________________________________ @@ -1187,8 +1328,7 @@ std::set OsmBuilder::getMatchingNds(const NodePL& s, NodeGrid* ng, std::set ret; double distor = webMercDistFactor(*s.getGeom()); std::set neighs; - BOX box = - util::geo::pad(util::geo::getBoundingBox(*s.getGeom()), d / distor); + BOX box = util::geo::pad(util::geo::getBoundingBox(*s.getGeom()), d / distor); ng->get(box, &neighs); for (auto* n : neighs) { @@ -1205,8 +1345,7 @@ std::set OsmBuilder::getMatchingNds(const NodePL& s, NodeGrid* ng, Node* OsmBuilder::getMatchingNd(const NodePL& s, NodeGrid* ng, double d) const { double distor = webMercDistFactor(*s.getGeom()); std::set neighs; - BOX box = - util::geo::pad(util::geo::getBoundingBox(*s.getGeom()), d / distor); + BOX box = util::geo::pad(util::geo::getBoundingBox(*s.getGeom()), d / distor); ng->get(box, &neighs); Node* ret = 0; @@ -1229,7 +1368,7 @@ Node* OsmBuilder::getMatchingNd(const NodePL& s, NodeGrid* ng, double d) const { std::set OsmBuilder::snapStation(Graph* g, NodePL* s, EdgeGrid* eg, NodeGrid* sng, const OsmReadOpts& opts, Restrictor* restor, bool surrHeur, - double d) const { + bool orphSnap, double d) const { assert(s->getSI()); std::set ret; @@ -1239,10 +1378,14 @@ std::set OsmBuilder::snapStation(Graph* g, NodePL* s, EdgeGrid* eg, if (pq.empty() && surrHeur) { // no station found in the first round, try again with the nearest - // surrounding - // station with matching name + // surrounding station with matching name const Node* best = getMatchingNd(*s, sng, opts.maxSnapFallbackHeurDistance); - if (best) getEdgCands(*best->pl().getGeom(), &pq, eg, d); + if (best) { + getEdgCands(*best->pl().getGeom(), &pq, eg, d); + } else { + // if still no luck, get edge cands in fallback snap distance + getEdgCands(*s->getGeom(), &pq, eg, opts.maxSnapFallbackHeurDistance); + } } while (!pq.empty()) { @@ -1254,7 +1397,7 @@ std::set OsmBuilder::snapStation(Graph* g, NodePL* s, EdgeGrid* eg, Node* eq = 0; if (!(eq = eqStatReach(e, s->getSI(), geom, 2 * d, 0, - opts.maxAngleSnapReach))) { + opts.maxAngleSnapReach, orphSnap))) { if (e->pl().lvl() > opts.maxSnapLevel) continue; if (isBlocked(e, s->getSI(), geom, opts.maxBlockDistance, 0, opts.maxAngleSnapReach)) { @@ -1309,11 +1452,6 @@ std::set OsmBuilder::snapStation(Graph* g, NodePL* s, EdgeGrid* eg, } } - // get surrounding nodes - // TODO(patrick): own distance configuration for this! - const auto& sur = getMatchingNds(*s, sng, opts.maxGroupSearchDistance); - ret.insert(sur.begin(), sur.end()); - return ret; } @@ -1336,7 +1474,10 @@ StatGroup* OsmBuilder::groupStats(const NodeSet& s) const { } } - if (!used) delete ret; + if (!used) { + delete ret; + return 0; + } return ret; } @@ -1507,7 +1648,7 @@ void OsmBuilder::getKeptAttrKeys(const OsmReadOpts& opts, } // _____________________________________________________________________________ -void OsmBuilder::deleteOrphEdgs(Graph* g) const { +void OsmBuilder::deleteOrphEdgs(Graph* g, const OsmReadOpts& opts) const { size_t ROUNDS = 3; for (size_t c = 0; c < ROUNDS; c++) { for (auto i = g->getNds()->begin(); i != g->getNds()->end();) { @@ -1515,6 +1656,15 @@ void OsmBuilder::deleteOrphEdgs(Graph* g) const { ++i; continue; } + + // check if the removal of this edge would transform a steep angle + // full turn at an intersection into a node 2 eligible for contraction + // if so, dont delete + if (keepFullTurn(*i, opts.fullTurnAngle)) { + ++i; + continue; + } + i = g->delNd(*i); continue; i++; @@ -1706,3 +1856,43 @@ void OsmBuilder::writeSelfEdgs(Graph* g) const { } } } + +// _____________________________________________________________________________ +bool OsmBuilder::keepFullTurn(const trgraph::Node* n, double ang) const { + if (n->getInDeg() + n->getOutDeg() != 1) return false; + + const trgraph::Edge* e = 0; + if (n->getOutDeg()) + e = n->getAdjListOut().front(); + else + e = n->getAdjListIn().front(); + + auto other = e->getOtherNd(n); + + if (other->getInDeg() + other->getOutDeg() == 3) { + const trgraph::Edge* a = 0; + const trgraph::Edge* b = 0; + for (auto f : other->getAdjListIn()) { + if (f != e && !a) + a = f; + else if (f != e && !b) + b = f; + } + + for (auto f : other->getAdjListOut()) { + if (f != e && !a) + a = f; + else if (f != e && !b) + b = f; + } + + auto ap = a->pl().backHop(); + auto bp = b->pl().backHop(); + if (a->getTo() != other) ap = a->pl().frontHop(); + if (b->getTo() != other) bp = b->pl().frontHop(); + + return router::angSmaller(ap, *other->pl().getGeom(), bp, ang); + } + + return false; +} diff --git a/src/pfaedle/osm/OsmBuilder.h b/src/pfaedle/osm/OsmBuilder.h index 01ed98c..2a2efa2 100644 --- a/src/pfaedle/osm/OsmBuilder.h +++ b/src/pfaedle/osm/OsmBuilder.h @@ -59,10 +59,10 @@ struct SearchFunc { }; struct EqSearch : public SearchFunc { + explicit EqSearch(bool orphanSnap) : orphanSnap(orphanSnap) {} double minSimi = 0.9; - bool operator()(const Node* cand, const StatInfo* si) const { - return cand->pl().getSI() && cand->pl().getSI()->simi(si) > minSimi; - } + bool orphanSnap; + bool operator()(const Node* cand, const StatInfo* si) const; }; struct BlockSearch : public SearchFunc { @@ -91,6 +91,11 @@ class OsmBuilder { const BBoxIdx& box, size_t gridSize, router::FeedStops* fs, Restrictor* res); + // Based on the list of options, output an overpass XML query for getting + // the data needed for routing + void overpassQryWrite(std::ostream* out, const std::vector& opts, + const BBoxIdx& latLngBox) const; + // Based on the list of options, read an OSM file from in and output an // OSM file to out which contains exactly the entities that are needed // from the file at in @@ -170,7 +175,7 @@ class OsmBuilder { void writeGeoms(Graph* g) const; void deleteOrphNds(Graph* g) const; - void deleteOrphEdgs(Graph* g) const; + void deleteOrphEdgs(Graph* g, const OsmReadOpts& opts) const; double dist(const Node* a, const Node* b) const; double webMercDist(const Node* a, const Node* b) const; double webMercDistFactor(const POINT& a) const; @@ -198,13 +203,14 @@ class OsmBuilder { NodeSet snapStation(Graph* g, NodePL* s, EdgeGrid* eg, NodeGrid* sng, const OsmReadOpts& opts, Restrictor* restor, bool surHeur, - double maxD) const; + bool orphSnap, double maxD) const; // Checks if from the edge e, a station similar to si can be reach with less // than maxD distance and less or equal to "maxFullTurns" full turns. If // such a station exists, it is returned. If not, 0 is returned. Node* eqStatReach(const Edge* e, const StatInfo* si, const POINT& p, - double maxD, int maxFullTurns, double maxAng) const; + double maxD, int maxFullTurns, double maxAng, + bool orph) const; Node* depthSearch(const Edge* e, const StatInfo* si, const POINT& p, double maxD, int maxFullTurns, double minAngle, @@ -243,6 +249,8 @@ class OsmBuilder { bool relKeep(osmid id, const RelMap& rels, const FlatRels& fl) const; + bool keepFullTurn(const trgraph::Node* n, double ang) const; + std::map _lines; std::map _relLines; }; diff --git a/src/pfaedle/osm/OsmFilter.cpp b/src/pfaedle/osm/OsmFilter.cpp index bb2860e..589cf5c 100644 --- a/src/pfaedle/osm/OsmFilter.cpp +++ b/src/pfaedle/osm/OsmFilter.cpp @@ -102,12 +102,12 @@ uint64_t OsmFilter::contained(const AttrMap& attrs, const Attr& attr) { // _____________________________________________________________________________ uint8_t OsmFilter::level(const AttrMap& attrs) const { // the best matching level is always returned - for (int16_t i = 0; i < 7; i++) { + for (int16_t i = 0; i < 8; i++) { for (const auto& kv : attrs) { const auto& lkv = (_levels + i)->find(kv.first); if (lkv != (_levels + i)->end()) { for (const auto& val : lkv->second) { - if (valMatches(kv.second, val.first)) return i + 1; + if (valMatches(kv.second, val.first)) return i; } } } @@ -169,7 +169,7 @@ std::vector OsmFilter::getAttrKeys() const { for (const auto& kv : _noRestr) { ret.push_back(kv.first); } - for (uint8_t i = 0; i < 7; i++) { + for (uint8_t i = 0; i < 8; i++) { for (const auto& kv : *(_levels + i)) { ret.push_back(kv.first); } @@ -191,27 +191,6 @@ OsmFilter OsmFilter::merge(const OsmFilter& other) const { keep[kv.first].insert(kv.second.begin(), kv.second.end()); } - // TODO(patrick): multi-level combination for filters. otherwise - // filter drop filters meant as a refinement for keep filters - // interfere with other keeps - - // const auto* d = &_drop; - - // for (size_t i = 0; i < 2; i++) { - // for (const auto& kv : *d) { - // if (keep.find(kv.first) != keep.end()) { - // for (const auto& val : kv.second) { - // if (keep[kv.first].find(val.first) == keep[kv.first].end()) { - // drop[kv.first].insert(val); - // } - // } - // } else { - // drop[kv.first].insert(kv.second.begin(), kv.second.end()); - // } - // } - // d = &other._drop; - // } - return OsmFilter(keep, drop); } @@ -258,3 +237,13 @@ uint64_t OsmFilter::posRestr(const AttrMap& attrs) const { if (contained(attrs, _noRestr, ALL)) return false; return (contained(attrs, _posRestr, ALL)); } + +// _____________________________________________________________________________ +const pfaedle::osm::MultAttrMap& OsmFilter::getKeepRules() const { + return _keep; +} + +// _____________________________________________________________________________ +const pfaedle::osm::MultAttrMap& OsmFilter::getDropRules() const { + return _drop; +} diff --git a/src/pfaedle/osm/OsmFilter.h b/src/pfaedle/osm/OsmFilter.h index 2fa9f82..daf353a 100644 --- a/src/pfaedle/osm/OsmFilter.h +++ b/src/pfaedle/osm/OsmFilter.h @@ -33,6 +33,9 @@ class OsmFilter { OsmFilter merge(const OsmFilter& other) const; + const MultAttrMap& getKeepRules() const; + const MultAttrMap& getDropRules() const; + std::string toString() const; static bool valMatches(const std::string& a, const std::string& b, bool m); diff --git a/src/pfaedle/osm/OsmReadOpts.h b/src/pfaedle/osm/OsmReadOpts.h index f4d3924..12e6d72 100644 --- a/src/pfaedle/osm/OsmReadOpts.h +++ b/src/pfaedle/osm/OsmReadOpts.h @@ -113,7 +113,7 @@ struct OsmReadOpts { MultAttrMap noHupFilter; MultAttrMap keepFilter; - MultAttrMap levelFilters[7]; + MultAttrMap levelFilters[8]; MultAttrMap dropFilter; MultAttrMap oneWayFilter; MultAttrMap oneWayFilterRev; @@ -136,7 +136,6 @@ struct OsmReadOpts { double maxAngleSnapReach; std::vector maxSnapDistances; double maxSnapFallbackHeurDistance; - double maxGroupSearchDistance; double maxBlockDistance; double maxOsmStationDistance; @@ -144,6 +143,8 @@ struct OsmReadOpts { // TODO(patrick): this is not implemented yet double levelSnapPunishFac[7] = {0, 0, 0, 0, 0, 0, 0}; + double fullTurnAngle; + // restriction system MultAttrMap restrPosRestr; MultAttrMap restrNegRestr; @@ -179,7 +180,6 @@ inline bool operator==(const OsmReadOpts& a, const OsmReadOpts& b) { fabs(a.maxOsmStationDistance - b.maxOsmStationDistance) < 0.1 && fabs(a.maxSnapFallbackHeurDistance - b.maxSnapFallbackHeurDistance) < 0.1 && - fabs(a.maxGroupSearchDistance - b.maxGroupSearchDistance) < 0.1 && fabs(a.maxBlockDistance - b.maxBlockDistance) < 0.1 && fabs(a.levelSnapPunishFac[0] - b.levelSnapPunishFac[0]) < 0.1 && fabs(a.levelSnapPunishFac[1] - b.levelSnapPunishFac[1]) < 0.1 && @@ -188,6 +188,7 @@ inline bool operator==(const OsmReadOpts& a, const OsmReadOpts& b) { fabs(a.levelSnapPunishFac[4] - b.levelSnapPunishFac[4]) < 0.1 && fabs(a.levelSnapPunishFac[5] - b.levelSnapPunishFac[5]) < 0.1 && fabs(a.levelSnapPunishFac[6] - b.levelSnapPunishFac[6]) < 0.1 && + fabs(a.fullTurnAngle - b.fullTurnAngle) < 0.1 && a.restrPosRestr == b.restrPosRestr && a.restrNegRestr == b.restrNegRestr && a.noRestrFilter == b.noRestrFilter; diff --git a/src/pfaedle/router/Comp.h b/src/pfaedle/router/Comp.h index 875c7d4..349ff04 100644 --- a/src/pfaedle/router/Comp.h +++ b/src/pfaedle/router/Comp.h @@ -60,11 +60,28 @@ inline double lineSimi(const std::string& a, const std::string& b) { if (a.empty() || b.empty()) return 0; - // if one of the lines is completely contained in the other, return 1 - if (a.find(b) != std::string::npos) { - return 1; - } else if (b.find(a) != std::string::npos) { - return 1; + if (a.size() > b.size() + 1) { + // check if a begins with b + if (a.compare(0, b.size() + 1, b + " ") == 0) { + return 1; + } + + // check if a ends with b + if (a.compare(a.size() - (b.size() + 1), b.size() + 1, " " + b) == 0) { + return 1; + } + } + + if (b.size() > a.size() + 1) { + // check if b begins with a + if (b.compare(0, a.size() + 1, a + " ") == 0) { + return 1; + } + + // check if b ends with a + if (b.compare(b.size() - (a.size() + 1), a.size() + 1, " " + a) == 0) { + return 1; + } } return 0; diff --git a/src/pfaedle/router/EdgePL.cpp b/src/pfaedle/router/EdgePL.cpp index 711f275..a166b5b 100644 --- a/src/pfaedle/router/EdgePL.cpp +++ b/src/pfaedle/router/EdgePL.cpp @@ -82,21 +82,6 @@ util::json::Dict EdgePL::getAttrs() const { obj["cost"] = std::to_string(_cost.getValue()); obj["from_edge"] = util::toString(_startE); obj["to_edge"] = util::toString(_endE); - obj["cost_m_lvl1"] = std::to_string(_cost.meterDistLvl1); - obj["cost_m_lvl0"] = std::to_string(_cost.meterDist); - obj["cost_m_lvl1"] = std::to_string(_cost.meterDistLvl1); - obj["cost_m_lvl2"] = std::to_string(_cost.meterDistLvl2); - obj["cost_m_lvl3"] = std::to_string(_cost.meterDistLvl3); - obj["cost_m_lvl4"] = std::to_string(_cost.meterDistLvl4); - obj["cost_m_lvl5"] = std::to_string(_cost.meterDistLvl5); - obj["cost_m_lvl6"] = std::to_string(_cost.meterDistLvl6); - obj["cost_m_lvl7"] = std::to_string(_cost.meterDistLvl7); - obj["cost_fullturn"] = std::to_string(_cost.fullTurns); - obj["cost_st_passthru"] = std::to_string(_cost.passThruStations); - obj["cost_m_oneway"] = std::to_string(_cost.oneWayMeters); - obj["cost_m_lineunmatch"] = std::to_string(_cost.lineUnmatchedMeters); - obj["cost_reach_node_pen"] = std::to_string(_cost.reachPen); - obj["cost_oneway_event"] = std::to_string(_cost.oneWayEdges); obj["dummy"] = _edges.size() ? "no" : "yes"; return obj; diff --git a/src/pfaedle/router/EdgePL.h b/src/pfaedle/router/EdgePL.h index dc9f851..024b9a2 100644 --- a/src/pfaedle/router/EdgePL.h +++ b/src/pfaedle/router/EdgePL.h @@ -17,7 +17,7 @@ using util::geograph::GeoEdgePL; namespace pfaedle { namespace router { -class EdgePL : public GeoEdgePL { +class EdgePL { public: EdgePL() : _cost(), _start(0), _end(0), _startE(0), _endE(0) {} const LINE* getGeom() const; diff --git a/src/pfaedle/router/Misc.h b/src/pfaedle/router/Misc.h index 5b4f0e2..07bf013 100644 --- a/src/pfaedle/router/Misc.h +++ b/src/pfaedle/router/Misc.h @@ -7,8 +7,8 @@ #include #include -#include #include +#include #include "ad/cppgtfs/gtfs/Feed.h" #include "ad/cppgtfs/gtfs/Route.h" #include "pfaedle/trgraph/Graph.h" @@ -67,90 +67,35 @@ inline bool operator==(const RoutingOpts& a, const RoutingOpts& b) { } struct EdgeCost { - EdgeCost() - : meterDist(0), - meterDistLvl1(0), - meterDistLvl2(0), - meterDistLvl3(0), - meterDistLvl4(0), - meterDistLvl5(0), - meterDistLvl6(0), - meterDistLvl7(0), - fullTurns(0), - passThruStations(0), - oneWayMeters(0), - oneWayEdges(0), - lineUnmatchedMeters(0), - reachPen(0), - o(0) {} + EdgeCost() : _cost(0) {} + explicit EdgeCost(double cost) : _cost(cost) {} EdgeCost(double mDist, double mDistLvl1, double mDistLvl2, double mDistLvl3, double mDistLvl4, double mDistLvl5, double mDistLvl6, double mDistLvl7, uint32_t fullTurns, int32_t passThru, double oneWayMeters, size_t oneWayEdges, double lineUnmatchedMeters, - double reachPen, const RoutingOpts* o) - : meterDist(mDist), - meterDistLvl1(mDistLvl1), - meterDistLvl2(mDistLvl2), - meterDistLvl3(mDistLvl3), - meterDistLvl4(mDistLvl4), - meterDistLvl5(mDistLvl5), - meterDistLvl6(mDistLvl6), - meterDistLvl7(mDistLvl7), - fullTurns(fullTurns), - passThruStations(passThru), - oneWayMeters(oneWayMeters), - oneWayEdges(oneWayEdges), - lineUnmatchedMeters(lineUnmatchedMeters), - reachPen(reachPen), - o(o) {} - double meterDist; - double meterDistLvl1; - double meterDistLvl2; - double meterDistLvl3; - double meterDistLvl4; - double meterDistLvl5; - double meterDistLvl6; - double meterDistLvl7; - uint32_t fullTurns; - int32_t passThruStations; - double oneWayMeters; - size_t oneWayEdges; - double lineUnmatchedMeters; - double reachPen; - const RoutingOpts* o; - - double getValue() const { - if (!o) return meterDist + reachPen; - return meterDist * o->levelPunish[0] + meterDistLvl1 * o->levelPunish[1] + - meterDistLvl2 * o->levelPunish[2] + - meterDistLvl3 * o->levelPunish[3] + - meterDistLvl4 * o->levelPunish[4] + - meterDistLvl5 * o->levelPunish[5] + - meterDistLvl6 * o->levelPunish[6] + - meterDistLvl7 * o->levelPunish[7] + - oneWayMeters * o->oneWayPunishFac + - oneWayEdges * o->oneWayEdgePunish + - lineUnmatchedMeters * o->lineUnmatchedPunishFact + - fullTurns * o->fullTurnPunishFac + - passThruStations * o->passThruStationsPunish + reachPen; + double reachPen, const RoutingOpts* o) { + if (!o) { + _cost = mDist + reachPen; + } else { + _cost = mDist * o->levelPunish[0] + mDistLvl1 * o->levelPunish[1] + + mDistLvl2 * o->levelPunish[2] + mDistLvl3 * o->levelPunish[3] + + mDistLvl4 * o->levelPunish[4] + mDistLvl5 * o->levelPunish[5] + + mDistLvl6 * o->levelPunish[6] + mDistLvl7 * o->levelPunish[7] + + oneWayMeters * o->oneWayPunishFac + + oneWayEdges * o->oneWayEdgePunish + + lineUnmatchedMeters * o->lineUnmatchedPunishFact + + fullTurns * o->fullTurnPunishFac + + passThru * o->passThruStationsPunish + reachPen; + } } - double getTotalMeters() const { - return meterDist + meterDistLvl1 + meterDistLvl2 + meterDistLvl3 + - meterDistLvl4 + meterDistLvl5 + meterDistLvl6 + meterDistLvl7; - } + float _cost; + + double getValue() const { return _cost; } }; inline EdgeCost operator+(const EdgeCost& a, const EdgeCost& b) { - return EdgeCost( - a.meterDist + b.meterDist, a.meterDistLvl1 + b.meterDistLvl1, - a.meterDistLvl2 + b.meterDistLvl2, a.meterDistLvl3 + b.meterDistLvl3, - a.meterDistLvl4 + b.meterDistLvl4, a.meterDistLvl5 + b.meterDistLvl5, - a.meterDistLvl6 + b.meterDistLvl6, a.meterDistLvl7 + b.meterDistLvl7, - a.fullTurns + b.fullTurns, a.passThruStations + b.passThruStations, - a.oneWayMeters + b.oneWayMeters, a.oneWayEdges + b.oneWayEdges, - a.lineUnmatchedMeters + b.lineUnmatchedMeters, a.reachPen + b.reachPen, - a.o ? a.o : b.o); + return EdgeCost(a.getValue() + b.getValue()); } inline bool operator<=(const EdgeCost& a, const EdgeCost& b) { @@ -165,9 +110,9 @@ inline bool operator>(const EdgeCost& a, const EdgeCost& b) { return a.getValue() > b.getValue(); } -template +template inline bool angSmaller(const Point& f, const Point& m, const Point& t, - double ang) { + double ang) { if (util::geo::innerProd(m, f, t) < ang) return 1; return 0; } diff --git a/src/pfaedle/router/NodePL.h b/src/pfaedle/router/NodePL.h index 6d02dbf..a9c2ea4 100644 --- a/src/pfaedle/router/NodePL.h +++ b/src/pfaedle/router/NodePL.h @@ -18,7 +18,7 @@ using util::geograph::GeoNodePL; namespace pfaedle { namespace router { -class NodePL : public GeoNodePL { +class NodePL { public: NodePL() : _n(0) {} NodePL(const pfaedle::trgraph::Node* n) : _n(n) {} // NOLINT diff --git a/src/pfaedle/router/Router.cpp b/src/pfaedle/router/Router.cpp index de2dd86..703825f 100644 --- a/src/pfaedle/router/Router.cpp +++ b/src/pfaedle/router/Router.cpp @@ -191,7 +191,8 @@ double CombCostFunc::operator()(const router::Edge* from, const router::Node* n, } // _____________________________________________________________________________ -Router::Router(size_t numThreads) : _cache(numThreads) { +Router::Router(size_t numThreads, bool caching) + : _cache(numThreads), _caching(caching) { for (size_t i = 0; i < numThreads; i++) { _cache[i] = new Cache(); } @@ -219,6 +220,9 @@ bool Router::compConned(const NodeCandGroup& a, const NodeCandGroup& b) const { HopBand Router::getHopBand(const NodeCandGroup& a, const NodeCandGroup& b, const RoutingAttrs& rAttrs, const RoutingOpts& rOpts, const osm::Restrictor& rest) const { + assert(a.size()); + assert(b.size()); + double pend = 0; for (size_t i = 0; i < a.size(); i++) { for (size_t j = 0; j < b.size(); j++) { @@ -231,6 +235,7 @@ HopBand Router::getHopBand(const NodeCandGroup& a, const NodeCandGroup& b, LOG(VDEBUG) << "Pending max hop distance is " << pend << " meters"; const trgraph::StatGroup* tgGrpTo = 0; + if (b.begin()->nd->pl().getSI()) tgGrpTo = b.begin()->nd->pl().getSI()->getGroup(); @@ -556,6 +561,7 @@ void Router::nestedCache(const EdgeList* el, const std::set& froms, const CostFunc& cost, const RoutingAttrs& rAttrs) const { + if (!_caching) return; if (el->size() == 0) return; // iterate over result edges backwards EdgeList curEdges; @@ -586,7 +592,7 @@ std::set Router::getCachedHops( const RoutingAttrs& rAttrs) const { std::set ret; for (auto to : tos) { - if ((*_cache[omp_get_thread_num()])[rAttrs][from].count(to)) { + if (_caching && (*_cache[omp_get_thread_num()])[rAttrs][from].count(to)) { const auto& cv = (*_cache[omp_get_thread_num()])[rAttrs][from][to]; (*rCosts)[to] = cv.first; *edgesRet.at(to) = cv.second; @@ -601,6 +607,7 @@ std::set Router::getCachedHops( // _____________________________________________________________________________ void Router::cache(trgraph::Edge* from, trgraph::Edge* to, const EdgeCost& c, EdgeList* edges, const RoutingAttrs& rAttrs) const { + if (!_caching) return; if (from == to) return; (*_cache[omp_get_thread_num()])[rAttrs][from][to] = std::pair(c, *edges); diff --git a/src/pfaedle/router/Router.h b/src/pfaedle/router/Router.h index 9d33368..76003d5 100644 --- a/src/pfaedle/router/Router.h +++ b/src/pfaedle/router/Router.h @@ -137,7 +137,7 @@ struct CombCostFunc class Router { public: // Init this router with caches for numThreads threads - explicit Router(size_t numThreads); + explicit Router(size_t numThreads, bool caching); ~Router(); // Find the most likely path through the graph for a node candidate route. @@ -163,6 +163,7 @@ class Router { private: mutable std::vector _cache; + bool _caching; HopBand getHopBand(const NodeCandGroup& a, const NodeCandGroup& b, const RoutingAttrs& rAttrs, const RoutingOpts& rOpts, const osm::Restrictor& rest) const; diff --git a/src/pfaedle/router/ShapeBuilder.cpp b/src/pfaedle/router/ShapeBuilder.cpp index 84b10dc..2beb663 100644 --- a/src/pfaedle/router/ShapeBuilder.cpp +++ b/src/pfaedle/router/ShapeBuilder.cpp @@ -9,6 +9,7 @@ #define omp_get_num_procs() 1 #endif +#include #include #include #include @@ -16,6 +17,8 @@ #include "ad/cppgtfs/gtfs/Feed.h" #include "pfaedle/Def.h" #include "pfaedle/eval/Collector.h" +#include "pfaedle/gtfs/Feed.h" +#include "pfaedle/gtfs/StopTime.h" #include "pfaedle/osm/OsmBuilder.h" #include "pfaedle/router/ShapeBuilder.h" #include "pfaedle/trgraph/StatGroup.h" @@ -43,38 +46,36 @@ using pfaedle::router::EdgeListHops; using pfaedle::router::Clusters; using pfaedle::osm::BBoxIdx; using ad::cppgtfs::gtfs::Stop; -using ad::cppgtfs::gtfs::Trip; -using ad::cppgtfs::gtfs::Feed; -using ad::cppgtfs::gtfs::StopTime; +using pfaedle::gtfs::Trip; +using pfaedle::gtfs::Feed; +using pfaedle::gtfs::StopTime; using ad::cppgtfs::gtfs::ShapePoint; // _____________________________________________________________________________ -ShapeBuilder::ShapeBuilder(Feed* feed, MOTs mots, - const config::MotConfig& motCfg, +ShapeBuilder::ShapeBuilder(Feed* feed, ad::cppgtfs::gtfs::Feed* evalFeed, + MOTs mots, const config::MotConfig& motCfg, eval::Collector* ecoll, const config::Config& cfg) : _feed(feed), + _evalFeed(evalFeed), _mots(mots), _motCfg(motCfg), _ecoll(ecoll), _cfg(cfg), - _crouter(omp_get_num_procs()), + _crouter(omp_get_num_procs(), cfg.useCaching), _curShpCnt(0) { _numThreads = _crouter.getCacheNumber(); writeMotStops(); - // TODO(patrick): maybe do this on demand to avoid graph filtering / reading - // for input where no routing is necessary (already shape'd) buildGraph(); } // _____________________________________________________________________________ void ShapeBuilder::writeMotStops() { for (auto t : _feed->getTrips()) { - if (!_cfg.shapeTripId.empty() && t.second->getId() != _cfg.shapeTripId) - continue; - if (_mots.count(t.second->getRoute()->getType()) && - _motCfg.mots.count(t.second->getRoute()->getType())) { - for (auto st : t.second->getStopTimes()) { + if (!_cfg.shapeTripId.empty() && t.getId() != _cfg.shapeTripId) continue; + if (_mots.count(t.getRoute()->getType()) && + _motCfg.mots.count(t.getRoute()->getType())) { + for (auto st : t.getStopTimes()) { _stops[st.getStop()] = 0; } } @@ -95,28 +96,34 @@ const NodeCandGroup& ShapeBuilder::getNodeCands(const Stop* s) const { // _____________________________________________________________________________ LINE ShapeBuilder::shapeL(const router::NodeCandRoute& ncr, const router::RoutingAttrs& rAttrs) { - const router::EdgeListHops& res = route(ncr, rAttrs); + try { + const router::EdgeListHops& res = route(ncr, rAttrs); - LINE l; - for (const auto& hop : res) { - const trgraph::Node* last = hop.start; - if (hop.edges.size() == 0) { - l.push_back(*hop.start->pl().getGeom()); - l.push_back(*hop.end->pl().getGeom()); - } - for (auto i = hop.edges.rbegin(); i != hop.edges.rend(); i++) { - const auto* e = *i; - if ((e->getFrom() == last) ^ e->pl().isRev()) { - l.insert(l.end(), e->pl().getGeom()->begin(), e->pl().getGeom()->end()); - } else { - l.insert(l.end(), e->pl().getGeom()->rbegin(), - e->pl().getGeom()->rend()); + LINE l; + for (const auto& hop : res) { + const trgraph::Node* last = hop.start; + if (hop.edges.size() == 0) { + l.push_back(*hop.start->pl().getGeom()); + l.push_back(*hop.end->pl().getGeom()); + } + for (auto i = hop.edges.rbegin(); i != hop.edges.rend(); i++) { + const auto* e = *i; + if ((e->getFrom() == last) ^ e->pl().isRev()) { + l.insert(l.end(), e->pl().getGeom()->begin(), + e->pl().getGeom()->end()); + } else { + l.insert(l.end(), e->pl().getGeom()->rbegin(), + e->pl().getGeom()->rend()); + } + last = e->getOtherNd(last); } - last = e->getOtherNd(last); } - } - return l; + return l; + } catch (const std::runtime_error& e) { + LOG(ERROR) << e.what(); + return LINE(); + } } // _____________________________________________________________________________ @@ -193,9 +200,9 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) { Clusters clusters = clusterTrips(_feed, _mots); LOG(DEBUG) << "Clustered trips into " << clusters.size() << " clusters."; - std::map shpUsage; + std::map shpUsage; for (auto t : _feed->getTrips()) { - if (t.second->getShape()) shpUsage[t.second->getShape()]++; + if (!t.getShape().empty()) shpUsage[t.getShape()]++; } // to avoid unfair load balance on threads @@ -223,7 +230,7 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) { << "%, " << (EDijkstra::ITERS - oiters) << " iters, " /** TODO: this is actually misleading. We are counting the - Dijkstra iterations, but the measuring them against + Dijkstra iterations, but measuring them against the total running time (including all overhead + HMM solve) << tput " << (static_cast(EDijkstra::ITERS - oiters)) / @@ -249,7 +256,7 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) { } std::vector distances; - ad::cppgtfs::gtfs::Shape* shp = + const ad::cppgtfs::gtfs::Shape& shp = getGtfsShape(cshp, clusters[i][0], &distances); LOG(DEBUG) << "Took " << EDijkstra::ITERS - iters << " iterations."; @@ -259,14 +266,14 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) { for (auto t : clusters[i]) { if (_cfg.evaluate) { - _ecoll->add(t, t->getShape(), shp, distances); + _ecoll->add(t, _evalFeed->getShapes().get(t->getShape()), shp, + distances); } - if (t->getShape() && shpUsage[t->getShape()] > 0) { + if (!t->getShape().empty() && shpUsage[t->getShape()] > 0) { shpUsage[t->getShape()]--; if (shpUsage[t->getShape()] == 0) { - _feed->getShapes().remove(t->getShape()->getId()); - delete t->getShape(); + _feed->getShapes().remove(t->getShape()); } } setShape(t, shp, distances); @@ -295,27 +302,25 @@ void ShapeBuilder::shape(pfaedle::netgraph::Graph* ng) { } // _____________________________________________________________________________ -void ShapeBuilder::setShape(Trip* t, ad::cppgtfs::gtfs::Shape* s, +void ShapeBuilder::setShape(Trip* t, const ad::cppgtfs::gtfs::Shape& s, const std::vector& distances) { assert(distances.size() == t->getStopTimes().size()); // set distances size_t i = 0; - for (const StopTime& st : t->getStopTimes()) { - const_cast(st).setShapeDistanceTravelled(distances[i]); + for (const auto& st : t->getStopTimes()) { + const_cast&>(st).setShapeDistanceTravelled(distances[i]); i++; } - t->setShape(s); - std::lock_guard guard(_shpMutex); - _feed->getShapes().add(s); + // TODO(patrick): + t->setShape(_feed->getShapes().add(s)); } // _____________________________________________________________________________ -ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape( +ad::cppgtfs::gtfs::Shape ShapeBuilder::getGtfsShape( const Shape& shp, Trip* t, std::vector* hopDists) { - ad::cppgtfs::gtfs::Shape* ret = - new ad::cppgtfs::gtfs::Shape(getFreeShapeId(t)); + ad::cppgtfs::gtfs::Shape ret(getFreeShapeId(t)); assert(shp.hops.size() == t->getStopTimes().size() - 1); @@ -338,7 +343,7 @@ ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape( last = *hop.start->pl().getGeom(); if (dist - lastDist > 0.01) { - ret->addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq)); + ret.addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq)); seq++; lastDist = dist; } @@ -349,11 +354,12 @@ ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape( if (dist - lastDist > 0.01) { ll = webMercToLatLng( hop.end->pl().getGeom()->getX(), hop.end->pl().getGeom()->getY()); - ret->addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq)); + ret.addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq)); seq++; lastDist = dist; } } + for (auto i = hop.edges.rbegin(); i != hop.edges.rend(); i++) { const auto* e = *i; if ((e->getFrom() == l) ^ e->pl().isRev()) { @@ -367,7 +373,7 @@ ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape( if (dist - lastDist > 0.01) { POINT ll = webMercToLatLng(cur.getX(), cur.getY()); - ret->addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq)); + ret.addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq)); seq++; lastDist = dist; } @@ -383,7 +389,7 @@ ad::cppgtfs::gtfs::Shape* ShapeBuilder::getGtfsShape( if (dist - lastDist > 0.01) { POINT ll = webMercToLatLng(cur.getX(), cur.getY()); - ret->addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq)); + ret.addPoint(ShapePoint(ll.getY(), ll.getX(), dist, seq)); seq++; lastDist = dist; } @@ -447,33 +453,30 @@ const RoutingAttrs& ShapeBuilder::getRAttrs(const Trip* trip) const { } // _____________________________________________________________________________ -BBoxIdx ShapeBuilder::getPaddedGtfsBox(const Feed* feed, double pad, - const MOTs& mots, const std::string& tid, - bool dropShapes) { - osm::BBoxIdx box(pad); +void ShapeBuilder::getGtfsBox(const Feed* feed, const MOTs& mots, + const std::string& tid, bool dropShapes, + osm::BBoxIdx* box) { for (const auto& t : feed->getTrips()) { - if (!tid.empty() && t.second->getId() != tid) continue; - if (tid.empty() && t.second->getShape() && !dropShapes) continue; - if (t.second->getStopTimes().size() < 2) continue; - if (mots.count(t.second->getRoute()->getType())) { + if (!tid.empty() && t.getId() != tid) continue; + if (tid.empty() && !t.getShape().empty() && !dropShapes) continue; + if (t.getStopTimes().size() < 2) continue; + if (mots.count(t.getRoute()->getType())) { DBox cur; - for (const auto& st : t.second->getStopTimes()) { + for (const auto& st : t.getStopTimes()) { cur = extendBox(DPoint(st.getStop()->getLng(), st.getStop()->getLat()), cur); } - box.add(cur); + box->add(cur); } } - - return box; } // _____________________________________________________________________________ void ShapeBuilder::buildGraph() { osm::OsmBuilder osmBuilder; - osm::BBoxIdx box = - getPaddedGtfsBox(_feed, 2500, _mots, _cfg.shapeTripId, _cfg.dropShapes); + osm::BBoxIdx box(BOX_PADDING); + getGtfsBox(_feed, _mots, _cfg.shapeTripId, _cfg.dropShapes, &box); osmBuilder.read(_cfg.osmPath, _motCfg.osmBuildOpts, &_g, box, _cfg.gridSize, getFeedStops(), &_restr); @@ -497,6 +500,11 @@ NodeCandRoute ShapeBuilder::getNCR(Trip* trip) const { for (const auto& st : trip->getStopTimes()) { ncr[i] = getNodeCands(st.getStop()); + if (ncr[i].size() == 0) { + throw std::runtime_error("No node candidate found for station '" + + st.getStop()->getName() + "' on trip '" + + trip->getId() + "'"); + } i++; } return ncr; @@ -535,29 +543,29 @@ Clusters ShapeBuilder::clusterTrips(Feed* f, MOTs mots) { size_t j = 0; Clusters ret; - for (const auto& trip : f->getTrips()) { - if (trip.second->getShape() && !_cfg.dropShapes) continue; - if (trip.second->getStopTimes().size() < 2) continue; - if (!mots.count(trip.second->getRoute()->getType()) || - !_motCfg.mots.count(trip.second->getRoute()->getType())) + for (auto& trip : f->getTrips()) { + if (!trip.getShape().empty() && !_cfg.dropShapes) continue; + if (trip.getStopTimes().size() < 2) continue; + if (!mots.count(trip.getRoute()->getType()) || + !_motCfg.mots.count(trip.getRoute()->getType())) continue; bool found = false; - auto spair = StopPair(trip.second->getStopTimes().begin()->getStop(), - trip.second->getStopTimes().rbegin()->getStop()); + auto spair = StopPair(trip.getStopTimes().begin()->getStop(), + trip.getStopTimes().rbegin()->getStop()); const auto& c = clusterIdx[spair]; for (size_t i = 0; i < c.size(); i++) { j++; - if (routingEqual(ret[c[i]][0], trip.second)) { - ret[c[i]].push_back(trip.second); + if (routingEqual(ret[c[i]][0], &trip)) { + ret[c[i]].push_back(&trip); found = true; break; } } if (!found) { - ret.push_back({trip.second}); + ret.push_back(Cluster{&trip}); // explicit call to write render attrs to cache - getRAttrs(trip.second); + getRAttrs(&trip); clusterIdx[spair].push_back(ret.size() - 1); } } diff --git a/src/pfaedle/router/ShapeBuilder.h b/src/pfaedle/router/ShapeBuilder.h index 949b297..15c006e 100644 --- a/src/pfaedle/router/ShapeBuilder.h +++ b/src/pfaedle/router/ShapeBuilder.h @@ -16,6 +16,7 @@ #include "pfaedle/config/MotConfig.h" #include "pfaedle/config/PfaedleConfig.h" #include "pfaedle/eval/Collector.h" +#include "pfaedle/gtfs/Feed.h" #include "pfaedle/netgraph/Graph.h" #include "pfaedle/osm/Restrictor.h" #include "pfaedle/router/Misc.h" @@ -27,8 +28,8 @@ namespace pfaedle { namespace router { using ad::cppgtfs::gtfs::Stop; -using ad::cppgtfs::gtfs::Trip; -using ad::cppgtfs::gtfs::Feed; +using pfaedle::gtfs::Trip; +using pfaedle::gtfs::Feed; struct Shape { router::EdgeListHops hops; @@ -48,8 +49,9 @@ typedef std::unordered_map> */ class ShapeBuilder { public: - ShapeBuilder(Feed* feed, MOTs mots, const config::MotConfig& motCfg, - eval::Collector* ecoll, const config::Config& cfg); + ShapeBuilder(Feed* feed, ad::cppgtfs::gtfs::Feed* evalFeed, MOTs mots, + const config::MotConfig& motCfg, eval::Collector* ecoll, + const config::Config& cfg); void shape(pfaedle::netgraph::Graph* ng); @@ -66,12 +68,13 @@ class ShapeBuilder { const trgraph::Graph* getGraph() const; - static osm::BBoxIdx getPaddedGtfsBox(const Feed* feed, double pad, - const MOTs& mots, const std::string& tid, - bool dropShapes); + static void getGtfsBox(const Feed* feed, const MOTs& mots, + const std::string& tid, bool dropShapes, + osm::BBoxIdx* box); private: Feed* _feed; + ad::cppgtfs::gtfs::Feed* _evalFeed; MOTs _mots; config::MotConfig _motCfg; eval::Collector* _ecoll; @@ -101,10 +104,10 @@ class ShapeBuilder { std::string getFreeShapeId(Trip* t); - ad::cppgtfs::gtfs::Shape* getGtfsShape(const Shape& shp, Trip* t, + ad::cppgtfs::gtfs::Shape getGtfsShape(const Shape& shp, Trip* t, std::vector* hopDists); - void setShape(Trip* t, ad::cppgtfs::gtfs::Shape* s, + void setShape(Trip* t, const ad::cppgtfs::gtfs::Shape& s, const std::vector& dists); router::NodeCandRoute getNCR(Trip* trip) const; diff --git a/src/pfaedle/trgraph/EdgePL.cpp b/src/pfaedle/trgraph/EdgePL.cpp index fe26873..cd6c241 100644 --- a/src/pfaedle/trgraph/EdgePL.cpp +++ b/src/pfaedle/trgraph/EdgePL.cpp @@ -39,7 +39,7 @@ EdgePL::EdgePL(const EdgePL& pl, bool geoflat) } _flines[_l]++; - for (auto l : _lines) addLine(l); + for (auto l : pl._lines) addLine(l); } // _____________________________________________________________________________ @@ -82,7 +82,9 @@ double EdgePL::getLength() const { return _length; } // _____________________________________________________________________________ void EdgePL::addLine(const TransitEdgeLine* l) { - if (_lines.insert(l).second) { + if (std::find(_lines.begin(), _lines.end(), l) == _lines.end()) { + _lines.reserve(_lines.size() + 1); + _lines.push_back(l); if (_tlines.count(l)) _tlines[l]++; else @@ -96,7 +98,7 @@ void EdgePL::addLines(const std::vector& l) { } // _____________________________________________________________________________ -const std::set& EdgePL::getLines() const { +const std::vector& EdgePL::getLines() const { return _lines; } diff --git a/src/pfaedle/trgraph/EdgePL.h b/src/pfaedle/trgraph/EdgePL.h index 2911ba1..368ac2f 100644 --- a/src/pfaedle/trgraph/EdgePL.h +++ b/src/pfaedle/trgraph/EdgePL.h @@ -45,7 +45,7 @@ inline bool operator<(const TransitEdgeLine& a, const TransitEdgeLine& b) { /* * An edge payload class for the transit graph. */ -class EdgePL : public GeoEdgePL { +class EdgePL { public: EdgePL(); ~EdgePL(); @@ -101,7 +101,7 @@ class EdgePL : public GeoEdgePL { void addLines(const std::vector& l); // Return the TransitEdgeLines stored for this payload - const std::set& getLines() const; + const std::vector& getLines() const; // Returns the last hop of the payload - this is the (n-2)th point in // the payload geometry of length n > 1 @@ -123,7 +123,7 @@ class EdgePL : public GeoEdgePL { LINE* _l; - std::set _lines; + std::vector _lines; static void unRefTLine(const TransitEdgeLine* l); diff --git a/src/pfaedle/trgraph/NodePL.h b/src/pfaedle/trgraph/NodePL.h index c37cde0..7c1cdb0 100644 --- a/src/pfaedle/trgraph/NodePL.h +++ b/src/pfaedle/trgraph/NodePL.h @@ -26,7 +26,7 @@ struct Component { /* * A node payload class for the transit graph. */ -class NodePL : public GeoNodePL { +class NodePL { public: NodePL(); NodePL(const NodePL& pl); // NOLINT diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 1c3f62f..7888cd6 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -2,4 +2,11 @@ file(GLOB_RECURSE util_SRC *.cpp) list(REMOVE_ITEM util_SRC TestMain.cpp) add_library(util ${util_SRC}) +find_package( ZLIB ) +if (ZLIB_FOUND) + include_directories( ${ZLIB_INCLUDE_DIRS} ) + target_link_libraries( util ${ZLIB_LIBRARIES} ) + add_definitions( -DZLIB_FOUND=${ZLIB_FOUND} ) +endif( ZLIB_FOUND ) + add_subdirectory(tests) diff --git a/src/util/Nullable.h b/src/util/Nullable.h index e993aaa..069426f 100644 --- a/src/util/Nullable.h +++ b/src/util/Nullable.h @@ -26,7 +26,7 @@ class Nullable { : val(other.val), null(other.isNull()) {} Nullable& operator=(const Nullable& other) { - val = other.get(); + if (!other.isNull()) val = other.get(); null = other.isNull(); return *this; } diff --git a/src/util/String.h b/src/util/String.h index ed03c17..fdb2ef3 100644 --- a/src/util/String.h +++ b/src/util/String.h @@ -142,6 +142,39 @@ inline size_t editDist(const std::string& s1, const std::string& s2) { return prev[len2]; } +// _____________________________________________________________________________ +inline size_t prefixEditDist(const std::string& prefix, const std::string& s, + size_t deltaMax) { + // https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#C++ + size_t len1 = prefix.size(); + size_t len2 = std::min(s.size(), prefix.size() + deltaMax + 1); + std::vector d((len1 + 1) * (len2 + 1)); + + d[0] = 0; + for (size_t i = 1; i <= len1; ++i) d[i * (len2 + 1)] = i; + for (size_t i = 1; i <= len2; ++i) d[ i] = i; + + for (size_t i = 1; i <= len1; i++) { + for (size_t j = 1; j <= len2; j++) { + d[i * (len2 + 1) + j] = std::min(std::min(d[(i - 1) * (len2 + 1) + j] + 1, d[i * (len2 + 1) + j - 1] + 1), + d[(i - 1) * (len2 + 1) + j - 1] + (prefix[i - 1] == s[j - 1] ? 0 : 1)); + } + } + + // take min of last row + size_t deltaMin = std::max(std::max(deltaMax + 1, prefix.size()), s.size()); + for (size_t i = 0; i <= len2; i++) { + if (d[len1 * (len2 + 1) + i] < deltaMin) deltaMin = d[len1 * (len2 + 1) + i]; + } + + return deltaMin; +} + +// _____________________________________________________________________________ +inline size_t prefixEditDist(const std::string& prefix, const std::string& s) { + return prefixEditDist(prefix, s, s.size()); +} + // _____________________________________________________________________________ template inline std::string implode(Iter begin, const Iter& end, const char* del) { diff --git a/src/util/geo/Geo.h b/src/util/geo/Geo.h index cd93423..5cfa6cf 100644 --- a/src/util/geo/Geo.h +++ b/src/util/geo/Geo.h @@ -47,7 +47,7 @@ typedef Polygon DPolygon; typedef Polygon FPolygon; typedef Polygon IPolygon; -const static double EPSILON = 0.00000000001; +const static double EPSILON = 0.00001; const static double RAD = 0.017453292519943295; // PI/180 // _____________________________________________________________________________ @@ -236,7 +236,7 @@ inline RotatedBox shrink(const RotatedBox& b, double d) { } // _____________________________________________________________________________ -inline bool doubleEq(double a, double b) { return fabs(a - b) < 0.000001; } +inline bool doubleEq(double a, double b) { return fabs(a - b) < EPSILON; } // _____________________________________________________________________________ template @@ -404,7 +404,7 @@ inline bool intersects(const LineSegment& ls1, const LineSegment& ls2) { // intersecting return intersects(getBoundingBox(ls1), getBoundingBox(ls2)) && (((contains(ls1.first, ls2) ^ contains(ls1.second, ls2)) ^ - (contains(ls2.first, ls1) ^ contains(ls2.second, ls1))) || + (contains(ls2.first, ls1) ^ contains(ls2.second, ls1))) || (((crossProd(ls1.first, ls2) < 0) ^ (crossProd(ls1.second, ls2) < 0)) && ((crossProd(ls2.first, ls1) < 0) ^ @@ -1153,7 +1153,7 @@ inline size_t convexHullImpl(const MultiPoint& a, size_t p1, size_t p2, for (const auto& p : a) { double tmpDist = distToSegment((*h)[p1], (*h)[p2], p); double cp = crossProd(p, LineSegment((*h)[p1], (*h)[p2])); - if (((cp > 0 && !d) || (cp < 0 && d)) && tmpDist > maxDist) { + if (((cp > 0 && !d) || (cp < 0 && d)) && tmpDist >= maxDist + EPSILON) { pa = p; found = true; maxDist = tmpDist; @@ -1462,7 +1462,8 @@ inline Point latLngToWebMerc(double lat, double lng) { // _____________________________________________________________________________ template inline Point webMercToLatLng(double x, double y) { - double lat = 114.591559026 * (atan(exp(y / 6378137.0)) - 0.78539825); + double lat = + (1.5707963267948966 - (2.0 * atan(exp(-y / 6378137.0)))) * (180.0 / M_PI); double lon = x / 111319.4907932735677; return Point(lon, lat); } diff --git a/src/util/geo/Point.h b/src/util/geo/Point.h index 923e9d4..65398dc 100644 --- a/src/util/geo/Point.h +++ b/src/util/geo/Point.h @@ -5,6 +5,8 @@ #ifndef UTIL_GEO_POINT_H_ #define UTIL_GEO_POINT_H_ +#include + namespace util { namespace geo { diff --git a/src/util/graph/Algorithm.h b/src/util/graph/Algorithm.h new file mode 100644 index 0000000..01b23f3 --- /dev/null +++ b/src/util/graph/Algorithm.h @@ -0,0 +1,33 @@ +// Copyright 2017, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#ifndef UTIL_GRAPH_ALGORITHM_H_ +#define UTIL_GRAPH_ALGORITHM_H_ + +#include +#include "util/graph/Edge.h" +#include "util/graph/UndirGraph.h" +#include "util/graph/Node.h" + +namespace util { +namespace graph { + +using util::graph::Graph; +using util::graph::Node; +using util::graph::Edge; + +// collection of general graph algorithms +class Algorithm { + public: + template + static std::vector*> > connectedComponents( + const UndirGraph& g); +}; + +#include "util/graph/Algorithm.tpp" + +} +} + +#endif // UTIL_GRAPH_ALGORITHM_H_ diff --git a/src/util/graph/Algorithm.tpp b/src/util/graph/Algorithm.tpp new file mode 100644 index 0000000..5c1f7c1 --- /dev/null +++ b/src/util/graph/Algorithm.tpp @@ -0,0 +1,32 @@ +// Copyright 2017, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +// _____________________________________________________________________________ +template +std::vector*> > Algorithm::connectedComponents( + const UndirGraph& g) { + std::vector*>> ret; + std::set*> visited; + + for (auto* n : g.getNds()) { + if (!visited.count(n)) { + ret.resize(ret.size() + 1); + std::stack*> q; + q.push(n); + while (!q.empty()) { + Node* cur = q.top(); + q.pop(); + + ret.back().insert(cur); + visited.insert(cur); + + for (auto* e : cur->getAdjList()) { + if (!visited.count(e->getOtherNd(cur))) q.push(e->getOtherNd(cur)); + } + } + } + } + + return ret; +} diff --git a/src/util/graph/DirNode.h b/src/util/graph/DirNode.h index 6dc532b..5274ebf 100644 --- a/src/util/graph/DirNode.h +++ b/src/util/graph/DirNode.h @@ -6,6 +6,7 @@ #define UTIL_GRAPH_DIRNODE_H_ #include +#include #include "util/graph/Node.h" namespace util { diff --git a/src/util/graph/Node.h b/src/util/graph/Node.h index ababb0e..62d5984 100644 --- a/src/util/graph/Node.h +++ b/src/util/graph/Node.h @@ -33,12 +33,15 @@ class Node { virtual void addEdge(Edge* e) = 0; virtual void removeEdge(Edge* e) = 0; - virtual ~Node() {}; + virtual ~Node() = 0; virtual N& pl() = 0; virtual const N& pl() const = 0; }; +template +inline Node::~Node() {} + }} #endif // UTIL_GRAPH_NODE_H_ diff --git a/src/util/graph/UndirNode.h b/src/util/graph/UndirNode.h index b591756..f5aba70 100644 --- a/src/util/graph/UndirNode.h +++ b/src/util/graph/UndirNode.h @@ -6,6 +6,7 @@ #define UTIL_GRAPH_UNDIRNODE_H_ #include +#include #include "util/graph/Node.h" namespace util { diff --git a/src/util/http/Server.cpp b/src/util/http/Server.cpp new file mode 100644 index 0000000..666396b --- /dev/null +++ b/src/util/http/Server.cpp @@ -0,0 +1,347 @@ +// Copyright 2018, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#ifndef ZLIB_CONST +#define ZLIB_CONST +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef ZLIB_FOUND +#include +#endif +#include +#include "Server.h" +#include "util/String.h" + +using util::http::Socket; +using util::http::Queue; +using util::http::Req; +using util::http::HttpErr; +using util::http::HttpServer; +using util::http::HeaderState; + +// _____________________________________________________________________________ +Socket::Socket(int port) { + int y = 1; + _sock = socket(PF_INET, SOCK_STREAM, 0); + if (_sock < 0) + throw std::runtime_error(std::string("Could not create socket (") + + std::strerror(errno) + ")"); + + struct sockaddr_in addr; + + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = INADDR_ANY; + memset(&(addr.sin_zero), '\0', 8); + + setsockopt(_sock, SOL_SOCKET, SO_REUSEADDR, &y, sizeof(y)); + // https://news.ycombinator.com/item?id=10608356 + setsockopt(_sock, IPPROTO_TCP, TCP_QUICKACK, &y, sizeof(y)); + + if (bind(_sock, reinterpret_cast(&addr), sizeof(addr)) < 0) { + throw std::runtime_error(std::string("Could not bind to port ") + + std::to_string(port) + " (" + + std::strerror(errno) + ")"); + } +} + +// _____________________________________________________________________________ +Socket::~Socket() { close(_sock); } + +// _____________________________________________________________________________ +int Socket::wait() { + if (listen(_sock, BLOG) < 0) + throw std::runtime_error(std::string("Cannot listen to socket (") + + std::strerror(errno) + ")"); + sockaddr_in cli_addr; + socklen_t clilen = sizeof(cli_addr); + int sock = accept(_sock, reinterpret_cast(&cli_addr), &clilen); + return sock; +} + +// _____________________________________________________________________________ +void HttpServer::send(int sock, Answer* aw) { + std::string enc = "identity"; + if (aw->gzip) aw->pl = compress(aw->pl, &enc); + + aw->params["Content-Encoding"] = enc; + aw->params["Content-Length"] = std::to_string(aw->pl.size()); + + std::stringstream ss; + ss << "HTTP/1.1 " << aw->status << "\r\n"; + for (const auto& kv : aw->params) + ss << kv.first << ": " << kv.second << "\r\n"; + ss << "\r\n" << aw->pl; + std::string buff = ss.str(); + + size_t writes = 0; + // https://news.ycombinator.com/item?id=10608356 + int y = 1; + setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, &y, sizeof(y)); + + while (writes != buff.size()) { + int64_t out = write(sock, buff.c_str() + writes, buff.size() - writes); + if (out < 0) { + if (errno == EWOULDBLOCK || errno == EAGAIN || errno == EINTR) continue; + throw std::runtime_error("Failed to write to socket"); + } + writes += out; + } +} + +// _____________________________________________________________________________ +void HttpServer::handle() { + int connection = -1; + while ((connection = _jobs.get()) != -1) { + Answer answ; + + try { + Req req = getReq(connection); + answ = _handler->handle(req, connection); + answ.gzip = gzipSupport(req); + } catch (HttpErr err) { + answ = Answer{err.what(), err.what(), false, {}}; + } catch (...) { + // catch everything to make sure the server continues running + answ = Answer{ + "500 Internal Server Error", "500 Internal Server Error", false, {}}; + } + + send(connection, &answ); + close(connection); + } +} + +// _____________________________________________________________________________ +bool HttpServer::gzipSupport(const Req& req) { + bool accepts = false; + // decide according to + // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html + for (const auto& kv : req.params) { + if (kv.first == "Accept-Encoding") { + for (const auto& encoding : split(kv.second, ',')) { + std::vector parts = split(encoding, ';'); + for (size_t i = 0; i < parts.size(); i++) { + parts[i] = trim(parts[i]); + } + if (parts[0] == "*" && ((parts.size() == 1) || parts[1] != "q=0")) + accepts = true; + if (parts[0] == "gzip") accepts = true; + if (parts.size() > 1 && parts[1] == "q=0") accepts = false; + } + } + } + return accepts; +} + +// _____________________________________________________________________________ +Req HttpServer::getReq(int connection) { + char buf[BSIZE + 1]; + size_t rcvd = 0; + int64_t curRcvd = 0; + HeaderState state = NONE; + Req ret{"", "", "", "", {}}; + char *tmp, *tmp2; + char* brk = 0; + + while ((curRcvd = read(connection, buf + rcvd, BSIZE - rcvd))) { + if (curRcvd < 0) { + if (errno == EAGAIN || errno == EINTR) continue; + throw HttpErr("500 Internal Server Error"); + } + + // parse request + for (int i = 0; i < curRcvd; i++) { + if (brk) break; + char* c = buf + rcvd + i; + switch (state) { + case NONE: + state = I_COM; + tmp = c; + continue; + case I_VER: + if (*c == '\n') { + *c = 0; + ret.ver = trim(tmp); + state = A_KEY; + } + continue; + case I_URL: + if (*c == ' ') { + *c = 0, ret.url = trim(tmp); + tmp = c + 1; + state = I_VER; + } else if (*c == '\n') { + *c = 0, ret.url = trim(tmp); + state = A_KEY; + } + continue; + case I_COM: + if (*c == ' ') { + *c = 0, ret.cmd = trim(tmp); + tmp = c + 1; + state = I_URL; + } else if (*c == '\n') { + *c = 0, ret.cmd = trim(tmp); + state = A_KEY; + } + continue; + case A_KEY: + if (*c == '\r') *c = ' '; + if (*c == '\n') + brk = c + 1; + else if (*c != ' ') { + state = I_KEY; + tmp = c; + } + continue; + case I_KEY: + if (*c == ':') { + *c = 0; + state = A_VAL; + } + continue; + case A_VAL: + if (*c != ' ') { + state = I_VAL; + tmp2 = c; + } + continue; + case I_VAL: + if (*c == '\r') *c = ' '; + if (*c == '\n') { + *c = 0; + ret.params[tmp] = trim(tmp2); + state = A_KEY; + } + continue; + } + } + + rcvd += curRcvd; + + // buffer is full + if (rcvd == BSIZE) throw HttpErr("431 Request Header Fields Too Large"); + if (brk) break; + } + + // POST payload + if (ret.cmd == "POST") { + size_t size = 0; + if (ret.params.count("Content-Length")) + size = atoi(ret.params["Content-Length"].c_str()); + if (size) { + char* postBuf = new char[size + 1]; + postBuf[size] = 0; + size_t rem = 0; + + // copy existing to new buffer + if ((int)rcvd > brk - buf) { + rem = std::min(size, rcvd - (brk - buf)); + memcpy(postBuf, brk, rem); + } + + rcvd = 0; + + if (rem < size) { + while ((curRcvd = read(connection, postBuf + rcvd + rem, size - rem))) { + if (curRcvd == -1 && (errno == EAGAIN || errno == EINTR)) continue; + if (curRcvd == -1) { + postBuf[rcvd + 1] = 0; + break; + } + rcvd += curRcvd; + if (rcvd == size - rem) break; + } + } + + ret.payload = postBuf; + delete[] postBuf; + } + } + + return ret; +} + +// _____________________________________________________________________________ +std::string HttpServer::compress(const std::string& str, std::string* enc) { +#ifdef ZLIB_FOUND + // do not compress small payloads + if (str.size() < 500) return str; + + std::string ret; + + // based on http://www.zlib.net/zlib_how.html + z_stream defStr; + defStr.zalloc = Z_NULL; + defStr.zfree = Z_NULL; + defStr.opaque = Z_NULL; + defStr.avail_in = 0; + defStr.next_in = Z_NULL; + + // fail silently with no compression at all + if (deflateInit2(&defStr, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, + Z_DEFAULT_STRATEGY) != Z_OK) + return str; + + defStr.next_in = reinterpret_cast(str.c_str()); + defStr.avail_in = static_cast(str.size()); + + size_t cSize = 0; + do { + if (ret.size() < (cSize + BSIZE_C)) ret.resize(cSize + BSIZE_C); + defStr.avail_out = BSIZE_C; + defStr.next_out = reinterpret_cast(&ret[0] + cSize); + deflate(&defStr, Z_FINISH); + cSize += BSIZE_C - defStr.avail_out; + } while (defStr.avail_out == 0); + + deflateEnd(&defStr); + ret.resize(cSize); + + if (ret.size() > str.size()) return str; + *enc = "gzip"; + return ret; +#else + return str; +#endif +} + +// _____________________________________________________________________________ +void HttpServer::run() { + Socket socket(_port); + + std::vector thrds(_threads); + for (auto& thr : thrds) thr = std::thread(&HttpServer::handle, this); + + while (1) _jobs.add(socket.wait()); +} + +// _____________________________________________________________________________ +void Queue::add(int c) { + if (c < 0) return; + { + std::unique_lock lock(_mut); + _jobs.push(c); + } + _hasNew.notify_one(); +} + +// _____________________________________________________________________________ +int Queue::get() { + std::unique_lock lock(_mut); + while (_jobs.empty()) _hasNew.wait(lock); + int next = _jobs.front(); + _jobs.pop(); + return next; +} diff --git a/src/util/http/Server.h b/src/util/http/Server.h new file mode 100644 index 0000000..4397d67 --- /dev/null +++ b/src/util/http/Server.h @@ -0,0 +1,134 @@ +// Copyright 2018, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Authors: Patrick Brosi + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifndef UTIL_HTTP_SERVER_H_ +#define UTIL_HTTP_SERVER_H_ + +namespace util { +namespace http { + +// socket backlog size +const static size_t BLOG = 128; +// socket read buffer size +const static size_t BSIZE = 4 * 1024; +// zlib compression buffer size +const size_t BSIZE_C = 128 * 1024; + +// states for HTTP header parser +enum HeaderState { NONE, I_COM, I_URL, I_VER, A_KEY, I_KEY, A_VAL, I_VAL }; + +/* + * HTTP Error + */ +class HttpErr : public std::exception { + public: + HttpErr(std::string msg) : _msg(msg) {} + ~HttpErr() throw() {} + + virtual const char* what() const throw() { return _msg.c_str(); } + + private: + std::string _msg; + uint16_t _code; +}; + +/* + * HTTP Request + */ +struct Req { + std::string cmd, url, ver, payload; + std::unordered_map params; +}; + +/* + * HTTP Answer + */ +struct Answer { + std::string status, pl; + bool gzip; + std::unordered_map params; +}; + +/* + * Virtual handler provider class + */ +class Handler { + public: + virtual Answer handle(const Req& request, int connection) const = 0; +}; + +/* + * Queue of connections to handle + */ +class Queue { + public: + void add(int c); + int get(); + + private: + std::mutex _mut; + std::queue _jobs; + std::condition_variable _hasNew; +}; + +/* + * Socket wrapper + */ +class Socket { + public: + Socket(int port); + ~Socket(); + int wait(); + + private: + int _sock; +}; + +/* + * Simple HTTP server, must provide a pointer to a class instance implementing + * virtual class Handler. + */ +class HttpServer { + public: + HttpServer(int port, const Handler* h) : HttpServer(port, h, 0) {} + HttpServer(int port, const Handler* h, size_t threads) + : _port(port), _handler(h), _threads(threads) { + if (!_threads) _threads = 8 * std::thread::hardware_concurrency(); + } + void run(); + + private: + int _port; + Queue _jobs; + const Handler* _handler; + size_t _threads; + + void handle(); + + static void send(int sock, Answer* aw); + static Req getReq(int connection); + static std::string compress(const std::string& str, std::string* enc); + static bool gzipSupport(const Req& req); +}; +} // http +} // util + +#endif // UTIL_HTTP_SERVER_H_ diff --git a/src/util/tests/TestMain.cpp b/src/util/tests/TestMain.cpp index 677d574..bc36557 100644 --- a/src/util/tests/TestMain.cpp +++ b/src/util/tests/TestMain.cpp @@ -8,6 +8,7 @@ #include "util/String.h" #include "util/geo/Geo.h" #include "util/json/Writer.h" +#include "util/graph/Algorithm.h" #include "util/graph/DirGraph.h" #include "util/graph/UndirGraph.h" #include "util/graph/Dijkstra.h" @@ -403,6 +404,46 @@ CASE("editdist") { EXPECT(util::editDist("hello", "hello") == (size_t)0); }}, +// ___________________________________________________________________________ +{ +CASE("prefixeditdist") { + EXPECT(util::prefixEditDist("hello", "hello", 0) == (size_t)0); + EXPECT(util::prefixEditDist("hello", "hello", 100) == (size_t)0); + EXPECT(util::prefixEditDist("hello", "hello") == (size_t)0); + EXPECT(util::prefixEditDist("hel", "hello") == (size_t)0); + EXPECT(util::prefixEditDist("hel", "hello", 0) == (size_t)0); + EXPECT(util::prefixEditDist("hel", "hello", 1) == (size_t)0); + EXPECT(util::prefixEditDist("hel", "hello", 2) == (size_t)0); + EXPECT(util::prefixEditDist("hal", "hello", 2) == (size_t)1); + EXPECT(util::prefixEditDist("hal", "hello", 1) == (size_t)1); + EXPECT(util::prefixEditDist("hal", "hello", 0) > (size_t)0); + EXPECT(util::prefixEditDist("fel", "hello", 0) > (size_t)0); + EXPECT(util::prefixEditDist("fel", "hello", 1) == (size_t)1); + EXPECT(util::prefixEditDist("fel", "hello", 2) == (size_t)1); + EXPECT(util::prefixEditDist("fal", "hello", 2) == (size_t)2); + EXPECT(util::prefixEditDist("fal", "hello", 1) > (size_t)1); + EXPECT(util::prefixEditDist("fal", "hello", 0) > (size_t)0); + EXPECT(util::prefixEditDist("far", "hello", 0) > (size_t)0); + EXPECT(util::prefixEditDist("far", "hello", 1) > (size_t)1); + EXPECT(util::prefixEditDist("far", "hello", 2) > (size_t)2); + EXPECT(util::prefixEditDist("far", "hello", 3) == (size_t)3); + EXPECT(util::prefixEditDist("far", "hello", 4) == (size_t)3); + EXPECT(util::prefixEditDist("far", "hello") == (size_t)3); + EXPECT(util::prefixEditDist("hefar", "hello") == (size_t)3); + EXPECT(util::prefixEditDist("hefaree", "hello") == (size_t)5); + EXPECT(util::prefixEditDist("helloo", "hello") == (size_t)1); + EXPECT(util::prefixEditDist("helloo", "hello", 0) > (size_t)0); + EXPECT(util::prefixEditDist("helloo", "hello", 1) == (size_t)1); + EXPECT(util::prefixEditDist("helloo", "hello", 2) == (size_t)1); + EXPECT(util::prefixEditDist("", "hello", 2) == (size_t)0); + EXPECT(util::prefixEditDist("e", "hello", 2) == (size_t)1); + EXPECT(util::prefixEditDist("el", "hello", 2) == (size_t)1); + EXPECT(util::prefixEditDist("ello", "hello", 2) == (size_t)1); + EXPECT(util::prefixEditDist("hell", "hello", 2) == (size_t)0); + EXPECT(util::prefixEditDist("hell", "", 2) > (size_t)2); + EXPECT(util::prefixEditDist("hell", "") == (size_t)4); +}}, + // ___________________________________________________________________________ { CASE("toString") { @@ -447,6 +488,51 @@ CASE("replace") { EXPECT(b == "loree aaaau aaaau loree"); }}, +// ___________________________________________________________________________ +{ +CASE("Connected components undirected") { + UndirGraph g; + + auto a = g.addNd("A"); + auto b = g.addNd("B"); + auto c = g.addNd("C"); + auto d = g.addNd("D"); + auto e = g.addNd("E"); + + g.addEdg(a, c, 1); + g.addEdg(a, b, 5); + g.addEdg(d, c, 1); + g.addEdg(d, b, 3); + g.addEdg(e, d, 1); + g.addEdg(e, b, 1); + + auto comps = util::graph::Algorithm::connectedComponents(g); + + EXPECT(comps.size() == static_cast(1)); + EXPECT(comps[0].count(a)); + EXPECT(comps[0].count(b)); + EXPECT(comps[0].count(c)); + EXPECT(comps[0].count(d)); + EXPECT(comps[0].count(e)); + + auto f = g.addNd("F"); + comps = util::graph::Algorithm::connectedComponents(g); + EXPECT(comps.size() == static_cast(2)); + + auto gn = g.addNd("G"); + comps = util::graph::Algorithm::connectedComponents(g); + EXPECT(comps.size() == static_cast(3)); + + g.addEdg(f, gn, 1); + comps = util::graph::Algorithm::connectedComponents(g); + EXPECT(comps.size() == static_cast(2)); + + g.addEdg(f, a, 1); + comps = util::graph::Algorithm::connectedComponents(g); + EXPECT(comps.size() == static_cast(1)); + +}}, + // ___________________________________________________________________________ { CASE("Edge-based Dijkstra directed, 1 to all") { diff --git a/src/xml b/src/xml index 00e19bf..5081d32 160000 --- a/src/xml +++ b/src/xml @@ -1 +1 @@ -Subproject commit 00e19bfbdc300eb064fbceef2efaed2ccedfda88 +Subproject commit 5081d32879c30456f6cb515342a3096c5a0d7de6