🚧 rollback to v3 of actions/upload-artifact

waiting for https://codeberg.org/forgejo/forgejo/issues/3485 to be fixed
🚧 fix OSM data format in test.yml workflow
2024-06-01 22:41:01 +02:00 · 2024-06-01 22:34:36 +02:00 · 2024-06-01 22:26:36 +02:00 · 2024-06-01 22:25:21 +02:00 · 2024-06-01 20:11:38 +02:00 · 2024-06-01 20:07:40 +02:00
23 changed files with 236 additions and 212 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,4 +1,4 @@
 /build

 /Dockerfile
-/push-docker-image.sh
+gtfs-out
--- a/.forgejo/workflows/test.yml
+++ b/.forgejo/workflows/test.yml
@ -0,0 +1,33 @@
+name: GTFS Shapes Generation
+
+on: [push]
+
+jobs:
+  modify_routes:
+    runs-on: docker
+    steps:
+      - name: Download sample feed
+        run: curl -o sample-feed.zip https://download.data.public.lu/resources/horaires-et-arrets-des-transport-publics-gtfs/20240530-080402/gtfs-20240529-20240621.zip
+
+      - name: Download OSM data
+        run: curl -o luxembourg-latest.osm.pbf https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf
+
+      - name: Convert OSM data to .osm
+        run: |
+          apt update && \
+          apt install -y osmctools && \
+          osmconvert luxembourg-latest.osm.pbf -o=luxembourg-latest.osm
+
+      - name: Run Shape Generation
+        uses: gtfs-actions/generate-shapes@main
+        with:
+          gtfs_file: sample-feed.zip
+          osm_file: luxembourg-latest.osm
+          mot: bus
+          output_file: modified-feed.zip
+
+      - name: Upload modified feed
+        uses: actions/upload-artifact@v3
+        with:
+          name: modified-feed
+          path: modified-feed.zip
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -1,110 +0,0 @@
-name: Build
-'on':
-  - push
-jobs:
-  ubuntu-20-04-build-gcc:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout repository code
-        uses: actions/checkout@v2
-      - name: Checkout submodules
-        run: git submodule update --init --recursive
-      - name: update apt
-        run: sudo apt update
-      - name: install dependencies
-        run: sudo apt install -y cmake gcc g++
-      - name: cmake
-        run: mkdir build && cd build && cmake ..
-      - name: make
-        run: cd build && make
-      - name: tests
-        run: cd build && ctest --output-on-failure
-  ubuntu-latest-build-gcc:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository code
-        uses: actions/checkout@v2
-      - name: Checkout submodules
-        run: git submodule update --init --recursive
-      - name: update apt
-        run: sudo apt update
-      - name: install dependencies
-        run: sudo apt install -y cmake gcc g++
-      - name: cmake
-        run: mkdir build && cd build && cmake ..
-      - name: make
-        run: cd build && make
-      - name: tests
-        run: cd build && ctest --output-on-failure
-  ubuntu-20-04-build-clang:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout repository code
-        uses: actions/checkout@v2
-      - name: Checkout submodules
-        run: git submodule update --init --recursive
-      - name: update apt
-        run: sudo apt update
-      - name: install dependencies
-        run: sudo apt install -y cmake clang
-      - name: cmake
-        run: mkdir build && cd build && cmake ..
-        shell: bash
-        env:
-          CC: clang
-          CXX: clang++
-      - name: make
-        run: cd build && make
-      - name: tests
-        run: cd build && ctest --output-on-failure
-  ubuntu-latest-build-clang:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository code
-        uses: actions/checkout@v2
-      - name: Checkout submodules
-        run: git submodule update --init --recursive
-      - name: update apt
-        run: sudo apt update
-      - name: install dependencies
-        run: sudo apt install -y cmake clang
-      - name: cmake
-        run: mkdir build && cd build && cmake ..
-        shell: bash
-        env:
-          CC: clang
-          CXX: clang++
-      - name: make
-        run: cd build && make
-      - name: tests
-        run: cd build && ctest --output-on-failure
-  macos-latest-build:
-    runs-on: macOS-latest
-    steps:
-      - name: Checkout repository code
-        uses: actions/checkout@v2
-      - name: Checkout submodules
-        run: git submodule update --init --recursive
-      - name: install dependencies
-        run: brew install cmake
-      - name: cmake
-        run: mkdir build && cd build && cmake ..
-      - name: make
-        run: cd build && make
-      - name: tests
-        run: cd build && ctest --output-on-failure
-  macos-12-build:
-    runs-on: macOS-12
-    steps:
-      - name: Checkout repository code
-        uses: actions/checkout@v2
-      - name: Checkout submodules
-        run: git submodule update --init --recursive
-      - name: install dependencies
-        run: brew install cmake
-      - name: cmake
-        run: mkdir build && cd build && cmake ..
-      - name: make
-        run: cd build && make
-      - name: tests
-        run: cd build && ctest --output-on-failure
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -1,4 +1,4 @@
-cmake_minimum_required (VERSION 3.1)
+cmake_minimum_required (VERSION 3.5)
 set(CMAKE_CXX_STANDARD 11)

 project (pfaedle)
--- a/8
+++ b/8
@ -1,9 +1,9 @@
-FROM debian:buster-slim AS builder
+FROM debian:bookworm-slim AS builder

 WORKDIR /app

 RUN apt-get update && \
-	apt-get install -y g++ cmake git
+	apt-get install -y g++ cmake git libzip-dev zlib1g-dev libbz2-dev

 ADD . /app
 RUN mkdir build && \
@ -13,10 +13,10 @@ RUN mkdir build && \
 	pwd && \
 	make install

-FROM debian:buster-slim
+FROM debian:bookworm-slim

 RUN apt-get update && \
-	apt-get install -y libgomp1 && \
+	apt-get install -y libzip4 zlib1g libbz2-1.0 && \
 	rm -rf /var/lib/apt/lists/*

 COPY --from=builder /usr/local/etc/pfaedle /usr/local/etc/pfaedle
--- a/README.md
+++ b/README.md
@ -1,7 +1,7 @@
-[![Left: station-to-station path of a single train through Switzerland obtained from schedule timetable data. Right: path of the same train map-matched by pfaedle.](geo/schweiz_ex_res.png?raw=true)](geo/schweiz_ex.png?raw=true)
+[![Left: station-to-station path of a single train through Switzerland obtained from schedule timetable data. Right: path of the same train map-matched by pfaedle.](geo/schweiz_ex_res.png?raw=true)](geo/schweiz_ex.png?raw=true)  
 *Left: station-to-station path of a single train through Switzerland obtained from official schedule data. Right: path of the same train map-matched by pfaedle.*

-[![Left: station-to-station path of a single bus through Stuttgart obtained from official schedule data. Right: path of the same bus map-matched by pfaedle.](geo/stuttgart_ex_res.png?raw=true)](geo/stuttgart_ex.png?raw=true)
+[![Left: station-to-station path of a single bus through Stuttgart obtained from official schedule data. Right: path of the same bus map-matched by pfaedle.](geo/stuttgart_ex_res.png?raw=true)](geo/stuttgart_ex.png?raw=true)  
 *Left: station-to-station path of a single bus through Stuttgart obtained from official schedule data. Right: path of the same bus map-matched by pfaedle.*

 [![Build](https://github.com/ad-freiburg/pfaedle/actions/workflows/build.yml/badge.svg)](https://github.com/ad-freiburg/pfaedle/actions/workflows/build.yml)
@ -25,27 +25,27 @@ For a quick visual inspection of the shape quality, see for example the schedule

 Fetch this repository and init submodules:

-```
-git clone --recurse-submodules https://github.com/ad-freiburg/pfaedle
+```shell
+$ git clone --recurse-submodules https://github.com/ad-freiburg/pfaedle
 ```

-```
-mkdir build && cd build
-cmake ..
-make -j
+```shell
+$ mkdir build && cd build
+$ cmake ..
+$ make -j
 ```

 To install, type
-```
-make install
+```shell
+$ make install
 ```

 # General Usage

 ## Generating shapes for a GTFS feed

-```
-pfaedle -x <OSM FILE> <GTFS INPUT FEED>
+```shell
+$ pfaedle -x <OSM FILE> <GTFS INPUT FEED>
 ```

 A shape'd version of the input GTFS feed will be written to `./gtfs-out`.
@ -55,7 +55,7 @@ input feed. To drop all existing shapes, use the `-D` flag.

 For example, you may generate (and replace existing, see `-D` flag) shapes for the GTFS dataset for Freiburg like this:

-```
+```shell
 $ wget https://fritz.freiburg.de/csv_Downloads/VAGFR.zip
 $ wget http://download.geofabrik.de/europe/germany/baden-wuerttemberg/freiburg-regbez-latest.osm.bz2
 $ pfaedle -D -x freiburg-regbez-latest.osm.bz2 VAGFR.zip
@ -81,16 +81,20 @@ run.

 ## via Docker

-You can use the [`adfreiburg/pfaedle` Docker image](https://hub.docker.com/r/adfreiburg/pfaedle) by mounting the OSM & GTFS data into the container:
+You can use the [Docker image](https://github.com/orgs/ad-freiburg/packages/container/package/pfaedle) by mounting the OSM & GTFS data into the container:

 ```shell
-docker run -i --rm \
+$ docker pull ghcr.io/ad-freiburg/pfaedle:latest
+$ docker run -i --rm \
 	# mount OSM data
 	--volume /path/to/osm/data:/osm \
 	# mount GTFS data
 	--volume /path/to/gtfs/data:/gtfs \
+	# mount default output folder gtfs-out
+	--volume /path/to/output-dir:/gtfs-out \
+	ghcr.io/ad-freiburg/pfaedle:latest \
 	# tell pfaedle where to find the data
-	adfreiburg/pfaedle -x /osm/osm-data.xml -i /gtfs
+	-x /osm/osm-data.xml.bz2 -i /gtfs/myfeed.zip
 ```

 ## Debugging
@ -105,3 +109,7 @@ The following flags may be useful for debugging:
 # Configuration

 A default configuration file `pfaedle.cfg` can be found in this repo and will be installed with `make install`. Custom configuration files can be specified with the `-c` flag. If no `-c` flag is set, `pfaedle` will parse and merge the following cfg files in the given order (if present): `<install prefix>/etc/pfaedle/pfaedle.cfg`, `$HOME/.config/pfaedle/pfaedle.cfg`, `<CWD>/pfaedle.cfg`. Values given in later files will overwrite earlier defined values.
+
+# Attribution
+
+Note that the `shapes.txt` produced by `pfaedle` is based on OpenStreetMap data, which is licensed under ODbL 1.0 (see [here](https://osm.org/copyright)). If you copy, distribute, transmit or adapt the shapefied GTFS feed, please credit the contributors of OpenStreetMap.
--- a/action.yml
+++ b/action.yml
@ -0,0 +1,30 @@
+name: 'Generate shapes from OSM data'
+description: 'Use pfaedle to generate shapes from OSM data for a GTFS feed.'
+
+inputs:
+  gtfs_file:
+    description: 'Path to GTFS .zip file.'
+    required: true
+  osm_file:
+    description: 'Path to OSM .pbf file.'
+    required: true
+  mot:
+    description: 'Mode of transport to generate shapes for.'
+    required: false
+    default: 'all'
+  output_file:
+    description: 'Path to output GTFS .zip file.'
+    required: true
+
+runs:
+    using: 'docker'
+    image: 'Dockerfile'
+    args:
+      - '-i'
+      - ${{ inputs.gtfs_file }}
+      - '-x'
+      - ${{ inputs.osm_file }}
+      - '-m'
+      - ${{ inputs.mot }}
+      - '-o'
+      - ${{ inputs.output_file }}
--- a/pfaedle.cfg
+++ b/pfaedle.cfg
@ -1341,6 +1341,9 @@ osm_filter_keep:
 	waterway=river
 	motorboat=yes
 	ferry=yes
+	amenity=ferry_terminal
+	mooring=ferry
+	station=ferry

 # Nodes that are stations.
 # Only nodes that have been kept during the filtering above will be
--- a/push-docker-image.sh
+++ b/push-docker-image.sh
@ -1,6 +0,0 @@
-#!/bin/sh
-set -ex
-set -o pipefail
-
-echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-docker push adfreiburg/pfaedle
--- a/src/cppgtfs
+++ b/src/cppgtfs
@ -1 +1 @@
-Subproject commit 8e1dd71097cbc2fa1136522a2e2ba3cb574efe96
+Subproject commit d26d5794d396141905d71ecb8cd4f45e0120cba7
--- a/src/pfaedle/PfaedleMain.cpp
+++ b/src/pfaedle/PfaedleMain.cpp
@ -86,6 +86,9 @@ enum class RetCode {
 std::string getFileNameMotStr(const MOTs& mots);
 std::vector<std::string> getCfgPaths(const Config& cfg);

+// _____________________________________________________________________________
+void gtfsWarnCb(std::string msg) { LOG(WARN) << msg; }
+
 // _____________________________________________________________________________
 int main(int argc, char** argv) {
  // disable output buffering for standard output
@ -135,7 +138,9 @@ int main(int argc, char** argv) {
    if (!cfg.writeOverpass && !cfg.writeOsmfilter)
      LOG(INFO) << "Reading GTFS feed " << cfg.feedPaths[0] << " ...";
    try {
-      ad::cppgtfs::Parser p(cfg.feedPaths[0]);
+      ad::cppgtfs::Parser p(cfg.feedPaths[0], false,
+                            cfg.parseAdditionalGTFSFields,
+                            cfg.verbosity ? gtfsWarnCb : 0);
      p.parse(&gtfs[0]);
    } catch (const ad::cppgtfs::ParserException& ex) {
      LOG(ERROR) << "Could not parse input GTFS feed, reason was:";
@ -192,7 +197,7 @@ int main(int argc, char** argv) {

    for (size_t i = 0; i < cfg.feedPaths.size(); i++) {
      ShapeBuilder::getGtfsBox(&gtfs[i], cmdCfgMots, cfg.shapeTripId, true,
-                               &box, maxSpeed, 0);
+                               &box, maxSpeed, 0, cfg.verbosity);
    }
    OsmBuilder osmBuilder;
    std::vector<pfaedle::osm::OsmReadOpts> opts;
@ -214,7 +219,7 @@ int main(int argc, char** argv) {
    BBoxIdx box(BOX_PADDING);
    for (size_t i = 0; i < cfg.feedPaths.size(); i++) {
      ShapeBuilder::getGtfsBox(&gtfs[i], cmdCfgMots, cfg.shapeTripId, true,
-                               &box, maxSpeed, 0);
+                               &box, maxSpeed, 0, cfg.verbosity);
    }
    OsmBuilder osmBuilder;
    std::vector<pfaedle::osm::OsmReadOpts> opts;
@ -269,9 +274,9 @@ int main(int argc, char** argv) {
      pfaedle::osm::OsmBuilder osmBuilder;

      pfaedle::osm::BBoxIdx box(BOX_PADDING);
-      ShapeBuilder::getGtfsBox(&gtfs[0], usedMots, cfg.shapeTripId,
-                               cfg.dropShapes, &box,
-                               motCfg.osmBuildOpts.maxSpeed, &hopDists);
+      ShapeBuilder::getGtfsBox(
+          &gtfs[0], usedMots, cfg.shapeTripId, cfg.dropShapes, &box,
+          motCfg.osmBuildOpts.maxSpeed, &hopDists, cfg.verbosity);

      T_START(osmBuild);

--- a/src/pfaedle/config/ConfigReader.cpp
+++ b/src/pfaedle/config/ConfigReader.cpp
@ -111,6 +111,8 @@ void ConfigReader::help(const char* bin) {
            << "Disable hop cache \n"
            << std::setw(35) << "  --stats"
            << "write stats to stats.json\n"
+            << std::setw(35) << "  -W [ --warn ]"
+            << "enable verbose warning messages\n"
            << std::setw(35) << "  -P"
            << "additional parameter string (in cfg file format)\n";
 }
@ -144,10 +146,12 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
                         {"stats", no_argument, 0, 14},
                         {"no-hop-cache", no_argument, 0, 15},
                         {"gaussian-noise", required_argument, 0, 16},
+                         {"warn", no_argument, 0, 'W'},
+                         {"keep-additional-gtfs-fields", no_argument, 0, 'F'},
                         {0, 0, 0, 0}};

-  char c;
-  while ((c = getopt_long(argc, argv, ":o:hvi:c:x:Dm:g:X:T:d:pP:", ops, 0)) !=
+  int c;
+  while ((c = getopt_long(argc, argv, ":o:hvi:c:x:Dm:g:X:T:d:pP:FW", ops, 0)) !=
         -1) {
    switch (c) {
      case 1:
@ -219,6 +223,12 @@ void ConfigReader::read(Config* cfg, int argc, char** argv) {
      case 16:
        cfg->gaussianNoise = atof(optarg);
        break;
+      case 'W':
+        cfg->verbosity = 1;
+        break;
+      case 'F':
+        cfg->parseAdditionalGTFSFields = true;
+        break;
      case 'v':
        std::cout << "pfaedle " << VERSION_FULL << std::endl;
        exit(0);
--- a/src/pfaedle/config/PfaedleConfig.h
+++ b/src/pfaedle/config/PfaedleConfig.h
@ -9,6 +9,7 @@
 #include <sstream>
 #include <string>
 #include <vector>
+
 #include "ad/cppgtfs/gtfs/Route.h"
 #include "util/geo/Geo.h"

@ -36,8 +37,10 @@ struct Config {
        noTrie(false),
        noHopCache(false),
        writeStats(false),
+        parseAdditionalGTFSFields(false),
        gridSize(2000 / util::geo::M_PER_DEG),
-        gaussianNoise(0) {}
+        gaussianNoise(0),
+        verbosity(0) {}
  std::string dbgOutputPath;
  std::string solveMethod;
  std::string shapeTripId;
@ -62,8 +65,10 @@ struct Config {
  bool noTrie;
  bool noHopCache;
  bool writeStats;
+  bool parseAdditionalGTFSFields;
  double gridSize;
  double gaussianNoise;
+  uint8_t verbosity;

  std::string toString() {
    std::stringstream ss;
@ -85,6 +90,8 @@ struct Config {
       << "no-a-star: " << noAStar << "\n"
       << "no-trie: " << noTrie << "\n"
       << "no-hop-cache: " << noHopCache << "\n"
+       << "verbosity: " << verbosity << "\n"
+       << "parse-additional-gtfs-fields: " << parseAdditionalGTFSFields << "\n"
       << "write-stats: " << writeStats << "\n"
       << "feed-paths: ";

--- a/src/pfaedle/gtfs/Writer.cpp
+++ b/src/pfaedle/gtfs/Writer.cpp
@ -345,6 +345,22 @@ void Writer::write(gtfs::Feed* sourceFeed, const std::string& path) const {
          cannotWrite(curFileTg);
      }
    }
+
+    curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "attributions.txt");
+    curFileTg = gtfsPath + "/attributions.txt";
+    fs.open(curFile.c_str());
+    if (!fs.good()) cannotWrite(curFile, curFileTg);
+    writeAttribution(sourceFeed, &fs);
+    fs.close();
+
+    if (toZip) {
+#ifdef LIBZIP_FOUND
+      moveIntoZip(za, curFile, "attributions.txt");
+#endif
+    } else {
+      if (std::rename(curFile.c_str(), curFileTg.c_str()))
+        cannotWrite(curFileTg);
+    }
  } catch (...) {
 #ifdef LIBZIP_FOUND
    zip_discard(za);
@ -363,6 +379,18 @@ void Writer::write(gtfs::Feed* sourceFeed, const std::string& path) const {
  }
 }

+// ____________________________________________________________________________
+void Writer::writeAttribution(gtfs::Feed*, std::ostream* os) const {
+  auto csvw = ad::cppgtfs::Writer::getAttributionCsvw(os);
+
+  csvw->flushLine();
+  csvw->writeString("OpenStreetMap contributors");
+  csvw->writeString("https://www.openstreetmap.org/copyright");
+  csvw->writeInt(1);
+
+  csvw->flushLine();
+}
+
 // ____________________________________________________________________________
 void Writer::writeFeedInfo(gtfs::Feed* f, std::ostream* os) const {
  auto csvw = ad::cppgtfs::Writer::getFeedInfoCsvw(os);
@ -426,14 +454,15 @@ void Writer::writeAgency(gtfs::Feed* sourceFeed, std::ostream* os) const {

  ad::cppgtfs::Writer w;

-  auto csvw = ad::cppgtfs::Writer::getAgencyCsvw(os);
+  auto csvw =
+      ad::cppgtfs::Writer::getAgencyCsvw(os, sourceFeed->getAgencyAddFlds());
  csvw->flushLine();

  ad::cppgtfs::gtfs::flat::Agency fa;
  auto flds = Parser::getAgencyFlds(csvp.get());

  while (p.nextAgency(csvp.get(), &fa, flds)) {
-    w.writeAgency(fa, csvw.get());
+    w.writeAgency(fa, csvw.get(), sourceFeed->getAgencyAddFlds());
  }
 }

@ -443,14 +472,15 @@ void Writer::writeStops(gtfs::Feed* sourceFeed, std::ostream* os) const {
  auto csvp = p.getCsvParser("stops.txt");
  ad::cppgtfs::Writer w;

-  auto csvw = ad::cppgtfs::Writer::getStopsCsvw(os);
+  auto csvw =
+      ad::cppgtfs::Writer::getStopsCsvw(os, sourceFeed->getStopAddFlds());
  csvw->flushLine();

  ad::cppgtfs::gtfs::flat::Stop s;
  auto flds = Parser::getStopFlds(csvp.get());

  while (p.nextStop(csvp.get(), &s, flds)) {
-    w.writeStop(s, csvw.get());
+    w.writeStop(s, csvw.get(), sourceFeed->getStopAddFlds());
  }
 }

@ -458,11 +488,13 @@ void Writer::writeStops(gtfs::Feed* sourceFeed, std::ostream* os) const {
 void Writer::writeRoutes(gtfs::Feed* sourceFeed, std::ostream* os) const {
  ad::cppgtfs::Writer w;

-  auto csvw = ad::cppgtfs::Writer::getRoutesCsvw(os);
+  auto csvw =
+      ad::cppgtfs::Writer::getRoutesCsvw(os, sourceFeed->getRouteAddFlds());
  csvw->flushLine();

  for (auto r : sourceFeed->getRoutes()) {
-    w.writeRoute(r.second->getFlat(), csvw.get());
+    w.writeRoute(r.second->getFlat(), csvw.get(),
+                 sourceFeed->getRouteAddFlds());
  }
 }

@ -611,12 +643,13 @@ bool Writer::writeTrips(gtfs::Feed* sourceFeed, std::ostream* os) const {
  ad::cppgtfs::Writer w;
  bool hasFreqs = false;

-  auto csvw = ad::cppgtfs::Writer::getTripsCsvw(os);
+  auto csvw =
+      ad::cppgtfs::Writer::getTripsCsvw(os, sourceFeed->getTripAddFlds());
  csvw->flushLine();

  for (auto t : sourceFeed->getTrips()) {
    if (t.getFrequencies().size()) hasFreqs = true;
-    w.writeTrip(t.getFlat(), csvw.get());
+    w.writeTrip(t.getFlat(), csvw.get(), sourceFeed->getTripAddFlds());
  }

  return hasFreqs;
--- a/src/pfaedle/gtfs/Writer.h
+++ b/src/pfaedle/gtfs/Writer.h
@ -40,6 +40,7 @@ class Writer {
  void writeStopTimes(Feed* f, std::ostream* os) const;
  void writeLevels(Feed* f, std::ostream* os) const;
  void writePathways(Feed* f, std::ostream* os) const;
+  void writeAttribution(Feed* f, std::ostream* os) const;

  static void cannotWrite(const std::string& file, const std::string& file2);
  static void cannotWrite(const std::string& file);
--- a/src/pfaedle/osm/Osm.h
+++ b/src/pfaedle/osm/Osm.h
@ -5,10 +5,12 @@
 #ifndef PFAEDLE_OSM_OSM_H_
 #define PFAEDLE_OSM_OSM_H_

+#include <stdint.h>
+
 #include <set>
 #include <string>
-#include <utility>
 #include <unordered_map>
+#include <utility>
 #include <vector>

 namespace pfaedle {
--- a/src/pfaedle/osm/OsmBuilder.cpp
+++ b/src/pfaedle/osm/OsmBuilder.cpp
@ -137,18 +137,12 @@ void OsmBuilder::read(const std::string& path, const OsmReadOpts& opts,
  LOG(DEBUG) << "Snapping stations...";
  snapStats(opts, g, bbox, gridSize, res, orphanStations);

-  LOG(DEBUG) << "Deleting orphan edges...";
-  deleteOrphEdgs(g, opts);
-
  LOG(DEBUG) << "Collapsing edges...";
  collapseEdges(g);

  LOG(DEBUG) << "Writing edge geoms...";
  writeGeoms(g, opts);

-  LOG(DEBUG) << "Deleting orphan edges...";
-  deleteOrphEdgs(g, opts);
-
  LOG(DEBUG) << "Deleting orphan nodes...";
  deleteOrphNds(g, opts);

@ -1566,30 +1560,6 @@ void OsmBuilder::deleteOrphNds(Graph* g, const OsmReadOpts& opts) {
  }
 }

-// _____________________________________________________________________________
-void OsmBuilder::deleteOrphEdgs(Graph* g, const OsmReadOpts& opts) {
-  size_t ROUNDS = 3;
-  for (size_t c = 0; c < ROUNDS; c++) {
-    for (auto i = g->getNds().begin(); i != g->getNds().end();) {
-      if ((*i)->getInDeg() + (*i)->getOutDeg() != 1 || (*i)->pl().getSI() ||
-          (*i)->pl().isTurnCycle()) {
-        ++i;
-        continue;
-      }
-
-      // check if the removal of this edge would transform a steep angle
-      // full turn at an intersection into a node 2 eligible for contraction
-      // if so, dont delete
-      if (keepFullTurn(*i, opts.fullTurnAngle)) {
-        ++i;
-        continue;
-      }
-
-      i = g->delNd(*i);
-    }
-  }
-}
-
 // _____________________________________________________________________________
 bool OsmBuilder::edgesSim(const Edge* a, const Edge* b) {
  if (static_cast<bool>(a->pl().oneWay()) ^ static_cast<bool>(b->pl().oneWay()))
--- a/src/pfaedle/osm/OsmBuilder.h
+++ b/src/pfaedle/osm/OsmBuilder.h
@ -180,7 +180,6 @@ class OsmBuilder {
                        double gridSize, Restrictor* res,
                        const NodeSet& orphanStations);
  static void writeGeoms(Graph* g, const OsmReadOpts& opts);
-  static void deleteOrphEdgs(Graph* g, const OsmReadOpts& opts);
  static void deleteOrphNds(Graph* g, const OsmReadOpts& opts);
  static double dist(const Node* a, const Node* b);

--- a/src/pfaedle/osm/OsmFilter.h
+++ b/src/pfaedle/osm/OsmFilter.h
@ -5,8 +5,11 @@
 #ifndef PFAEDLE_OSM_OSMFILTER_H_
 #define PFAEDLE_OSM_OSMFILTER_H_

+#include <stdint.h>
+
 #include <string>
 #include <vector>
+
 #include "pfaedle/osm/Osm.h"
 #include "pfaedle/osm/OsmReadOpts.h"

--- a/src/pfaedle/router/ShapeBuilder.cpp
+++ b/src/pfaedle/router/ShapeBuilder.cpp
@ -73,7 +73,7 @@ ShapeBuilder::ShapeBuilder(
      _router(router) {
  pfaedle::osm::BBoxIdx box(BOX_PADDING);
  ShapeBuilder::getGtfsBox(feed, mots, cfg.shapeTripId, cfg.dropShapes, &box,
-                           _motCfg.osmBuildOpts.maxSpeed, 0);
+                           _motCfg.osmBuildOpts.maxSpeed, 0, cfg.verbosity);

  _eGrid = EdgeGrid(cfg.gridSize, cfg.gridSize, box.getFullBox(), false);
  _nGrid = NodeGrid(cfg.gridSize, cfg.gridSize, box.getFullBox(), false);
@ -270,6 +270,11 @@ EdgeCandGroup ShapeBuilder::getEdgCands(const Stop* s) const {
                   {}});
  }

+  if (ret.size() == 1 && _cfg.verbosity) {
+    LOG(WARN) << "No snapping candidate found for stop '" << s->getName()
+              << "' (" << s->getId() << ")";
+  }
+
  return ret;
 }

@ -321,7 +326,7 @@ std::pair<std::vector<LINE>, Stats> ShapeBuilder::shapeL(Trip* trip) {
    LOG(INFO) << "Matched 1 trip in " << std::fixed << std::setprecision(2)
              << stats.solveTime << " ms.";
    // print to line
-    return {getGeom(hops, getRAttrs(trip), &colors), stats};
+    return {getGeom(hops, getRAttrs(trip), &colors, trip, 1), stats};
  } catch (const std::runtime_error& e) {
    LOG(ERROR) << e.what();
    return {std::vector<LINE>(), stats};
@ -341,7 +346,6 @@ std::map<size_t, EdgeListHops> ShapeBuilder::shapeify(
    const TripTrie<pfaedle::gtfs::Trip>* trie, HopCache* hopCache) const {
  LOG(VDEBUG) << "Map-matching trie " << trie;

-  // TODO(patrick): assumes the trie is not empty, check this!
  assert(trie->getNdTrips().size());
  assert(trie->getNdTrips().begin()->second.size());
  RoutingAttrs rAttrs = getRAttrs(trie->getNdTrips().begin()->second[0]);
@ -558,15 +562,16 @@ void ShapeBuilder::setShape(Trip* t, const ad::cppgtfs::gtfs::Shape& s,

 // _____________________________________________________________________________
 ad::cppgtfs::gtfs::Shape ShapeBuilder::getGtfsShape(
-    const EdgeListHops& hops, Trip* t, const RoutingAttrs& rAttrs,
-    std::vector<float>* hopDists, uint32_t* bestColor) {
+    const EdgeListHops& hops, Trip* t, size_t numOthers,
+    const RoutingAttrs& rAttrs, std::vector<float>* hopDists,
+    uint32_t* bestColor) {
  ad::cppgtfs::gtfs::Shape ret(getFreeShapeId(t));

  assert(hops.size() == t->getStopTimes().size() - 1);

  std::map<uint32_t, double> colors;

-  const std::vector<LINE>& gl = getGeom(hops, rAttrs, &colors);
+  const std::vector<LINE>& gl = getGeom(hops, rAttrs, &colors, t, numOthers);
  const std::vector<float>& measures = getMeasure(gl);

  size_t seq = 0;
@ -651,7 +656,8 @@ const RoutingAttrs& ShapeBuilder::getRAttrs(const Trip* trip) const {
 void ShapeBuilder::getGtfsBox(const Feed* feed, const MOTs& mots,
                              const std::string& tid, bool dropShapes,
                              osm::BBoxIdx* box, double maxSpeed,
-                              std::vector<double>* hopDists) {
+                              std::vector<double>* hopDists,
+                              uint8_t verbosity) {
  for (const auto& t : feed->getTrips()) {
    if (!tid.empty() && t.getId() != tid) continue;
    if (tid.empty() && !t.getShape().empty() && !dropShapes) continue;
@ -695,11 +701,21 @@ void ShapeBuilder::getGtfsBox(const Feed* feed, const MOTs& mots,
        if (reqToTime > (BUFFER + toTime) * 3 * MAX_ROUTE_COST_DOUBLING_STEPS &&
            reqFromTime >
                (BUFFER + fromTime) * 3 * MAX_ROUTE_COST_DOUBLING_STEPS) {
-          LOG(DEBUG) << "Skipping station " << st.getStop()->getId() << " ("
-                     << st.getStop()->getName() << ") @ "
-                     << st.getStop()->getLat() << ", " << st.getStop()->getLng()
-                     << " for bounding box as the vehicle cannot realistically "
-                        "reach and leave it in the scheduled time";
+          if (verbosity) {
+            LOG(WARN)
+                << "Skipping station '" << st.getStop()->getName() << "' ("
+                << st.getStop()->getId() << ") @ " << st.getStop()->getLat()
+                << ", " << st.getStop()->getLng()
+                << " for bounding box as the vehicle cannot realistically "
+                   "reach and leave it in the scheduled time";
+          } else {
+            LOG(DEBUG)
+                << "Skipping station '" << st.getStop()->getName() << "' ("
+                << st.getStop()->getId() << ") @ " << st.getStop()->getLat()
+                << ", " << st.getStop()->getLng()
+                << " for bounding box as the vehicle cannot realistically "
+                   "reach and leave it in the scheduled time";
+          }
          continue;
        }

@ -821,9 +837,8 @@ double ShapeBuilder::timePen(int candTime, int schedTime) const {
 // _____________________________________________________________________________
 EdgeCandGroup ShapeBuilder::timeExpand(const EdgeCand& ec, int time) const {
  EdgeCandGroup ret;
-  // TODO(patrick): heuristic for time expansion variance
-  // for (int i = -5; i < 6; i++) {
-  // for (int i = -10; i < 1; i++) {
+  // TODO(patrick): heuristic for time expansion variance, currently
+  // unused
  for (int i = 0; i < 1; i++) {
    EdgeCand ecNew = ec;
    // in 30 sec steps
@ -954,9 +969,10 @@ void ShapeBuilder::buildNetGraph(TrGraphEdgs* edgs,
 }

 // _____________________________________________________________________________
-std::vector<LINE> ShapeBuilder::getGeom(
-    const EdgeListHops& hops, const RoutingAttrs& rAttrs,
-    std::map<uint32_t, double>* colors) const {
+std::vector<LINE> ShapeBuilder::getGeom(const EdgeListHops& hops,
+                                        const RoutingAttrs& rAttrs,
+                                        std::map<uint32_t, double>* colors,
+                                        Trip* t, size_t numOthers) const {
  std::vector<LINE> ret;

  for (size_t i = hops.size(); i > 0; i--) {
@ -964,6 +980,23 @@ std::vector<LINE> ShapeBuilder::getGeom(
    if (!hop.start || !hop.end) {
      // no hop was found, use the fallback geometry

+      if (_cfg.verbosity) {
+        const auto stopFr = t->getStopTimes()[hops.size() - i].getStop();
+        const auto stopTo = t->getStopTimes()[hops.size() - i + 1].getStop();
+
+        LOG(WARN) << "No viable hop found between stops '" << stopFr->getName()
+                  << "' (" << stopFr->getId() << ") and '" << stopTo->getName()
+                  << "' (" << stopTo->getId() << ") for trip " << t->getId()
+                  << " of type '"
+                  << ad::cppgtfs::gtfs::flat::Route::getTypeString(
+                         t->getRoute()->getType())
+                  << "'"
+                  << (numOthers > 1 ? " (and " + std::to_string(numOthers) +
+                                          " similar trips)"
+                                    : "")
+                  << ", falling back to straight line";
+      }
+
      if (hop.start) {
        if (hop.progrStart > 0) {
          auto l = getLine(hop.start);
@ -1159,8 +1192,9 @@ void ShapeBuilder::shapeWorker(

        uint32_t color;

-        const ad::cppgtfs::gtfs::Shape& shp = getGtfsShape(
-            hops.at(leaf.first), leaf.second[0], rAttrs, &distances, &color);
+        const ad::cppgtfs::gtfs::Shape& shp =
+            getGtfsShape(hops.at(leaf.first), leaf.second[0],
+                         leaf.second.size(), rAttrs, &distances, &color);

        if (_cfg.buildTransitGraph) {
          writeTransitGraph(hops.at(leaf.first), gtfsGraph, leaf.second);
--- a/src/pfaedle/router/ShapeBuilder.h
+++ b/src/pfaedle/router/ShapeBuilder.h
@ -12,6 +12,7 @@
 #include <unordered_map>
 #include <utility>
 #include <vector>
+
 #include "ad/cppgtfs/gtfs/Feed.h"
 #include "pfaedle/Def.h"
 #include "pfaedle/config/MotConfig.h"
@ -73,7 +74,7 @@ class ShapeBuilder {
  static void getGtfsBox(const pfaedle::gtfs::Feed* feed, const MOTs& mots,
                         const std::string& tid, bool dropShapes,
                         osm::BBoxIdx* box, double maxSpeed,
-                         std::vector<double>* hopDists);
+                         std::vector<double>* hopDists, uint8_t verbosity);

 private:
  pfaedle::gtfs::Feed* _feed;
@ -103,6 +104,7 @@ class ShapeBuilder {
  std::string getFreeShapeId(pfaedle::gtfs::Trip* t);
  ad::cppgtfs::gtfs::Shape getGtfsShape(const EdgeListHops& shp,
                                        pfaedle::gtfs::Trip* t,
+                                        size_t numOthers,
                                        const RoutingAttrs& rAttrs,
                                        std::vector<float>* hopDists,
                                        uint32_t* bestColor);
@ -126,7 +128,8 @@ class ShapeBuilder {
  void buildIndex();

  std::vector<LINE> getGeom(const EdgeListHops& shp, const RoutingAttrs& rAttrs,
-                            std::map<uint32_t, double>* colors) const;
+                            std::map<uint32_t, double>* colors, Trip* t,
+                            size_t numOthers) const;
  double timePen(int candTime, int schedTime) const;

  LINE getLine(const EdgeListHop& hop, const RoutingAttrs&,
--- a/src/pfaedle/router/TripTrie.tpp
+++ b/src/pfaedle/router/TripTrie.tpp
@ -193,7 +193,6 @@ size_t TripTrie<TRIP>::getMatchChild(size_t parentNid,
                                     const std::string& platform, POINT pos,
                                     int time, bool timeEx) const {
  for (size_t child : _nds[parentNid].childs) {
-    // TODO(patrick): use similarity classification here?
    if (_nds[child].stopName == stopName && _nds[child].platform == platform &&
        util::geo::dist(_nds[child].pos, pos) < 1 &&
        (!timeEx || _nds[child].time == time)) {
--- a/src/util
+++ b/src/util
@ -1 +1 @@
-Subproject commit af1ced1c82539675b8c9d49c19136224e4af07a9
+Subproject commit d1c30e9ec4cb68803be073d35beb6af2b860bda4
Author	SHA1	Message	Date
Charles P.	e2f9679c22	🚧 rollback to v3 of actions/upload-artifact All checks were successful GTFS Shapes Generation / modify_routes (push) Successful in 58s Details waiting for https://codeberg.org/forgejo/forgejo/issues/3485 to be fixed	2024-06-01 22:41:01 +02:00
Charles P.	172f7eafdb	🚧 fix OSM data format in test.yml workflow Some checks failed GTFS Shapes Generation / modify_routes (push) Failing after 49s Details	2024-06-01 22:34:36 +02:00
Charles P.	eb2a556276	🚧 add test.yml Some checks failed GTFS Shapes Generation / modify_routes (push) Failing after 2m41s Details	2024-06-01 22:26:36 +02:00
Charles P.	0a2f4eecda	🚧 add test.yml Some checks failed GTFS Shapes Generation / modify_routes (push) Failing after 2s Details	2024-06-01 22:25:21 +02:00
Charles P.	e779970387	🚧 make pfaedle an action	2024-06-01 20:11:38 +02:00
Charles P.	ea8185a784	🚧 remove github actions	2024-06-01 20:07:40 +02:00
Paul	33508404e1	Update minimum cmake version to 3.5 (#54 ) Signed-off-by: paulober <44974737+paulober@users.noreply.github.com>	2024-04-05 18:41:44 +02:00
Patrick Brosi	e503fba5b6	Update README.md	2024-03-04 13:47:13 +01:00
Patrick Brosi	6a1ae5fc77	Update README.md	2024-03-04 13:45:28 +01:00
Patrick Brosi	e643166555	typo	2024-03-04 13:45:05 +01:00
Patrick Brosi	7ccc303022	write attributions.txt crediting OSM contributors, mention OSM copyright in README, closes #53	2024-03-04 13:41:19 +01:00
Patrick Brosi	0037a83f68	dont install OMP in docker - not needed anymore	2024-02-15 13:41:41 +01:00
Patrick Brosi	67b9f7f6d2	remove TODOs	2024-02-15 12:53:33 +01:00
Patrick Brosi	eda6f50fbb	Update README.md	2024-02-15 09:08:36 +01:00
Patrick Brosi	5ced5e5a50	mount gtfs-out in Docker example, closes #45	2024-02-15 09:07:17 +01:00
Patrick Brosi	76c1324353	Consistent use of `$` in README	2024-02-15 09:01:17 +01:00
Patrick Brosi	1752da6147	update .dockerignore	2024-02-15 08:57:47 +01:00
Patrick Brosi	a6b09de921	fix Docker file (build and run on bookworm), install libzip, zlib and libbz2, delete obsolete push-docker-image.sh, update README	2024-02-15 08:53:23 +01:00
Leonard Ehrenfried	9a5fc2abdf	Automatically publish container image to Github Container Repo (#49 ) * Update Debian base image to newest version * Add workflow for building container image * Use date-based tags	2024-02-15 08:23:13 +01:00
Patrick Brosi	ad0252695f	add -W [ --warn ] flag to output warning messages, update cppgtfs	2024-01-30 12:51:22 +01:00
Patrick Brosi	a078b22e18	Revert "enable flto link time optimization" This reverts commit `81a8d9d65b`.	2024-01-25 14:19:51 +01:00
Patrick Brosi	81a8d9d65b	enable flto link time optimization	2024-01-25 14:08:53 +01:00
Patrick Brosi	d87eacb6e0	catch more ferry terminals explicitly	2024-01-24 14:51:10 +01:00
Patrick Brosi	22bc04aa9d	dont delete orphan edges anymore, might remove some candidates for edge snapping	2024-01-24 14:06:54 +01:00
Patrick Brosi	136fae1cf1	update cppgtfs	2024-01-21 16:29:45 +01:00
Patrick Brosi	3b35ef638d	update cppgtfs	2024-01-20 18:41:30 +01:00
Patrick Brosi	80504b21fe	use int instead of char to hold return value of getopt_long to avoid issues if the default char is unsigned (arm64), update util and cppgtfs to fix further issues on such architectures	2024-01-02 16:26:07 +01:00
Patrick Brosi	6bbb461b02	update cppgtfs	2023-11-22 11:04:21 +01:00
Patrick Brosi	83851eed49	update util	2023-11-22 10:36:14 +01:00
Patrick Brosi	6adfc5feff	update cppgtfs	2023-11-21 17:33:43 +01:00
Patrick Brosi	de9583877c	update cppgtfs	2023-11-07 01:14:05 +01:00
Patrick Brosi	be3c41b784	include stdint.h in Osm.h	2023-11-07 01:02:25 +01:00
Patrick Brosi	9d00fa2d75	update util	2023-11-07 00:58:53 +01:00
Patrick Brosi	62e2ea36e1	update util	2023-11-07 00:47:55 +01:00
Patrick Brosi	dd05506c42	update cppgtfs, add (hidden as of now, as the mode is not complete) option -F to keep additional non-standard GTFS fields	2023-11-06 17:59:29 +01:00