From 9a8e5d4e311c1e51e6219ab020ed768c2b52fbe9 Mon Sep 17 00:00:00 2001 From: Patrick Brosi Date: Tue, 11 Oct 2022 15:04:24 +0200 Subject: [PATCH] update cppgtfs, support ZIP reading and writing for GTFS feeds --- README.md | 6 +- src/cppgtfs | 2 +- src/pfaedle/Def.h | 29 -- src/pfaedle/PfaedleMain.cpp | 9 +- src/pfaedle/gtfs/Feed.h | 9 +- src/pfaedle/gtfs/ShapeContainer.tpp | 2 +- src/pfaedle/gtfs/Writer.cpp | 643 +++++++++++++++++----------- src/pfaedle/gtfs/Writer.h | 11 +- src/pfaedle/osm/OsmIdSet.cpp | 2 +- src/util/Misc.h | 26 ++ 10 files changed, 441 insertions(+), 298 deletions(-) diff --git a/README.md b/README.md index 7810e03..e4cd8bc 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ make install ## Generating shapes for a GTFS feed ``` -pfaedle -x +pfaedle -x ``` A shape'd version of the input GTFS feed will be written to `./gtfs-out`. @@ -53,9 +53,9 @@ input feed. To drop all existing shapes, use the `-D` flag. For example, you may generate (and replace existing, see -D parameter) shapes for the GTFS dataset for Freiburg like this: ``` -$ wget https://fritz.freiburg.de/csv_Downloads/VAGFR.zip && unzip VAGFR.zip +$ wget https://fritz.freiburg.de/csv_Downloads/VAGFR.zip $ wget http://download.geofabrik.de/europe/germany/baden-wuerttemberg/freiburg-regbez-latest.osm.bz2 && bunzip2 freiburg-regbez-latest.osm.bz2 -$ pfaedle -D -x freiburg-regbez-latest.osm . +$ pfaedle -D -x freiburg-regbez-latest.osm VAGFR.zip ``` ## Generating shapes for a specific MOT diff --git a/src/cppgtfs b/src/cppgtfs index aa27140..d79469f 160000 --- a/src/cppgtfs +++ b/src/cppgtfs @@ -1 +1 @@ -Subproject commit aa2714065e1ee8a4773f25aec598d781985086b4 +Subproject commit d79469fa692d52cae56f5e797561451edcf3016a diff --git a/src/pfaedle/Def.h b/src/pfaedle/Def.h index 9308161..397dc9b 100644 --- a/src/pfaedle/Def.h +++ b/src/pfaedle/Def.h @@ -31,33 +31,4 @@ #define BOX_PADDING 2500 -namespace pfaedle { - -// _____________________________________________________________________________ -inline std::string getTmpFName(std::string dir, std::string postf) { - if (postf.size()) postf = "-" + postf; - if (!dir.size()) dir = util::getTmpDir(); - if (dir.size() && dir.back() != '/') dir = dir + "/"; - - std::string f = dir + ".pfaedle-tmp" + postf; - - size_t c = 0; - - while (access(f.c_str(), F_OK) != -1) { - c++; - if (c > 10000) { - // giving up... - LOG(ERROR) << "Could not find temporary file name!"; - exit(1); - } - std::stringstream ss; - ss << dir << ".pfaedle-tmp" << postf << "-" << std::rand(); - f = ss.str().c_str(); - } - - return f; -} - -} // namespace pfaedle - #endif // PFAEDLE_DEF_H_ diff --git a/src/pfaedle/PfaedleMain.cpp b/src/pfaedle/PfaedleMain.cpp index 35c10ef..a8d1a41 100644 --- a/src/pfaedle/PfaedleMain.cpp +++ b/src/pfaedle/PfaedleMain.cpp @@ -135,8 +135,8 @@ int main(int argc, char** argv) { if (!cfg.writeOverpass && !cfg.writeOsmfilter) LOG(INFO) << "Reading GTFS feed " << cfg.feedPaths[0] << " ..."; try { - ad::cppgtfs::Parser p; - p.parse(>fs[0], cfg.feedPaths[0]); + ad::cppgtfs::Parser p(cfg.feedPaths[0]); + p.parse(>fs[0]); } catch (const ad::cppgtfs::ParserException& ex) { LOG(ERROR) << "Could not parse input GTFS feed, reason was:"; std::cerr << ex.what() << std::endl; @@ -146,9 +146,9 @@ int main(int argc, char** argv) { for (size_t i = 0; i < cfg.feedPaths.size(); i++) { if (!cfg.writeOverpass && !cfg.writeOsmfilter) LOG(INFO) << "Reading GTFS feed " << cfg.feedPaths[i] << " ..."; - ad::cppgtfs::Parser p; try { - p.parse(>fs[i], cfg.feedPaths[i]); + ad::cppgtfs::Parser p(cfg.feedPaths[i]); + p.parse(>fs[i]); } catch (const ad::cppgtfs::ParserException& ex) { LOG(ERROR) << "Could not parse input GTFS feed, reason was:"; std::cerr << ex.what() << std::endl; @@ -427,7 +427,6 @@ int main(int argc, char** argv) { if (cfg.feedPaths.size()) { try { - mkdir(cfg.outputPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); LOG(INFO) << "Writing output GTFS to " << cfg.outputPath << " ..."; pfaedle::gtfs::Writer w; w.write(>fs[0], cfg.outputPath); diff --git a/src/pfaedle/gtfs/Feed.h b/src/pfaedle/gtfs/Feed.h index dd4abdd..f640760 100644 --- a/src/pfaedle/gtfs/Feed.h +++ b/src/pfaedle/gtfs/Feed.h @@ -6,6 +6,7 @@ #define PFAEDLE_GTFS_FEED_H_ #include + #include "Service.h" #include "ShapeContainer.h" #include "StopTime.h" @@ -22,10 +23,10 @@ namespace gtfs { typedef ad::cppgtfs::gtfs::FeedB< ad::cppgtfs::gtfs::Agency, ad::cppgtfs::gtfs::Route, ad::cppgtfs::gtfs::Stop, Service, StopTime, Shape, ad::cppgtfs::gtfs::Fare, - ad::cppgtfs::gtfs::Container, ad::cppgtfs::gtfs::Container, - ad::cppgtfs::gtfs::NullContainer, ad::cppgtfs::gtfs::ContContainer, - ad::cppgtfs::gtfs::ContContainer, ShapeContainer, - ad::cppgtfs::gtfs::Container> + ad::cppgtfs::gtfs::Level, ad::cppgtfs::gtfs::Container, + ad::cppgtfs::gtfs::Container, ad::cppgtfs::gtfs::NullContainer, + ad::cppgtfs::gtfs::ContContainer, ad::cppgtfs::gtfs::ContContainer, + ShapeContainer, ad::cppgtfs::gtfs::Container, ad::cppgtfs::gtfs::Container> Feed; typedef ad::cppgtfs::gtfs::TripB, Service, ad::cppgtfs::gtfs::Route, Shape> diff --git a/src/pfaedle/gtfs/ShapeContainer.tpp b/src/pfaedle/gtfs/ShapeContainer.tpp index 9366028..7d3d305 100644 --- a/src/pfaedle/gtfs/ShapeContainer.tpp +++ b/src/pfaedle/gtfs/ShapeContainer.tpp @@ -8,7 +8,7 @@ // ____________________________________________________________________________ template ShapeContainer::ShapeContainer() : _lastBuff(0) { - std::string f = pfaedle::getTmpFName("", ""); + std::string f = util::getTmpFName("", ".pfaedle-tmp", ""); _storage.open(f, std::fstream::in | std::fstream::out | std::fstream::trunc); // immediately unlink diff --git a/src/pfaedle/gtfs/Writer.cpp b/src/pfaedle/gtfs/Writer.cpp index 26dd51d..8f51fa4 100644 --- a/src/pfaedle/gtfs/Writer.cpp +++ b/src/pfaedle/gtfs/Writer.cpp @@ -2,11 +2,16 @@ // Chair of Algorithms and Data Structures. // Authors: Patrick Brosi +#include +#include + #include #include #include +#include #include #include + #include "ad/cppgtfs/Parser.h" #include "ad/cppgtfs/Writer.h" #include "ad/cppgtfs/gtfs/flat/Agency.h" @@ -15,373 +20,490 @@ using ad::cppgtfs::Parser; using ad::util::CsvWriter; -using pfaedle::getTmpFName; +using ad::util::ZipCsvParser; using pfaedle::gtfs::Writer; +using util::getTmpFName; // ____________________________________________________________________________ void Writer::write(gtfs::Feed* sourceFeed, const std::string& path) const { + bool toZip = + (path.size() > 3 && 0 == path.compare(path.size() - 4, 4, ".zip")); + std::ofstream fs; - std::ifstream is; std::string gtfsPath(path); std::string curFile; std::string curFileTg; - curFile = getTmpFName(gtfsPath, "agency.txt"); - curFileTg = gtfsPath + "/agency.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeAgency(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); + std::string tmpZip; + std::string zipFileName; - curFile = getTmpFName(gtfsPath, "stops.txt"); - curFileTg = gtfsPath + "/stops.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeStops(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); + if (gtfsPath.size() == 0) gtfsPath = "."; - curFile = getTmpFName(gtfsPath, "routes.txt"); - curFileTg = gtfsPath + "/routes.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeRoutes(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); + zip* za = 0; - is.open((sourceFeed->getPath() + "/calendar.txt").c_str()); - if (is.good()) { - is.close(); - curFile = getTmpFName(gtfsPath, "calendar.txt"); - curFileTg = gtfsPath + "/calendar.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeCalendar(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); + if (toZip) { + const size_t slashIdx = path.rfind('/'); + if (slashIdx != std::string::npos) { + zipFileName = path.substr(slashIdx, -1); + gtfsPath = path.substr(0, slashIdx); + } else { + zipFileName = path; + gtfsPath = "."; + } + + tmpZip = getTmpFName(gtfsPath, ".pfaedle-tmp", zipFileName); + + int zipErr = 0; + za = zip_open(tmpZip.c_str(), ZIP_CREATE | ZIP_TRUNCATE, &zipErr); + + if (zipErr != 0) { + char errBuf[100]; + zip_error_to_str(errBuf, sizeof(errBuf), zipErr, errno); + cannotWrite(tmpZip, gtfsPath + "/" + zipFileName); + std::stringstream ss; + ss << "(temporary file for " << (gtfsPath + "/" + zipFileName) + << ") Could not open ZIP file, reason was: " << errBuf; + throw ad::cppgtfs::WriterException(ss.str(), tmpZip); + } + } else { + mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); } - is.open((sourceFeed->getPath() + "/calendar_dates.txt").c_str()); - if (is.good()) { - is.close(); - curFile = getTmpFName(gtfsPath, "calendar_dates.txt"); - curFileTg = gtfsPath + "/calendar_dates.txt"; + try { + Parser ip(sourceFeed->getPath()); + + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "agency.txt"); + curFileTg = gtfsPath + "/agency.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); - writeCalendarDates(sourceFeed, &fs); + writeAgency(sourceFeed, &fs); fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); + + if (toZip) { + moveIntoZip(za, curFile, "agency.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "stops.txt"); + curFileTg = gtfsPath + "/stops.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeStops(sourceFeed, &fs); + fs.close(); + + if (toZip) { + moveIntoZip(za, curFile, "stops.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "routes.txt"); + curFileTg = gtfsPath + "/routes.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeRoutes(sourceFeed, &fs); + fs.close(); + + if (toZip) { + moveIntoZip(za, curFile, "routes.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + + auto csvp = ip.getCsvParser("calendar.txt"); + if (csvp->isGood()) { + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "calendar.txt"); + curFileTg = gtfsPath + "/calendar.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeCalendar(sourceFeed, &fs); + fs.close(); + if (toZip) { + moveIntoZip(za, curFile, "calendar.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + } + + csvp = ip.getCsvParser("calendar_dates.txt"); + if (csvp->isGood()) { + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "calendar_dates.txt"); + curFileTg = gtfsPath + "/calendar_dates.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeCalendarDates(sourceFeed, &fs); + fs.close(); + if (toZip) { + moveIntoZip(za, curFile, "calendar_dates.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + } + + csvp = ip.getCsvParser("transfers.txt"); + if (csvp->isGood()) { + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "transfers.txt"); + curFileTg = gtfsPath + "/transfers.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeTransfers(sourceFeed, &fs); + fs.close(); + if (toZip) { + moveIntoZip(za, curFile, "transfers.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + } + + csvp = ip.getCsvParser("fare_attributes.txt"); + if (csvp->isGood()) { + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "fare_attributes.txt"); + curFileTg = gtfsPath + "/fare_attributes.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeFares(sourceFeed, &fs); + fs.close(); + if (toZip) { + moveIntoZip(za, curFile, "fare_attributes.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + } + + csvp = ip.getCsvParser("fare_rules.txt"); + if (csvp->isGood()) { + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "fare_rules.txt"); + curFileTg = gtfsPath + "/fare_rules.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeFareRules(sourceFeed, &fs); + fs.close(); + if (toZip) { + moveIntoZip(za, curFile, "fare_rules.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + } + + csvp = ip.getCsvParser("levels.txt"); + if (csvp->isGood()) { + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "levels.txt"); + curFileTg = gtfsPath + "/levels.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeLevels(sourceFeed, &fs); + fs.close(); + + if (toZip) { + moveIntoZip(za, curFile, "levels.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + } + + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "shapes.txt"); + curFileTg = gtfsPath + "/shapes.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeShapes(sourceFeed, &fs); + fs.close(); + + if (toZip) { + moveIntoZip(za, curFile, "shapes.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "trips.txt"); + curFileTg = gtfsPath + "/trips.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + bool hasFreqs = writeTrips(sourceFeed, &fs); + fs.close(); + + if (toZip) { + moveIntoZip(za, curFile, "trips.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + + csvp = ip.getCsvParser("frequencies.txt"); + if (hasFreqs && csvp->isGood()) { + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "frequencies.txt"); + curFileTg = gtfsPath + "/frequencies.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeFrequencies(sourceFeed, &fs); + fs.close(); + + if (toZip) { + moveIntoZip(za, curFile, "frequencies.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + } + + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "stop_times.txt"); + curFileTg = gtfsPath + "/stop_times.txt"; + fs.open(curFile.c_str()); + + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeStopTimes(sourceFeed, &fs); + fs.close(); + + if (toZip) { + moveIntoZip(za, curFile, "stop_times.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + + if (!sourceFeed->getPublisherUrl().empty() && + !sourceFeed->getPublisherName().empty()) { + curFile = getTmpFName(gtfsPath, ".pfaedle-tmp", "feed_info.txt"); + curFileTg = gtfsPath + "/feed_info.txt"; + fs.open(curFile.c_str()); + if (!fs.good()) cannotWrite(curFile, curFileTg); + writeFeedInfo(sourceFeed, &fs); + fs.close(); + + if (toZip) { + moveIntoZip(za, curFile, "feed_info.txt"); + } else { + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); + } + } + } catch (...) { + zip_discard(za); + throw; } - is.open((sourceFeed->getPath() + "/transfers.txt").c_str()); - if (is.good()) { - is.close(); - curFile = getTmpFName(gtfsPath, "transfers.txt"); - curFileTg = gtfsPath + "/transfers.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeTransfers(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); - } - - is.open((sourceFeed->getPath() + "/fare_attributes.txt").c_str()); - if (is.good()) { - is.close(); - curFile = getTmpFName(gtfsPath, "fare_attributes.txt"); - curFileTg = gtfsPath + "/fare_attributes.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeFares(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); - } - - is.open((sourceFeed->getPath() + "/fare_rules.txt").c_str()); - if (is.good()) { - is.close(); - curFile = getTmpFName(gtfsPath, "fare_rules.txt"); - curFileTg = gtfsPath + "/fare_rules.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeFareRules(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); - } - - is.close(); - curFile = getTmpFName(gtfsPath, "shapes.txt"); - curFileTg = gtfsPath + "/shapes.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeShapes(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); - - is.close(); - curFile = getTmpFName(gtfsPath, "trips.txt"); - curFileTg = gtfsPath + "/trips.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - bool hasFreqs = writeTrips(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); - - is.open((sourceFeed->getPath() + "/frequencies.txt").c_str()); - if (hasFreqs && is.good()) { - is.close(); - curFile = getTmpFName(gtfsPath, "frequencies.txt"); - curFileTg = gtfsPath + "/frequencies.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeFrequencies(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); - } - - is.close(); - curFile = getTmpFName(gtfsPath, "stop_times.txt"); - curFileTg = gtfsPath + "/stop_times.txt"; - fs.open(curFile.c_str()); - - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeStopTimes(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); - - if (!sourceFeed->getPublisherUrl().empty() && - !sourceFeed->getPublisherName().empty()) { - curFile = getTmpFName(gtfsPath, "feed_info.txt"); - curFileTg = gtfsPath + "/feed_info.txt"; - fs.open(curFile.c_str()); - if (!fs.good()) cannotWrite(curFile, curFileTg); - writeFeedInfo(sourceFeed, &fs); - fs.close(); - if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); + if (toZip) { + std::string targetZipPath = gtfsPath + "/" + zipFileName; + if (!za) cannotWrite(targetZipPath); + zip_close(za); + if (std::rename(tmpZip.c_str(), targetZipPath.c_str())) + cannotWrite(targetZipPath); } } // ____________________________________________________________________________ void Writer::writeFeedInfo(gtfs::Feed* f, std::ostream* os) const { auto csvw = ad::cppgtfs::Writer::getFeedInfoCsvw(os); - csvw.flushLine(); - csvw.writeString(f->getPublisherName()); - csvw.writeString(f->getPublisherUrl()); - csvw.writeString(f->getLang()); + csvw->flushLine(); + csvw->writeString(f->getPublisherName()); + csvw->writeString(f->getPublisherUrl()); + csvw->writeString(f->getLang()); if (!f->getStartDate().empty()) - csvw.writeInt(f->getStartDate().getYYYYMMDD()); + csvw->writeInt(f->getStartDate().getYYYYMMDD()); else - csvw.skip(); + csvw->skip(); if (!f->getEndDate().empty()) - csvw.writeInt(f->getEndDate().getYYYYMMDD()); + csvw->writeInt(f->getEndDate().getYYYYMMDD()); else - csvw.skip(); - csvw.writeString(f->getVersion()); - csvw.flushLine(); + csvw->skip(); + csvw->writeString(f->getVersion()); + csvw->flushLine(); +} + +// ____________________________________________________________________________ +void Writer::writeLevels(gtfs::Feed* sourceFeed, std::ostream* os) const { + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("levels.txt"); + ad::cppgtfs::Writer w; + + auto csvw = ad::cppgtfs::Writer::getLevelCsvw(os); + csvw->flushLine(); + + ad::cppgtfs::gtfs::flat::Level fa; + auto flds = Parser::getLevelFlds(csvp.get()); + + while (p.nextLevel(csvp.get(), &fa, flds)) { + w.writeLevel(fa, csvw.get()); + } } // ____________________________________________________________________________ void Writer::writeAgency(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/agency.txt").c_str()); + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("agency.txt"); - CsvParser csvp(&fs); - Parser p; ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getAgencyCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getAgencyCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::Agency fa; - auto flds = Parser::getAgencyFlds(&csvp); + auto flds = Parser::getAgencyFlds(csvp.get()); - while (p.nextAgency(&csvp, &fa, flds)) { - w.writeAgency(fa, &csvw); + while (p.nextAgency(csvp.get(), &fa, flds)) { + w.writeAgency(fa, csvw.get()); } - fs.close(); } // ____________________________________________________________________________ void Writer::writeStops(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/stops.txt").c_str()); - - CsvParser csvp(&fs); - Parser p; + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("stops.txt"); ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getStopsCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getStopsCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::Stop s; - auto flds = Parser::getStopFlds(&csvp); + auto flds = Parser::getStopFlds(csvp.get()); - while (p.nextStop(&csvp, &s, flds)) { - w.writeStop(s, &csvw); + while (p.nextStop(csvp.get(), &s, flds)) { + w.writeStop(s, csvw.get()); } - fs.close(); } // ____________________________________________________________________________ void Writer::writeRoutes(gtfs::Feed* sourceFeed, std::ostream* os) const { ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getRoutesCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getRoutesCsvw(os); + csvw->flushLine(); for (auto r : sourceFeed->getRoutes()) { - w.writeRoute(r.second->getFlat(), &csvw); + w.writeRoute(r.second->getFlat(), csvw.get()); } } // ____________________________________________________________________________ void Writer::writeCalendar(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/calendar.txt").c_str()); - - CsvParser csvp(&fs); - Parser p; + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("calendar.txt"); ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getCalendarCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getCalendarCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::Calendar c; - auto flds = Parser::getCalendarFlds(&csvp); + auto flds = Parser::getCalendarFlds(csvp.get()); - while (p.nextCalendar(&csvp, &c, flds)) { - w.writeCalendar(c, &csvw); + while (p.nextCalendar(csvp.get(), &c, flds)) { + w.writeCalendar(c, csvw.get()); } - fs.close(); } // ____________________________________________________________________________ void Writer::writeCalendarDates(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/calendar_dates.txt").c_str()); - - CsvParser csvp(&fs); - Parser p; + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("calendar_dates.txt"); ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getCalendarDatesCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getCalendarDatesCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::CalendarDate c; - auto flds = Parser::getCalendarDateFlds(&csvp); + auto flds = Parser::getCalendarDateFlds(csvp.get()); - while (p.nextCalendarDate(&csvp, &c, flds)) { - w.writeCalendarDate(c, &csvw); + while (p.nextCalendarDate(csvp.get(), &c, flds)) { + w.writeCalendarDate(c, csvw.get()); } - fs.close(); } // ____________________________________________________________________________ void Writer::writeFrequencies(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/frequencies.txt").c_str()); - - CsvParser csvp(&fs); - Parser p; + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("frequencies.txt"); ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getFrequencyCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getFrequencyCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::Frequency f; - auto flds = Parser::getFrequencyFlds(&csvp); + auto flds = Parser::getFrequencyFlds(csvp.get()); - while (p.nextFrequency(&csvp, &f, flds)) { - w.writeFrequency(f, &csvw); + while (p.nextFrequency(csvp.get(), &f, flds)) { + w.writeFrequency(f, csvw.get()); } - fs.close(); } // ____________________________________________________________________________ void Writer::writeTransfers(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/transfers.txt").c_str()); - - CsvParser csvp(&fs); - Parser p; + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("transfers.txt"); ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getTransfersCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getTransfersCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::Transfer t; - auto flds = Parser::getTransfersFlds(&csvp); + auto flds = Parser::getTransfersFlds(csvp.get()); - while (p.nextTransfer(&csvp, &t, flds)) { - w.writeTransfer(t, &csvw); + while (p.nextTransfer(csvp.get(), &t, flds)) { + w.writeTransfer(t, csvw.get()); } - fs.close(); } // ____________________________________________________________________________ void Writer::writeFares(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/fare_attributes.txt").c_str()); - - CsvParser csvp(&fs); - Parser p; + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("fare_attributes.txt"); ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getFaresCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getFaresCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::Fare f; - auto flds = Parser::getFareFlds(&csvp); + auto flds = Parser::getFareFlds(csvp.get()); - while (p.nextFare(&csvp, &f, flds)) { - w.writeFare(f, &csvw); + while (p.nextFare(csvp.get(), &f, flds)) { + w.writeFare(f, csvw.get()); } - fs.close(); } // ____________________________________________________________________________ void Writer::writeFareRules(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/fare_rules.txt").c_str()); - - CsvParser csvp(&fs); - Parser p; + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("fare_rules.txt"); ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getFareRulesCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getFareRulesCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::FareRule f; - auto flds = Parser::getFareRuleFlds(&csvp); + auto flds = Parser::getFareRuleFlds(csvp.get()); - while (p.nextFareRule(&csvp, &f, flds)) { - w.writeFareRule(f, &csvw); + while (p.nextFareRule(csvp.get(), &f, flds)) { + w.writeFareRule(f, csvw.get()); } - fs.close(); } // ____________________________________________________________________________ void Writer::writeShapes(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/shapes.txt").c_str()); - - CsvWriter csvw = ad::cppgtfs::Writer::getShapesCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getShapesCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::ShapePoint sp; ad::cppgtfs::Writer w; - if (fs.good()) { - CsvParser csvp(&fs); - Parser p; + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("shapes.txt"); - auto flds = Parser::getShapeFlds(&csvp); + if (csvp->isGood()) { + auto flds = Parser::getShapeFlds(csvp.get()); std::string curShapeId; std::string curSkipShapeId; - while (p.nextShapePoint(&csvp, &sp, flds)) { + while (p.nextShapePoint(csvp.get(), &sp, flds)) { if (sp.id == curSkipShapeId) continue; if (sp.id != curShapeId) { if (sourceFeed->getShapes().has(sp.id)) { @@ -392,16 +514,14 @@ void Writer::writeShapes(gtfs::Feed* sourceFeed, std::ostream* os) const { } } - w.writeShapePoint(sp, &csvw); + w.writeShapePoint(sp, csvw.get()); } } sourceFeed->getShapes().open(); while (sourceFeed->getShapes().nextStoragePt(&sp)) { - w.writeShapePoint(sp, &csvw); + w.writeShapePoint(sp, csvw.get()); } - - fs.close(); } // ____________________________________________________________________________ @@ -409,12 +529,12 @@ bool Writer::writeTrips(gtfs::Feed* sourceFeed, std::ostream* os) const { ad::cppgtfs::Writer w; bool hasFreqs = false; - CsvWriter csvw = ad::cppgtfs::Writer::getTripsCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getTripsCsvw(os); + csvw->flushLine(); for (auto t : sourceFeed->getTrips()) { if (t.getFrequencies().size()) hasFreqs = true; - w.writeTrip(t.getFlat(), &csvw); + w.writeTrip(t.getFlat(), csvw.get()); } return hasFreqs; @@ -422,23 +542,20 @@ bool Writer::writeTrips(gtfs::Feed* sourceFeed, std::ostream* os) const { // ____________________________________________________________________________ void Writer::writeStopTimes(gtfs::Feed* sourceFeed, std::ostream* os) const { - std::ifstream fs; - fs.open((sourceFeed->getPath() + "/stop_times.txt").c_str()); - - CsvParser csvp(&fs); - Parser p; + Parser p(sourceFeed->getPath()); + auto csvp = p.getCsvParser("stop_times.txt"); ad::cppgtfs::Writer w; - CsvWriter csvw = ad::cppgtfs::Writer::getStopTimesCsvw(os); - csvw.flushLine(); + auto csvw = ad::cppgtfs::Writer::getStopTimesCsvw(os); + csvw->flushLine(); ad::cppgtfs::gtfs::flat::StopTime st; - auto flds = Parser::getStopTimeFlds(&csvp); + auto flds = Parser::getStopTimeFlds(csvp.get()); std::string curTripId; Trip* cur = 0; - while (p.nextStopTime(&csvp, &st, flds)) { + while (p.nextStopTime(csvp.get(), &st, flds)) { // we may have changed to distance field if (curTripId != st.trip) { cur = sourceFeed->getTrips().get(st.trip); @@ -449,9 +566,8 @@ void Writer::writeStopTimes(gtfs::Feed* sourceFeed, std::ostream* os) const { st.shapeDistTravelled = stN.getShapeDistanceTravelled(); } - w.writeStopTime(st, &csvw); + w.writeStopTime(st, csvw.get()); } - fs.close(); } // ___________________________________________________________________________ @@ -467,3 +583,24 @@ void Writer::cannotWrite(const std::string& file, const std::string& file2) { ss << "(temporary file for " << file2 << ") Could not write to file"; throw ad::cppgtfs::WriterException(ss.str(), file); } + +// ___________________________________________________________________________ +void Writer::moveIntoZip(zip* za, const std::string& sourcePath, + const std::string& targetPath) { + zip_source_t* s; + FILE* fp = fopen(sourcePath.c_str(), "r"); + if (fp == 0) { + std::stringstream ss; + ss << "(temporary file for " << targetPath << ") Could not open file"; + throw ad::cppgtfs::WriterException(ss.str(), sourcePath); + } + + // immediately unlink + unlink(sourcePath.c_str()); + + if ((s = zip_source_filep(za, fp, 0, -1)) == 0 || + zip_file_add(za, targetPath.c_str(), s, ZIP_FL_ENC_UTF_8) < 0) { + zip_source_free(s); + cannotWrite(targetPath); + } +} diff --git a/src/pfaedle/gtfs/Writer.h b/src/pfaedle/gtfs/Writer.h index 9b894c9..d3ef995 100644 --- a/src/pfaedle/gtfs/Writer.h +++ b/src/pfaedle/gtfs/Writer.h @@ -5,9 +5,12 @@ #ifndef PFAEDLE_GTFS_WRITER_H_ #define PFAEDLE_GTFS_WRITER_H_ +#include #include -#include "ad/cppgtfs/Writer.h" + #include "Feed.h" +#include "ad/cppgtfs/Parser.h" +#include "ad/cppgtfs/Writer.h" namespace pfaedle { namespace gtfs { @@ -32,9 +35,15 @@ class Writer { void writeShapes(Feed* f, std::ostream* os) const; bool writeTrips(Feed* f, std::ostream* os) const; void writeStopTimes(Feed* f, std::ostream* os) const; + void writeLevels(Feed* f, std::ostream* os) const; static void cannotWrite(const std::string& file, const std::string& file2); static void cannotWrite(const std::string& file); + + static void moveIntoZip(zip* za, const std::string& sourcePath, + const std::string& targetPath); + + mutable std::ifstream _ifs; }; } // namespace gtfs diff --git a/src/pfaedle/osm/OsmIdSet.cpp b/src/pfaedle/osm/OsmIdSet.cpp index 3f79e71..753c810 100644 --- a/src/pfaedle/osm/OsmIdSet.cpp +++ b/src/pfaedle/osm/OsmIdSet.cpp @@ -306,7 +306,7 @@ uint32_t OsmIdSet::jenkins(uint32_t in) const { // _____________________________________________________________________________ int OsmIdSet::openTmpFile() const { - const std::string& fname = getTmpFName("", ""); + const std::string& fname = util::getTmpFName("", ".pfaedle-tmp", ""); int file = open(fname.c_str(), O_RDWR | O_CREAT, 0666); // immediately unlink diff --git a/src/util/Misc.h b/src/util/Misc.h index 66bc65f..089bef7 100644 --- a/src/util/Misc.h +++ b/src/util/Misc.h @@ -304,6 +304,32 @@ inline std::string getTmpDir() { return getHomeDir(); } +// _____________________________________________________________________________ +inline std::string getTmpFName(std::string dir, std::string name, + std::string postf) { + if (postf.size()) postf = "-" + postf; + if (dir == "") dir = util::getTmpDir(); + if (dir.size() && dir.back() != '/') dir = dir + "/"; + + std::string f = dir + name + postf; + + size_t c = 0; + + while (access(f.c_str(), F_OK) != -1) { + c++; + if (c > 10000) { + // giving up... + std::cerr << "Could not find temporary file name!" << std::endl; + exit(1); + } + std::stringstream ss; + ss << dir << name << postf << "-" << std::rand(); + f = ss.str().c_str(); + } + + return f; +} + // _____________________________________________________________________________ class approx { public: