From 67308d02e93b8075d9f67fc43f2dd2f8ca4618bf Mon Sep 17 00:00:00 2001 From: Patrick Brosi Date: Mon, 13 May 2019 18:56:43 +0200 Subject: [PATCH] centralize code for tmp storage file creation --- src/pfaedle/Def.h | 30 +++++++++++ src/pfaedle/gtfs/ShapeContainer.h | 3 ++ src/pfaedle/gtfs/ShapeContainer.tpp | 16 ++---- src/pfaedle/gtfs/Writer.cpp | 81 +++++++++++++++-------------- src/pfaedle/gtfs/Writer.h | 2 +- src/pfaedle/osm/OsmIdSet.cpp | 28 +++------- src/pfaedle/osm/OsmIdSet.h | 3 +- 7 files changed, 87 insertions(+), 76 deletions(-) diff --git a/src/pfaedle/Def.h b/src/pfaedle/Def.h index 02e8002..cde8959 100644 --- a/src/pfaedle/Def.h +++ b/src/pfaedle/Def.h @@ -5,6 +5,8 @@ #ifndef PFAEDLE_DEF_H_ #define PFAEDLE_DEF_H_ +#include +#include "util/log/Log.h" #include "util/geo/Geo.h" #include "util/geo/PolyLine.h" @@ -27,4 +29,32 @@ #define BOX_PADDING 2500 +namespace pfaedle { + +// _____________________________________________________________________________ +inline std::string getTmpFName(std::string dir, std::string postf) { + if (postf.size()) postf = "-" + postf; + if (dir.size() && dir.back() != '/') dir = dir + "/"; + + std::string f = dir + ".pfaedle-tmp" + postf; + + size_t c = 0; + + while (access(f.c_str(), F_OK) != -1) { + c++; + if (c > 10000) { + // giving up... + LOG(ERROR) << "Could not find temporary file name!"; + exit(1); + } + std::stringstream ss; + ss << dir << ".pfaedle-tmp" << postf << "-" << std::rand(); + f = ss.str().c_str(); + } + + return f; +} + +} + #endif // PFAEDLE_DEF_H_ diff --git a/src/pfaedle/gtfs/ShapeContainer.h b/src/pfaedle/gtfs/ShapeContainer.h index 6a675b8..0e25cb1 100644 --- a/src/pfaedle/gtfs/ShapeContainer.h +++ b/src/pfaedle/gtfs/ShapeContainer.h @@ -5,13 +5,16 @@ #ifndef PFAEDLE_GTFS_SHAPECONTAINER_H_ #define PFAEDLE_GTFS_SHAPECONTAINER_H_ +#include #include #include #include #include #include +#include #include "ad/cppgtfs/gtfs/Shape.h" #include "ad/cppgtfs/gtfs/flat/Shape.h" +#include "pfaedle/Def.h" #include "util/Misc.h" namespace pfaedle { diff --git a/src/pfaedle/gtfs/ShapeContainer.tpp b/src/pfaedle/gtfs/ShapeContainer.tpp index 081a47b..e4e5395 100644 --- a/src/pfaedle/gtfs/ShapeContainer.tpp +++ b/src/pfaedle/gtfs/ShapeContainer.tpp @@ -2,28 +2,18 @@ // Chair of Algorithms and Data Structures. // Authors: Patrick Brosi -#include -#include - // ____________________________________________________________________________ template ShapeContainer::ShapeContainer() { - std::string f = ".pfaedle-tmp"; - - while (access(f.c_str(), F_OK) != -1) { - std::stringstream ss; - ss << ".pfaedle-tmp-"; - ss << std::rand(); - f = ss.str().c_str(); - } - + std::string f = pfaedle::getTmpFName("", ""); _storage.open(f, std::fstream::in | std::fstream::out | std::fstream::trunc); // immediately unlink unlink(f.c_str()); if (!_storage.good()) { - std::cerr << "Could not open temporary file " << f << std::endl; + std::cerr << "Could not open temporary file " << f + << std::endl; exit(1); } } diff --git a/src/pfaedle/gtfs/Writer.cpp b/src/pfaedle/gtfs/Writer.cpp index 4042da6..7a0c5b3 100644 --- a/src/pfaedle/gtfs/Writer.cpp +++ b/src/pfaedle/gtfs/Writer.cpp @@ -16,6 +16,7 @@ using ad::util::CsvWriter; using ad::cppgtfs::Parser; using pfaedle::gtfs::Writer; +using pfaedle::getTmpFName; // ____________________________________________________________________________ bool Writer::write(gtfs::Feed* sourceFeed, const std::string& path) const { @@ -25,138 +26,145 @@ bool Writer::write(gtfs::Feed* sourceFeed, const std::string& path) const { std::string curFile; std::string curFileTg; - curFile = getTmpFName("agency.txt"); + curFile = getTmpFName(gtfsPath, "agency.txt"); curFileTg = gtfsPath + "/agency.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeAgency(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); - curFile = getTmpFName("stops.txt"); + curFile = getTmpFName(gtfsPath, "stops.txt"); curFileTg = gtfsPath + "/stops.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeStops(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); - curFile = getTmpFName("routes.txt"); + curFile = getTmpFName(gtfsPath, "routes.txt"); curFileTg = gtfsPath + "/routes.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeRoutes(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); is.open((sourceFeed->getPath() + "/calendar.txt").c_str()); if (is.good()) { is.close(); - curFile = getTmpFName("calendar.txt"); + curFile = getTmpFName(gtfsPath, "calendar.txt"); curFileTg = gtfsPath + "/calendar.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeCalendar(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); } is.open((sourceFeed->getPath() + "/calendar_dates.txt").c_str()); if (is.good()) { is.close(); - curFile = getTmpFName("calendar_dates.txt"); + curFile = getTmpFName(gtfsPath, "calendar_dates.txt"); curFileTg = gtfsPath + "/calendar_dates.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeCalendarDates(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); } is.open((sourceFeed->getPath() + "/transfers.txt").c_str()); if (is.good()) { is.close(); - curFile = getTmpFName("transfers.txt"); + curFile = getTmpFName(gtfsPath, "transfers.txt"); curFileTg = gtfsPath + "/transfers.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeTransfers(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); } is.open((sourceFeed->getPath() + "/fare_attributes.txt").c_str()); if (is.good()) { is.close(); - curFile = getTmpFName("fare_attributes.txt"); + curFile = getTmpFName(gtfsPath, "fare_attributes.txt"); curFileTg = gtfsPath + "/fare_attributes.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeFares(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); } is.open((sourceFeed->getPath() + "/fare_rules.txt").c_str()); if (is.good()) { is.close(); - curFile = getTmpFName("fare_rules.txt"); + curFile = getTmpFName(gtfsPath, "fare_rules.txt"); curFileTg = gtfsPath + "/fare_rules.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeFareRules(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); } is.close(); - curFile = getTmpFName("shapes.txt"); + curFile = getTmpFName(gtfsPath, "shapes.txt"); curFileTg = gtfsPath + "/shapes.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeShapes(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); is.close(); - curFile = getTmpFName("trips.txt"); + curFile = getTmpFName(gtfsPath, "trips.txt"); curFileTg = gtfsPath + "/trips.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); bool hasFreqs = writeTrips(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); is.open((sourceFeed->getPath() + "/frequencies.txt").c_str()); if (hasFreqs && is.good()) { is.close(); - curFile = getTmpFName("frequencies.txt"); + curFile = getTmpFName(gtfsPath, "frequencies.txt"); curFileTg = gtfsPath + "/frequencies.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeFrequencies(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); } is.close(); - curFile = getTmpFName("stop_times.txt"); + curFile = getTmpFName(gtfsPath, "stop_times.txt"); curFileTg = gtfsPath + "/stop_times.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeStopTimes(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg); if (!sourceFeed->getPublisherUrl().empty() && !sourceFeed->getPublisherName().empty()) { - curFile = getTmpFName("feed_info.txt"); + curFile = getTmpFName(gtfsPath, "feed_info.txt"); curFileTg = gtfsPath + "/feed_info.txt"; fs.open(curFile.c_str()); if (!fs.good()) cannotWrite(curFile, curFileTg); writeFeedInfo(sourceFeed, &fs); fs.close(); - std::rename(curFile.c_str(), curFileTg.c_str()); + if (std::rename(curFile.c_str(), curFileTg.c_str())) + cannotWrite(curFileTg); } return true; @@ -487,6 +495,13 @@ bool Writer::writeStopTimes(gtfs::Feed* sourceFeed, std::ostream* os) const { return true; } +// ___________________________________________________________________________ +void Writer::cannotWrite(const std::string& file) { + std::stringstream ss; + ss << "Could not write to file"; + throw ad::cppgtfs::WriterException(ss.str(), file); +} + // ___________________________________________________________________________ void Writer::cannotWrite(const std::string& file, const std::string& file2) { std::stringstream ss; @@ -494,17 +509,3 @@ void Writer::cannotWrite(const std::string& file, const std::string& file2) { throw ad::cppgtfs::WriterException(ss.str(), file); } -// _____________________________________________________________________________ -std::string Writer::getTmpFName(const std::string& postf) { - std::string f = ".pfaedle-tmp-" + postf; - - while (access(f.c_str(), F_OK) != -1) { - std::stringstream ss; - ss << ".pfaedle-tmp-"; - ss << postf << "-"; - ss << std::rand(); - f = ss.str().c_str(); - } - - return f; -} diff --git a/src/pfaedle/gtfs/Writer.h b/src/pfaedle/gtfs/Writer.h index deb25f8..917195d 100644 --- a/src/pfaedle/gtfs/Writer.h +++ b/src/pfaedle/gtfs/Writer.h @@ -34,7 +34,7 @@ class Writer { bool writeStopTimes(Feed* f, std::ostream* os) const; static void cannotWrite(const std::string& file, const std::string& file2); - static std::string getTmpFName(const std::string& postf); + static void cannotWrite(const std::string& file); }; } // namespace gtfs diff --git a/src/pfaedle/osm/OsmIdSet.cpp b/src/pfaedle/osm/OsmIdSet.cpp index 5390def..63a12fc 100644 --- a/src/pfaedle/osm/OsmIdSet.cpp +++ b/src/pfaedle/osm/OsmIdSet.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -13,6 +12,8 @@ #include #include #include +#include +#include "pfaedle/Def.h" #include "pfaedle/osm/OsmIdSet.h" using pfaedle::osm::OsmIdSet; @@ -48,7 +49,6 @@ OsmIdSet::~OsmIdSet() { void OsmIdSet::add(osmid id) { if (_closed) throw std::exception(); diskAdd(id); - // _set.insert(id); if (_last > id) _sorted = false; _last = id; @@ -134,7 +134,6 @@ bool OsmIdSet::has(osmid id) const { } bool has = diskHas(id); - // assert(has == (bool)_set.count(id)); return has; } @@ -232,7 +231,7 @@ void OsmIdSet::sort() const { size_t OsmIdSet::cwrite(int f, const void* buf, size_t n) const { ssize_t w = write(f, buf, n); if (w < 0) { - throw std::runtime_error("OSMIDSET: could not write to tmp file.\n"); + throw std::runtime_error("Could not write to tmp file.\n"); } return w; @@ -242,7 +241,7 @@ size_t OsmIdSet::cwrite(int f, const void* buf, size_t n) const { size_t OsmIdSet::cread(int f, void* buf, size_t n) const { ssize_t w = read(f, buf, n); if (w < 0) { - throw std::runtime_error("OSMIDSET: could not read from tmp file.\n"); + throw std::runtime_error("Could not read from tmp file.\n"); } return w; @@ -272,14 +271,15 @@ uint32_t OsmIdSet::hash(uint32_t in, int i) const { // _____________________________________________________________________________ int OsmIdSet::openTmpFile() const { - const std::string& fname = getFName(); + const std::string& fname = getTmpFName(_tmpPath, ""); int file = open(fname.c_str(), O_RDWR | O_CREAT, 0666); // immediately unlink unlink(fname.c_str()); if (file < 0) { - std::cerr << "Could not open temporary file " << fname << std::endl; + std::cerr << "Could not open temporary file " << fname + << std::endl; exit(1); } @@ -288,17 +288,3 @@ int OsmIdSet::openTmpFile() const { #endif return file; } - -// _____________________________________________________________________________ -std::string OsmIdSet::getFName() const { - std::string f = ".pfaedle-tmp"; - - while (access(f.c_str(), F_OK) != -1) { - std::stringstream ss; - ss << ".pfaedle-tmp-"; - ss << std::rand(); - f = ss.str().c_str(); - } - - return f; -} diff --git a/src/pfaedle/osm/OsmIdSet.h b/src/pfaedle/osm/OsmIdSet.h index 24d356d..8eb703e 100644 --- a/src/pfaedle/osm/OsmIdSet.h +++ b/src/pfaedle/osm/OsmIdSet.h @@ -34,6 +34,7 @@ static const size_t OBUFFER_S = 8 * 1024 * 1024; class OsmIdSet { public: OsmIdSet(); + OsmIdSet(const std::string& tmpPath) : _tmpPath(tmpPath) {}; ~OsmIdSet(); // Add an OSM id @@ -47,7 +48,7 @@ class OsmIdSet { static size_t FLOOKUPS; private: - std::set _set; + std::string _tmpPath; mutable bool _closed; mutable int _file; unsigned char* _buffer;