gtfs-actions/generate-shapes
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

centralize code for tmp storage file creation

Browse source
This commit is contained in:
Patrick Brosi 2019-05-13 18:56:43 +02:00
parent ee948a8810
commit 67308d02e9
7 changed files with 87 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/pfaedle/Def.h
View file

@ -5,6 +5,8 @@
#ifndef PFAEDLE_DEF_H_
#define PFAEDLE_DEF_H_
#include <unistd.h>
#include "util/log/Log.h"
#include "util/geo/Geo.h"
#include "util/geo/PolyLine.h"
@ -27,4 +29,32 @@
#define BOX_PADDING 2500
namespace pfaedle {
// _____________________________________________________________________________
inline std::string getTmpFName(std::string dir, std::string postf) {
if (postf.size()) postf = "-" + postf;
if (dir.size() && dir.back() != '/') dir = dir + "/";
std::string f = dir + ".pfaedle-tmp" + postf;
size_t c = 0;
while (access(f.c_str(), F_OK) != -1) {
c++;
if (c > 10000) {
// giving up...
LOG(ERROR) << "Could not find temporary file name!";
exit(1);
}
std::stringstream ss;
ss << dir << ".pfaedle-tmp" << postf << "-" << std::rand();
f = ss.str().c_str();
}
return f;
}
}
#endif // PFAEDLE_DEF_H_

3
src/pfaedle/gtfs/ShapeContainer.h
View file

@ -5,13 +5,16 @@
#ifndef PFAEDLE_GTFS_SHAPECONTAINER_H_
#define PFAEDLE_GTFS_SHAPECONTAINER_H_
#include <unistd.h>
#include <fstream>
#include <iostream>
#include <set>
#include <sstream>
#include <string>
#include <string>
#include "ad/cppgtfs/gtfs/Shape.h"
#include "ad/cppgtfs/gtfs/flat/Shape.h"
#include "pfaedle/Def.h"
#include "util/Misc.h"
namespace pfaedle {

16
src/pfaedle/gtfs/ShapeContainer.tpp
View file

@ -2,28 +2,18 @@
// Chair of Algorithms and Data Structures.
// Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
#include <unistd.h>
#include <string>
// ____________________________________________________________________________
template <typename T>
ShapeContainer<T>::ShapeContainer() {
std::string f = ".pfaedle-tmp";
while (access(f.c_str(), F_OK) != -1) {
std::stringstream ss;
ss << ".pfaedle-tmp-";
ss << std::rand();
f = ss.str().c_str();
}
std::string f = pfaedle::getTmpFName("", "");
_storage.open(f, std::fstream::in | std::fstream::out | std::fstream::trunc);
// immediately unlink
unlink(f.c_str());
if (!_storage.good()) {
std::cerr << "Could not open temporary file " << f << std::endl;
std::cerr << "Could not open temporary file " << f
<< std::endl;
exit(1);
}
}

81
src/pfaedle/gtfs/Writer.cpp
View file

@ -16,6 +16,7 @@
using ad::util::CsvWriter;
using ad::cppgtfs::Parser;
using pfaedle::gtfs::Writer;
using pfaedle::getTmpFName;
// ____________________________________________________________________________
bool Writer::write(gtfs::Feed* sourceFeed, const std::string& path) const {
@ -25,138 +26,145 @@ bool Writer::write(gtfs::Feed* sourceFeed, const std::string& path) const {
std::string curFile;
std::string curFileTg;
curFile = getTmpFName("agency.txt");
curFile = getTmpFName(gtfsPath, "agency.txt");
curFileTg = gtfsPath + "/agency.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeAgency(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg);
curFile = getTmpFName("stops.txt");
curFile = getTmpFName(gtfsPath, "stops.txt");
curFileTg = gtfsPath + "/stops.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeStops(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg);
curFile = getTmpFName("routes.txt");
curFile = getTmpFName(gtfsPath, "routes.txt");
curFileTg = gtfsPath + "/routes.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeRoutes(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg);
is.open((sourceFeed->getPath() + "/calendar.txt").c_str());
if (is.good()) {
is.close();
curFile = getTmpFName("calendar.txt");
curFile = getTmpFName(gtfsPath, "calendar.txt");
curFileTg = gtfsPath + "/calendar.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeCalendar(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str()))
cannotWrite(curFileTg);
}
is.open((sourceFeed->getPath() + "/calendar_dates.txt").c_str());
if (is.good()) {
is.close();
curFile = getTmpFName("calendar_dates.txt");
curFile = getTmpFName(gtfsPath, "calendar_dates.txt");
curFileTg = gtfsPath + "/calendar_dates.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeCalendarDates(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str()))
cannotWrite(curFileTg);
}
is.open((sourceFeed->getPath() + "/transfers.txt").c_str());
if (is.good()) {
is.close();
curFile = getTmpFName("transfers.txt");
curFile = getTmpFName(gtfsPath, "transfers.txt");
curFileTg = gtfsPath + "/transfers.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeTransfers(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str()))
cannotWrite(curFileTg);
}
is.open((sourceFeed->getPath() + "/fare_attributes.txt").c_str());
if (is.good()) {
is.close();
curFile = getTmpFName("fare_attributes.txt");
curFile = getTmpFName(gtfsPath, "fare_attributes.txt");
curFileTg = gtfsPath + "/fare_attributes.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeFares(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str()))
cannotWrite(curFileTg);
}
is.open((sourceFeed->getPath() + "/fare_rules.txt").c_str());
if (is.good()) {
is.close();
curFile = getTmpFName("fare_rules.txt");
curFile = getTmpFName(gtfsPath, "fare_rules.txt");
curFileTg = gtfsPath + "/fare_rules.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeFareRules(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str()))
cannotWrite(curFileTg);
}
is.close();
curFile = getTmpFName("shapes.txt");
curFile = getTmpFName(gtfsPath, "shapes.txt");
curFileTg = gtfsPath + "/shapes.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeShapes(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg);
is.close();
curFile = getTmpFName("trips.txt");
curFile = getTmpFName(gtfsPath, "trips.txt");
curFileTg = gtfsPath + "/trips.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
bool hasFreqs = writeTrips(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg);
is.open((sourceFeed->getPath() + "/frequencies.txt").c_str());
if (hasFreqs && is.good()) {
is.close();
curFile = getTmpFName("frequencies.txt");
curFile = getTmpFName(gtfsPath, "frequencies.txt");
curFileTg = gtfsPath + "/frequencies.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeFrequencies(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str()))
cannotWrite(curFileTg);
}
is.close();
curFile = getTmpFName("stop_times.txt");
curFile = getTmpFName(gtfsPath, "stop_times.txt");
curFileTg = gtfsPath + "/stop_times.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeStopTimes(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str())) cannotWrite(curFileTg);
if (!sourceFeed->getPublisherUrl().empty() &&
!sourceFeed->getPublisherName().empty()) {
curFile = getTmpFName("feed_info.txt");
curFile = getTmpFName(gtfsPath, "feed_info.txt");
curFileTg = gtfsPath + "/feed_info.txt";
fs.open(curFile.c_str());
if (!fs.good()) cannotWrite(curFile, curFileTg);
writeFeedInfo(sourceFeed, &fs);
fs.close();
std::rename(curFile.c_str(), curFileTg.c_str());
if (std::rename(curFile.c_str(), curFileTg.c_str()))
cannotWrite(curFileTg);
}
return true;
@ -487,6 +495,13 @@ bool Writer::writeStopTimes(gtfs::Feed* sourceFeed, std::ostream* os) const {
return true;
}
// ___________________________________________________________________________
void Writer::cannotWrite(const std::string& file) {
std::stringstream ss;
ss << "Could not write to file";
throw ad::cppgtfs::WriterException(ss.str(), file);
}
// ___________________________________________________________________________
void Writer::cannotWrite(const std::string& file, const std::string& file2) {
std::stringstream ss;
@ -494,17 +509,3 @@ void Writer::cannotWrite(const std::string& file, const std::string& file2) {
throw ad::cppgtfs::WriterException(ss.str(), file);
}
// _____________________________________________________________________________
std::string Writer::getTmpFName(const std::string& postf) {
std::string f = ".pfaedle-tmp-" + postf;
while (access(f.c_str(), F_OK) != -1) {
std::stringstream ss;
ss << ".pfaedle-tmp-";
ss << postf << "-";
ss << std::rand();
f = ss.str().c_str();
}
return f;
}

2
src/pfaedle/gtfs/Writer.h
View file

@ -34,7 +34,7 @@ class Writer {
bool writeStopTimes(Feed* f, std::ostream* os) const;
static void cannotWrite(const std::string& file, const std::string& file2);
static std::string getTmpFName(const std::string& postf);
static void cannotWrite(const std::string& file);
};
} // namespace gtfs

28
src/pfaedle/osm/OsmIdSet.cpp
View file

@ -4,7 +4,6 @@
#include <stdio.h>
#include <unistd.h>
#include <string>
#include <algorithm>
#include <cassert>
#include <climits>
@ -13,6 +12,8 @@
#include <exception>
#include <iostream>
#include <sstream>
#include <string>
#include "pfaedle/Def.h"
#include "pfaedle/osm/OsmIdSet.h"
using pfaedle::osm::OsmIdSet;
@ -48,7 +49,6 @@ OsmIdSet::~OsmIdSet() {
void OsmIdSet::add(osmid id) {
if (_closed) throw std::exception();
diskAdd(id);
// _set.insert(id);
if (_last > id) _sorted = false;
_last = id;
@ -134,7 +134,6 @@ bool OsmIdSet::has(osmid id) const {
}
bool has = diskHas(id);
// assert(has == (bool)_set.count(id));
return has;
}
@ -232,7 +231,7 @@ void OsmIdSet::sort() const {
size_t OsmIdSet::cwrite(int f, const void* buf, size_t n) const {
ssize_t w = write(f, buf, n);
if (w < 0) {
throw std::runtime_error("OSMIDSET: could not write to tmp file.\n");
throw std::runtime_error("Could not write to tmp file.\n");
}
return w;
@ -242,7 +241,7 @@ size_t OsmIdSet::cwrite(int f, const void* buf, size_t n) const {
size_t OsmIdSet::cread(int f, void* buf, size_t n) const {
ssize_t w = read(f, buf, n);
if (w < 0) {
throw std::runtime_error("OSMIDSET: could not read from tmp file.\n");
throw std::runtime_error("Could not read from tmp file.\n");
}
return w;
@ -272,14 +271,15 @@ uint32_t OsmIdSet::hash(uint32_t in, int i) const {
// _____________________________________________________________________________
int OsmIdSet::openTmpFile() const {
const std::string& fname = getFName();
const std::string& fname = getTmpFName(_tmpPath, "");
int file = open(fname.c_str(), O_RDWR | O_CREAT, 0666);
// immediately unlink
unlink(fname.c_str());
if (file < 0) {
std::cerr << "Could not open temporary file " << fname << std::endl;
std::cerr << "Could not open temporary file " << fname
<< std::endl;
exit(1);
}
@ -288,17 +288,3 @@ int OsmIdSet::openTmpFile() const {
#endif
return file;
}
// _____________________________________________________________________________
std::string OsmIdSet::getFName() const {
std::string f = ".pfaedle-tmp";
while (access(f.c_str(), F_OK) != -1) {
std::stringstream ss;
ss << ".pfaedle-tmp-";
ss << std::rand();
f = ss.str().c_str();
}
return f;
}

3
src/pfaedle/osm/OsmIdSet.h
View file

@ -34,6 +34,7 @@ static const size_t OBUFFER_S = 8 * 1024 * 1024;
class OsmIdSet {
public:
OsmIdSet();
OsmIdSet(const std::string& tmpPath) : _tmpPath(tmpPath) {};
~OsmIdSet();
// Add an OSM id
@ -47,7 +48,7 @@ class OsmIdSet {
static size_t FLOOKUPS;
private:
std::set<osmid> _set;
std::string _tmpPath;
mutable bool _closed;
mutable int _file;
unsigned char* _buffer;