gtfs-actions/generate-shapes
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

bz2 support for OSM reading and writing

Browse source
This commit is contained in:
Patrick Brosi 2023-08-24 17:29:01 +02:00
parent 3d6ed3e2fd
commit 58c47bb1a5
8 changed files with 101 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
CMakeLists.txt
View file

@ -49,6 +49,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFAEDLE_PRECISION=${PFAEDLE_PRECISION}
find_package(LibZip)
find_package(ZLIB)
find_package(BZip2)
if (LIBZIP_FOUND)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLIBZIP_FOUND=1")
@ -60,6 +61,12 @@ else ()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFXML_NO_ZLIB=1")
endif()
if (BZIP2_FOUND)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBZLIB_FOUND=1")
else ()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFXML_NO_BZLIB=1")
endif()
# http://brianmilco.blogspot.de/2012/11/cmake-automatically-use-git-tags-as.html
include(GetGitRevisionDescription)
git_get_tag(VERSION_GIT)

1
README.md
View file

@ -19,6 +19,7 @@ For a quick visual inspection of the shape quality, see for example the schedule
* `gcc >= 5.0` (or `clang >= 3.9`)
* `libzip` (optional, for ZIP support)
* `zlib` (optional, for gzip support)
* `libbz2` (optional, for bzip2 support)
## Building and Installation

3
src/pfaedle/CMakeLists.txt
View file

@ -6,6 +6,7 @@ list(REMOVE_ITEM pfaedle_SRC ${pfaedle_main})
include_directories(
${PFAEDLE_INCLUDE_DIR}
SYSTEM ${BZIP2_INCLUDE_DIR}
SYSTEM ${LIBZIP_INCLUDE_DIR}
SYSTEM ${LIBZIP_CONF_INCLUDE_DIR}
)
@ -19,6 +20,6 @@ add_executable(pfaedle ${pfaedle_main})
add_library(pfaedle_dep ${pfaedle_SRC})
include_directories(pfaedle_dep PUBLIC ${PROJECT_SOURCE_DIR}/src/xml/include/ ${PROJECT_SOURCE_DIR}/src/cppgtfs/src)
target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread ${LIBZIP_LIBRARY})
target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread ${LIBZIP_LIBRARY} ${BZIP2_LIBRARIES})
add_subdirectory(tests)

2
src/pfaedle/osm/OsmBuilder.cpp
View file

@ -379,7 +379,7 @@ void OsmBuilder::filterWrite(const std::string& in, const std::string& out,
latLngBox.add(Box<double>({minlon, minlat}, {maxlon, maxlat}));
}
util::xml::XmlWriter wr(out, true, 4);
util::xml::XmlWriter wr(out, false, 0);
wr.put("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
wr.openTag("osm", {{"version", "0.6"},

16
src/util/CMakeLists.txt
View file

@ -2,8 +2,20 @@ file(GLOB_RECURSE util_SRC *.cpp)
list(REMOVE_ITEM util_SRC TestMain.cpp)
add_library(util ${util_SRC})
if (!BZIP2_FOUND)
find_package(BZip2)
if (ZLIB_FOUND)
add_definitions( -DBZLIB_FOUND=${BZIP2_FOUND} )
endif()
endif()
if (BZIP2_FOUND)
include_directories( ${BZIP2_INCLUDE_DIR} )
target_link_libraries( util ${BZIP2_LIBRARIES} )
endif(BZIP2_FOUND)
if (!ZLIB_FOUND)
find_package( ZLIB )
find_package(ZLIB)
if (ZLIB_FOUND)
add_definitions( -DZLIB_FOUND=${ZLIB_FOUND} )
endif()
@ -12,6 +24,6 @@ endif()
if (ZLIB_FOUND)
include_directories( ${ZLIB_INCLUDE_DIRS} )
target_link_libraries( util ${ZLIB_LIBRARIES} )
endif( ZLIB_FOUND )
endif(ZLIB_FOUND)
add_subdirectory(tests)

57
src/util/xml/XmlWriter.cpp
View file

@ -5,9 +5,13 @@
#include <algorithm>
#include <fstream>
#include <map>
#include <cstring>
#include <ostream>
#include <stack>
#include <string>
#ifdef BZLIB_FOUND
#include <bzlib.h>
#endif
#include "XmlWriter.h"
@ -20,15 +24,15 @@ using std::string;
// _____________________________________________________________________________
XmlWriter::XmlWriter(std::ostream* out)
: _out(out), _pretty(false), _indent(4), _gzfile(0) {}
: _out(out), _pretty(false), _indent(4), _gzfile(0), _bzfile(0) {}
// _____________________________________________________________________________
XmlWriter::XmlWriter(std::ostream* out, bool pret)
: _out(out), _pretty(pret), _indent(4), _gzfile(0) {}
: _out(out), _pretty(pret), _indent(4), _gzfile(0), _bzfile(0) {}
// _____________________________________________________________________________
XmlWriter::XmlWriter(std::ostream* out, bool pret, size_t indent)
: _out(out), _pretty(pret), _indent(indent), _gzfile(0) {}
: _out(out), _pretty(pret), _indent(indent), _gzfile(0), _bzfile(0) {}
// _____________________________________________________________________________
XmlWriter::XmlWriter(const std::string& file)
@ -40,7 +44,7 @@ XmlWriter::XmlWriter(const std::string& file, bool pret)
// _____________________________________________________________________________
XmlWriter::XmlWriter(const std::string& file, bool pret, size_t indent)
: _out(0), _pretty(pret), _indent(indent), _gzfile(0) {
: _out(0), _pretty(pret), _indent(indent), _gzfile(0), _bzfile(0) {
if (file.size() > 2 && file[file.size() - 1] == 'z' &&
file[file.size() - 2] == 'g' && file[file.size() - 3] == '.') {
#ifdef ZLIB_FOUND
@ -52,6 +56,26 @@ XmlWriter::XmlWriter(const std::string& file, bool pret, size_t indent)
throw std::runtime_error(
"Could not open gzip file for writing, was compiled without gzip "
"support");
#endif
} else if (file.size() > 3 && file[file.size() - 1] == '2' &&
file[file.size() - 2] == 'z' && file[file.size() - 3] == 'b' &&
file[file.size() - 4] == '.') {
#ifdef BZLIB_FOUND
_bzbuf = new char[BUFFER_S];
FILE* f = fopen(file.c_str(), "w");
int err;
if (!f) throw std::runtime_error("Could not open file for writing.");
_bzfile = BZ2_bzWriteOpen(&err, f, 9, 0, 30);
if (err != BZ_OK) {
throw std::runtime_error("Could not open bzip file for writing.");
}
#else
throw std::runtime_error(
"Could not open bzip file for writing, was compiled without bzip "
"support");
#endif
} else {
_outs.open(file);
@ -176,18 +200,43 @@ void XmlWriter::put(const string& str) {
if (_gzfile) {
#ifdef ZLIB_FOUND
gzwrite(_gzfile, str.c_str(), str.size());
#endif
} else if (_bzfile) {
#ifdef BZLIB_FOUND
if (_bzbufpos == BUFFER_S || _bzbufpos + str.size() > BUFFER_S) flushBzip();
memcpy( _bzbuf + _bzbufpos, str.c_str(), str.size());
_bzbufpos += str.size();
#endif
} else {
_out->write(str.c_str(), str.size());
}
}
// _____________________________________________________________________________
void XmlWriter::flushBzip() {
#ifdef BZLIB_FOUND
int err = 0;
BZ2_bzWrite(&err, _bzfile, _bzbuf, _bzbufpos);
if (err == BZ_IO_ERROR) {
BZ2_bzWriteClose(&err, _bzfile, 0, 0, 0);
throw std::runtime_error("Could not write to file.");
}
_bzbufpos = 0;
#endif
}
// _____________________________________________________________________________
void XmlWriter::put(const char c) {
if (_gzfile) {
#ifdef ZLIB_FOUND
gzputc(_gzfile, c);
#endif
} else if (_bzfile) {
#ifdef BZLIB_FOUND
_bzbuf[_bzbufpos++] = c;
if (_bzbufpos == BUFFER_S) flushBzip();
#endif
} else {
_out->put(c);

22
src/util/xml/XmlWriter.h
View file

@ -8,6 +8,9 @@
#ifdef ZLIB_FOUND
#include <zlib.h>
#endif
#ifdef BZLIB_FOUND
#include <bzlib.h>
#endif
#include <map>
#include <ostream>
#include <fstream>
@ -17,6 +20,8 @@
namespace util {
namespace xml {
static const size_t BUFFER_S = 32 * 1024 * 1024;
class XmlWriterException : public std::exception {
public:
XmlWriterException(std::string msg) : _msg(msg) {}
@ -41,6 +46,13 @@ class XmlWriter {
~XmlWriter(){
#ifdef ZLIB_FOUND
if (_gzfile) gzclose(_gzfile);
#endif
#ifdef BZLIB_FOUND
int err;
if (_bzfile) {
flushBzip();
BZ2_bzWriteClose(&err, _bzfile, 0, 0, 0);
}
#endif
};
@ -83,6 +95,8 @@ class XmlWriter {
bool hanging;
};
void flushBzip();
std::ostream* _out;
std::ofstream _outs;
std::stack<XmlNode> _nstack;
@ -96,6 +110,14 @@ class XmlWriter {
int _gzfile;
#endif
char* _bzbuf;
size_t _bzbufpos = 0;
#ifdef BZLIB_FOUND
BZFILE* _bzfile;
#else
int _bzfile;
#endif
// handles indentation
void doIndent();

2
src/xml

@ -1 +1 @@
Subproject commit f0152258dccd2962a21fff2658b7740b595b83ed
Subproject commit c4c3551acbb2343706b1c80feadec81b3bb5e941