From 58c47bb1a5a4b9ca90c5e05da8cd40b9454f7f6d Mon Sep 17 00:00:00 2001 From: Patrick Brosi Date: Thu, 24 Aug 2023 17:29:01 +0200 Subject: [PATCH] bz2 support for OSM reading and writing --- CMakeLists.txt | 7 +++++ README.md | 1 + src/pfaedle/CMakeLists.txt | 3 +- src/pfaedle/osm/OsmBuilder.cpp | 2 +- src/util/CMakeLists.txt | 16 ++++++++-- src/util/xml/XmlWriter.cpp | 57 +++++++++++++++++++++++++++++++--- src/util/xml/XmlWriter.h | 22 +++++++++++++ src/xml | 2 +- 8 files changed, 101 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 328163f..1e86646 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,6 +49,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFAEDLE_PRECISION=${PFAEDLE_PRECISION} find_package(LibZip) find_package(ZLIB) +find_package(BZip2) if (LIBZIP_FOUND) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLIBZIP_FOUND=1") @@ -60,6 +61,12 @@ else () set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFXML_NO_ZLIB=1") endif() +if (BZIP2_FOUND) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBZLIB_FOUND=1") +else () + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFXML_NO_BZLIB=1") +endif() + # http://brianmilco.blogspot.de/2012/11/cmake-automatically-use-git-tags-as.html include(GetGitRevisionDescription) git_get_tag(VERSION_GIT) diff --git a/README.md b/README.md index 7d7b041..96f3220 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ For a quick visual inspection of the shape quality, see for example the schedule * `gcc >= 5.0` (or `clang >= 3.9`) * `libzip` (optional, for ZIP support) * `zlib` (optional, for gzip support) + * `libbz2` (optional, for bzip2 support) ## Building and Installation diff --git a/src/pfaedle/CMakeLists.txt b/src/pfaedle/CMakeLists.txt index 95574c2..124e5d5 100644 --- a/src/pfaedle/CMakeLists.txt +++ b/src/pfaedle/CMakeLists.txt @@ -6,6 +6,7 @@ list(REMOVE_ITEM pfaedle_SRC ${pfaedle_main}) include_directories( ${PFAEDLE_INCLUDE_DIR} + SYSTEM ${BZIP2_INCLUDE_DIR} SYSTEM ${LIBZIP_INCLUDE_DIR} SYSTEM ${LIBZIP_CONF_INCLUDE_DIR} ) @@ -19,6 +20,6 @@ add_executable(pfaedle ${pfaedle_main}) add_library(pfaedle_dep ${pfaedle_SRC}) include_directories(pfaedle_dep PUBLIC ${PROJECT_SOURCE_DIR}/src/xml/include/ ${PROJECT_SOURCE_DIR}/src/cppgtfs/src) -target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread ${LIBZIP_LIBRARY}) +target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread ${LIBZIP_LIBRARY} ${BZIP2_LIBRARIES}) add_subdirectory(tests) diff --git a/src/pfaedle/osm/OsmBuilder.cpp b/src/pfaedle/osm/OsmBuilder.cpp index 8fcb7b3..2c6d728 100644 --- a/src/pfaedle/osm/OsmBuilder.cpp +++ b/src/pfaedle/osm/OsmBuilder.cpp @@ -379,7 +379,7 @@ void OsmBuilder::filterWrite(const std::string& in, const std::string& out, latLngBox.add(Box({minlon, minlat}, {maxlon, maxlat})); } - util::xml::XmlWriter wr(out, true, 4); + util::xml::XmlWriter wr(out, false, 0); wr.put("\n"); wr.openTag("osm", {{"version", "0.6"}, diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 4eb3701..fef7557 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -2,8 +2,20 @@ file(GLOB_RECURSE util_SRC *.cpp) list(REMOVE_ITEM util_SRC TestMain.cpp) add_library(util ${util_SRC}) +if (!BZIP2_FOUND) + find_package(BZip2) + if (ZLIB_FOUND) + add_definitions( -DBZLIB_FOUND=${BZIP2_FOUND} ) + endif() +endif() + +if (BZIP2_FOUND) + include_directories( ${BZIP2_INCLUDE_DIR} ) + target_link_libraries( util ${BZIP2_LIBRARIES} ) +endif(BZIP2_FOUND) + if (!ZLIB_FOUND) - find_package( ZLIB ) + find_package(ZLIB) if (ZLIB_FOUND) add_definitions( -DZLIB_FOUND=${ZLIB_FOUND} ) endif() @@ -12,6 +24,6 @@ endif() if (ZLIB_FOUND) include_directories( ${ZLIB_INCLUDE_DIRS} ) target_link_libraries( util ${ZLIB_LIBRARIES} ) -endif( ZLIB_FOUND ) +endif(ZLIB_FOUND) add_subdirectory(tests) diff --git a/src/util/xml/XmlWriter.cpp b/src/util/xml/XmlWriter.cpp index 99f0803..9c75b4d 100644 --- a/src/util/xml/XmlWriter.cpp +++ b/src/util/xml/XmlWriter.cpp @@ -5,9 +5,13 @@ #include #include #include +#include #include #include #include +#ifdef BZLIB_FOUND +#include +#endif #include "XmlWriter.h" @@ -20,15 +24,15 @@ using std::string; // _____________________________________________________________________________ XmlWriter::XmlWriter(std::ostream* out) - : _out(out), _pretty(false), _indent(4), _gzfile(0) {} + : _out(out), _pretty(false), _indent(4), _gzfile(0), _bzfile(0) {} // _____________________________________________________________________________ XmlWriter::XmlWriter(std::ostream* out, bool pret) - : _out(out), _pretty(pret), _indent(4), _gzfile(0) {} + : _out(out), _pretty(pret), _indent(4), _gzfile(0), _bzfile(0) {} // _____________________________________________________________________________ XmlWriter::XmlWriter(std::ostream* out, bool pret, size_t indent) - : _out(out), _pretty(pret), _indent(indent), _gzfile(0) {} + : _out(out), _pretty(pret), _indent(indent), _gzfile(0), _bzfile(0) {} // _____________________________________________________________________________ XmlWriter::XmlWriter(const std::string& file) @@ -40,7 +44,7 @@ XmlWriter::XmlWriter(const std::string& file, bool pret) // _____________________________________________________________________________ XmlWriter::XmlWriter(const std::string& file, bool pret, size_t indent) - : _out(0), _pretty(pret), _indent(indent), _gzfile(0) { + : _out(0), _pretty(pret), _indent(indent), _gzfile(0), _bzfile(0) { if (file.size() > 2 && file[file.size() - 1] == 'z' && file[file.size() - 2] == 'g' && file[file.size() - 3] == '.') { #ifdef ZLIB_FOUND @@ -52,6 +56,26 @@ XmlWriter::XmlWriter(const std::string& file, bool pret, size_t indent) throw std::runtime_error( "Could not open gzip file for writing, was compiled without gzip " "support"); +#endif + } else if (file.size() > 3 && file[file.size() - 1] == '2' && + file[file.size() - 2] == 'z' && file[file.size() - 3] == 'b' && + file[file.size() - 4] == '.') { +#ifdef BZLIB_FOUND + _bzbuf = new char[BUFFER_S]; + + FILE* f = fopen(file.c_str(), "w"); + int err; + if (!f) throw std::runtime_error("Could not open file for writing."); + + _bzfile = BZ2_bzWriteOpen(&err, f, 9, 0, 30); + + if (err != BZ_OK) { + throw std::runtime_error("Could not open bzip file for writing."); + } +#else + throw std::runtime_error( + "Could not open bzip file for writing, was compiled without bzip " + "support"); #endif } else { _outs.open(file); @@ -176,18 +200,43 @@ void XmlWriter::put(const string& str) { if (_gzfile) { #ifdef ZLIB_FOUND gzwrite(_gzfile, str.c_str(), str.size()); +#endif + } else if (_bzfile) { +#ifdef BZLIB_FOUND + if (_bzbufpos == BUFFER_S || _bzbufpos + str.size() > BUFFER_S) flushBzip(); + memcpy( _bzbuf + _bzbufpos, str.c_str(), str.size()); + _bzbufpos += str.size(); #endif } else { _out->write(str.c_str(), str.size()); } } +// _____________________________________________________________________________ +void XmlWriter::flushBzip() { +#ifdef BZLIB_FOUND + int err = 0; + BZ2_bzWrite(&err, _bzfile, _bzbuf, _bzbufpos); + if (err == BZ_IO_ERROR) { + BZ2_bzWriteClose(&err, _bzfile, 0, 0, 0); + throw std::runtime_error("Could not write to file."); + } + + _bzbufpos = 0; + #endif +} + // _____________________________________________________________________________ void XmlWriter::put(const char c) { if (_gzfile) { #ifdef ZLIB_FOUND gzputc(_gzfile, c); +#endif + } else if (_bzfile) { +#ifdef BZLIB_FOUND + _bzbuf[_bzbufpos++] = c; + if (_bzbufpos == BUFFER_S) flushBzip(); #endif } else { _out->put(c); diff --git a/src/util/xml/XmlWriter.h b/src/util/xml/XmlWriter.h index cd8ea4b..12ddd17 100644 --- a/src/util/xml/XmlWriter.h +++ b/src/util/xml/XmlWriter.h @@ -8,6 +8,9 @@ #ifdef ZLIB_FOUND #include #endif +#ifdef BZLIB_FOUND +#include +#endif #include #include #include @@ -17,6 +20,8 @@ namespace util { namespace xml { +static const size_t BUFFER_S = 32 * 1024 * 1024; + class XmlWriterException : public std::exception { public: XmlWriterException(std::string msg) : _msg(msg) {} @@ -41,6 +46,13 @@ class XmlWriter { ~XmlWriter(){ #ifdef ZLIB_FOUND if (_gzfile) gzclose(_gzfile); +#endif +#ifdef BZLIB_FOUND + int err; + if (_bzfile) { + flushBzip(); + BZ2_bzWriteClose(&err, _bzfile, 0, 0, 0); + } #endif }; @@ -83,6 +95,8 @@ class XmlWriter { bool hanging; }; + void flushBzip(); + std::ostream* _out; std::ofstream _outs; std::stack _nstack; @@ -96,6 +110,14 @@ class XmlWriter { int _gzfile; #endif + char* _bzbuf; + size_t _bzbufpos = 0; +#ifdef BZLIB_FOUND + BZFILE* _bzfile; +#else + int _bzfile; +#endif + // handles indentation void doIndent(); diff --git a/src/xml b/src/xml index f015225..c4c3551 160000 --- a/src/xml +++ b/src/xml @@ -1 +1 @@ -Subproject commit f0152258dccd2962a21fff2658b7740b595b83ed +Subproject commit c4c3551acbb2343706b1c80feadec81b3bb5e941