From d43d9fc9083fd7ba1865446825274cf282bbdd36 Mon Sep 17 00:00:00 2001 From: Patrick Brosi Date: Wed, 23 Aug 2023 13:46:23 +0200 Subject: [PATCH] gz support for OSM xml reading and writing --- CMakeLists.txt | 7 +++ src/CMakeLists.txt | 9 +-- src/cppgtfs | 2 +- src/pfaedle/CMakeLists.txt | 2 +- src/pfaedle/osm/OsmBuilder.cpp | 8 +-- src/pfaedle/osm/OsmBuilder.h | 2 +- src/util/CMakeLists.txt | 9 ++- src/util/xml/XmlWriter.cpp | 112 +++++++++++++++++++++++++-------- src/util/xml/XmlWriter.h | 28 ++++++++- src/xml | 2 +- 10 files changed, 137 insertions(+), 44 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ce5e3e7..328163f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,11 +48,18 @@ endif () set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFAEDLE_PRECISION=${PFAEDLE_PRECISION}") find_package(LibZip) +find_package(ZLIB) if (LIBZIP_FOUND) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLIBZIP_FOUND=1") endif() +if (ZLIB_FOUND) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DZLIB_FOUND=1") +else () + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFXML_NO_ZLIB=1") +endif() + # http://brianmilco.blogspot.de/2012/11/cmake-automatically-use-git-tags-as.html include(GetGitRevisionDescription) git_get_tag(VERSION_GIT) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fbf3449..d0e7dad 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,15 +1,16 @@ set(PFAEDLE_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -if (COMMAND cpplint_add_subdirectory) - cpplint_add_subdirectory(pfaedle) -endif() - include_directories( ${PFAEDLE_INCLUDE_DIR} ) +if (ZLIB_FOUND) + include_directories( ${ZLIB_INCLUDE_DIRS} ) +endif( ZLIB_FOUND ) + add_subdirectory(util) add_subdirectory(pfaedle) add_subdirectory(cppgtfs) add_subdirectory(configparser) add_subdirectory(shapevl) +add_subdirectory(xml) diff --git a/src/cppgtfs b/src/cppgtfs index 3cd9b7e..a27c5fe 160000 --- a/src/cppgtfs +++ b/src/cppgtfs @@ -1 +1 @@ -Subproject commit 3cd9b7ecdaab216c07dda65a670352be22271fca +Subproject commit a27c5fee863d1a99fd081d20d1cb57861fb263e7 diff --git a/src/pfaedle/CMakeLists.txt b/src/pfaedle/CMakeLists.txt index e36ccf1..95574c2 100644 --- a/src/pfaedle/CMakeLists.txt +++ b/src/pfaedle/CMakeLists.txt @@ -18,7 +18,7 @@ configure_file ( add_executable(pfaedle ${pfaedle_main}) add_library(pfaedle_dep ${pfaedle_SRC}) -include_directories(pfaedle_dep PUBLIC ${PROJECT_SOURCE_DIR}/src/cppgtfs/src) +include_directories(pfaedle_dep PUBLIC ${PROJECT_SOURCE_DIR}/src/xml/include/ ${PROJECT_SOURCE_DIR}/src/cppgtfs/src) target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread ${LIBZIP_LIBRARY}) add_subdirectory(tests) diff --git a/src/pfaedle/osm/OsmBuilder.cpp b/src/pfaedle/osm/OsmBuilder.cpp index 425e283..8fcb7b3 100644 --- a/src/pfaedle/osm/OsmBuilder.cpp +++ b/src/pfaedle/osm/OsmBuilder.cpp @@ -24,7 +24,7 @@ #include "util/Misc.h" #include "util/Nullable.h" #include "util/log/Log.h" -#include "xml/pfxml.h" +#include "pfxml/pfxml.h" using ad::cppgtfs::gtfs::Stop; using pfaedle::osm::BlockSearch; @@ -333,8 +333,6 @@ void OsmBuilder::filterWrite(const std::string& in, const std::string& out, NIdMultMap multNodes; pfxml::file xml(in); - std::ofstream outstr; - outstr.open(out); BBoxIdx latLngBox = box; @@ -381,9 +379,9 @@ void OsmBuilder::filterWrite(const std::string& in, const std::string& out, latLngBox.add(Box({minlon, minlat}, {maxlon, maxlat})); } - util::xml::XmlWriter wr(&outstr, true, 4); + util::xml::XmlWriter wr(out, true, 4); - outstr << "\n"; + wr.put("\n"); wr.openTag("osm", {{"version", "0.6"}, {"generator", std::string("pfaedle/") + VERSION_FULL}}); wr.openTag( diff --git a/src/pfaedle/osm/OsmBuilder.h b/src/pfaedle/osm/OsmBuilder.h index 9cf1505..73b8e9e 100644 --- a/src/pfaedle/osm/OsmBuilder.h +++ b/src/pfaedle/osm/OsmBuilder.h @@ -25,7 +25,7 @@ #include "util/Nullable.h" #include "util/geo/Geo.h" #include "util/xml/XmlWriter.h" -#include "xml/pfxml.h" +#include "pfxml/pfxml.h" namespace pfaedle { namespace osm { diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 7888cd6..4eb3701 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -2,11 +2,16 @@ file(GLOB_RECURSE util_SRC *.cpp) list(REMOVE_ITEM util_SRC TestMain.cpp) add_library(util ${util_SRC}) -find_package( ZLIB ) +if (!ZLIB_FOUND) + find_package( ZLIB ) + if (ZLIB_FOUND) + add_definitions( -DZLIB_FOUND=${ZLIB_FOUND} ) + endif() +endif() + if (ZLIB_FOUND) include_directories( ${ZLIB_INCLUDE_DIRS} ) target_link_libraries( util ${ZLIB_LIBRARIES} ) - add_definitions( -DZLIB_FOUND=${ZLIB_FOUND} ) endif( ZLIB_FOUND ) add_subdirectory(tests) diff --git a/src/util/xml/XmlWriter.cpp b/src/util/xml/XmlWriter.cpp index 5595f0f..99f0803 100644 --- a/src/util/xml/XmlWriter.cpp +++ b/src/util/xml/XmlWriter.cpp @@ -3,30 +3,64 @@ // Authors: Patrick Brosi #include +#include #include #include #include #include + #include "XmlWriter.h" using namespace util; using namespace xml; +using std::map; using std::ostream; using std::string; -using std::map; // _____________________________________________________________________________ XmlWriter::XmlWriter(std::ostream* out) - : _out(out), _pretty(false), _indent(4) {} + : _out(out), _pretty(false), _indent(4), _gzfile(0) {} // _____________________________________________________________________________ XmlWriter::XmlWriter(std::ostream* out, bool pret) - : _out(out), _pretty(pret), _indent(4) {} + : _out(out), _pretty(pret), _indent(4), _gzfile(0) {} // _____________________________________________________________________________ XmlWriter::XmlWriter(std::ostream* out, bool pret, size_t indent) - : _out(out), _pretty(pret), _indent(indent) {} + : _out(out), _pretty(pret), _indent(indent), _gzfile(0) {} + +// _____________________________________________________________________________ +XmlWriter::XmlWriter(const std::string& file) + : XmlWriter::XmlWriter(file, false, 4) {} + +// _____________________________________________________________________________ +XmlWriter::XmlWriter(const std::string& file, bool pret) + : XmlWriter::XmlWriter(file, pret, 4) {} + +// _____________________________________________________________________________ +XmlWriter::XmlWriter(const std::string& file, bool pret, size_t indent) + : _out(0), _pretty(pret), _indent(indent), _gzfile(0) { + if (file.size() > 2 && file[file.size() - 1] == 'z' && + file[file.size() - 2] == 'g' && file[file.size() - 3] == '.') { +#ifdef ZLIB_FOUND + _gzfile = gzopen(file.c_str(), "w"); + if (_gzfile == Z_NULL) { + throw std::runtime_error("Could not open file for writing."); + } +#else + throw std::runtime_error( + "Could not open gzip file for writing, was compiled without gzip " + "support"); +#endif + } else { + _outs.open(file); + if (_outs.fail()) { + throw std::runtime_error("Could not open file for writing."); + } + _out = &_outs; + } +} // _____________________________________________________________________________ void XmlWriter::openTag(const string& tag, const map& attrs) { @@ -38,14 +72,15 @@ void XmlWriter::openTag(const string& tag, const map& attrs) { closeHanging(); doIndent(); - *_out << "<" << tag; + put("<"); + put(tag); for (auto kv : attrs) { - *_out << " "; - putEsced(_out, kv.first, '"'); - *_out << "=\""; - putEsced(_out, kv.second, '"'); - *_out << "\""; + put(" "); + putEsced(kv.first, '"'); + put("=\""); + putEsced(kv.second, '"'); + put("\""); } _nstack.push(XmlNode(TAG, tag, true)); @@ -71,7 +106,7 @@ void XmlWriter::openComment() { closeHanging(); doIndent(); - *_out << ""; + put(" -->"); } else if (_nstack.top().t == TAG) { if (_nstack.top().hanging) { - *_out << " />"; + put(" />"); _nstack.pop(); } else { string tag = _nstack.top().pload; _nstack.pop(); doIndent(); - *_out << ""; + put(""); } } } @@ -117,8 +154,8 @@ void XmlWriter::closeTags() { // _____________________________________________________________________________ void XmlWriter::doIndent() { if (_pretty) { - *_out << std::endl; - for (size_t i = 0; i < _nstack.size() * _indent; i++) *_out << " "; + put("\n"); + for (size_t i = 0; i < _nstack.size() * _indent; i++) put(" "); } } @@ -127,7 +164,7 @@ void XmlWriter::closeHanging() { if (_nstack.empty()) return; if (_nstack.top().hanging) { - *_out << ">"; + put(">"); _nstack.top().hanging = false; } else if (_nstack.top().t == TEXT) { _nstack.pop(); @@ -135,25 +172,48 @@ void XmlWriter::closeHanging() { } // _____________________________________________________________________________ -void XmlWriter::putEsced(ostream* out, const string& str, char quot) { +void XmlWriter::put(const string& str) { + if (_gzfile) { +#ifdef ZLIB_FOUND + gzwrite(_gzfile, str.c_str(), str.size()); +#endif + } else { + _out->write(str.c_str(), str.size()); + } +} + +// _____________________________________________________________________________ +void XmlWriter::put(const char c) { + if (_gzfile) { +#ifdef ZLIB_FOUND + gzputc(_gzfile, c); + +#endif + } else { + _out->put(c); + } +} + +// _____________________________________________________________________________ +void XmlWriter::putEsced(const string& str, char quot) { if (!_nstack.empty() && _nstack.top().t == COMMENT) { - *out << str; + put(str); return; } for (const char& c : str) { if (quot == '"' && c == '"') - *out << """; + put("""); else if (quot == '\'' && c == '\'') - *out << "'"; + put("'"); else if (c == '<') - *out << "<"; + put("<"); else if (c == '>') - *out << ">"; + put(">"); else if (c == '&') - *out << "&"; + put("&"); else - *out << c; + put(c); } } diff --git a/src/util/xml/XmlWriter.h b/src/util/xml/XmlWriter.h index ee6d5e8..cd8ea4b 100644 --- a/src/util/xml/XmlWriter.h +++ b/src/util/xml/XmlWriter.h @@ -5,8 +5,12 @@ #ifndef UTIL_XML_XMLWRITER_H_ #define UTIL_XML_XMLWRITER_H_ +#ifdef ZLIB_FOUND +#include +#endif #include #include +#include #include #include @@ -30,7 +34,15 @@ class XmlWriter { explicit XmlWriter(std::ostream* out); XmlWriter(std::ostream* out, bool pretty); XmlWriter(std::ostream* out, bool pretty, size_t indent); - ~XmlWriter(){}; + + explicit XmlWriter(const std::string& file); + XmlWriter(const std::string& file, bool pretty); + XmlWriter(const std::string& file, bool pretty, size_t indent); + ~XmlWriter(){ +#ifdef ZLIB_FOUND + if (_gzfile) gzclose(_gzfile); +#endif + }; // open tag without attributes void openTag(const std::string& tag); @@ -55,6 +67,11 @@ class XmlWriter { // close all open tags, essentially closing the document void closeTags(); + // pushes XML escaped text to stream + void putEsced(const std::string& str, char quot); + void put(const std::string& str); + void put(const char c); + private: enum XML_NODE_T { TAG, TEXT, COMMENT }; @@ -67,19 +84,24 @@ class XmlWriter { }; std::ostream* _out; + std::ofstream _outs; std::stack _nstack; bool _pretty; size_t _indent; +#ifdef ZLIB_FOUND + gzFile _gzfile; +#else + int _gzfile; +#endif + // handles indentation void doIndent(); // close "hanging" tags void closeHanging(); - // pushes XML escaped text to stream - void putEsced(std::ostream* out, const std::string& str, char quot); // checks tag names for validiy void checkTagName(const std::string& str) const; diff --git a/src/xml b/src/xml index ee92108..f015225 160000 --- a/src/xml +++ b/src/xml @@ -1 +1 @@ -Subproject commit ee9210828b1d1f81fb9ff9f3b7da87327c3390e6 +Subproject commit f0152258dccd2962a21fff2658b7740b595b83ed