gz support for OSM xml reading and writing

CMakeLists.txt

@@ -48,11 +48,18 @@ endif ()
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFAEDLE_PRECISION=${PFAEDLE_PRECISION}")
 
 find_package(LibZip)
+find_package(ZLIB)
 
 if (LIBZIP_FOUND)
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLIBZIP_FOUND=1")
 endif()
 
+if (ZLIB_FOUND)
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DZLIB_FOUND=1")
+else ()
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFXML_NO_ZLIB=1")
+endif()
+
 # http://brianmilco.blogspot.de/2012/11/cmake-automatically-use-git-tags-as.html
 include(GetGitRevisionDescription)
 git_get_tag(VERSION_GIT)

src/CMakeLists.txt

@@ -1,15 +1,16 @@
 set(PFAEDLE_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 
-if (COMMAND cpplint_add_subdirectory)
-	cpplint_add_subdirectory(pfaedle)
-endif()
-
 include_directories(
 	${PFAEDLE_INCLUDE_DIR}
 )
 
+if (ZLIB_FOUND)
+	include_directories( ${ZLIB_INCLUDE_DIRS} )
+endif( ZLIB_FOUND )
+
 add_subdirectory(util)
 add_subdirectory(pfaedle)
 add_subdirectory(cppgtfs)
 add_subdirectory(configparser)
 add_subdirectory(shapevl)
+add_subdirectory(xml)

src/cppgtfs

@@ -1 +1 @@
-Subproject commit 3cd9b7ecdaab216c07dda65a670352be22271fca
+Subproject commit a27c5fee863d1a99fd081d20d1cb57861fb263e7

src/pfaedle/CMakeLists.txt

@@ -18,7 +18,7 @@ configure_file (
 add_executable(pfaedle ${pfaedle_main})
 add_library(pfaedle_dep ${pfaedle_SRC})
 
-include_directories(pfaedle_dep PUBLIC ${PROJECT_SOURCE_DIR}/src/cppgtfs/src)
+include_directories(pfaedle_dep PUBLIC ${PROJECT_SOURCE_DIR}/src/xml/include/ ${PROJECT_SOURCE_DIR}/src/cppgtfs/src)
 target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread ${LIBZIP_LIBRARY})
 
 add_subdirectory(tests)

src/pfaedle/osm/OsmBuilder.cpp

@@ -24,7 +24,7 @@
 #include "util/Misc.h"
 #include "util/Nullable.h"
 #include "util/log/Log.h"
-#include "xml/pfxml.h"
+#include "pfxml/pfxml.h"
 
 using ad::cppgtfs::gtfs::Stop;
 using pfaedle::osm::BlockSearch;
@@ -333,8 +333,6 @@ void OsmBuilder::filterWrite(const std::string& in, const std::string& out,
   NIdMultMap multNodes;
 
   pfxml::file xml(in);
-  std::ofstream outstr;
-  outstr.open(out);
 
   BBoxIdx latLngBox = box;
 
@@ -381,9 +379,9 @@ void OsmBuilder::filterWrite(const std::string& in, const std::string& out,
     latLngBox.add(Box<double>({minlon, minlat}, {maxlon, maxlat}));
   }
 
-  util::xml::XmlWriter wr(&outstr, true, 4);
+  util::xml::XmlWriter wr(out, true, 4);
 
-  outstr << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+  wr.put("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
   wr.openTag("osm", {{"version", "0.6"},
                      {"generator", std::string("pfaedle/") + VERSION_FULL}});
   wr.openTag(

src/pfaedle/osm/OsmBuilder.h

@@ -25,7 +25,7 @@
 #include "util/Nullable.h"
 #include "util/geo/Geo.h"
 #include "util/xml/XmlWriter.h"
-#include "xml/pfxml.h"
+#include "pfxml/pfxml.h"
 
 namespace pfaedle {
 namespace osm {

src/util/CMakeLists.txt

@@ -2,11 +2,16 @@ file(GLOB_RECURSE util_SRC *.cpp)
 list(REMOVE_ITEM util_SRC TestMain.cpp)
 add_library(util ${util_SRC})
 
+if (!ZLIB_FOUND)
     find_package( ZLIB )
+    if (ZLIB_FOUND)
+		add_definitions( -DZLIB_FOUND=${ZLIB_FOUND} )
+	endif()
+endif()
+
 if (ZLIB_FOUND)
 	include_directories( ${ZLIB_INCLUDE_DIRS} )
 	target_link_libraries( util ${ZLIB_LIBRARIES} )
-	add_definitions( -DZLIB_FOUND=${ZLIB_FOUND} )
 endif( ZLIB_FOUND )
 
 add_subdirectory(tests)

src/util/xml/XmlWriter.cpp

@@ -3,30 +3,64 @@
 // Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
 
 #include <algorithm>
+#include <fstream>
 #include <map>
 #include <ostream>
 #include <stack>
 #include <string>
+
 #include "XmlWriter.h"
 
 using namespace util;
 using namespace xml;
 
+using std::map;
 using std::ostream;
 using std::string;
-using std::map;
 
 // _____________________________________________________________________________
 XmlWriter::XmlWriter(std::ostream* out)
-    : _out(out), _pretty(false), _indent(4) {}
+    : _out(out), _pretty(false), _indent(4), _gzfile(0) {}
 
 // _____________________________________________________________________________
 XmlWriter::XmlWriter(std::ostream* out, bool pret)
-    : _out(out), _pretty(pret), _indent(4) {}
+    : _out(out), _pretty(pret), _indent(4), _gzfile(0) {}
 
 // _____________________________________________________________________________
 XmlWriter::XmlWriter(std::ostream* out, bool pret, size_t indent)
-    : _out(out), _pretty(pret), _indent(indent) {}
+    : _out(out), _pretty(pret), _indent(indent), _gzfile(0) {}
+
+// _____________________________________________________________________________
+XmlWriter::XmlWriter(const std::string& file)
+    : XmlWriter::XmlWriter(file, false, 4) {}
+
+// _____________________________________________________________________________
+XmlWriter::XmlWriter(const std::string& file, bool pret)
+    : XmlWriter::XmlWriter(file, pret, 4) {}
+
+// _____________________________________________________________________________
+XmlWriter::XmlWriter(const std::string& file, bool pret, size_t indent)
+    : _out(0), _pretty(pret), _indent(indent), _gzfile(0) {
+  if (file.size() > 2 && file[file.size() - 1] == 'z' &&
+      file[file.size() - 2] == 'g' && file[file.size() - 3] == '.') {
+#ifdef ZLIB_FOUND
+    _gzfile = gzopen(file.c_str(), "w");
+    if (_gzfile == Z_NULL) {
+      throw std::runtime_error("Could not open file for writing.");
+    }
+#else
+    throw std::runtime_error(
+        "Could not open gzip file for writing, was compiled without gzip "
+        "support");
+#endif
+  } else {
+    _outs.open(file);
+    if (_outs.fail()) {
+      throw std::runtime_error("Could not open file for writing.");
+    }
+    _out = &_outs;
+  }
+}
 
 // _____________________________________________________________________________
 void XmlWriter::openTag(const string& tag, const map<string, string>& attrs) {
@@ -38,14 +72,15 @@ void XmlWriter::openTag(const string& tag, const map<string, string>& attrs) {
   closeHanging();
   doIndent();
 
-  *_out << "<" << tag;
+  put("<");
+  put(tag);
 
   for (auto kv : attrs) {
-    *_out << " ";
+    put(" ");
-    putEsced(_out, kv.first, '"');
+    putEsced(kv.first, '"');
-    *_out << "=\"";
+    put("=\"");
-    putEsced(_out, kv.second, '"');
+    putEsced(kv.second, '"');
-    *_out << "\"";
+    put("\"");
   }
 
   _nstack.push(XmlNode(TAG, tag, true));
@@ -71,7 +106,7 @@ void XmlWriter::openComment() {
   closeHanging();
   doIndent();
 
-  *_out << "<!-- ";
+  put("<!-- ");
 
   _nstack.push(XmlNode(COMMENT, "", false));
 }
@@ -83,7 +118,7 @@ void XmlWriter::writeText(const string& text) {
   }
   closeHanging();
   doIndent();
-  putEsced(_out, text, ' ');
+  putEsced(text, ' ');
 }
 
 // _____________________________________________________________________________
@@ -95,16 +130,18 @@ void XmlWriter::closeTag() {
   if (_nstack.top().t == COMMENT) {
     _nstack.pop();
     doIndent();
-    *_out << " -->";
+    put(" -->");
   } else if (_nstack.top().t == TAG) {
     if (_nstack.top().hanging) {
-      *_out << " />";
+      put(" />");
       _nstack.pop();
     } else {
       string tag = _nstack.top().pload;
       _nstack.pop();
       doIndent();
-      *_out << "</" << tag << ">";
+      put("</");
+      put(tag);
+      put(">");
     }
   }
 }
@@ -117,8 +154,8 @@ void XmlWriter::closeTags() {
 // _____________________________________________________________________________
 void XmlWriter::doIndent() {
   if (_pretty) {
-    *_out << std::endl;
+    put("\n");
-    for (size_t i = 0; i < _nstack.size() * _indent; i++) *_out << " ";
+    for (size_t i = 0; i < _nstack.size() * _indent; i++) put(" ");
   }
 }
 
@@ -127,7 +164,7 @@ void XmlWriter::closeHanging() {
   if (_nstack.empty()) return;
 
   if (_nstack.top().hanging) {
-    *_out << ">";
+    put(">");
     _nstack.top().hanging = false;
   } else if (_nstack.top().t == TEXT) {
     _nstack.pop();
@@ -135,25 +172,48 @@ void XmlWriter::closeHanging() {
 }
 
 // _____________________________________________________________________________
-void XmlWriter::putEsced(ostream* out, const string& str, char quot) {
+void XmlWriter::put(const string& str) {
+  if (_gzfile) {
+#ifdef ZLIB_FOUND
+    gzwrite(_gzfile, str.c_str(), str.size());
+#endif
+  } else {
+    _out->write(str.c_str(), str.size());
+  }
+}
+
+// _____________________________________________________________________________
+void XmlWriter::put(const char c) {
+  if (_gzfile) {
+#ifdef ZLIB_FOUND
+    gzputc(_gzfile, c);
+
+#endif
+  } else {
+    _out->put(c);
+  }
+}
+
+// _____________________________________________________________________________
+void XmlWriter::putEsced(const string& str, char quot) {
   if (!_nstack.empty() && _nstack.top().t == COMMENT) {
-    *out << str;
+    put(str);
     return;
   }
 
   for (const char& c : str) {
     if (quot == '"' && c == '"')
-      *out << "&quot;";
+      put("&quot;");
     else if (quot == '\'' && c == '\'')
-      *out << "&apos;";
+      put("&apos;");
     else if (c == '<')
-      *out << "&lt;";
+      put("&lt;");
     else if (c == '>')
-      *out << "&gt;";
+      put("&gt;");
     else if (c == '&')
-      *out << "&amp;";
+      put("&amp;");
     else
-      *out << c;
+      put(c);
   }
 }
 

src/util/xml/XmlWriter.h

@@ -5,8 +5,12 @@
 #ifndef UTIL_XML_XMLWRITER_H_
 #define UTIL_XML_XMLWRITER_H_
 
+#ifdef ZLIB_FOUND
+#include <zlib.h>
+#endif
 #include <map>
 #include <ostream>
+#include <fstream>
 #include <stack>
 #include <string>
 
@@ -30,7 +34,15 @@ class XmlWriter {
   explicit XmlWriter(std::ostream* out);
   XmlWriter(std::ostream* out, bool pretty);
   XmlWriter(std::ostream* out, bool pretty, size_t indent);
-  ~XmlWriter(){};
+
+  explicit XmlWriter(const std::string& file);
+  XmlWriter(const std::string& file, bool pretty);
+  XmlWriter(const std::string& file, bool pretty, size_t indent);
+  ~XmlWriter(){
+#ifdef ZLIB_FOUND
+    if (_gzfile) gzclose(_gzfile);
+#endif
+  };
 
   // open tag without attributes
   void openTag(const std::string& tag);
@@ -55,6 +67,11 @@ class XmlWriter {
   // close all open tags, essentially closing the document
   void closeTags();
 
+  // pushes XML escaped text to stream
+  void putEsced(const std::string& str, char quot);
+  void put(const std::string& str);
+  void put(const char c);
+
  private:
   enum XML_NODE_T { TAG, TEXT, COMMENT };
 
@@ -67,19 +84,24 @@ class XmlWriter {
   };
 
   std::ostream* _out;
+  std::ofstream _outs;
   std::stack<XmlNode> _nstack;
 
   bool _pretty;
   size_t _indent;
 
+#ifdef ZLIB_FOUND
+  gzFile _gzfile;
+#else
+  int _gzfile;
+#endif
+
   // handles indentation
   void doIndent();
 
   // close "hanging" tags
   void closeHanging();
 
-  // pushes XML escaped text to stream
-  void putEsced(std::ostream* out, const std::string& str, char quot);
 
   // checks tag names for validiy
   void checkTagName(const std::string& str) const;

src/xml

@@ -1 +1 @@
-Subproject commit ee9210828b1d1f81fb9ff9f3b7da87327c3390e6
+Subproject commit f0152258dccd2962a21fff2658b7740b595b83ed