Libosmium  2.15.0
Fast and flexible C++ library for working with OpenStreetMap data
bzip2_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_BZIP2_COMPRESSION_HPP
2 #define OSMIUM_IO_BZIP2_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2018 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
46 #include <osmium/io/detail/read_write.hpp>
47 #include <osmium/io/error.hpp>
51 #include <osmium/util/file.hpp>
52 
53 #include <bzlib.h>
54 
55 #include <cassert>
56 #include <cerrno>
57 #include <cstdio>
58 #include <limits>
59 #include <string>
60 #include <system_error>
61 
62 #ifndef _MSC_VER
63 # include <unistd.h>
64 #endif
65 
66 namespace osmium {
67 
72  struct bzip2_error : public io_error {
73 
75  int system_errno = 0;
76 
77  bzip2_error(const std::string& what, const int error_code) :
78  io_error(what),
79  bzip2_error_code(error_code) {
80  if (error_code == BZ_IO_ERROR) {
81  system_errno = errno;
82  }
83  }
84 
85  }; // struct bzip2_error
86 
87  namespace io {
88 
89  namespace detail {
90 
91  OSMIUM_NORETURN inline void throw_bzip2_error(BZFILE* bzfile, const char* msg, const int bzlib_error) {
92  std::string error{"bzip2 error: "};
93  error += msg;
94  error += ": ";
95  int errnum = bzlib_error;
96  if (bzlib_error) {
97  error += std::to_string(bzlib_error);
98  } else if (bzfile) {
99  error += ::BZ2_bzerror(bzfile, &errnum);
100  }
101  throw osmium::bzip2_error{error, errnum};
102  }
103 
104  class file_wrapper {
105 
106  FILE* m_file = nullptr;
107 
108  public:
109 
110  file_wrapper() noexcept = default;
111 
112  file_wrapper(const int fd, const char* mode) {
113 #ifdef _MSC_VER
114  osmium::detail::disable_invalid_parameter_handler diph;
115 #endif
116  m_file = fdopen(fd, mode);
117  if (!m_file) {
118  ::close(fd);
119  throw std::system_error{errno, std::system_category(), "fdopen failed"};
120  }
121  }
122 
123  file_wrapper(const file_wrapper&) = delete;
124  file_wrapper& operator=(const file_wrapper&) = delete;
125 
126  file_wrapper(file_wrapper&&) = delete;
127  file_wrapper& operator=(file_wrapper&&) = delete;
128 
129  ~file_wrapper() noexcept {
130 #ifdef _MSC_VER
131  osmium::detail::disable_invalid_parameter_handler diph;
132 #endif
133  if (m_file) {
134  fclose(m_file);
135  }
136  }
137 
138  FILE* file() const noexcept {
139  return m_file;
140  }
141 
142  void close() {
143 #ifdef _MSC_VER
144  osmium::detail::disable_invalid_parameter_handler diph;
145 #endif
146  if (m_file) {
147  if (fclose(m_file) != 0) {
148  m_file = nullptr;
149  throw std::system_error{errno, std::system_category(), "fclose failed"};
150  }
151  m_file = nullptr;
152  }
153  }
154 
155  }; // class file_wrapper
156 
157  } // namespace detail
158 
159  class Bzip2Compressor : public Compressor {
160 
161  detail::file_wrapper m_file;
162  BZFILE* m_bzfile = nullptr;
163 
164  public:
165 
166  explicit Bzip2Compressor(const int fd, const fsync sync) :
167  Compressor(sync),
168  m_file(fd, "wb") {
169 #ifdef _MSC_VER
170  osmium::detail::disable_invalid_parameter_handler diph;
171 #endif
172  int bzerror = BZ_OK;
173  m_bzfile = ::BZ2_bzWriteOpen(&bzerror, m_file.file(), 6, 0, 0);
174  if (!m_bzfile) {
175  throw bzip2_error{"bzip2 error: write open failed", bzerror};
176  }
177  }
178 
179  Bzip2Compressor(const Bzip2Compressor&) = delete;
180  Bzip2Compressor& operator=(const Bzip2Compressor&) = delete;
181 
182  Bzip2Compressor(Bzip2Compressor&&) = delete;
183  Bzip2Compressor& operator=(Bzip2Compressor&&) = delete;
184 
185  ~Bzip2Compressor() noexcept final {
186  try {
187  close();
188  } catch (...) {
189  // Ignore any exceptions because destructor must not throw.
190  }
191  }
192 
193  void write(const std::string& data) final {
194  assert(data.size() < std::numeric_limits<int>::max());
195  assert(m_bzfile);
196 #ifdef _MSC_VER
197  osmium::detail::disable_invalid_parameter_handler diph;
198 #endif
199  int bzerror = BZ_OK;
200  ::BZ2_bzWrite(&bzerror, m_bzfile, const_cast<char*>(data.data()), static_cast<int>(data.size()));
201  if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) {
202  detail::throw_bzip2_error(m_bzfile, "write failed", bzerror);
203  }
204  }
205 
206  void close() final {
207  if (m_bzfile) {
208 #ifdef _MSC_VER
209  osmium::detail::disable_invalid_parameter_handler diph;
210 #endif
211  int bzerror = BZ_OK;
212  ::BZ2_bzWriteClose(&bzerror, m_bzfile, 0, nullptr, nullptr);
213  m_bzfile = nullptr;
214  if (do_fsync() && m_file.file()) {
215  osmium::io::detail::reliable_fsync(fileno(m_file.file()));
216  }
217  m_file.close();
218  if (bzerror != BZ_OK) {
219  throw bzip2_error{"bzip2 error: write close failed", bzerror};
220  }
221  }
222  }
223 
224  }; // class Bzip2Compressor
225 
227 
228  detail::file_wrapper m_file;
229  BZFILE* m_bzfile = nullptr;
230  bool m_stream_end = false;
231 
232  public:
233 
234  explicit Bzip2Decompressor(const int fd) :
235  m_file(fd, "rb") {
236 #ifdef _MSC_VER
237  osmium::detail::disable_invalid_parameter_handler diph;
238 #endif
239  int bzerror = BZ_OK;
240  m_bzfile = ::BZ2_bzReadOpen(&bzerror, m_file.file(), 0, 0, nullptr, 0);
241  if (!m_bzfile) {
242  throw bzip2_error{"bzip2 error: read open failed", bzerror};
243  }
244  }
245 
246  Bzip2Decompressor(const Bzip2Decompressor&) = delete;
247  Bzip2Decompressor& operator=(const Bzip2Decompressor&) = delete;
248 
250  Bzip2Decompressor& operator=(Bzip2Decompressor&&) = delete;
251 
252  ~Bzip2Decompressor() noexcept final {
253  try {
254  close();
255  } catch (...) {
256  // Ignore any exceptions because destructor must not throw.
257  }
258  }
259 
260  std::string read() final {
261 #ifdef _MSC_VER
262  osmium::detail::disable_invalid_parameter_handler diph;
263 #endif
264  assert(m_bzfile);
265  std::string buffer;
266 
267  if (!m_stream_end) {
269  int bzerror = BZ_OK;
270  assert(buffer.size() < std::numeric_limits<int>::max());
271  const int nread = ::BZ2_bzRead(&bzerror, m_bzfile, &*buffer.begin(), static_cast<int>(buffer.size()));
272  if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) {
273  detail::throw_bzip2_error(m_bzfile, "read failed", bzerror);
274  }
275  if (bzerror == BZ_STREAM_END) {
276  void* unused;
277  int nunused;
278  if (!feof(m_file.file())) {
279  ::BZ2_bzReadGetUnused(&bzerror, m_bzfile, &unused, &nunused);
280  if (bzerror != BZ_OK) {
281  detail::throw_bzip2_error(m_bzfile, "get unused failed", bzerror);
282  }
283  std::string unused_data{static_cast<const char*>(unused), static_cast<std::string::size_type>(nunused)};
284  ::BZ2_bzReadClose(&bzerror, m_bzfile);
285  if (bzerror != BZ_OK) {
286  throw bzip2_error{"bzip2 error: read close failed", bzerror};
287  }
288  assert(unused_data.size() < std::numeric_limits<int>::max());
289  m_bzfile = ::BZ2_bzReadOpen(&bzerror, m_file.file(), 0, 0, &*unused_data.begin(), static_cast<int>(unused_data.size()));
290  if (!m_bzfile) {
291  throw bzip2_error{"bzip2 error: read open failed", bzerror};
292  }
293  } else {
294  m_stream_end = true;
295  }
296  }
297  buffer.resize(static_cast<std::string::size_type>(nread));
298  }
299 
300  set_offset(static_cast<std::size_t>(ftell(m_file.file())));
301 
302  return buffer;
303  }
304 
305  void close() final {
306  if (m_bzfile) {
307 #ifdef _MSC_VER
308  osmium::detail::disable_invalid_parameter_handler diph;
309 #endif
310  int bzerror = BZ_OK;
311  ::BZ2_bzReadClose(&bzerror, m_bzfile);
312  m_bzfile = nullptr;
313  m_file.close();
314  if (bzerror != BZ_OK) {
315  throw bzip2_error{"bzip2 error: read close failed", bzerror};
316  }
317  }
318  }
319 
320  }; // class Bzip2Decompressor
321 
323 
324  const char* m_buffer;
325  std::size_t m_buffer_size;
326  bz_stream m_bzstream;
327 
328  public:
329 
330  Bzip2BufferDecompressor(const char* buffer, const std::size_t size) :
331  m_buffer(buffer),
332  m_buffer_size(size),
333  m_bzstream() {
334  m_bzstream.next_in = const_cast<char*>(buffer);
335  assert(size < std::numeric_limits<unsigned int>::max());
336  m_bzstream.avail_in = static_cast<unsigned int>(size);
337  const int result = BZ2_bzDecompressInit(&m_bzstream, 0, 0);
338  if (result != BZ_OK) {
339  throw bzip2_error{"bzip2 error: decompression init failed: ", result};
340  }
341  }
342 
344  Bzip2BufferDecompressor& operator=(const Bzip2BufferDecompressor&) = delete;
345 
347  Bzip2BufferDecompressor& operator=(Bzip2BufferDecompressor&&) = delete;
348 
349  ~Bzip2BufferDecompressor() noexcept final {
350  try {
351  close();
352  } catch (...) {
353  // Ignore any exceptions because destructor must not throw.
354  }
355  }
356 
357  std::string read() final {
358  std::string output;
359 
360  if (m_buffer) {
361  const std::size_t buffer_size = 10240;
362  output.resize(buffer_size);
363  m_bzstream.next_out = &*output.begin();
364  m_bzstream.avail_out = buffer_size;
365  const int result = BZ2_bzDecompress(&m_bzstream);
366 
367  if (result != BZ_OK) {
368  m_buffer = nullptr;
369  m_buffer_size = 0;
370  }
371 
372  if (result != BZ_OK && result != BZ_STREAM_END) {
373  throw bzip2_error{"bzip2 error: decompress failed: ", result};
374  }
375 
376  output.resize(static_cast<std::size_t>(m_bzstream.next_out - output.data()));
377  }
378 
379  return output;
380  }
381 
382  void close() final {
383  BZ2_bzDecompressEnd(&m_bzstream);
384  }
385 
386  }; // class Bzip2BufferDecompressor
387 
388  namespace detail {
389 
390  // we want the register_compression() function to run, setting
391  // the variable is only a side-effect, it will never be used
393  [](const int fd, const fsync sync) { return new osmium::io::Bzip2Compressor{fd, sync}; },
394  [](const int fd) { return new osmium::io::Bzip2Decompressor{fd}; },
395  [](const char* buffer, const std::size_t size) { return new osmium::io::Bzip2BufferDecompressor{buffer, size}; }
396  );
397 
398  // dummy function to silence the unused variable warning from above
399  inline bool get_registered_bzip2_compression() noexcept {
400  return registered_bzip2_compression;
401  }
402 
403  } // namespace detail
404 
405  } // namespace io
406 
407 } // namespace osmium
408 
409 #endif // OSMIUM_IO_BZIP2_COMPRESSION_HPP
bz_stream m_bzstream
Definition: bzip2_compression.hpp:326
Definition: bzip2_compression.hpp:226
Definition: compression.hpp:95
int bzip2_error_code
Definition: bzip2_compression.hpp:74
#define OSMIUM_NORETURN
Definition: compatibility.hpp:41
~Bzip2Decompressor() noexcept final
Definition: bzip2_compression.hpp:252
void close() final
Definition: bzip2_compression.hpp:382
void write(const std::string &data) final
Definition: bzip2_compression.hpp:193
const char * m_buffer
Definition: bzip2_compression.hpp:324
static CompressionFactory & instance()
Definition: compression.hpp:180
~Bzip2BufferDecompressor() noexcept final
Definition: bzip2_compression.hpp:349
std::string read() final
Definition: bzip2_compression.hpp:260
Definition: bzip2_compression.hpp:159
int system_errno
Definition: bzip2_compression.hpp:75
Definition: compression.hpp:87
Bzip2Decompressor(const int fd)
Definition: bzip2_compression.hpp:234
Definition: bzip2_compression.hpp:322
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: attr.hpp:333
Bzip2BufferDecompressor(const char *buffer, const std::size_t size)
Definition: bzip2_compression.hpp:330
fsync
Definition: writer_options.hpp:51
Definition: error.hpp:44
void close() final
Definition: bzip2_compression.hpp:206
void close() final
Definition: bzip2_compression.hpp:305
detail::file_wrapper m_file
Definition: bzip2_compression.hpp:228
std::string read() final
Definition: bzip2_compression.hpp:357
Definition: compression.hpp:57
bzip2_error(const std::string &what, const int error_code)
Definition: bzip2_compression.hpp:77
Bzip2Compressor(const int fd, const fsync sync)
Definition: bzip2_compression.hpp:166
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:185
detail::file_wrapper m_file
Definition: bzip2_compression.hpp:161
std::size_t m_buffer_size
Definition: bzip2_compression.hpp:325
~Bzip2Compressor() noexcept final
Definition: bzip2_compression.hpp:185
Definition: bzip2_compression.hpp:72