KOSMIndoorMap

osmpbfwriter.cpp
1/*
2 SPDX-FileCopyrightText: 2023 Volker Krause <vkrause@kde.org>
3 SPDX-License-Identifier: LGPL-2.0-or-later
4*/
5
6#include "osmpbfwriter.h"
7#include "datatypes.h"
8
9#include "fileformat.pb.h"
10#include "osmformat.pb.h"
11
12#include <QIODevice>
13#include <QtEndian>
14
15#include <zlib.h>
16
17using namespace OSM;
18
19static constexpr const std::size_t BLOCK_SIZE_LIMIT = 16'777'216;
20
21OsmPbfWriter::OsmPbfWriter() = default;
22OsmPbfWriter::~OsmPbfWriter() = default;
23
24void OsmPbfWriter::writeToIODevice(const OSM::DataSet &dataSet, QIODevice *io)
25{
26 m_io = io;
27 writeNodes(dataSet);
28 writeWays(dataSet);
29 writeRelations(dataSet);
30 if (m_block) {
31 writeBlob();
32 }
33 m_io = nullptr;
34}
35
36void OsmPbfWriter::writeNodes(const OSM::DataSet &dataSet)
37{
38 if (dataSet.nodes.empty()) {
39 return;
40 }
41
42 int64_t prevId = 0;
43 int64_t prevLat = 900'000'000ll;
44 int64_t prevLon = 1'800'000'000ll;
45
46 OSMPBF::DenseNodes *denseBlock = nullptr;
47 for(auto const &node: dataSet.nodes) {
48 createBlockIfNeeded();
49 if (!denseBlock) {
50 denseBlock = m_block->add_primitivegroup()->mutable_dense();
51 }
52
53 denseBlock->add_id(node.id - prevId);
54 prevId = node.id;
55
56 denseBlock->add_lat((int64_t)node.coordinate.latitude - prevLat);
57 prevLat = node.coordinate.latitude;
58 denseBlock->add_lon((int64_t)node.coordinate.longitude - prevLon);
59 prevLon = node.coordinate.longitude;
60
61 for (const auto &tag : node.tags) {
62 denseBlock->add_keys_vals(stringTableEntry(tag.key.name()));
63 denseBlock->add_keys_vals(stringTableEntry(tag.value.constData()));
64 m_blockSizeEstimate += 2 * sizeof(int32_t);
65 }
66
67 denseBlock->add_keys_vals(0);
68 m_blockSizeEstimate += 3 * sizeof(int64_t) + sizeof(int32_t);
69
70 if (blockSizeLimitReached()) {
71 denseBlock = nullptr;
72 writeBlob();
73 }
74 }
75}
76
77void OsmPbfWriter::writeWays(const OSM::DataSet &dataSet)
78{
79 OSMPBF::PrimitiveGroup *group = nullptr;
80 for(auto const &way : dataSet.ways) {
81 createBlockIfNeeded();
82 if (!group) {
83 group = m_block->add_primitivegroup();
84 }
85
86 auto w = group->add_ways();
87 w->set_id(way.id);
88 m_blockSizeEstimate += sizeof(int64_t);
89
90 int64_t prevId = 0;
91 for (const auto &id : way.nodes) {
92 w->add_refs(id - prevId);
93 prevId = id;
94 m_blockSizeEstimate += sizeof(int64_t);
95 }
96 for (const auto &tag : way.tags) {
97 w->add_keys(stringTableEntry(tag.key.name()));
98 w->add_vals(stringTableEntry(tag.value.constData()));
99 m_blockSizeEstimate += 2 * sizeof(int32_t);
100 }
101
102 if (blockSizeLimitReached()) {
103 group = nullptr;
104 writeBlob();
105 }
106 }
107}
108
109static OSMPBF::Relation_MemberType pbfMemberType(OSM::Type t)
110{
111 switch (t) {
112 case OSM::Type::Null:
113 Q_UNREACHABLE();
114 case OSM::Type::Node:
115 return OSMPBF::Relation_MemberType::Relation_MemberType_NODE;
116 case OSM::Type::Way:
117 return OSMPBF::Relation_MemberType::Relation_MemberType_WAY;
118 case OSM::Type::Relation:
119 return OSMPBF::Relation_MemberType::Relation_MemberType_RELATION;
120 }
121 return OSMPBF::Relation_MemberType::Relation_MemberType_NODE;
122}
123
124void OsmPbfWriter::writeRelations(const OSM::DataSet &dataSet)
125{
126 OSMPBF::PrimitiveGroup *group = nullptr;
127 for (const auto &rel :dataSet.relations) {
128 createBlockIfNeeded();
129 if (!group) {
130 group = m_block->add_primitivegroup();
131 }
132
133 auto r = group->add_relations();
134 r->set_id(rel.id);
135 m_blockSizeEstimate += sizeof(int64_t);
136
137 for (const auto &tag : rel.tags) {
138 r->add_keys(stringTableEntry(tag.key.name()));
139 r->add_vals(stringTableEntry(tag.value.constData()));
140 m_blockSizeEstimate += 2 * sizeof(int32_t);
141 }
142
143 int64_t prevMemId = 0;
144 for (const auto &mem : rel.members) {
145 r->add_roles_sid(stringTableEntry(mem.role().name()));
146 r->add_memids(mem.id - prevMemId);
147 prevMemId = mem.id;
148 r->add_types(pbfMemberType(mem.type()));
149 m_blockSizeEstimate += 2* sizeof(int32_t) + sizeof(int64_t);
150 }
151
152 if (blockSizeLimitReached()) {
153 group = nullptr;
154 writeBlob();
155 }
156 }
157}
158
159int32_t OsmPbfWriter::stringTableEntry(const char *s)
160{
161 assert(m_block);
162 const auto it = m_stringTable.find(s);
163 if (it == m_stringTable.end()) {
164 auto st = m_block->mutable_stringtable();
165 st->add_s(s);
166 m_stringTable[s] = st->s_size() - 1;
167 m_blockSizeEstimate += std::strlen(s) + 1 + sizeof(int32_t);
168 return st->s_size() - 1;
169 }
170
171 return (*it).second;
172}
173
174void OsmPbfWriter::createBlockIfNeeded()
175{
176 if (!m_block) {
177 m_block = std::make_unique<OSMPBF::PrimitiveBlock>();
178 m_blockSizeEstimate = 0;
179 m_block->mutable_stringtable()->add_s(""); // dense node block tag separation marker
180 }
181}
182
183bool OsmPbfWriter::blockSizeLimitReached() const
184{
185 return m_blockSizeEstimate >BLOCK_SIZE_LIMIT;
186}
187
188void OsmPbfWriter::writeBlob()
189{
190 OSMPBF::Blob blob;
191 auto rawBlobData = m_block->SerializeAsString();
192 auto zlibBlobData = blob.mutable_zlib_data();
193 zlibBlobData->resize(32 * 1024ul * 1024ul);
194
195 z_stream zStream;
196 zStream.next_in = (uint8_t*)rawBlobData.data();
197 zStream.avail_in = rawBlobData.size();
198 zStream.next_out = (uint8_t*)zlibBlobData->data();
199 zStream.avail_out = zlibBlobData->size();
200 zStream.zalloc = nullptr;
201 zStream.zfree = nullptr;
202 zStream.opaque = nullptr;
203 deflateInit(&zStream, Z_DEFAULT_COMPRESSION);
204 while (true) {
205 const auto ret = deflate(&zStream, Z_FINISH);
206 if (ret == Z_STREAM_END) {
207 break;
208 }
209 if (ret != Z_OK) {
210 qWarning() << "zlib compression error!" << ret;
211 break;
212 }
213 if (zStream.avail_out == 0) {
214 // we could dynamically resize the output buffer here, but that
215 // is already about twice the size of the data we want to compress
216 // so we really shouldn't end up here
217 qFatal("zlib output buffer underrun");
218 break;
219 }
220 }
221 zlibBlobData->resize(zlibBlobData->size() - zStream.avail_out);
222
223 deflateEnd(&zStream);
224 blob.set_raw_size((int32_t)rawBlobData.size());
225
226 OSMPBF::BlobHeader header;
227 header.set_type("OSMData");
228 header.set_datasize((int32_t)blob.ByteSizeLong());
229
230 auto blobHeaderSize = (int32_t)header.ByteSizeLong();
231 blobHeaderSize = qToBigEndian(blobHeaderSize);
232 m_io->write(reinterpret_cast<const char*>(&blobHeaderSize), sizeof(blobHeaderSize));
233 m_io->write(header.SerializeAsString().c_str(), header.ByteSizeLong()); // TODO do this copy-free
234 m_io->write(blob.SerializeAsString().c_str(), blob.ByteSizeLong()); // TODO do this copy-free
235
236 m_block.reset();
237 m_blockSizeEstimate = 0;
238 m_stringTable.clear();
239}
A set of nodes, ways and relations.
Definition datatypes.h:340
Low-level types and functions to work with raw OSM data as efficiently as possible.
Type
Element type.
Definition datatypes.h:262
qint64 write(const QByteArray &data)
This file is part of the KDE documentation.
Documentation copyright © 1996-2024 The KDE developers.
Generated on Tue Mar 26 2024 11:20:03 by doxygen 1.10.0 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.