KOSMIndoorMap

o5mparser.cpp
1 /*
2  SPDX-FileCopyrightText: 2020 Volker Krause <[email protected]>
3 
4  SPDX-License-Identifier: LGPL-2.0-or-later
5 */
6 
7 #include "o5mparser.h"
8 #include "datatypes.h"
9 #include "datasetmergebuffer.h"
10 
11 #include <QDebug>
12 
13 #include <cstdlib>
14 #include <cstring>
15 
16 using namespace OSM;
17 
18 enum : uint8_t {
19  O5M_BLOCK_RESET = 0xff,
20  O5M_BLOCK_NODE = 0x10,
21  O5M_BLOCK_WAY = 0x11,
22  O5M_BLOCK_RELATION = 0x12,
23  O5M_BLOCK_BOUNDING_BOX = 0xdb,
24  O5M_BLOCK_TIMESTAMP = 0xdc,
25  O5M_BLOCK_HEADER = 0xe0,
26 
27  O5M_NUMBER_CONTINUATION = 0b1000'0000,
28  O5M_NUMBER_MASK = 0b0111'1111,
29  O5M_NUMBER_SIGNED_BIT = 0b1,
30 
31  O5M_MEMTYPE_NODE = 0x30,
32  O5M_MEMTYPE_WAY = 0x31,
33  O5M_MEMTYPE_RELATION = 0x32,
34 };
35 
36 enum : uint16_t {
37  O5M_STRING_TABLE_SIZE = 15000,
38  O5M_STRING_TABLE_MAXLEN = 250,
39 };
40 
41 O5mParser::O5mParser(DataSet *dataSet)
42  : m_dataSet(dataSet)
43 {
44  m_stringLookupTable.resize(O5M_STRING_TABLE_SIZE);
45 }
46 
47 void O5mParser::setMergeBuffer(DataSetMergeBuffer *buffer)
48 {
49  m_mergeBuffer = buffer;
50 }
51 
52 void O5mParser::parse(const uint8_t* data, std::size_t len)
53 {
54  std::fill(m_stringLookupTable.begin(), m_stringLookupTable.end(), nullptr);
55  resetDeltaCodingState();
56 
57  const auto endIt = data + len;
58  for (auto it = data; it < endIt - 1;) {
59  const auto blockType = (*it);
60  if (blockType == O5M_BLOCK_RESET) {
61  resetDeltaCodingState();
62  ++it;
63  continue;
64  }
65 
66  auto blockSize = readUnsigned(++it, endIt);
67  if (blockSize >= (uint64_t)(endIt - it)) {
68  qWarning() << "premature end of file, or blocksize too large" << (endIt - it) << blockType << blockSize;
69  break;
70  }
71  switch (blockType) {
72  case O5M_BLOCK_HEADER:
73  if (blockSize != 4 || std::strncmp(reinterpret_cast<const char*>(it), "o5m2", 4) != 0) {
74  qWarning() << "Invalid file header";
75  return;
76  }
77  break;
78  case O5M_BLOCK_BOUNDING_BOX:
79  case O5M_BLOCK_TIMESTAMP:
80  // not of interest at the moment
81  break;
82  case O5M_BLOCK_NODE:
83  readNode(it, it + blockSize);
84  break;
85  case O5M_BLOCK_WAY:
86  readWay(it, it + blockSize);
87  break;
88  case O5M_BLOCK_RELATION:
89  readRelation(it, it + blockSize);
90  break;
91  default:
92  qDebug() << "unhandled o5m block type:" << (it - data) << blockType << blockSize;
93  }
94 
95  it += blockSize;
96  }
97 }
98 
99 uint64_t O5mParser::readUnsigned(const uint8_t *&it, const uint8_t *endIt) const
100 {
101  uint64_t result = 0;
102  int i = 0;
103  for (; it < endIt && ((*it) & O5M_NUMBER_CONTINUATION); ++it, ++i) {
104  result |= ((*it) & O5M_NUMBER_MASK) << (i * 7);
105  }
106  result |= ((uint64_t)(*it++) & O5M_NUMBER_MASK) << (i * 7);
107  return result;
108 }
109 
110 int64_t O5mParser::readSigned(const uint8_t *&it, const uint8_t *endIt) const
111 {
112  const uint64_t u = readUnsigned(it, endIt);
113  return (u & O5M_NUMBER_SIGNED_BIT) ? (-(u >> 1) -1) : (u >> 1);
114 }
115 
116 template <typename T>
117 T O5mParser::readDelta(const uint8_t *&it, const uint8_t *endIt, T &deltaState)
118 {
119  deltaState += (T)readSigned(it, endIt);
120  return deltaState;
121 }
122 
123 const char* O5mParser::readString(const uint8_t *&it, const uint8_t *endIt)
124 {
125  auto ref = readUnsigned(it, endIt);
126  if (ref) {
127  return m_stringLookupTable[(m_stringLookupPosition + O5M_STRING_TABLE_SIZE - ref) % O5M_STRING_TABLE_SIZE];
128  } else {
129  const auto s = reinterpret_cast<const char*>(it);
130  const auto len = std::strlen(s);
131  if (len <= O5M_STRING_TABLE_MAXLEN) {
132  m_stringLookupTable[m_stringLookupPosition] = s;
133  m_stringLookupPosition = (m_stringLookupPosition + 1) % O5M_STRING_TABLE_SIZE;
134  }
135  it += len + 1;
136  return s;
137  }
138 }
139 
140 std::pair<const char*, const char*> O5mParser::readStringPair(const uint8_t *&it, const uint8_t *endIt)
141 {
142  auto ref = readUnsigned(it, endIt);
143  if (ref) {
144  const auto s = m_stringLookupTable[(m_stringLookupPosition + O5M_STRING_TABLE_SIZE - ref) % O5M_STRING_TABLE_SIZE];
145  if (!s) {
146  return {};
147  }
148  const auto len1 = std::strlen(s);
149  return std::make_pair(s, s + len1 + 1);
150  } else {
151  const auto s = reinterpret_cast<const char*>(it);
152  const auto len1 = std::strlen(s);
153  const auto len2 = std::strlen(s + len1 + 1);
154 
155  if (len1 + len2 <= O5M_STRING_TABLE_MAXLEN) {
156  m_stringLookupTable[m_stringLookupPosition] = s;
157  m_stringLookupPosition = (m_stringLookupPosition + 1) % O5M_STRING_TABLE_SIZE;
158  }
159 
160  it += len1 + len2 + 2;
161  return std::make_pair(s, s + len1 + 1);
162  }
163 }
164 
165 void O5mParser::skipVersionInformation(const uint8_t *&it, const uint8_t *end)
166 {
167  if (it >= end) { return; }
168  const auto version = readUnsigned(it, end);
169  if (version > 0) {
170  qWarning() << "skipping changeset data not implemented yet!";
171  // timestamp (seconds since 1970, signed, delta-coded)
172  // author information – only if timestamp is not 0:
173  // changeset (signed, delta-coded)
174  // uid, user (string pair)
175  it = end;
176  }
177 }
178 
179 template<typename Elem>
180 void O5mParser::readTagOrBbox(Elem &e, const uint8_t *&it, const uint8_t *endIt)
181 {
182  const auto tagData = readStringPair(it, endIt);
183  if (!tagData.first) {
184  return;
185  }
186  if (std::strcmp(tagData.first, "bBox") == 0) {
187  char *next = nullptr;
188  const auto lon1 = std::strtod(tagData.second, &next);
189  ++next;
190  const auto lat1 = std::strtod(next, &next);
191  ++next;
192  const auto lon2 = std::strtod(next, &next);
193  ++next;
194  const auto lat2 = std::strtod(next, &next);
195  e.bbox = OSM::BoundingBox(OSM::Coordinate(lat1, lon1), OSM::Coordinate(lat2, lon2));
196  return;
197  }
198 
199  OSM::Tag tag;
200  tag.key = m_dataSet->makeTagKey(tagData.first, OSM::StringMemory::Transient); // TODO make use of mmap'ed data for this
201  tag.value = QByteArray(tagData.second);
202  OSM::setTag(e, std::move(tag));
203 }
204 
205 void O5mParser::readNode(const uint8_t *begin, const uint8_t *end)
206 {
207  OSM::Node node;
208 
209  auto it = begin;
210  node.id = readDelta(it, end, m_nodeIdDelta);
211  skipVersionInformation(it, end);
212  if (it >= end) { return; }
213 
214  node.coordinate.longitude = (int64_t)readDelta(it, end, m_lonDelta) + 1'800'000'000ll;
215  node.coordinate.latitude = (int64_t)readDelta(it, end, m_latDelata) + 900'000'000ll;
216 
217  while (it < end) {
218  OSM::Tag tag;
219  const auto tagData = readStringPair(it, end);
220  if (tagData.first) {
221  tag.key = m_dataSet->makeTagKey(tagData.first, OSM::StringMemory::Transient); // TODO use the fact this is mmap'ed data here
222  tag.value = QByteArray(tagData.second);
223  OSM::setTag(node, std::move(tag));
224  }
225  }
226 
227  m_mergeBuffer ? m_mergeBuffer->nodes.push_back(std::move(node)) : m_dataSet->addNode(std::move(node));
228 }
229 
230 void O5mParser::readWay(const uint8_t *begin, const uint8_t *end)
231 {
232  OSM::Way way;
233 
234  auto it = begin;
235  way.id = readDelta(it, end, m_wayIdDelta);
236  skipVersionInformation(it, end);
237  if (it >= end) { return; }
238 
239  const auto nodesBlockSize = readUnsigned(it, end);
240  if (it + nodesBlockSize > end) { return; }
241 
242  const auto nodesBlockEnd = it + nodesBlockSize;
243  while(it < nodesBlockEnd) {
244  way.nodes.push_back(readDelta(it, end, m_wayNodeIdDelta));
245  }
246 
247  while (it < end) {
248  readTagOrBbox(way, it, end);
249  }
250 
251  m_mergeBuffer ? m_mergeBuffer->ways.push_back(std::move(way)) : m_dataSet->addWay(std::move(way));
252 }
253 
254 void O5mParser::readRelation(const uint8_t *begin, const uint8_t *end)
255 {
256  OSM::Relation rel;
257 
258  auto it = begin;
259  rel.id = readDelta(it, end, m_relIdDelta);
260  skipVersionInformation(it, end);
261  if (it >= end) { return; }
262 
263  const auto relBlockSize = readUnsigned(it, end);
264  if (it + relBlockSize > end) { return; }
265 
266  const auto relBlockEnd = it + relBlockSize;
267  while (it < relBlockEnd) {
268  const int64_t memId = readSigned(it, end);
269  OSM::Member mem;
270  const auto typeAndRole = readString(it, end);
271  switch (typeAndRole[0]) {
272  case O5M_MEMTYPE_NODE:
273  mem.id = m_relNodeMemberIdDelta += memId;
274  mem.setType(OSM::Type::Node);
275  break;
276  case O5M_MEMTYPE_WAY:
277  mem.id = m_relWayMemberIdDelta += memId;
278  mem.setType(OSM::Type::Way);
279  break;
280  case O5M_MEMTYPE_RELATION:
281  mem.id = m_relRelMemberIdDelta += memId;
282  mem.setType(OSM::Type::Relation);
283  break;
284  }
285  mem.setRole(m_dataSet->makeRole(typeAndRole + 1, OSM::StringMemory::Transient));
286 
287  rel.members.push_back(std::move(mem));
288  }
289 
290 
291 
292  while (it < end) {
293  readTagOrBbox(rel, it, end);
294  }
295 
296  m_mergeBuffer ? m_mergeBuffer->relations.push_back(std::move(rel)) : m_dataSet->addRelation(std::move(rel));
297 }
298 
299 void O5mParser::resetDeltaCodingState()
300 {
301  m_nodeIdDelta = 0;
302  m_latDelata = 0;
303  m_lonDelta = 0;
304 
305  m_wayIdDelta = 0;
306  m_wayNodeIdDelta = 0;
307 
308  m_relIdDelta = 0;
309  m_relNodeMemberIdDelta = 0;
310  m_relWayMemberIdDelta = 0;
311  m_relRelMemberIdDelta = 0;
312 }
void setTag(Elem &elem, Tag &&tag)
Inserts a new tag, or replaces an existing one with the same key.
Definition: datatypes.h:421
Coordinate, stored as 1e7 * degree to avoid floating point precision issues, and offset to unsigned v...
Definition: datatypes.h:37
Holds OSM elements produced by a parser prior to merging into OSM::DataSet.
An OSM node.
Definition: datatypes.h:194
An OSM element tag.
Definition: datatypes.h:182
An OSM way.
Definition: datatypes.h:206
An OSM relation.
Definition: datatypes.h:270
A set of nodes, ways and relations.
Definition: datatypes.h:283
Bounding box, ie.
Definition: datatypes.h:95
Low-level types and functions to work with raw OSM data as efficiently as possible.
A member in a relation.
Definition: datatypes.h:241
This file is part of the KDE documentation.
Documentation copyright © 1996-2021 The KDE developers.
Generated on Mon Oct 25 2021 23:04:00 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.