KI18n

isocodescache.cpp
1 /*
2  SPDX-FileCopyrightText: 2021 Volker Krause <[email protected]>
3 
4  SPDX-License-Identifier: LGPL-2.0-or-later
5 */
6 
7 #include "config-localedata.h"
8 
9 #include "isocodes_p.h"
10 #include "isocodescache_p.h"
11 #include "logging.h"
12 
13 #include <QDir>
14 #include <QFile>
15 #include <QFileInfo>
16 #include <QJsonArray>
17 #include <QJsonDocument>
18 #include <QJsonObject>
19 #include <QStandardPaths>
20 
21 // increment those when changing the format
22 enum : uint32_t {
23  Iso3166_1CacheHeader = 0x4B493101,
24  Iso3166_2CacheHeader = 0x4B493201,
25 };
26 
27 static QString isoCodesPath(QStringView file)
28 {
29 #ifndef Q_OS_ANDROID
31  if (!path.isEmpty()) {
32  return path;
33  }
34 
35  // search manually in the compile-time determined prefix
36  // needed for example for non-installed Windows binaries to work, such as unit tests
37  for (const char *installLocation : {"/share", "/bin/data"}) {
38  path = QLatin1String(ISO_CODES_PREFIX) + QLatin1String(installLocation) + QLatin1String("/iso-codes/json/") + file;
39  if (QFileInfo::exists(path)) {
40  return path;
41  }
42  }
43 
44  return {};
45 #else
46  return QLatin1String("assets:/share/iso-codes/json/") + file;
47 #endif
48 }
49 
50 static QString cachePath()
51 {
53 }
54 
55 static QString cacheFilePath(QStringView file)
56 {
57  return cachePath() + file;
58 }
59 
60 IsoCodesCache::~IsoCodesCache() = default;
61 
62 IsoCodesCache *IsoCodesCache::instance()
63 {
64  static IsoCodesCache s_cache;
65  return &s_cache;
66 }
67 
68 void IsoCodesCache::loadIso3166_1()
69 {
70  if (!m_iso3166_1CacheData && !loadIso3166_1Cache()) {
71  createIso3166_1Cache();
72  loadIso3166_1Cache();
73  }
74 }
75 
76 bool IsoCodesCache::loadIso3166_1Cache()
77 {
78  QFileInfo jsonFi(isoCodesPath(u"iso_3166-1.json"));
79  auto f = std::make_unique<QFile>(cacheFilePath(u"iso_3166-1"));
80  if (!f->open(QFile::ReadOnly) || f->fileTime(QFile::FileModificationTime) < jsonFi.lastModified() || f->size() < 8) {
81  return false;
82  }
83  m_iso3166_1CacheSize = f->size();
84 
85  // validate cache file is usable
86  // header matches
87  const auto data = f->map(0, m_iso3166_1CacheSize);
88  if (*reinterpret_cast<const uint32_t *>(data) != Iso3166_1CacheHeader) {
89  return false;
90  }
91  // lookup tables fit into the available size
92  const auto size = *(reinterpret_cast<const uint32_t *>(data) + 1);
93  if (sizeof(Iso3166_1CacheHeader) + sizeof(size) + size * sizeof(MapEntry<uint16_t>) * 2 >= m_iso3166_1CacheSize) {
94  return false;
95  }
96  // string table is 0 terminated
97  if (data[m_iso3166_1CacheSize - 1] != '\0') {
98  return false;
99  }
100 
101  m_iso3166_1CacheFile = std::move(f);
102  m_iso3166_1CacheData = data;
103  return true;
104 }
105 
106 uint32_t IsoCodesCache::countryCount() const
107 {
108  return m_iso3166_1CacheData ? *(reinterpret_cast<const uint32_t *>(m_iso3166_1CacheData) + 1) : 0;
109 }
110 
111 const MapEntry<uint16_t> *IsoCodesCache::countryNameMapBegin() const
112 {
113  return m_iso3166_1CacheData ? reinterpret_cast<const MapEntry<uint16_t> *>(m_iso3166_1CacheData + sizeof(uint32_t) * 2) : nullptr;
114 }
115 
116 const MapEntry<uint16_t> *IsoCodesCache::countryAlpha3MapBegin() const
117 {
118  return m_iso3166_1CacheData ? countryNameMapBegin() + countryCount() : nullptr;
119 }
120 
121 const char *IsoCodesCache::countryStringTableLookup(uint16_t offset) const
122 {
123  if (m_iso3166_1CacheData) {
124  const auto pos = offset + 2 * sizeof(uint32_t) + 2 * countryCount() * sizeof(MapEntry<uint16_t>);
125  return m_iso3166_1CacheSize > pos ? reinterpret_cast<const char *>(m_iso3166_1CacheData + pos) : nullptr;
126  }
127  return nullptr;
128 }
129 
130 void IsoCodesCache::createIso3166_1Cache()
131 {
132  qCDebug(KI18NLD) << "Rebuilding ISO 3166-1 cache";
133  const auto path = isoCodesPath(u"iso_3166-1.json");
134 
135  QFile file(path);
136  if (!file.open(QFile::ReadOnly)) {
137  qCWarning(KI18NLD) << "Unable to open iso_3166-1.json" << path << file.errorString();
138  return;
139  }
140 
141  std::vector<MapEntry<uint16_t>> alpha2NameMap;
142  std::vector<MapEntry<uint16_t>> alpha3alpha2Map;
143  QByteArray iso3166_1stringTable;
144 
145  const auto doc = QJsonDocument::fromJson(file.readAll());
146  const auto array = doc.object().value(QLatin1String("3166-1")).toArray();
147  for (const auto &entryVal : array) {
148  const auto entry = entryVal.toObject();
149  const auto alpha2 = entry.value(QLatin1String("alpha_2")).toString();
150  if (alpha2.size() != 2) {
151  continue;
152  }
153  const auto alpha2Key = IsoCodes::alpha2CodeToKey(alpha2);
154 
155  assert(std::numeric_limits<uint16_t>::max() > iso3166_1stringTable.size());
156  alpha2NameMap.push_back({alpha2Key, (uint16_t)iso3166_1stringTable.size()});
157  iso3166_1stringTable.append(entry.value(QLatin1String("name")).toString().toUtf8());
158  iso3166_1stringTable.append('\0');
159 
160  const auto alpha3Key = IsoCodes::alpha3CodeToKey(entry.value(QLatin1String("alpha_3")).toString());
161  alpha3alpha2Map.push_back({alpha3Key, alpha2Key});
162  }
163 
164  std::sort(alpha2NameMap.begin(), alpha2NameMap.end());
165  std::sort(alpha3alpha2Map.begin(), alpha3alpha2Map.end());
166 
167  // write out binary cache file
168  QDir().mkpath(cachePath());
169  QFile cache(cacheFilePath(u"iso_3166-1"));
170  if (!cache.open(QFile::WriteOnly)) {
171  qCWarning(KI18NLD) << "Failed to write ISO 3166-1 cache:" << cache.errorString() << cache.fileName();
172  return;
173  }
174 
175  uint32_t n = Iso3166_1CacheHeader;
176  cache.write(reinterpret_cast<const char *>(&n), 4); // header
177  n = alpha2NameMap.size();
178  cache.write(reinterpret_cast<const char *>(&n), 4); // size
179  for (auto entry : alpha2NameMap) {
180  cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry));
181  }
182  for (auto entry : alpha3alpha2Map) {
183  cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry));
184  }
185  cache.write(iso3166_1stringTable);
186 }
187 
188 void IsoCodesCache::loadIso3166_2()
189 {
190  if (!m_iso3166_2CacheData && !loadIso3166_2Cache()) {
191  createIso3166_2Cache();
192  loadIso3166_2Cache();
193  }
194 }
195 
196 bool IsoCodesCache::loadIso3166_2Cache()
197 {
198  QFileInfo jsonFi(isoCodesPath(u"iso_3166-2.json"));
199  auto f = std::make_unique<QFile>(cacheFilePath(u"iso_3166-2"));
200  if (!f->open(QFile::ReadOnly) || f->fileTime(QFile::FileModificationTime) < jsonFi.lastModified() || f->size() < 8) {
201  return false;
202  }
203  m_iso3166_2CacheSize = f->size();
204 
205  // validate cache file is usable
206  // header matches
207  const auto data = f->map(0, m_iso3166_2CacheSize);
208  if (*reinterpret_cast<const uint32_t *>(data) != Iso3166_2CacheHeader) {
209  return false;
210  }
211  // name lookup table fits into the available size
212  auto size = *(reinterpret_cast<const uint32_t *>(data) + 1);
213  auto offset = 3 * sizeof(uint32_t) + size * sizeof(MapEntry<uint32_t>);
214  if (offset >= m_iso3166_2CacheSize) {
215  return false;
216  }
217  // hierarchy map boundary check
218  size = *(reinterpret_cast<const uint32_t *>(data + offset) - 1);
219  offset += size * sizeof(MapEntry<uint32_t>);
220  if (offset >= m_iso3166_2CacheSize) {
221  return false;
222  }
223  // string table is 0 terminated
224  if (data[m_iso3166_2CacheSize - 1] != '\0') {
225  return false;
226  }
227 
228  m_iso3166_2CacheFile = std::move(f);
229  m_iso3166_2CacheData = data;
230  return true;
231 }
232 
233 uint32_t IsoCodesCache::subdivisionCount() const
234 {
235  return m_iso3166_2CacheData ? *(reinterpret_cast<const uint32_t *>(m_iso3166_2CacheData) + 1) : 0;
236 }
237 
238 const MapEntry<uint32_t> *IsoCodesCache::subdivisionNameMapBegin() const
239 {
240  return m_iso3166_2CacheData ? reinterpret_cast<const MapEntry<uint32_t> *>(m_iso3166_2CacheData + 2 * sizeof(uint32_t)) : nullptr;
241 }
242 
243 uint32_t IsoCodesCache::subdivisionHierachyMapSize() const
244 {
245  return m_iso3166_2CacheData
246  ? *(reinterpret_cast<const uint32_t *>(m_iso3166_2CacheData + 2 * sizeof(uint32_t) + subdivisionCount() * sizeof(MapEntry<uint32_t>)))
247  : 0;
248 }
249 
250 const MapEntry<uint32_t> *IsoCodesCache::subdivisionParentMapBegin() const
251 {
252  return m_iso3166_2CacheData
253  ? reinterpret_cast<const MapEntry<uint32_t> *>(m_iso3166_2CacheData + 3 * sizeof(uint32_t) + subdivisionCount() * sizeof(MapEntry<uint32_t>))
254  : nullptr;
255 }
256 
257 const char *IsoCodesCache::subdivisionStringTableLookup(uint16_t offset) const
258 {
259  if (m_iso3166_2CacheData) {
260  const auto pos = offset + 3 * sizeof(uint32_t) + (subdivisionCount() + subdivisionHierachyMapSize()) * sizeof(MapEntry<uint32_t>);
261  return m_iso3166_2CacheSize > pos ? reinterpret_cast<const char *>(m_iso3166_2CacheData + pos) : nullptr;
262  }
263  return nullptr;
264 }
265 
266 void IsoCodesCache::createIso3166_2Cache()
267 {
268  qCDebug(KI18NLD) << "Rebuilding ISO 3166-2 cache";
269  const auto path = isoCodesPath(u"iso_3166-2.json");
270 
271  QFile file(path);
272  if (!file.open(QFile::ReadOnly)) {
273  qCWarning(KI18NLD) << "Unable to open iso_3166-2.json" << path << file.errorString();
274  return;
275  }
276 
277  std::vector<MapEntry<uint32_t>> subdivNameMap;
278  std::vector<MapEntry<uint32_t>> subdivParentMap;
279  QByteArray iso3166_2stringTable;
280 
281  const auto doc = QJsonDocument::fromJson(file.readAll());
282  const auto array = doc.object().value(QLatin1String("3166-2")).toArray();
283  for (const auto &entryVal : array) {
284  const auto entry = entryVal.toObject();
285  const auto key = IsoCodes::subdivisionCodeToKey(entry.value(QLatin1String("code")).toString());
286 
287  assert(std::numeric_limits<uint16_t>::max() > iso3166_2stringTable.size());
288  subdivNameMap.push_back({key, (uint16_t)iso3166_2stringTable.size()});
289  iso3166_2stringTable.append(entry.value(QLatin1String("name")).toString().toUtf8());
290  iso3166_2stringTable.append('\0');
291 
292  const auto parentKey = IsoCodes::alphaNum3CodeToKey(entry.value(QLatin1String("parent")).toString());
293  if (parentKey) {
294  subdivParentMap.push_back({key, parentKey});
295  }
296  }
297 
298  std::sort(subdivNameMap.begin(), subdivNameMap.end());
299  std::sort(subdivParentMap.begin(), subdivParentMap.end());
300 
301  // write out binary cache file
302  QDir().mkpath(cachePath());
303  QFile cache(cacheFilePath(u"iso_3166-2"));
304  if (!cache.open(QFile::WriteOnly)) {
305  qCWarning(KI18NLD) << "Failed to write ISO 3166-2 cache:" << cache.errorString() << cache.fileName();
306  return;
307  }
308 
309  uint32_t n = Iso3166_2CacheHeader;
310  cache.write(reinterpret_cast<const char *>(&n), 4); // header
311  n = subdivNameMap.size();
312  cache.write(reinterpret_cast<const char *>(&n), 4); // size of the name map
313  for (auto entry : subdivNameMap) {
314  cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry));
315  }
316  n = subdivParentMap.size();
317  cache.write(reinterpret_cast<const char *>(&n), 4); // size of the hierarchy map
318  for (auto entry : subdivParentMap) {
319  cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry));
320  }
321  cache.write(iso3166_2stringTable);
322 }
QJsonDocument fromJson(const QByteArray &json, QJsonParseError *error)
QByteArray & append(char ch)
QString writableLocation(QStandardPaths::StandardLocation type)
QString locate(QStandardPaths::StandardLocation type, const QString &fileName, QStandardPaths::LocateOptions options)
bool exists() const const
char * toString(const T &value)
bool mkpath(const QString &dirPath) const const
int size() const const
This file is part of the KDE documentation.
Documentation copyright © 1996-2022 The KDE developers.
Generated on Tue Aug 16 2022 04:07:13 by doxygen 1.8.17 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.