KArchive

kcompressiondevice.cpp
1 /* This file is part of the KDE libraries
2  SPDX-FileCopyrightText: 2000 David Faure <[email protected]>
3  SPDX-FileCopyrightText: 2011 Mario Bensi <[email protected]>
4 
5  SPDX-License-Identifier: LGPL-2.0-or-later
6 */
7 
8 #include "kcompressiondevice.h"
9 #include "kcompressiondevice_p.h"
10 #include "kfilterbase.h"
11 #include "loggingcategory.h"
12 #include "kgzipfilter.h"
13 #include "knonefilter.h"
14 
15 #include "config-compression.h"
16 
17 #if HAVE_BZIP2_SUPPORT
18 #include "kbzip2filter.h"
19 #endif
20 #if HAVE_XZ_SUPPORT
21 #include "kxzfilter.h"
22 #endif
23 #if HAVE_ZSTD_SUPPORT
24 #include "kzstdfilter.h"
25 #endif
26 
27 #include <QDebug>
28 #include <QFile>
29 #include <QMimeDatabase>
30 
31 #include <assert.h>
32 #include <stdio.h> // for EOF
33 #include <stdlib.h>
34 
35 class KCompressionDevicePrivate
36 {
37 public:
38  KCompressionDevicePrivate(KCompressionDevice *qq)
39  : bNeedHeader(true)
40  , bSkipHeaders(false)
41  , bOpenedUnderlyingDevice(false)
43  , errorCode(QFileDevice::NoError)
44  , deviceReadPos(0)
45  , q(qq)
46  {
47  }
48 
49  void propagateErrorCode();
50 
51  bool bNeedHeader;
52  bool bSkipHeaders;
53  bool bOpenedUnderlyingDevice;
54  QByteArray buffer; // Used as 'input buffer' when reading, as 'output buffer' when writing
55  QByteArray origFileName;
56  KFilterBase::Result result;
59  QFileDevice::FileError errorCode;
60  qint64 deviceReadPos;
62 };
63 
64 void KCompressionDevicePrivate::propagateErrorCode()
65 {
66  QIODevice *dev = filter->device();
67  if (QFileDevice *fileDev = qobject_cast<QFileDevice *>(dev)) {
68  if (fileDev->error() != QFileDevice::NoError) {
69  errorCode = fileDev->error();
70  q->setErrorString(dev->errorString());
71  }
72  }
73  // ... we have no generic way to propagate errors from other kinds of iodevices. Sucks, heh? :(
74 }
75 
76 static KCompressionDevice::CompressionType findCompressionByFileName(const QString &fileName)
77 {
78  if (fileName.endsWith(QLatin1String(".gz"), Qt::CaseInsensitive)) {
79  return KCompressionDevice::GZip;
80  }
81 #if HAVE_BZIP2_SUPPORT
82  if (fileName.endsWith(QLatin1String(".bz2"), Qt::CaseInsensitive)) {
83  return KCompressionDevice::BZip2;
84  }
85 #endif
86 #if HAVE_XZ_SUPPORT
87  if (fileName.endsWith(QLatin1String(".lzma"), Qt::CaseInsensitive) || fileName.endsWith(QLatin1String(".xz"), Qt::CaseInsensitive)) {
88  return KCompressionDevice::Xz;
89  }
90 #endif
91 #if HAVE_ZSTD_SUPPORT
92  if (fileName.endsWith(QLatin1String(".zst"), Qt::CaseInsensitive)) {
94  }
95 #endif
96  else {
97  // not a warning, since this is called often with other MIME types (see #88574)...
98  // maybe we can avoid that though?
99  // qCDebug(KArchiveLog) << "findCompressionByFileName : no compression found for " << fileName;
100  }
101 
102  return KCompressionDevice::None;
103 }
104 
106 {
107  if (mimeType == QLatin1String("application/gzip") //
108  || mimeType == QLatin1String("application/x-gzip") // legacy name, kept for compatibility
109  ) {
110  return KCompressionDevice::GZip;
111  }
112 #if HAVE_BZIP2_SUPPORT
113  if (mimeType == QLatin1String("application/x-bzip") //
114  || mimeType == QLatin1String("application/x-bzip2") // old name, kept for compatibility
115  ) {
116  return KCompressionDevice::BZip2;
117  }
118 #endif
119 #if HAVE_XZ_SUPPORT
120  if (mimeType == QLatin1String("application/x-lzma") // legacy name, still used
121  || mimeType == QLatin1String("application/x-xz") // current naming
122  ) {
123  return KCompressionDevice::Xz;
124  }
125 #endif
126 #if HAVE_ZSTD_SUPPORT
127  if (mimeType == QLatin1String("application/zstd")) {
129  }
130 #endif
131  QMimeDatabase db;
132  const QMimeType mime = db.mimeTypeForName(mimeType);
133  if (mime.isValid()) {
134  // use legacy MIME type for now, see comment in impl. of KTar(const QString &, const QString &_mimetype)
135  if (mime.inherits(QStringLiteral("application/x-gzip"))) {
136  return KCompressionDevice::GZip;
137  }
138 #if HAVE_BZIP2_SUPPORT
139  if (mime.inherits(QStringLiteral("application/x-bzip"))) {
140  return KCompressionDevice::BZip2;
141  }
142 #endif
143 #if HAVE_XZ_SUPPORT
144  if (mime.inherits(QStringLiteral("application/x-lzma"))) {
145  return KCompressionDevice::Xz;
146  }
147 
148  if (mime.inherits(QStringLiteral("application/x-xz"))) {
149  return KCompressionDevice::Xz;
150  }
151 #endif
152  }
153 
154  // not a warning, since this is called often with other MIME types (see #88574)...
155  // maybe we can avoid that though?
156  // qCDebug(KArchiveLog) << "no compression found for" << mimeType;
157  return KCompressionDevice::None;
158 }
159 
161 {
162  switch (type) {
163  case KCompressionDevice::GZip:
164  return new KGzipFilter;
165  case KCompressionDevice::BZip2:
166 #if HAVE_BZIP2_SUPPORT
167  return new KBzip2Filter;
168 #else
169  return nullptr;
170 #endif
171  case KCompressionDevice::Xz:
172 #if HAVE_XZ_SUPPORT
173  return new KXzFilter;
174 #else
175  return nullptr;
176 #endif
177  case KCompressionDevice::None:
178  return new KNoneFilter;
179 #if HAVE_ZSTD_SUPPORT
181  return new KZstdFilter;
182 #endif
183  }
184  return nullptr;
185 }
186 
187 KCompressionDevice::KCompressionDevice(QIODevice *inputDevice, bool autoDeleteInputDevice, CompressionType type)
188  : d(new KCompressionDevicePrivate(this))
189 {
190  assert(inputDevice);
191  d->filter = filterForCompressionType(type);
192  if (d->filter) {
193  d->type = type;
194  d->filter->setDevice(inputDevice, autoDeleteInputDevice);
195  }
196 }
197 
199  : d(new KCompressionDevicePrivate(this))
200 {
201  QFile *f = new QFile(fileName);
202  d->filter = filterForCompressionType(type);
203  if (d->filter) {
204  d->type = type;
205  d->filter->setDevice(f, true);
206  } else {
207  delete f;
208  }
209 }
210 
212  : KCompressionDevice(fileName, findCompressionByFileName(fileName))
213 {
214 }
215 
217 {
218  if (isOpen()) {
219  close();
220  }
221  delete d->filter;
222  delete d;
223 }
224 
226 {
227  return d->type;
228 }
229 
231 {
232  if (isOpen()) {
233  // qCWarning(KArchiveLog) << "KCompressionDevice::open: device is already open";
234  return true; // QFile returns false, but well, the device -is- open...
235  }
236  if (!d->filter) {
237  return false;
238  }
239  d->bOpenedUnderlyingDevice = false;
240  // qCDebug(KArchiveLog) << mode;
241  if (mode == QIODevice::ReadOnly) {
242  d->buffer.resize(0);
243  } else {
244  d->buffer.resize(BUFFER_SIZE);
245  d->filter->setOutBuffer(d->buffer.data(), d->buffer.size());
246  }
247  if (!d->filter->device()->isOpen()) {
248  if (!d->filter->device()->open(mode)) {
249  // qCWarning(KArchiveLog) << "KCompressionDevice::open: Couldn't open underlying device";
250  d->propagateErrorCode();
251  return false;
252  }
253  d->bOpenedUnderlyingDevice = true;
254  }
255  d->bNeedHeader = !d->bSkipHeaders;
256  d->filter->setFilterFlags(d->bSkipHeaders ? KFilterBase::NoHeaders : KFilterBase::WithHeaders);
257  if (!d->filter->init(mode)) {
258  return false;
259  }
260  d->result = KFilterBase::Ok;
261  setOpenMode(mode);
262  return true;
263 }
264 
266 {
267  if (!isOpen()) {
268  return;
269  }
270  if (d->filter->mode() == QIODevice::WriteOnly && d->errorCode == QFileDevice::NoError) {
271  write(nullptr, 0); // finish writing
272  }
273  // qCDebug(KArchiveLog) << "Calling terminate().";
274 
275  if (!d->filter->terminate()) {
276  // qCWarning(KArchiveLog) << "KCompressionDevice::close: terminate returned an error";
277  d->errorCode = QFileDevice::UnspecifiedError;
278  }
279  if (d->bOpenedUnderlyingDevice) {
280  QIODevice *dev = d->filter->device();
281  dev->close();
282  d->propagateErrorCode();
283  }
285 }
286 
288 {
289  return d->errorCode;
290 }
291 
292 bool KCompressionDevice::seek(qint64 pos)
293 {
294  if (d->deviceReadPos == pos) {
295  return QIODevice::seek(pos);
296  }
297 
298  // qCDebug(KArchiveLog) << "seek(" << pos << ") called, current pos=" << QIODevice::pos();
299 
300  Q_ASSERT(d->filter->mode() == QIODevice::ReadOnly);
301 
302  if (pos == 0) {
303  if (!QIODevice::seek(pos)) {
304  return false;
305  }
306 
307  // We can forget about the cached data
308  d->bNeedHeader = !d->bSkipHeaders;
309  d->result = KFilterBase::Ok;
310  d->filter->setInBuffer(nullptr, 0);
311  d->filter->reset();
312  d->deviceReadPos = 0;
313  return d->filter->device()->reset();
314  }
315 
316  qint64 bytesToRead;
317  if (d->deviceReadPos < pos) { // we can start from here
318  bytesToRead = pos - d->deviceReadPos;
319  // Since we're going to do a read() below
320  // we need to reset the internal QIODevice pos to the real position we are
321  // so that after read() we are indeed pointing to the pos seek
322  // asked us to be in
323  if (!QIODevice::seek(d->deviceReadPos)) {
324  return false;
325  }
326  } else {
327  // we have to start from 0 ! Ugly and slow, but better than the previous
328  // solution (KTarGz was allocating everything into memory)
329  if (!seek(0)) { // recursive
330  return false;
331  }
332  bytesToRead = pos;
333  }
334 
335  // qCDebug(KArchiveLog) << "reading " << bytesToRead << " dummy bytes";
336  QByteArray dummy(qMin(bytesToRead, qint64(SEEK_BUFFER_SIZE)), 0);
337  while (bytesToRead > 0) {
338  const qint64 bytesToReadThisTime = qMin(bytesToRead, qint64(dummy.size()));
339  const bool result = (read(dummy.data(), bytesToReadThisTime) == bytesToReadThisTime);
340  if (!result) {
341  return false;
342  }
343  bytesToRead -= bytesToReadThisTime;
344  }
345  return true;
346 }
347 
348 bool KCompressionDevice::atEnd() const
349 {
350  return (d->type == KCompressionDevice::None || d->result == KFilterBase::End) //
351  && QIODevice::atEnd() // take QIODevice's internal buffer into account
352  && d->filter->device()->atEnd();
353 }
354 
355 qint64 KCompressionDevice::readData(char *data, qint64 maxlen)
356 {
357  Q_ASSERT(d->filter->mode() == QIODevice::ReadOnly);
358  // qCDebug(KArchiveLog) << "maxlen=" << maxlen;
359  KFilterBase *filter = d->filter;
360 
361  uint dataReceived = 0;
362 
363  // We came to the end of the stream
364  if (d->result == KFilterBase::End) {
365  return dataReceived;
366  }
367 
368  // If we had an error, return -1.
369  if (d->result != KFilterBase::Ok) {
370  return -1;
371  }
372 
373  qint64 availOut = maxlen;
374  filter->setOutBuffer(data, maxlen);
375 
376  while (dataReceived < maxlen) {
377  if (filter->inBufferEmpty()) {
378  // Not sure about the best size to set there.
379  // For sure, it should be bigger than the header size (see comment in readHeader)
380  d->buffer.resize(BUFFER_SIZE);
381  // Request data from underlying device
382  int size = filter->device()->read(d->buffer.data(), d->buffer.size());
383  // qCDebug(KArchiveLog) << "got" << size << "bytes from device";
384  if (size) {
385  filter->setInBuffer(d->buffer.data(), size);
386  } else {
387  // Not enough data available in underlying device for now
388  break;
389  }
390  }
391  if (d->bNeedHeader) {
392  (void)filter->readHeader();
393  d->bNeedHeader = false;
394  }
395 
396  d->result = filter->uncompress();
397 
398  if (d->result == KFilterBase::Error) {
399  // qCWarning(KArchiveLog) << "KCompressionDevice: Error when uncompressing data";
400  break;
401  }
402 
403  // We got that much data since the last time we went here
404  uint outReceived = availOut - filter->outBufferAvailable();
405  // qCDebug(KArchiveLog) << "avail_out = " << filter->outBufferAvailable() << " result=" << d->result << " outReceived=" << outReceived;
406  if (availOut < uint(filter->outBufferAvailable())) {
407  // qCWarning(KArchiveLog) << " last availOut " << availOut << " smaller than new avail_out=" << filter->outBufferAvailable() << " !";
408  }
409 
410  dataReceived += outReceived;
411  data += outReceived;
412  availOut = maxlen - dataReceived;
413  if (d->result == KFilterBase::End) {
414  // We're actually at the end, no more data to check
415  if (filter->device()->atEnd()) {
416  break;
417  }
418 
419  // Still not done, re-init and try again
420  filter->init(filter->mode());
421  }
422  filter->setOutBuffer(data, availOut);
423  }
424 
425  d->deviceReadPos += dataReceived;
426  return dataReceived;
427 }
428 
429 qint64 KCompressionDevice::writeData(const char *data /*0 to finish*/, qint64 len)
430 {
431  KFilterBase *filter = d->filter;
432  Q_ASSERT(filter->mode() == QIODevice::WriteOnly);
433  // If we had an error, return 0.
434  if (d->result != KFilterBase::Ok) {
435  return 0;
436  }
437 
438  bool finish = (data == nullptr);
439  if (!finish) {
440  filter->setInBuffer(data, len);
441  if (d->bNeedHeader) {
442  (void)filter->writeHeader(d->origFileName);
443  d->bNeedHeader = false;
444  }
445  }
446 
447  uint dataWritten = 0;
448  uint availIn = len;
449  while (dataWritten < len || finish) {
450  d->result = filter->compress(finish);
451 
452  if (d->result == KFilterBase::Error) {
453  // qCWarning(KArchiveLog) << "KCompressionDevice: Error when compressing data";
454  // What to do ?
455  break;
456  }
457 
458  // Wrote everything ?
459  if (filter->inBufferEmpty() || (d->result == KFilterBase::End)) {
460  // We got that much data since the last time we went here
461  uint wrote = availIn - filter->inBufferAvailable();
462 
463  // qCDebug(KArchiveLog) << " Wrote everything for now. avail_in=" << filter->inBufferAvailable() << "result=" << d->result << "wrote=" << wrote;
464 
465  // Move on in the input buffer
466  data += wrote;
467  dataWritten += wrote;
468 
469  availIn = len - dataWritten;
470  // qCDebug(KArchiveLog) << " availIn=" << availIn << "dataWritten=" << dataWritten << "pos=" << pos();
471  if (availIn > 0) {
472  filter->setInBuffer(data, availIn);
473  }
474  }
475 
476  if (filter->outBufferFull() || (d->result == KFilterBase::End) || finish) {
477  // qCDebug(KArchiveLog) << " writing to underlying. avail_out=" << filter->outBufferAvailable();
478  int towrite = d->buffer.size() - filter->outBufferAvailable();
479  if (towrite > 0) {
480  // Write compressed data to underlying device
481  int size = filter->device()->write(d->buffer.data(), towrite);
482  if (size != towrite) {
483  // qCWarning(KArchiveLog) << "KCompressionDevice::write. Could only write " << size << " out of " << towrite << " bytes";
484  d->errorCode = QFileDevice::WriteError;
485  setErrorString(tr("Could not write. Partition full?"));
486  return 0; // indicate an error
487  }
488  // qCDebug(KArchiveLog) << " wrote " << size << " bytes";
489  }
490  if (d->result == KFilterBase::End) {
491  Q_ASSERT(finish); // hopefully we don't get end before finishing
492  break;
493  }
494  d->buffer.resize(BUFFER_SIZE);
495  filter->setOutBuffer(d->buffer.data(), d->buffer.size());
496  }
497  }
498 
499  return dataWritten;
500 }
501 
503 {
504  d->origFileName = fileName;
505 }
506 
508 {
509  d->bSkipHeaders = true;
510 }
511 
512 KFilterBase *KCompressionDevice::filterBase()
513 {
514  return d->filter;
515 }
bool inherits(const QString &mimeTypeName) const const
bool endsWith(const QString &s, Qt::CaseSensitivity cs) const const
QString errorString() const const
void setOpenMode(QIODevice::OpenMode openMode)
void setOrigFileName(const QByteArray &fileName)
For writing gzip compressed files only: set the name of the original file, to be used in the gzip hea...
typedef OpenMode
CaseInsensitive
Type type(const QSqlDatabase &db)
virtual bool seek(qint64 pos)
void setSkipHeaders()
Call this let this device skip the gzip headers when reading/writing.
void setErrorString(const QString &str)
bool seek(qint64) override
That one can be quite slow, when going back.
static KFilterBase * filterForCompressionType(CompressionType type)
Call this to create the appropriate filter for the CompressionType named type.
static CompressionType compressionTypeForMimeType(const QString &mimetype)
Returns the compression type for the given MIME type, if possible.
virtual qint64 pos() const const
QMimeType mimeTypeForName(const QString &nameOrAlias) const const
bool isOpen() const const
KCompressionDevice(QIODevice *inputDevice, bool autoDeleteInputDevice, CompressionType type)
Constructs a KCompressionDevice for a given CompressionType (e.g.
virtual qint64 size() const const
QFuture< void > filter(Sequence &sequence, KeepFunctor filterFunction)
~KCompressionDevice() override
Destructs the KCompressionDevice.
bool isValid() const const
CompressionType compressionType() const
The compression actually used by this device.
Internal class used by KCompressionDevice.
Definition: kgzipfilter.h:19
qint64 read(char *data, qint64 maxSize)
bool open(QIODevice::OpenMode mode) override
Open for reading or writing.
Internal class used by KCompressionDevice.
Definition: knonefilter.h:22
virtual bool atEnd() const const
void close() override
Close after reading or writing.
QFileDevice::FileError error() const
Returns the error code from the last failing operation.
int size() const const
virtual void close()
QString tr(const char *sourceText, const char *disambiguation, int n)
char * data()
qint64 write(const char *data, qint64 maxSize)
This file is part of the KDE documentation.
Documentation copyright © 1996-2023 The KDE developers.
Generated on Thu Mar 23 2023 04:15:17 by doxygen 1.8.17 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.