MailImporter

filteroe.cpp
1 /***************************************************************************
2  filter_oe.cxx - Outlook Express mail import
3  -------------------
4  begin : Sat Feb 1 2003
5  copyright : (C) 2003 by Laurence Anderson <[email protected]>
6  (C) 2005 by Danny Kukawka <[email protected]>
7  ***************************************************************************/
8 
9 /***************************************************************************
10  * *
11  * This program is free software; you can redistribute it and/or modify *
12  * it under the terms of the GNU General Public License as published by *
13  * the Free Software Foundation; either version 2 of the License, or *
14  * (at your option) any later version. *
15  * *
16  ***************************************************************************/
17 
18 // This filter was created by looking at libdbx &liboe
19 
20 #include "filteroe.h"
21 #include "mailimporter_debug.h"
22 
23 #include <QFileDialog>
24 #include <KLocalizedString>
25 
26 #include <QTemporaryFile>
27 
28 #define OE4_SIG_1 0x36464d4a
29 #define OE4_SIG_2 0x00010003
30 #define OE5_SIG_1 0xfe12adcf
31 #define OE5_EMAIL_SIG_2 0x6f74fdc5
32 #define OE5_FOLDER_SIG_2 0x6f74fdc6
33 #define OE5_SIG_3 0x11d1e366
34 #define OE5_SIG_4 0xc0004e9a
35 #define MBX_MAILMAGIC 0x7F007F00
36 
37 using namespace MailImporter;
38 
39 FilterOE::FilterOE()
40  : Filter(i18n("Import Outlook Express Emails"),
41  i18n("Laurence Anderson <br>( Filter enhanced by Danny Kukawka )</p>"),
42  i18n("<p><b>Outlook Express 4/5/6 import filter</b></p>"
43  "<p>You will need to locate the folder where the mailbox has been "
44  "stored by searching for .dbx or .mbx files under "
45  "<ul><li><i>C:\\Windows\\Application Data</i> in Windows 9x</li>"
46  "<li><i>Documents and Settings</i> in Windows 2000 or later</li></ul></p>"
47  "<p><b>Note:</b> Since it is possible to recreate the folder structure, the folders from "
48  "Outlook Express 5 and 6 will be stored under: \"OE-Import\" in your local folder.</p>"))
49 {
50 }
51 
52 FilterOE::~FilterOE()
53 {
54 }
55 
56 void FilterOE::import()
57 {
58  // Select directory containing plain text emails
59  const QString maildir = QFileDialog::getExistingDirectory(filterInfo()->parentWidget(), QString(), QDir::homePath());
60  importMails(maildir);
61 }
62 
63 void FilterOE::importMails(const QString &maildir)
64 {
65  if (maildir.isEmpty()) { // No directory selected
66  filterInfo()->alert(i18n("No directory selected."));
67  return;
68  }
69  setMailDir(maildir);
70 
71  QDir dir(mailDir());
72  QStringList files = dir.entryList(QStringList(QStringLiteral("*.[dDmM][bB][xX]")), QDir::Files, QDir::Name);
73  if (files.isEmpty()) {
74  filterInfo()->alert(i18n("No Outlook Express mailboxes found in directory %1.", mailDir()));
75  return;
76  }
77 
78  totalFiles = files.count();
79  currentFile = 0;
80  count0x04 = 0;
81  count0x84 = 0;
82  parsedFolder = false;
83 
84  filterInfo()->setOverall(0);
85 
88  for (QStringList::Iterator mailFile = files.begin(); mailFile != files.end(); ++mailFile) {
89  if (*mailFile == QLatin1String("Folders.dbx")) {
90  filterInfo()->addInfoLogEntry(i18n("Import folder structure..."));
91  importMailBox(dir.filePath(*mailFile));
92  if (!folderStructure.isEmpty()) {
93  parsedFolder = true;
94  }
95  // remove file from QStringList::files, no longer needed
96  files.erase(mailFile);
97  currentIsFolderFile = false;
98  break;
99  }
100  }
101 
102  int n = 0;
104  for (QStringList::ConstIterator mailFile = files.constBegin(); mailFile != end; ++mailFile) {
105  if (filterInfo()->shouldTerminate()) {
106  break;
107  }
108  importMailBox(dir.filePath(*mailFile));
109  filterInfo()->setOverall(100 * ++n / files.count());
110  }
111 
112  filterInfo()->setOverall(100);
113  filterInfo()->setCurrent(100);
114  filterInfo()->addInfoLogEntry(i18n("Finished importing Outlook Express emails"));
115  if (filterInfo()->shouldTerminate()) {
116  filterInfo()->addInfoLogEntry(i18n("Finished import, canceled by user."));
117  }
118 
119  qCDebug(MAILIMPORTER_LOG) << "total emails in current file:" << totalEmails;
120  qCDebug(MAILIMPORTER_LOG) << "0x84 Mails:" << count0x84;
121  qCDebug(MAILIMPORTER_LOG) << "0x04 Mails:" << count0x04;
122 }
123 
124 void FilterOE::importMailBox(const QString &fileName)
125 {
126  QFile mailfile(fileName);
127  QFileInfo mailfileinfo(fileName);
128  QString _nameOfFile = fileName;
129  _nameOfFile.remove(mailDir());
130  _nameOfFile.remove(QLatin1Char('/'));
131  filterInfo()->setFrom(mailfileinfo.fileName());
132 
133  if (!mailfile.open(QIODevice::ReadOnly)) {
134  filterInfo()->addErrorLogEntry(i18n("Unable to open mailbox %1", fileName));
135  return;
136  }
137  QDataStream mailbox(&mailfile);
138  mailbox.setByteOrder(QDataStream::LittleEndian);
139 
140  // Parse magic
141  quint32 sig_block1, sig_block2;
142  mailbox >> sig_block1 >> sig_block2;
143  if (sig_block1 == OE4_SIG_1 && sig_block2 == OE4_SIG_2) {
144  folderName = QLatin1String("OE-Import/") + mailfileinfo.completeBaseName();
145  filterInfo()->addInfoLogEntry(i18n("Importing OE4 Mailbox %1", QStringLiteral("../") + _nameOfFile));
146  filterInfo()->setTo(folderName);
147  mbxImport(mailbox);
148  return;
149  } else {
150  quint32 sig_block3, sig_block4;
151  mailbox >> sig_block3 >> sig_block4;
152  if (sig_block1 == OE5_SIG_1 && sig_block3 == OE5_SIG_3 && sig_block4 == OE5_SIG_4) {
153  if (sig_block2 == OE5_EMAIL_SIG_2) {
154  folderName = QLatin1String("OE-Import/") + mailfileinfo.completeBaseName();
155  if (parsedFolder) {
156  const QString _tmpFolder = getFolderName(_nameOfFile);
157  if (!_tmpFolder.isEmpty()) {
158  folderName = QLatin1String("OE-Import/") + _tmpFolder;
159  }
160  }
161  filterInfo()->addInfoLogEntry(i18n("Importing OE5+ Mailbox %1", QStringLiteral("../") + _nameOfFile));
162  filterInfo()->setTo(folderName);
163  dbxImport(mailbox);
164  return;
165  } else if (sig_block2 == OE5_FOLDER_SIG_2) {
166  if (!parsedFolder) {
167  filterInfo()->addInfoLogEntry(i18n("Importing OE5+ Folder file %1", QStringLiteral("../") + _nameOfFile));
168  currentIsFolderFile = true;
169  dbxImport(mailbox);
170  currentIsFolderFile = false;
171  }
172  return;
173  }
174  }
175  }
176 }
177 
178 /* ------------------- MBX support ------------------- */
179 
180 void FilterOE::mbxImport(QDataStream &ds)
181 {
182  quint32 msgCount, lastMsgNum, fileSize;
183 
184  // Read the header
185  ds >> msgCount >> lastMsgNum >> fileSize;
186  ds.device()->seek(ds.device()->pos() + 64); // Skip 0's
187  qCDebug(MAILIMPORTER_LOG) << "This mailbox has" << msgCount << " messages";
188  if (msgCount == 0) {
189  return; // Don't import empty mailbox
190  }
191 
192  quint32 msgMagic;
193  ds >> msgMagic; // Read first magic
194 
195  while (!ds.atEnd()) {
196  quint32 msgNumber, msgSize, msgTextSize;
197  QTemporaryFile tmp;
198  tmp.open();
199  QDataStream dataStream(&tmp);
200  dataStream.setByteOrder(QDataStream::LittleEndian);
201 
202  // Read the messages
203  ds >> msgNumber >> msgSize >> msgTextSize; // All seem to be lies...?
204  do {
205  ds >> msgMagic;
206  if (msgMagic != MBX_MAILMAGIC) {
207  dataStream << msgMagic;
208  } else {
209  break;
210  }
211  } while (!ds.atEnd());
212  tmp.flush();
213  if (!importMessage(folderName, tmp.fileName(), filterInfo()->removeDupMessage())) {
214  filterInfo()->addErrorLogEntry(i18n("Could not import %1", tmp.fileName()));
215  }
216 
217  if (filterInfo()->shouldTerminate()) {
218  return;
219  }
220  }
221 }
222 
223 /* ------------------- DBX support ------------------- */
224 
225 void FilterOE::dbxImport(QDataStream &ds)
226 {
227  // Get item count &offset of index
228  quint32 itemCount, indexPtr;
229  ds.device()->seek(0xc4);
230  ds >> itemCount;
231  ds.device()->seek(0xe4);
232  ds >> indexPtr;
233  qCDebug(MAILIMPORTER_LOG) << "Item count is" << itemCount << ", Index at" << indexPtr;
234 
235  if (itemCount == 0) {
236  return; // Empty file
237  }
238  totalEmails = itemCount;
239  currentEmail = 0;
240  // Parse the indexes
241  ds.device()->seek(indexPtr);
242  dbxReadIndex(ds, indexPtr);
243 }
244 
245 void FilterOE::dbxReadIndex(QDataStream &ds, int filePos)
246 {
247  if (filterInfo()->shouldTerminate()) {
248  return;
249  }
250  quint32 self, unknown, nextIndexPtr, parent, indexCount;
251  quint8 unknown2, ptrCount;
252  quint16 unknown3;
253  int wasAt = ds.device()->pos();
254  ds.device()->seek(filePos);
255 
256  qCDebug(MAILIMPORTER_LOG) << "Reading index of file" << folderName;
257  ds >> self >> unknown >> nextIndexPtr >> parent >> unknown2 >> ptrCount >> unknown3 >> indexCount; // _dbx_tableindexstruct
258 
259  qCDebug(MAILIMPORTER_LOG) << "This index has" << (int)ptrCount << " data pointers";
260  for (int count = 0; count < ptrCount; ++count) {
261  if (filterInfo()->shouldTerminate()) {
262  return;
263  }
264  quint32 dataIndexPtr, anotherIndexPtr, anotherIndexCount; // _dbx_indexstruct
265  ds >> dataIndexPtr >> anotherIndexPtr >> anotherIndexCount;
266 
267  if (anotherIndexCount > 0) {
268  qCDebug(MAILIMPORTER_LOG) << "Recursing to another table @" << anotherIndexPtr;
269  dbxReadIndex(ds, anotherIndexPtr);
270  }
271  qCDebug(MAILIMPORTER_LOG) << "Data index @" << dataIndexPtr;
272  dbxReadDataBlock(ds, dataIndexPtr);
273  }
274 
275  if (indexCount > 0) { // deal with nextTablePtr
276  qCDebug(MAILIMPORTER_LOG) << "Recuring to next table @" << nextIndexPtr;
277  dbxReadIndex(ds, nextIndexPtr);
278  }
279 
280  ds.device()->seek(wasAt); // Restore file position to same as when function called
281 }
282 
283 void FilterOE::dbxReadDataBlock(QDataStream &ds, int filePos)
284 {
285  quint32 curOffset, blockSize;
286  quint16 unknown;
287  quint8 count, unknown2;
288  int wasAt = ds.device()->pos();
289 
290  QString folderEntry[4];
291 
292  ds.device()->seek(filePos);
293 
294  ds >> curOffset >> blockSize >> unknown >> count >> unknown2; // _dbx_email_headerstruct
295  qCDebug(MAILIMPORTER_LOG) << "Data block has" << (int)count << " elements";
296 
297  for (int c = 0; c < count; c++) {
298  if (filterInfo()->shouldTerminate()) {
299  return;
300  }
301  quint8 type; // _dbx_email_pointerstruct
302  quint32 value; // Actually 24 bit
303 
304  ds >> type >> value;
305  value &= 0xffffff;
306  ds.device()->seek(ds.device()->pos() - 1); // We only wanted 3 bytes
307 
308  if (!currentIsFolderFile) {
309  if (type == 0x84) { // It's an email!
310  qCDebug(MAILIMPORTER_LOG) << "**** Offset of emaildata (0x84)" << value << " ****";
311  dbxReadEmail(ds, value);
312  ++count0x84;
313  } else if (type == 0x04) {
314  int currentFilePos = ds.device()->pos();
315  ds.device()->seek(filePos + 12 + value + (count * 4));
316  quint32 newOFF;
317  ds >> newOFF;
318  qCDebug(MAILIMPORTER_LOG) << "**** Offset of emaildata (0x04)" << newOFF;
319  ds.device()->seek(currentFilePos);
320  dbxReadEmail(ds, newOFF);
321  ++count0x04;
322  }
323  } else {
324  // this is a folderfile
325  if (type == 0x02) {
326  // qCDebug(MAILIMPORTER_LOG) <<"**** FOLDER: descriptive name ****";
327  folderEntry[0] = parseFolderOEString(ds, filePos + 12 + value + (count * 4));
328  } else if (type == 0x03) {
329  // qCDebug(MAILIMPORTER_LOG) <<"**** FOLDER: filename ****";
330  folderEntry[1] = parseFolderOEString(ds, filePos + 12 + value + (count * 4));
331  } else if (type == 0x80) {
332  // qCDebug(MAILIMPORTER_LOG) <<"**** FOLDER: current ID ****";
333  folderEntry[2] = QString::number(value);
334  } else if (type == 0x81) {
335  // qCDebug(MAILIMPORTER_LOG) <<"**** FOLDER: parent ID ****";
336  folderEntry[3] = QString::number(value);
337  }
338  }
339  }
340  if (currentIsFolderFile) {
341  folderStructure.append(FolderStructure(folderEntry));
342  }
343  ds.device()->seek(wasAt); // Restore file position to same as when function called
344 }
345 
346 void FilterOE::dbxReadEmail(QDataStream &ds, int filePos)
347 {
348  if (filterInfo()->shouldTerminate()) {
349  return;
350  }
351  quint32 self, nextAddressOffset, nextAddress = 0;
352  quint16 blockSize;
353  quint8 intCount, unknown;
354  QTemporaryFile tmp;
355  tmp.open();
356  bool _break = false;
357  int wasAt = ds.device()->pos();
358  ds.device()->seek(filePos);
359  QDataStream tempDs(&tmp);
360 
361  do {
362  ds >> self >> nextAddressOffset >> blockSize >> intCount >> unknown >> nextAddress; // _dbx_block_hdrstruct
363  QByteArray blockBuffer(blockSize, '\0');
364  ds.readRawData(blockBuffer.data(), blockSize);
365  tempDs.writeRawData(blockBuffer.data(), blockSize);
366  // to detect incomplete mails or corrupted archives. See Bug #86119
367  if (ds.atEnd()) {
368  _break = true;
369  break;
370  }
371  ds.device()->seek(nextAddress);
372  } while (nextAddress != 0);
373  tmp.flush();
374 
375  if (!_break) {
376  if (!importMessage(folderName, tmp.fileName(), filterInfo()->removeDupMessage())) {
377  filterInfo()->addErrorLogEntry(i18n("Could not import %1", tmp.fileName()));
378  }
379 
380  currentEmail++;
381  int currentPercentage = (int)(((float)currentEmail / totalEmails) * 100);
382  filterInfo()->setCurrent(currentPercentage);
383  ds.device()->seek(wasAt);
384  }
385 }
386 
387 /* ------------------- FolderFile support ------------------- */
388 QString FilterOE::parseFolderOEString(QDataStream &ds, int filePos)
389 {
390  char tmp;
391  QString returnString;
392  int wasAt = ds.device()->pos();
393  ds.device()->seek(filePos);
394 
395  // read while != 0x00
396  while (!ds.device()->atEnd()) {
397  ds.device()->getChar(&tmp);
398  if (tmp != 0x00) {
399  returnString += QLatin1Char(tmp);
400  } else {
401  break;
402  }
403  }
404  ds.device()->seek(wasAt);
405  return returnString;
406 }
407 
409 QString FilterOE::getFolderName(const QString &filename)
410 {
411  bool found = false;
412  bool foundFilename = false;
413  QString folder;
414  // we must do this because folder with more than one upper letter
415  // at start have maybe not a file named like the folder !!!
416  QString search = filename.toLower();
417 
418  while (!found) {
419  for (FolderStructureIterator it = folderStructure.begin(); it != folderStructure.end(); ++it) {
420  FolderStructure tmp = *it;
421  if (foundFilename == false) {
422  QString _tmpFileName = tmp[1];
423  _tmpFileName = _tmpFileName.toLower();
424  if (_tmpFileName == search) {
425  folder.prepend(tmp[0] + QLatin1String("/"));
426  search = tmp[3];
427  foundFilename = true;
428  }
429  } else {
430  QString _currentID = tmp[2];
431  QString _parentID = tmp[3];
432  if (_currentID == search) {
433  if (_parentID.isEmpty()) { // this is the root of the folder
434  found = true;
435  break;
436  } else {
437  folder.prepend(tmp[0] + QLatin1String("/"));
438  search = tmp[3];
439  }
440  }
441  }
442  }
443  // need to break the while loop maybe in some cases
444  if ((foundFilename == false) && (folder.isEmpty())) {
445  return folder;
446  }
447  }
448  return folder;
449 }
QString getExistingDirectory(QWidget *parent, const QString &caption, const QString &dir, QFileDialog::Options options)
virtual bool atEnd() const const
virtual bool seek(qint64 pos)
bool flush()
QString & prepend(QChar ch)
QString filePath(const QString &fileName) const const
QList::iterator erase(QList::iterator pos)
void importMails(const QString &maildir)
Definition: filteroe.cpp:63
QString & remove(int position, int n)
The Filter class.
Definition: filters.h:39
QString homePath()
int readRawData(char *s, int len)
virtual qint64 pos() const const
bool getChar(char *c)
QString number(int n, int base)
int count(const T &value) const const
QString fileName() const const
bool isEmpty() const const
bool isEmpty() const const
typedef Iterator
virtual bool open(QIODevice::OpenMode mode) override
bool atEnd() const const
QList::iterator end()
QString toLower() const const
virtual QString fileName() const const override
QString i18n(const char *text, const TYPE &arg...)
void setByteOrder(QDataStream::ByteOrder bo)
QStringList entryList(QDir::Filters filters, QDir::SortFlags sort) const const
typedef ConstIterator
int writeRawData(const char *s, int len)
char * data()
QString completeBaseName() const const
QIODevice * device() const const
QList::const_iterator constEnd() const const
QList::const_iterator constBegin() const const
QList::iterator begin()
Glorified QString[N] for (a) understandability (b) older gcc compatibility.
This file is part of the KDE documentation.
Documentation copyright © 1996-2020 The KDE developers.
Generated on Mon Jun 1 2020 23:02:54 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.