Skip to content

Commit 97e2025

Browse files
rp-rp-
committed
cvsparser: Newly implemented CSV Parser
Moved parser into it's own class This parser now proper supports new lines in quoted text and returns a QVector<QStringList> result.
1 parent 1430033 commit 97e2025

File tree

9 files changed

+387
-180
lines changed

9 files changed

+387
-180
lines changed

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ endif()
2929
set(SQLB_HDR
3030
src/gen_version.h
3131
src/sqlitetypes.h
32+
src/csvparser.h
3233
src/grammar/sqlite3TokenTypes.hpp
3334
src/grammar/Sqlite3Lexer.hpp
3435
src/grammar/Sqlite3Parser.hpp
@@ -73,6 +74,7 @@ set(SQLB_SRC
7374
src/sqlitetablemodel.cpp
7475
src/sqlitetypes.cpp
7576
src/sqltextedit.cpp
77+
src/csvparser.cpp
7678
src/DbStructureModel.cpp
7779
src/grammar/Sqlite3Lexer.cpp
7880
src/grammar/Sqlite3Parser.cpp

‎src/ImportCsvDialog.cpp‎

Lines changed: 116 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "ImportCsvDialog.h"
22
#include "ui_ImportCsvDialog.h"
33
#include "sqlitedb.h"
4+
#include "csvparser.h"
45

56
#include <QMessageBox>
67
#include <QProgressDialog>
@@ -9,6 +10,9 @@
910
#include <QTextCodec>
1011
#include <QCompleter>
1112
#include <sqlite3.h>
13+
#include <QFile>
14+
#include <QTextStream>
15+
#include <memory>
1216

1317
ImportCsvDialog::ImportCsvDialog(const QString& filename, DBBrowserDB* db, QWidget* parent)
1418
: QDialog(parent),
@@ -45,27 +49,77 @@ void rollback(ImportCsvDialog* dialog, DBBrowserDB* pdb, QProgressDialog& progre
4549
}
4650
}
4751

52+
class CSVImportProgress : public CSVProgress
53+
{
54+
public:
55+
CSVImportProgress(size_t filesize)
56+
{
57+
m_pProgressDlg = new QProgressDialog(
58+
QObject::tr("Decoding CSV file..."),
59+
QObject::tr("Cancel"),
60+
0,
61+
filesize);
62+
m_pProgressDlg->setWindowModality(Qt::ApplicationModal);
63+
}
64+
65+
~CSVImportProgress()
66+
{
67+
delete m_pProgressDlg;
68+
}
69+
70+
void start()
71+
{
72+
m_pProgressDlg->show();
73+
}
74+
75+
bool update(size_t pos)
76+
{
77+
m_pProgressDlg->setValue(pos);
78+
qApp->processEvents();
79+
80+
return !m_pProgressDlg->wasCanceled();
81+
}
82+
83+
void end()
84+
{
85+
m_pProgressDlg->hide();
86+
}
87+
88+
private:
89+
QProgressDialog* m_pProgressDlg;
90+
};
91+
4892
void ImportCsvDialog::accept()
4993
{
5094
QString sql;
5195

5296
// Parse all csv data
53-
int numfields;
54-
QStringList curList = pdb->decodeCSV(csvFilename, currentSeparatorChar(), currentQuoteChar(), currentEncoding(), -1, &numfields);
97+
QFile file(csvFilename);
98+
file.open(QIODevice::ReadOnly | QIODevice::Text);
99+
100+
CSVParser csv(true, currentSeparatorChar(), currentQuoteChar());
101+
csv.setCSVProgress(new CSVImportProgress(file.size()));
102+
103+
QTextStream tstream(&file);
104+
tstream.setCodec(currentEncoding().toUtf8());
105+
csv.parse(tstream);
106+
file.close();
55107

56-
// Can not operate on an empty result
57-
if(numfields == 0)
108+
if(csv.csv().size() == 0)
58109
return;
59110

60111
// Generate field names. These are either taken from the first CSV row or are generated in the format of "fieldXY" depending on the user input
61112
sqlb::FieldVector fieldList;
113+
CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin();
62114
if(ui->checkboxHeader->isChecked())
63115
{
64-
int cfieldnum = 0;
65-
while(!curList.empty() && cfieldnum != numfields)
116+
++itBegin;
117+
for(QStringList::const_iterator it = csv.csv().at(0).begin();
118+
it != csv.csv().at(0).end();
119+
++it)
66120
{
67121
// Remove invalid characters
68-
QString thisfield = curList.front();
122+
QString thisfield = *it;
69123
thisfield.replace("`", "");
70124
thisfield.replace(" ", "");
71125
thisfield.replace('"', "");
@@ -75,32 +129,27 @@ void ImportCsvDialog::accept()
75129

76130
// Avoid empty field names
77131
if(thisfield.isEmpty())
78-
thisfield = QString("field%1").arg(cfieldnum+1);
132+
thisfield = QString("field%1").arg(std::distance(csv.csv().at(0).begin(), it) + 1);
79133

80134
fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(thisfield, "")));
81-
cfieldnum++;
82-
curList.pop_front();
83135
}
84136
} else {
85-
for(int i=0; i < numfields; ++i)
137+
for(int i=0; i < csv.columns(); ++i)
86138
fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(QString("field%1").arg(i+1), "")));
87139
}
88140

89141
// Show progress dialog
90-
QProgressDialog progress(tr("Inserting data..."), tr("Cancel"), 0, curList.size());
142+
QProgressDialog progress(tr("Inserting data..."), tr("Cancel"), 0, csv.csv().size());
91143
progress.setWindowModality(Qt::ApplicationModal);
92144

93-
// declare local variables we will need before the rollback jump
94-
int colNum = 0;
95-
96145
// Are we importing into an existing table?
97146
bool importToExistingTable = false;
98147
objectMap objects = pdb->getBrowsableObjects();
99148
for(objectMap::ConstIterator i=objects.begin();i!=objects.end();++i)
100149
{
101150
if(i.value().gettype() == "table" && i.value().getname() == ui->editName->text())
102151
{
103-
if(i.value().table.fields().size() != numfields)
152+
if(i.value().table.fields().size() != csv.columns())
104153
{
105154
QMessageBox::warning(this, QApplication::applicationName(),
106155
tr("There is already a table of that name and an import into an existing table is only possible if the number of columns match."));
@@ -131,28 +180,30 @@ void ImportCsvDialog::accept()
131180
}
132181

133182
// now lets import all data, one row at a time
134-
for(int i=0;i<curList.size();++i)
183+
for(CSVParser::TCSVResult::const_iterator it = itBegin;
184+
it != csv.csv().end();
185+
++it)
135186
{
136-
if(colNum == 0)
137-
sql = QString("INSERT INTO `%1` VALUES(").arg(ui->editName->text());
138-
139-
// need to mprintf here
140-
char* formSQL = sqlite3_mprintf("%Q", (const char*)curList[i].toUtf8());
141-
sql.append(formSQL);
142-
if(formSQL)
143-
sqlite3_free(formSQL);
187+
sql = QString("INSERT INTO `%1` VALUES(").arg(ui->editName->text());
144188

145-
colNum++;
146-
if(colNum < numfields)
189+
for(QStringList::const_iterator jt = it->begin(); jt != it->end(); ++jt)
147190
{
148-
sql.append(",");
149-
} else {
150-
colNum = 0;
151-
sql.append(");");
152-
if(!pdb->executeSQL(sql, false, false))
153-
return rollback(this, pdb, progress, restorepointName);
191+
// need to mprintf here
192+
char* formSQL = sqlite3_mprintf("%Q", (const char*)jt->toUtf8());
193+
sql.append(formSQL);
194+
if(formSQL)
195+
sqlite3_free(formSQL);
196+
197+
if(jt != (it->end() - 1))
198+
sql.append((','));
154199
}
155-
progress.setValue(i);
200+
201+
sql.append(");");
202+
203+
if(!pdb->executeSQL(sql, false, false))
204+
return rollback(this, pdb, progress, restorepointName);
205+
206+
progress.setValue(std::distance(csv.csv().begin(), it));
156207
if(progress.wasCanceled())
157208
return rollback(this, pdb, progress, restorepointName);
158209
}
@@ -169,42 +220,52 @@ void ImportCsvDialog::updatePreview()
169220
ui->editCustomEncoding->setVisible(ui->comboEncoding->currentIndex() == ui->comboEncoding->count()-1);
170221

171222
// Get preview data
172-
int numfields;
173-
int maxrecs = 20;
174-
QStringList curList = pdb->decodeCSV(csvFilename, currentSeparatorChar(), currentQuoteChar(), currentEncoding(), maxrecs, &numfields);
223+
QFile file(csvFilename);
224+
file.open(QIODevice::ReadOnly | QIODevice::Text);
225+
226+
CSVParser csv(true, currentSeparatorChar(), currentQuoteChar());
227+
228+
QTextStream tstream(&file);
229+
tstream.setCodec(currentEncoding().toUtf8());
230+
csv.parse(tstream, 20);
231+
file.close();
175232

176233
// Reset preview widget
177234
ui->tablePreview->clear();
178-
ui->tablePreview->setColumnCount(numfields);
235+
ui->tablePreview->setColumnCount(csv.columns());
179236

180237
// Exit if there are no lines to preview at all
181-
if(numfields == 0)
238+
if(csv.columns() == 0)
182239
return;
183240

184241
// Use first row as header if necessary
242+
CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin();
185243
if(ui->checkboxHeader->isChecked())
186244
{
187-
ui->tablePreview->setHorizontalHeaderLabels(curList);
188-
189-
// Remove this row to not show it in the data section
190-
for(int e=0;e < numfields; ++e)
191-
curList.pop_front();
245+
ui->tablePreview->setHorizontalHeaderLabels(*itBegin);
246+
++itBegin;
192247
}
193248

194249
// Fill data section
195-
ui->tablePreview->setRowCount(curList.count() / numfields);
196-
int rowNum = 0;
197-
int colNum = 0;
198-
for(QStringList::Iterator ct=curList.begin();ct!=curList.end();++ct)
250+
ui->tablePreview->setRowCount(std::distance(itBegin, csv.csv().end()));
251+
252+
for(CSVParser::TCSVResult::const_iterator ct = itBegin;
253+
ct != csv.csv().end();
254+
++ct)
199255
{
200-
if(colNum == 0)
201-
ui->tablePreview->setVerticalHeaderItem(rowNum, new QTableWidgetItem(QString::number(rowNum + 1)));
202-
ui->tablePreview->setItem(rowNum, colNum, new QTableWidgetItem(*ct));
203-
colNum++;
204-
if(colNum == numfields)
256+
for(QStringList::const_iterator it = ct->begin(); it != ct->end(); ++it)
205257
{
206-
colNum = 0;
207-
rowNum++;
258+
int rowNum = std::distance(itBegin, ct);
259+
if(it == ct->begin())
260+
{
261+
ui->tablePreview->setVerticalHeaderItem(
262+
rowNum,
263+
new QTableWidgetItem(QString::number(rowNum + 1)));
264+
}
265+
ui->tablePreview->setItem(
266+
rowNum,
267+
std::distance(ct->begin(), it),
268+
new QTableWidgetItem(*it));
208269
}
209270
}
210271
}

0 commit comments

Comments
 (0)