From 4ff74a66459e8682a212b66eead35a51bf324406 Mon Sep 17 00:00:00 2001 From: iKlsR Date: Sun, 21 May 2017 11:53:14 -0500 Subject: [PATCH 1/7] Extend FileDialog to accommodate multiple file selection --- src/FileDialog.cpp | 11 +++++++++++ src/FileDialog.h | 3 +++ 2 files changed, 14 insertions(+) diff --git a/src/FileDialog.cpp b/src/FileDialog.cpp index 3ec070dcfd..e7d4b1886e 100644 --- a/src/FileDialog.cpp +++ b/src/FileDialog.cpp @@ -9,6 +9,17 @@ QString FileDialog::getOpenFileName(QWidget* parent, const QString& caption, con return result; } +QStringList FileDialog::getOpenFileNames(QWidget *parent, const QString &caption, const QString &filter, QString *selectedFilter, QFileDialog::Options options) +{ + QStringList result = QFileDialog::getOpenFileNames(parent, caption, getFileDialogPath(), filter, selectedFilter, options); + if(!result.isEmpty()) + { + QFileInfo path = QFileInfo(result.first()); + setFileDialogPath(path.absolutePath()); + } + return result; +} + QString FileDialog::getSaveFileName(QWidget* parent, const QString& caption, const QString& filter, const QString& defaultFileName, QString* selectedFilter, Options options) { QString dir = getFileDialogPath(); diff --git a/src/FileDialog.h b/src/FileDialog.h index a6fbbd4ef0..b5ca282987 100644 --- a/src/FileDialog.h +++ b/src/FileDialog.h @@ -11,6 +11,9 @@ class FileDialog : public QFileDialog static QString getOpenFileName(QWidget* parent = 0, const QString& caption = QString(), const QString& filter = QString(), QString* selectedFilter = 0, Options options = 0); + static QStringList getOpenFileNames(QWidget* parent = 0, const QString& caption = QString(), + const QString& filter = QString(), QString* selectedFilter = 0, + Options options = 0); static QString getSaveFileName(QWidget* parent = 0, const QString& caption = QString(), const QString& filter = QString(), const QString& defaultFileName = QString(), QString* selectedFilter = 0, Options options = 0); From c44ab26d87753ba9365e8b7e8811651bd5cf500a Mon Sep 17 00:00:00 2001 From: iKlsR Date: Tue, 23 May 2017 16:06:09 -0500 Subject: [PATCH 2/7] Allow for interactive CSV file import - Tweak input checker - Preserve old file import as not to cause any unforeseen breaks - Allow ignoring file name when importing multiple files to tables - Mass toggle several files for import --- src/ImportCsvDialog.cpp | 243 +++++++++++++++++++++++++++------------- src/ImportCsvDialog.h | 11 +- src/ImportCsvDialog.ui | 237 ++++++++++++++++++++++++++++----------- src/MainWindow.cpp | 22 ++-- 4 files changed, 355 insertions(+), 158 deletions(-) diff --git a/src/ImportCsvDialog.cpp b/src/ImportCsvDialog.cpp index bac0e82537..e974c13df9 100644 --- a/src/ImportCsvDialog.cpp +++ b/src/ImportCsvDialog.cpp @@ -18,16 +18,17 @@ #include #include -ImportCsvDialog::ImportCsvDialog(const QString& filename, DBBrowserDB* db, QWidget* parent) +ImportCsvDialog::ImportCsvDialog(const QStringList &filenames, DBBrowserDB* db, QWidget* parent) : QDialog(parent), ui(new Ui::ImportCsvDialog), - csvFilename(filename), + csvFilenames(filenames), pdb(db) { ui->setupUi(this); // Get the actual file name out of the provided path and use it as the default table name for import - QFileInfo file(filename); + // For importing several files at once, the fields have to be the same so we can safely use the first + QFileInfo file(filenames.first()); ui->editName->setText(file.baseName()); // Create a list of all available encodings and create an auto completion list from them @@ -46,9 +47,24 @@ ImportCsvDialog::ImportCsvDialog(const QString& filename, DBBrowserDB* db, QWidg setQuoteChar(QChar(settings.value("importcsv/quotecharacter", '"').toInt())); setEncoding(settings.value("importcsv/encoding", "UTF-8").toString()); - // Initialise user interface - checkInput(); + // Prepare and show interface depending on how many files are selected + if (csvFilenames.length() > 1) + { + ui->separateTables->setVisible(true); + ui->checkBoxSeparateTables->setVisible(true); + ui->filePickerBlock->setVisible(true); + selectFiles(); + } + else if (csvFilenames.length() == 1) + { + ui->separateTables->setVisible(false); + ui->checkBoxSeparateTables->setVisible(false); + ui->filePickerBlock->setVisible(false); + } + + selectedFile = csvFilenames.first(); updatePreview(); + checkInput(); } ImportCsvDialog::~ImportCsvDialog() @@ -131,8 +147,140 @@ void ImportCsvDialog::accept() settings.setValue("encoding", currentEncoding()); settings.endGroup(); + // Get all the selected files and start the import + if (ui->filePickerBlock->isVisible()) + { + QStringList selectedFiles; + for (int i = 0; i < ui->filePicker->count(); i++) { + auto item = ui->filePicker->item(i); + if (item->checkState() == Qt::Checked) + selectedFiles.append(item->data(Qt::DisplayRole).toString()); + } + + for (auto file : selectedFiles) + importCsv(file); + } + else + { + importCsv(csvFilenames.first()); + } + + QApplication::restoreOverrideCursor(); // restore original cursor + QDialog::accept(); +} + +void ImportCsvDialog::updatePreview() +{ + // Show/hide custom quote/separator input fields + ui->editCustomQuote->setVisible(ui->comboQuote->currentIndex() == ui->comboQuote->count()-1); + ui->editCustomSeparator->setVisible(ui->comboSeparator->currentIndex() == ui->comboSeparator->count()-1); + ui->editCustomEncoding->setVisible(ui->comboEncoding->currentIndex() == ui->comboEncoding->count()-1); + + // Get preview data + QFile file(selectedFile); + file.open(QIODevice::ReadOnly); + + CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar()); + + QTextStream tstream(&file); + tstream.setCodec(currentEncoding().toUtf8()); + csv.parse(tstream, 20); + file.close(); + + // Reset preview widget + ui->tablePreview->clear(); + ui->tablePreview->setColumnCount(csv.columns()); + + // Exit if there are no lines to preview at all + if(csv.columns() == 0) + return; + + // Use first row as header if necessary + CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); + if(ui->checkboxHeader->isChecked()) + { + ui->tablePreview->setHorizontalHeaderLabels(*itBegin); + ++itBegin; + } + + // Fill data section + ui->tablePreview->setRowCount(std::distance(itBegin, csv.csv().end())); + + for(CSVParser::TCSVResult::const_iterator ct = itBegin; + ct != csv.csv().end(); + ++ct) + { + for(QStringList::const_iterator it = ct->begin(); it != ct->end(); ++it) + { + int rowNum = std::distance(itBegin, ct); + if(it == ct->begin()) + { + ui->tablePreview->setVerticalHeaderItem( + rowNum, + new QTableWidgetItem(QString::number(rowNum + 1))); + } + ui->tablePreview->setItem( + rowNum, + std::distance(ct->begin(), it), + new QTableWidgetItem(*it)); + } + } +} + +void ImportCsvDialog::checkInput() +{ + bool checkedItem = false; + for (int i = 0; i < ui->filePicker->count(); i++) { + if (ui->filePicker->item(i)->checkState() == Qt::Checked) checkedItem = true; + } + + bool allowImporting = !ui->editName->text().isEmpty() && checkedItem; + + ui->editName->setEnabled(!ui->checkBoxSeparateTables->isChecked()); + ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(allowImporting); +} + +void ImportCsvDialog::selectFiles() +{ + for (auto filename : csvFilenames) { + auto item = new QListWidgetItem(); + item->setText(filename); + item->setCheckState(Qt::Checked); + ui->filePicker->addItem(item); + } + + connect(ui->filePicker, SIGNAL(itemClicked(QListWidgetItem*)), SLOT(updateSelectedFilePreview(QListWidgetItem*))); +} + +void ImportCsvDialog::updateSelectedFilePreview(QListWidgetItem* item) +{ + selectedFile = item->data(Qt::DisplayRole).toString(); + QFileInfo fileInfo(selectedFile); + ui->editName->setText(fileInfo.baseName()); + updatePreview(); + checkInput(); +} + +void ImportCsvDialog::updateSelection(bool selected) +{ + for (int i = 0; i < ui->filePicker->count(); i++) + ui->filePicker->item(i)->setCheckState(selected ? Qt::Checked : Qt::Unchecked); + ui->toggleSelected->setText(selected ? tr("Deselect All") : tr("Select All")); + checkInput(); +} + +void ImportCsvDialog::importCsv(const QString& fileName) +{ + QString tableName; + if (ui->checkBoxSeparateTables->isChecked()) { + QFileInfo fileInfo(fileName); + tableName = fileInfo.baseName(); + } else { + tableName = ui->editName->text(); + } + // Parse all csv data - QFile file(csvFilename); + QFile file(fileName); file.open(QIODevice::ReadOnly); CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar()); @@ -186,7 +334,7 @@ void ImportCsvDialog::accept() objectMap objects = pdb->getBrowsableObjects(); for(auto it=objects.constBegin();it!=objects.constEnd();++it) { - if((*it)->type() == sqlb::Object::Types::Table && (*it)->name() == ui->editName->text()) + if((*it)->type() == sqlb::Object::Types::Table && (*it)->name() == tableName) { if((size_t)(*it).dynamicCast()->fields().size() != csv.columns()) { @@ -194,6 +342,13 @@ void ImportCsvDialog::accept() tr("There is already a table of that name and an import into an existing table is only possible if the number of columns match.")); return; } else { + // If we are importing multiple files, we can skip the warning and perform the inserts + // To omit further warnings there is a filter button to select all conforming sql + if (!ui->checkBoxSeparateTables->isChecked()) { + importToExistingTable = true; + break; + } + if(QMessageBox::question(this, QApplication::applicationName(), tr("There is already a table of that name. Do you want to import the data into it?"), QMessageBox::Yes, QMessageBox::No) == QMessageBox::Yes) { importToExistingTable = true; @@ -214,7 +369,7 @@ void ImportCsvDialog::accept() // Create table if(!importToExistingTable) { - if(!pdb->createTable(ui->editName->text(), fieldList)) + if(!pdb->createTable(tableName, fieldList)) return rollback(this, pdb, progress, restorepointName, 0, tr("Creating the table failed: %1").arg(pdb->lastError())); } @@ -223,7 +378,7 @@ void ImportCsvDialog::accept() it != csv.csv().end(); ++it) { - QString sql = QString("INSERT INTO %1 VALUES(").arg(sqlb::escapeIdentifier(ui->editName->text())); + QString sql = QString("INSERT INTO %1 VALUES(").arg(sqlb::escapeIdentifier(tableName)); QStringList insertlist; for(QStringList::const_iterator jt = it->begin(); jt != it->end(); ++jt) @@ -255,76 +410,6 @@ void ImportCsvDialog::accept() if(progress.wasCanceled()) return rollback(this, pdb, progress, restorepointName, std::distance(itBegin, it) + 1, ""); } - - QApplication::restoreOverrideCursor(); // restore original cursor - QDialog::accept(); -} - -void ImportCsvDialog::updatePreview() -{ - // Show/hide custom quote/separator input fields - ui->editCustomQuote->setVisible(ui->comboQuote->currentIndex() == ui->comboQuote->count()-1); - ui->editCustomSeparator->setVisible(ui->comboSeparator->currentIndex() == ui->comboSeparator->count()-1); - ui->editCustomEncoding->setVisible(ui->comboEncoding->currentIndex() == ui->comboEncoding->count()-1); - - // Get preview data - QFile file(csvFilename); - file.open(QIODevice::ReadOnly); - - CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar()); - - QTextStream tstream(&file); - tstream.setCodec(currentEncoding().toUtf8()); - csv.parse(tstream, 20); - file.close(); - - // Reset preview widget - ui->tablePreview->clear(); - ui->tablePreview->setColumnCount(csv.columns()); - - // Exit if there are no lines to preview at all - if(csv.columns() == 0) - return; - - // Use first row as header if necessary - CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); - if(ui->checkboxHeader->isChecked()) - { - ui->tablePreview->setHorizontalHeaderLabels(*itBegin); - ++itBegin; - } - - // Fill data section - ui->tablePreview->setRowCount(std::distance(itBegin, csv.csv().end())); - - for(CSVParser::TCSVResult::const_iterator ct = itBegin; - ct != csv.csv().end(); - ++ct) - { - for(QStringList::const_iterator it = ct->begin(); it != ct->end(); ++it) - { - int rowNum = std::distance(itBegin, ct); - if(it == ct->begin()) - { - ui->tablePreview->setVerticalHeaderItem( - rowNum, - new QTableWidgetItem(QString::number(rowNum + 1))); - } - ui->tablePreview->setItem( - rowNum, - std::distance(ct->begin(), it), - new QTableWidgetItem(*it)); - } - } -} - -void ImportCsvDialog::checkInput() -{ - bool valid = true; - if(ui->editName->text().isEmpty()) - valid = false; - - ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(valid); } void ImportCsvDialog::setQuoteChar(const QChar& c) diff --git a/src/ImportCsvDialog.h b/src/ImportCsvDialog.h index 6797d10a70..481e67153d 100644 --- a/src/ImportCsvDialog.h +++ b/src/ImportCsvDialog.h @@ -5,6 +5,7 @@ class DBBrowserDB; class QCompleter; +class QListWidgetItem; namespace Ui { class ImportCsvDialog; @@ -15,20 +16,26 @@ class ImportCsvDialog : public QDialog Q_OBJECT public: - explicit ImportCsvDialog(const QString& filename, DBBrowserDB* db, QWidget* parent = 0); + explicit ImportCsvDialog(const QStringList& filenames, DBBrowserDB* db, QWidget* parent = 0); ~ImportCsvDialog(); private slots: virtual void accept(); void updatePreview(); void checkInput(); + void selectFiles(); + void updateSelectedFilePreview(QListWidgetItem*); + void updateSelection(bool); private: Ui::ImportCsvDialog* ui; - QString csvFilename; + QStringList csvFilenames; + QString selectedFile; DBBrowserDB* pdb; QCompleter* encodingCompleter; + void importCsv(const QString& f); + void setQuoteChar(const QChar& c); char currentQuoteChar() const; diff --git a/src/ImportCsvDialog.ui b/src/ImportCsvDialog.ui index 7cd61be29d..3aea331e8d 100644 --- a/src/ImportCsvDialog.ui +++ b/src/ImportCsvDialog.ui @@ -13,7 +13,7 @@ Import CSV file - + @@ -241,8 +241,76 @@ + + + + + + + Separate tables + + + + + + + + + + + + + + true + + + + + + + + + 7 + + + QLayout::SetDefaultConstraint + + + 0 + + + + + Deselect All + + + true + + + true + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + @@ -273,174 +341,206 @@ - buttonBox - accepted() + comboSeparator + currentIndexChanged(int) ImportCsvDialog - accept() + updatePreview() - 261 - 480 + 232 + 99 - 157 - 274 + 445 + 64 - buttonBox - rejected() + editName + textChanged(QString) ImportCsvDialog - reject() + checkInput() - 329 - 480 + 609 + 13 - 286 - 274 + 631 + 45 - checkboxHeader - toggled(bool) + editCustomSeparator + textChanged(QString) ImportCsvDialog updatePreview() - 187 - 46 + 478 + 99 - 354 - 45 + 577 + 76 - comboSeparator - currentIndexChanged(int) + editCustomQuote + textChanged(QString) ImportCsvDialog updatePreview() - 216 - 71 + 478 + 132 - 445 - 64 + 530 + 90 - comboQuote - currentIndexChanged(int) + editCustomEncoding + textChanged(QString) ImportCsvDialog updatePreview() - 224 - 100 + 495 + 165 - 350 - 88 + 540 + 133 - editName - textChanged(QString) + checkBoxTrimFields + toggled(bool) ImportCsvDialog - checkInput() + updatePreview() - 609 - 13 + 184 + 191 - 631 - 45 + 368 + 244 - editCustomSeparator - textChanged(QString) + comboEncoding + currentIndexChanged(int) ImportCsvDialog updatePreview() - 325 - 70 + 266 + 165 - 577 - 76 + 572 + 121 - editCustomQuote - textChanged(QString) + checkboxHeader + toggled(bool) ImportCsvDialog updatePreview() - 467 - 101 + 184 + 60 - 530 - 90 + 354 + 45 - comboEncoding - currentIndexChanged(int) + toggleSelected + toggled(bool) ImportCsvDialog - updatePreview() + updateSelection(bool) - 198 - 131 + 674 + 258 - 572 - 121 + 368 + 244 - editCustomEncoding - textChanged(QString) + comboQuote + currentIndexChanged(int) ImportCsvDialog updatePreview() - 378 + 232 132 - 540 - 133 + 350 + 88 - checkBoxTrimFields + buttonBox + accepted() + ImportCsvDialog + accept() + + + 272 + 478 + + + 157 + 274 + + + + + buttonBox + rejected() + ImportCsvDialog + reject() + + + 340 + 478 + + + 286 + 274 + + + + + checkBoxSeparateTables toggled(bool) ImportCsvDialog - updatePreview() + checkInput() - 191 - 189 + 176 + 216 368 @@ -452,5 +552,6 @@ updatePreview() checkInput() + updateSelection(bool) diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index 83f21bd879..286a059300 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -1094,16 +1094,20 @@ void MainWindow::mainTabSelected(int tabindex) void MainWindow::importTableFromCSV() { - QString wFile = FileDialog::getOpenFileName( - this, - tr("Choose a text file"), - tr("Text files(*.csv *.txt);;All files(*)")); + QStringList wFiles = FileDialog::getOpenFileNames( + this, + tr("Choose text files"), + tr("Text files(*.csv *.txt);;All files(*)")); + + QStringList validFiles; + foreach(auto file, wFiles) { + if (QFile::exists(file)) + validFiles.append(file); + } - if (QFile::exists(wFile) ) - { - ImportCsvDialog dialog(wFile, &db, this); - if(dialog.exec()) - { + if (!validFiles.isEmpty()) { + ImportCsvDialog dialog(validFiles, &db, this); + if (dialog.exec()) { populateTable(); QMessageBox::information(this, QApplication::applicationName(), tr("Import completed")); } From 9c354a54226177b71c928df5a1266a461f65fb18 Mon Sep 17 00:00:00 2001 From: iKlsR Date: Tue, 23 May 2017 16:54:59 -0500 Subject: [PATCH 3/7] Allow matching similar csv files for batch import --- src/ImportCsvDialog.cpp | 96 ++++++++++++++++++++++++++++++++++++++--- src/ImportCsvDialog.h | 3 ++ src/ImportCsvDialog.ui | 39 +++++++++++++++++ 3 files changed, 133 insertions(+), 5 deletions(-) diff --git a/src/ImportCsvDialog.cpp b/src/ImportCsvDialog.cpp index e974c13df9..4426f06455 100644 --- a/src/ImportCsvDialog.cpp +++ b/src/ImportCsvDialog.cpp @@ -229,13 +229,18 @@ void ImportCsvDialog::updatePreview() void ImportCsvDialog::checkInput() { - bool checkedItem = false; - for (int i = 0; i < ui->filePicker->count(); i++) { - if (ui->filePicker->item(i)->checkState() == Qt::Checked) checkedItem = true; + bool allowImporting = false; + if (ui->filePickerBlock->isVisible()) { + bool checkedItem = false; + for (int i = 0; i < ui->filePicker->count(); i++) { + if (ui->filePicker->item(i)->checkState() == Qt::Checked) checkedItem = true; + } + allowImporting = !ui->editName->text().isEmpty() && checkedItem; + } else { + allowImporting = !ui->editName->text().isEmpty(); } - bool allowImporting = !ui->editName->text().isEmpty() && checkedItem; - + ui->matchSimilar->setEnabled(ui->filePicker->currentItem() != nullptr); ui->editName->setEnabled(!ui->checkBoxSeparateTables->isChecked()); ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(allowImporting); } @@ -269,6 +274,38 @@ void ImportCsvDialog::updateSelection(bool selected) checkInput(); } +void ImportCsvDialog::matchSimilar() +{ + auto item = ui->filePicker->currentItem(); + auto selectedHeader = fetchCsvHeader(item->data(Qt::DisplayRole).toString()); + + for (int i = 0; i < ui->filePicker->count(); i++) + { + auto item = ui->filePicker->item(i); + auto headers = fetchCsvHeader(item->data(Qt::DisplayRole).toString()); + bool matchingHeader = false; + + if (selectedHeader.count() == headers.count()) + { + matchingHeader = std::equal(selectedHeader.begin(), selectedHeader.end(), headers.begin(), + [](const sqlb::FieldPtr& item1, const sqlb::FieldPtr& item2) -> bool { + return (item1->name() == item2->name()); + }); + if (matchingHeader) { + item->setCheckState(Qt::Checked); + item->setBackgroundColor(Qt::green); + } + } + else + { + item->setCheckState(Qt::Unchecked); + item->setBackground(Qt::white); + } + } + + checkInput(); +} + void ImportCsvDialog::importCsv(const QString& fileName) { QString tableName; @@ -412,6 +449,55 @@ void ImportCsvDialog::importCsv(const QString& fileName) } } +sqlb::FieldVector ImportCsvDialog::fetchCsvHeader(const QString& fileName) +{ + // Parse all csv data + QFile file(fileName); + file.open(QIODevice::ReadOnly); + + CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar()); + csv.setCSVProgress(new CSVImportProgress(file.size())); + + QTextStream tstream(&file); + tstream.setCodec(currentEncoding().toUtf8()); + csv.parse(tstream); + file.close(); + + if (csv.csv().size() == 0) return sqlb::FieldVector(); + + // Generate field names. These are either taken from the first CSV row or are generated in the format of "fieldXY" depending on the user input + sqlb::FieldVector fieldList; + CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); + if(ui->checkboxHeader->isChecked()) + { + ++itBegin; + for(QStringList::const_iterator it = csv.csv().at(0).begin(); + it != csv.csv().at(0).end(); + ++it) + { + // Remove invalid characters + QString thisfield = *it; + thisfield.replace("`", ""); + thisfield.replace(" ", ""); + thisfield.replace('"', ""); + thisfield.replace("'",""); + thisfield.replace(",",""); + thisfield.replace(";",""); + + // Avoid empty field names + if(thisfield.isEmpty()) + thisfield = QString("field%1").arg(std::distance(csv.csv().at(0).begin(), it) + 1); + + fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(thisfield, ""))); + } + } else { + for(size_t i=0; i < csv.columns(); ++i) + fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(QString("field%1").arg(i+1), ""))); + } + + return fieldList; +} + void ImportCsvDialog::setQuoteChar(const QChar& c) { QComboBox* combo = ui->comboQuote; diff --git a/src/ImportCsvDialog.h b/src/ImportCsvDialog.h index 481e67153d..d4f8af7e3d 100644 --- a/src/ImportCsvDialog.h +++ b/src/ImportCsvDialog.h @@ -1,6 +1,7 @@ #ifndef IMPORTCSVDIALOG_H #define IMPORTCSVDIALOG_H +#include "sqlitetypes.h" #include class DBBrowserDB; @@ -26,6 +27,7 @@ private slots: void selectFiles(); void updateSelectedFilePreview(QListWidgetItem*); void updateSelection(bool); + void matchSimilar(); private: Ui::ImportCsvDialog* ui; @@ -35,6 +37,7 @@ private slots: QCompleter* encodingCompleter; void importCsv(const QString& f); + sqlb::FieldVector fetchCsvHeader(const QString& f); void setQuoteChar(const QChar& c); char currentQuoteChar() const; diff --git a/src/ImportCsvDialog.ui b/src/ImportCsvDialog.ui index 3aea331e8d..84bc1ee9ff 100644 --- a/src/ImportCsvDialog.ui +++ b/src/ImportCsvDialog.ui @@ -266,6 +266,18 @@ true + + 0 + + + 0 + + + 0 + + + 0 + @@ -293,6 +305,16 @@ + + + + false + + + Match Similar + + + @@ -548,10 +570,27 @@ + + matchSimilar + pressed() + ImportCsvDialog + matchSimilar() + + + 682 + 279 + + + 368 + 244 + + + updatePreview() checkInput() updateSelection(bool) + matchSimilar() From da31620dfa74aa0b865f39dcbaa2e82539ebf7a9 Mon Sep 17 00:00:00 2001 From: iKlsR Date: Tue, 23 May 2017 16:58:47 -0500 Subject: [PATCH 4/7] Append Separate Tables state to import settings --- src/ImportCsvDialog.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ImportCsvDialog.cpp b/src/ImportCsvDialog.cpp index 4426f06455..201fc175bb 100644 --- a/src/ImportCsvDialog.cpp +++ b/src/ImportCsvDialog.cpp @@ -43,6 +43,7 @@ ImportCsvDialog::ImportCsvDialog(const QStringList &filenames, DBBrowserDB* db, QSettings settings(QApplication::organizationName(), QApplication::organizationName()); ui->checkboxHeader->setChecked(settings.value("importcsv/firstrowheader", false).toBool()); ui->checkBoxTrimFields->setChecked(settings.value("importcsv/trimfields", true).toBool()); + ui->checkBoxSeparateTables->setChecked(settings.value("importcsv/separatetables", false).toBool()); setSeparatorChar(QChar(settings.value("importcsv/separator", ',').toInt())); setQuoteChar(QChar(settings.value("importcsv/quotecharacter", '"').toInt())); setEncoding(settings.value("importcsv/encoding", "UTF-8").toString()); @@ -144,6 +145,7 @@ void ImportCsvDialog::accept() settings.setValue("separator", currentSeparatorChar()); settings.setValue("quotecharacter", currentQuoteChar()); settings.setValue("trimfields", ui->checkBoxTrimFields->isChecked()); + settings.setValue("separatetables", ui->checkBoxSeparateTables->isChecked()); settings.setValue("encoding", currentEncoding()); settings.endGroup(); From 7597153199be800c58b44567437fc50967640f82 Mon Sep 17 00:00:00 2001 From: iKlsR Date: Wed, 14 Jun 2017 19:22:07 -0500 Subject: [PATCH 5/7] Edit widget tab order --- src/ImportCsvDialog.ui | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ImportCsvDialog.ui b/src/ImportCsvDialog.ui index 84bc1ee9ff..60e464188a 100644 --- a/src/ImportCsvDialog.ui +++ b/src/ImportCsvDialog.ui @@ -358,6 +358,10 @@ comboEncoding editCustomEncoding checkBoxTrimFields + checkBoxSeparateTables + filePicker + toggleSelected + matchSimilar tablePreview From 45e133c492a89b99f3662f1a776920a2e4f2aefa Mon Sep 17 00:00:00 2001 From: iKlsR Date: Thu, 15 Jun 2017 00:18:40 -0500 Subject: [PATCH 6/7] Cleanup logic, code reuse, faster header picking --- src/ImportCsvDialog.cpp | 109 ++++++++++++++-------------------------- src/ImportCsvDialog.h | 5 +- 2 files changed, 42 insertions(+), 72 deletions(-) diff --git a/src/ImportCsvDialog.cpp b/src/ImportCsvDialog.cpp index 201fc175bb..be6bbf226b 100644 --- a/src/ImportCsvDialog.cpp +++ b/src/ImportCsvDialog.cpp @@ -279,17 +279,17 @@ void ImportCsvDialog::updateSelection(bool selected) void ImportCsvDialog::matchSimilar() { auto item = ui->filePicker->currentItem(); - auto selectedHeader = fetchCsvHeader(item->data(Qt::DisplayRole).toString()); + auto selectedHeader = generateFieldList(parseCSV(item->data(Qt::DisplayRole).toString(), 1)); for (int i = 0; i < ui->filePicker->count(); i++) { auto item = ui->filePicker->item(i); - auto headers = fetchCsvHeader(item->data(Qt::DisplayRole).toString()); + auto header = generateFieldList(parseCSV(item->data(Qt::DisplayRole).toString(), 1)); bool matchingHeader = false; - if (selectedHeader.count() == headers.count()) + if (selectedHeader.count() == header.count()) { - matchingHeader = std::equal(selectedHeader.begin(), selectedHeader.end(), headers.begin(), + matchingHeader = std::equal(selectedHeader.begin(), selectedHeader.end(), header.begin(), [](const sqlb::FieldPtr& item1, const sqlb::FieldPtr& item2) -> bool { return (item1->name() == item2->name()); }); @@ -308,39 +308,36 @@ void ImportCsvDialog::matchSimilar() checkInput(); } -void ImportCsvDialog::importCsv(const QString& fileName) +CSVParser ImportCsvDialog::parseCSV(const QString &fileName, qint64 count) { - QString tableName; - if (ui->checkBoxSeparateTables->isChecked()) { - QFileInfo fileInfo(fileName); - tableName = fileInfo.baseName(); - } else { - tableName = ui->editName->text(); - } - // Parse all csv data QFile file(fileName); file.open(QIODevice::ReadOnly); CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar()); - csv.setCSVProgress(new CSVImportProgress(file.size())); + // If count is one, we only want the header, no need to see progress + if (count != 1) csv.setCSVProgress(new CSVImportProgress(file.size())); QTextStream tstream(&file); tstream.setCodec(currentEncoding().toUtf8()); - csv.parse(tstream); + csv.parse(tstream, count); file.close(); - if(csv.csv().size() == 0) - return; + return csv; +} + +sqlb::FieldVector ImportCsvDialog::generateFieldList(const CSVParser &parser) +{ + if (parser.csv().size() == 0) return sqlb::FieldVector(); // Generate field names. These are either taken from the first CSV row or are generated in the format of "fieldXY" depending on the user input sqlb::FieldVector fieldList; - CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); + CSVParser::TCSVResult::const_iterator itBegin = parser.csv().begin(); if(ui->checkboxHeader->isChecked()) { ++itBegin; - for(QStringList::const_iterator it = csv.csv().at(0).begin(); - it != csv.csv().at(0).end(); + for(QStringList::const_iterator it = parser.csv().at(0).begin(); + it != parser.csv().at(0).end(); ++it) { // Remove invalid characters @@ -354,15 +351,33 @@ void ImportCsvDialog::importCsv(const QString& fileName) // Avoid empty field names if(thisfield.isEmpty()) - thisfield = QString("field%1").arg(std::distance(csv.csv().at(0).begin(), it) + 1); + thisfield = QString("field%1").arg(std::distance(parser.csv().at(0).begin(), it) + 1); fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(thisfield, ""))); } } else { - for(size_t i=0; i < csv.columns(); ++i) + for(size_t i=0; i < parser.columns(); ++i) fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(QString("field%1").arg(i+1), ""))); } + return fieldList; +} + +void ImportCsvDialog::importCsv(const QString& fileName) +{ + QString tableName; + if (ui->checkBoxSeparateTables->isChecked()) { + QFileInfo fileInfo(fileName); + tableName = fileInfo.baseName(); + } else { + tableName = ui->editName->text(); + } + + CSVParser csv = parseCSV(fileName); + if (csv.csv().size() == 0) return; + + sqlb::FieldVector fieldList = generateFieldList(csv); + // Show progress dialog QProgressDialog progress(tr("Inserting data..."), tr("Cancel"), 0, csv.csv().size()); progress.setWindowModality(Qt::ApplicationModal); @@ -413,6 +428,7 @@ void ImportCsvDialog::importCsv(const QString& fileName) } // now lets import all data, one row at a time + CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); for(CSVParser::TCSVResult::const_iterator it = itBegin; it != csv.csv().end(); ++it) @@ -451,55 +467,6 @@ void ImportCsvDialog::importCsv(const QString& fileName) } } -sqlb::FieldVector ImportCsvDialog::fetchCsvHeader(const QString& fileName) -{ - // Parse all csv data - QFile file(fileName); - file.open(QIODevice::ReadOnly); - - CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar()); - csv.setCSVProgress(new CSVImportProgress(file.size())); - - QTextStream tstream(&file); - tstream.setCodec(currentEncoding().toUtf8()); - csv.parse(tstream); - file.close(); - - if (csv.csv().size() == 0) return sqlb::FieldVector(); - - // Generate field names. These are either taken from the first CSV row or are generated in the format of "fieldXY" depending on the user input - sqlb::FieldVector fieldList; - CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); - if(ui->checkboxHeader->isChecked()) - { - ++itBegin; - for(QStringList::const_iterator it = csv.csv().at(0).begin(); - it != csv.csv().at(0).end(); - ++it) - { - // Remove invalid characters - QString thisfield = *it; - thisfield.replace("`", ""); - thisfield.replace(" ", ""); - thisfield.replace('"', ""); - thisfield.replace("'",""); - thisfield.replace(",",""); - thisfield.replace(";",""); - - // Avoid empty field names - if(thisfield.isEmpty()) - thisfield = QString("field%1").arg(std::distance(csv.csv().at(0).begin(), it) + 1); - - fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(thisfield, ""))); - } - } else { - for(size_t i=0; i < csv.columns(); ++i) - fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(QString("field%1").arg(i+1), ""))); - } - - return fieldList; -} - void ImportCsvDialog::setQuoteChar(const QChar& c) { QComboBox* combo = ui->comboQuote; diff --git a/src/ImportCsvDialog.h b/src/ImportCsvDialog.h index d4f8af7e3d..625ce5c272 100644 --- a/src/ImportCsvDialog.h +++ b/src/ImportCsvDialog.h @@ -6,6 +6,7 @@ class DBBrowserDB; class QCompleter; +class CSVParser; class QListWidgetItem; namespace Ui { @@ -36,8 +37,10 @@ private slots: DBBrowserDB* pdb; QCompleter* encodingCompleter; + CSVParser parseCSV(const QString &f, qint64 count = -1); + sqlb::FieldVector generateFieldList(const CSVParser& parser); + void importCsv(const QString& f); - sqlb::FieldVector fetchCsvHeader(const QString& f); void setQuoteChar(const QChar& c); char currentQuoteChar() const; From 18460af2dee07b0ed12156d4ad057ffba956c3c0 Mon Sep 17 00:00:00 2001 From: iKlsR Date: Thu, 15 Jun 2017 01:16:50 -0500 Subject: [PATCH 7/7] Allow specifying unique table names for separate imports --- src/ImportCsvDialog.cpp | 37 ++++++++++++++++++++++++------------- src/ImportCsvDialog.h | 2 +- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/ImportCsvDialog.cpp b/src/ImportCsvDialog.cpp index be6bbf226b..2e8188b7fc 100644 --- a/src/ImportCsvDialog.cpp +++ b/src/ImportCsvDialog.cpp @@ -152,15 +152,12 @@ void ImportCsvDialog::accept() // Get all the selected files and start the import if (ui->filePickerBlock->isVisible()) { - QStringList selectedFiles; for (int i = 0; i < ui->filePicker->count(); i++) { auto item = ui->filePicker->item(i); - if (item->checkState() == Qt::Checked) - selectedFiles.append(item->data(Qt::DisplayRole).toString()); + if (item->checkState() == Qt::Checked) { + importCsv(item->data(Qt::DisplayRole).toString(), item->data(Qt::UserRole).toString()); + } } - - for (auto file : selectedFiles) - importCsv(file); } else { @@ -242,16 +239,21 @@ void ImportCsvDialog::checkInput() allowImporting = !ui->editName->text().isEmpty(); } + if (ui->filePicker->currentItem()) { + ui->filePicker->currentItem()->setData(Qt::UserRole, ui->editName->text()); + } + ui->matchSimilar->setEnabled(ui->filePicker->currentItem() != nullptr); - ui->editName->setEnabled(!ui->checkBoxSeparateTables->isChecked()); ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(allowImporting); } void ImportCsvDialog::selectFiles() { - for (auto filename : csvFilenames) { + for (auto fileName : csvFilenames) { + auto fInfo = QFileInfo(fileName); auto item = new QListWidgetItem(); - item->setText(filename); + item->setText(fileName); + item->setData(Qt::UserRole, fInfo.baseName()); item->setCheckState(Qt::Checked); ui->filePicker->addItem(item); } @@ -263,7 +265,11 @@ void ImportCsvDialog::updateSelectedFilePreview(QListWidgetItem* item) { selectedFile = item->data(Qt::DisplayRole).toString(); QFileInfo fileInfo(selectedFile); - ui->editName->setText(fileInfo.baseName()); + if (ui->checkBoxSeparateTables->isChecked()) { + ui->editName->setText(item->data(Qt::UserRole).toString()); + } else { + ui->editName->setText(fileInfo.baseName()); + } updatePreview(); checkInput(); } @@ -363,12 +369,17 @@ sqlb::FieldVector ImportCsvDialog::generateFieldList(const CSVParser &parser) return fieldList; } -void ImportCsvDialog::importCsv(const QString& fileName) +void ImportCsvDialog::importCsv(const QString& fileName, const QString &name) { QString tableName; + if (ui->checkBoxSeparateTables->isChecked()) { - QFileInfo fileInfo(fileName); - tableName = fileInfo.baseName(); + if (name.isEmpty()) { + QFileInfo fileInfo(fileName); + tableName = fileInfo.baseName(); + } else { + tableName = name; + } } else { tableName = ui->editName->text(); } diff --git a/src/ImportCsvDialog.h b/src/ImportCsvDialog.h index 625ce5c272..b4e355fe10 100644 --- a/src/ImportCsvDialog.h +++ b/src/ImportCsvDialog.h @@ -40,7 +40,7 @@ private slots: CSVParser parseCSV(const QString &f, qint64 count = -1); sqlb::FieldVector generateFieldList(const CSVParser& parser); - void importCsv(const QString& f); + void importCsv(const QString& f, const QString &n = QString()); void setQuoteChar(const QChar& c); char currentQuoteChar() const;