diff --git a/src/FileDialog.cpp b/src/FileDialog.cpp index 3ec070dcfd..e7d4b1886e 100644 --- a/src/FileDialog.cpp +++ b/src/FileDialog.cpp @@ -9,6 +9,17 @@ QString FileDialog::getOpenFileName(QWidget* parent, const QString& caption, con return result; } +QStringList FileDialog::getOpenFileNames(QWidget *parent, const QString &caption, const QString &filter, QString *selectedFilter, QFileDialog::Options options) +{ + QStringList result = QFileDialog::getOpenFileNames(parent, caption, getFileDialogPath(), filter, selectedFilter, options); + if(!result.isEmpty()) + { + QFileInfo path = QFileInfo(result.first()); + setFileDialogPath(path.absolutePath()); + } + return result; +} + QString FileDialog::getSaveFileName(QWidget* parent, const QString& caption, const QString& filter, const QString& defaultFileName, QString* selectedFilter, Options options) { QString dir = getFileDialogPath(); diff --git a/src/FileDialog.h b/src/FileDialog.h index a6fbbd4ef0..b5ca282987 100644 --- a/src/FileDialog.h +++ b/src/FileDialog.h @@ -11,6 +11,9 @@ class FileDialog : public QFileDialog static QString getOpenFileName(QWidget* parent = 0, const QString& caption = QString(), const QString& filter = QString(), QString* selectedFilter = 0, Options options = 0); + static QStringList getOpenFileNames(QWidget* parent = 0, const QString& caption = QString(), + const QString& filter = QString(), QString* selectedFilter = 0, + Options options = 0); static QString getSaveFileName(QWidget* parent = 0, const QString& caption = QString(), const QString& filter = QString(), const QString& defaultFileName = QString(), QString* selectedFilter = 0, Options options = 0); diff --git a/src/ImportCsvDialog.cpp b/src/ImportCsvDialog.cpp index bac0e82537..2e8188b7fc 100644 --- a/src/ImportCsvDialog.cpp +++ b/src/ImportCsvDialog.cpp @@ -18,16 +18,17 @@ #include #include -ImportCsvDialog::ImportCsvDialog(const QString& filename, DBBrowserDB* db, QWidget* parent) +ImportCsvDialog::ImportCsvDialog(const QStringList &filenames, DBBrowserDB* db, QWidget* parent) : QDialog(parent), ui(new Ui::ImportCsvDialog), - csvFilename(filename), + csvFilenames(filenames), pdb(db) { ui->setupUi(this); // Get the actual file name out of the provided path and use it as the default table name for import - QFileInfo file(filename); + // For importing several files at once, the fields have to be the same so we can safely use the first + QFileInfo file(filenames.first()); ui->editName->setText(file.baseName()); // Create a list of all available encodings and create an auto completion list from them @@ -42,13 +43,29 @@ ImportCsvDialog::ImportCsvDialog(const QString& filename, DBBrowserDB* db, QWidg QSettings settings(QApplication::organizationName(), QApplication::organizationName()); ui->checkboxHeader->setChecked(settings.value("importcsv/firstrowheader", false).toBool()); ui->checkBoxTrimFields->setChecked(settings.value("importcsv/trimfields", true).toBool()); + ui->checkBoxSeparateTables->setChecked(settings.value("importcsv/separatetables", false).toBool()); setSeparatorChar(QChar(settings.value("importcsv/separator", ',').toInt())); setQuoteChar(QChar(settings.value("importcsv/quotecharacter", '"').toInt())); setEncoding(settings.value("importcsv/encoding", "UTF-8").toString()); - // Initialise user interface - checkInput(); + // Prepare and show interface depending on how many files are selected + if (csvFilenames.length() > 1) + { + ui->separateTables->setVisible(true); + ui->checkBoxSeparateTables->setVisible(true); + ui->filePickerBlock->setVisible(true); + selectFiles(); + } + else if (csvFilenames.length() == 1) + { + ui->separateTables->setVisible(false); + ui->checkBoxSeparateTables->setVisible(false); + ui->filePickerBlock->setVisible(false); + } + + selectedFile = csvFilenames.first(); updatePreview(); + checkInput(); } ImportCsvDialog::~ImportCsvDialog() @@ -128,32 +145,205 @@ void ImportCsvDialog::accept() settings.setValue("separator", currentSeparatorChar()); settings.setValue("quotecharacter", currentQuoteChar()); settings.setValue("trimfields", ui->checkBoxTrimFields->isChecked()); + settings.setValue("separatetables", ui->checkBoxSeparateTables->isChecked()); settings.setValue("encoding", currentEncoding()); settings.endGroup(); - // Parse all csv data - QFile file(csvFilename); + // Get all the selected files and start the import + if (ui->filePickerBlock->isVisible()) + { + for (int i = 0; i < ui->filePicker->count(); i++) { + auto item = ui->filePicker->item(i); + if (item->checkState() == Qt::Checked) { + importCsv(item->data(Qt::DisplayRole).toString(), item->data(Qt::UserRole).toString()); + } + } + } + else + { + importCsv(csvFilenames.first()); + } + + QApplication::restoreOverrideCursor(); // restore original cursor + QDialog::accept(); +} + +void ImportCsvDialog::updatePreview() +{ + // Show/hide custom quote/separator input fields + ui->editCustomQuote->setVisible(ui->comboQuote->currentIndex() == ui->comboQuote->count()-1); + ui->editCustomSeparator->setVisible(ui->comboSeparator->currentIndex() == ui->comboSeparator->count()-1); + ui->editCustomEncoding->setVisible(ui->comboEncoding->currentIndex() == ui->comboEncoding->count()-1); + + // Get preview data + QFile file(selectedFile); file.open(QIODevice::ReadOnly); CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar()); - csv.setCSVProgress(new CSVImportProgress(file.size())); QTextStream tstream(&file); tstream.setCodec(currentEncoding().toUtf8()); - csv.parse(tstream); + csv.parse(tstream, 20); file.close(); - if(csv.csv().size() == 0) + // Reset preview widget + ui->tablePreview->clear(); + ui->tablePreview->setColumnCount(csv.columns()); + + // Exit if there are no lines to preview at all + if(csv.columns() == 0) return; + // Use first row as header if necessary + CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); + if(ui->checkboxHeader->isChecked()) + { + ui->tablePreview->setHorizontalHeaderLabels(*itBegin); + ++itBegin; + } + + // Fill data section + ui->tablePreview->setRowCount(std::distance(itBegin, csv.csv().end())); + + for(CSVParser::TCSVResult::const_iterator ct = itBegin; + ct != csv.csv().end(); + ++ct) + { + for(QStringList::const_iterator it = ct->begin(); it != ct->end(); ++it) + { + int rowNum = std::distance(itBegin, ct); + if(it == ct->begin()) + { + ui->tablePreview->setVerticalHeaderItem( + rowNum, + new QTableWidgetItem(QString::number(rowNum + 1))); + } + ui->tablePreview->setItem( + rowNum, + std::distance(ct->begin(), it), + new QTableWidgetItem(*it)); + } + } +} + +void ImportCsvDialog::checkInput() +{ + bool allowImporting = false; + if (ui->filePickerBlock->isVisible()) { + bool checkedItem = false; + for (int i = 0; i < ui->filePicker->count(); i++) { + if (ui->filePicker->item(i)->checkState() == Qt::Checked) checkedItem = true; + } + allowImporting = !ui->editName->text().isEmpty() && checkedItem; + } else { + allowImporting = !ui->editName->text().isEmpty(); + } + + if (ui->filePicker->currentItem()) { + ui->filePicker->currentItem()->setData(Qt::UserRole, ui->editName->text()); + } + + ui->matchSimilar->setEnabled(ui->filePicker->currentItem() != nullptr); + ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(allowImporting); +} + +void ImportCsvDialog::selectFiles() +{ + for (auto fileName : csvFilenames) { + auto fInfo = QFileInfo(fileName); + auto item = new QListWidgetItem(); + item->setText(fileName); + item->setData(Qt::UserRole, fInfo.baseName()); + item->setCheckState(Qt::Checked); + ui->filePicker->addItem(item); + } + + connect(ui->filePicker, SIGNAL(itemClicked(QListWidgetItem*)), SLOT(updateSelectedFilePreview(QListWidgetItem*))); +} + +void ImportCsvDialog::updateSelectedFilePreview(QListWidgetItem* item) +{ + selectedFile = item->data(Qt::DisplayRole).toString(); + QFileInfo fileInfo(selectedFile); + if (ui->checkBoxSeparateTables->isChecked()) { + ui->editName->setText(item->data(Qt::UserRole).toString()); + } else { + ui->editName->setText(fileInfo.baseName()); + } + updatePreview(); + checkInput(); +} + +void ImportCsvDialog::updateSelection(bool selected) +{ + for (int i = 0; i < ui->filePicker->count(); i++) + ui->filePicker->item(i)->setCheckState(selected ? Qt::Checked : Qt::Unchecked); + ui->toggleSelected->setText(selected ? tr("Deselect All") : tr("Select All")); + checkInput(); +} + +void ImportCsvDialog::matchSimilar() +{ + auto item = ui->filePicker->currentItem(); + auto selectedHeader = generateFieldList(parseCSV(item->data(Qt::DisplayRole).toString(), 1)); + + for (int i = 0; i < ui->filePicker->count(); i++) + { + auto item = ui->filePicker->item(i); + auto header = generateFieldList(parseCSV(item->data(Qt::DisplayRole).toString(), 1)); + bool matchingHeader = false; + + if (selectedHeader.count() == header.count()) + { + matchingHeader = std::equal(selectedHeader.begin(), selectedHeader.end(), header.begin(), + [](const sqlb::FieldPtr& item1, const sqlb::FieldPtr& item2) -> bool { + return (item1->name() == item2->name()); + }); + if (matchingHeader) { + item->setCheckState(Qt::Checked); + item->setBackgroundColor(Qt::green); + } + } + else + { + item->setCheckState(Qt::Unchecked); + item->setBackground(Qt::white); + } + } + + checkInput(); +} + +CSVParser ImportCsvDialog::parseCSV(const QString &fileName, qint64 count) +{ + // Parse all csv data + QFile file(fileName); + file.open(QIODevice::ReadOnly); + + CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar()); + // If count is one, we only want the header, no need to see progress + if (count != 1) csv.setCSVProgress(new CSVImportProgress(file.size())); + + QTextStream tstream(&file); + tstream.setCodec(currentEncoding().toUtf8()); + csv.parse(tstream, count); + file.close(); + + return csv; +} + +sqlb::FieldVector ImportCsvDialog::generateFieldList(const CSVParser &parser) +{ + if (parser.csv().size() == 0) return sqlb::FieldVector(); + // Generate field names. These are either taken from the first CSV row or are generated in the format of "fieldXY" depending on the user input sqlb::FieldVector fieldList; - CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); + CSVParser::TCSVResult::const_iterator itBegin = parser.csv().begin(); if(ui->checkboxHeader->isChecked()) { ++itBegin; - for(QStringList::const_iterator it = csv.csv().at(0).begin(); - it != csv.csv().at(0).end(); + for(QStringList::const_iterator it = parser.csv().at(0).begin(); + it != parser.csv().at(0).end(); ++it) { // Remove invalid characters @@ -167,15 +357,38 @@ void ImportCsvDialog::accept() // Avoid empty field names if(thisfield.isEmpty()) - thisfield = QString("field%1").arg(std::distance(csv.csv().at(0).begin(), it) + 1); + thisfield = QString("field%1").arg(std::distance(parser.csv().at(0).begin(), it) + 1); fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(thisfield, ""))); } } else { - for(size_t i=0; i < csv.columns(); ++i) + for(size_t i=0; i < parser.columns(); ++i) fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(QString("field%1").arg(i+1), ""))); } + return fieldList; +} + +void ImportCsvDialog::importCsv(const QString& fileName, const QString &name) +{ + QString tableName; + + if (ui->checkBoxSeparateTables->isChecked()) { + if (name.isEmpty()) { + QFileInfo fileInfo(fileName); + tableName = fileInfo.baseName(); + } else { + tableName = name; + } + } else { + tableName = ui->editName->text(); + } + + CSVParser csv = parseCSV(fileName); + if (csv.csv().size() == 0) return; + + sqlb::FieldVector fieldList = generateFieldList(csv); + // Show progress dialog QProgressDialog progress(tr("Inserting data..."), tr("Cancel"), 0, csv.csv().size()); progress.setWindowModality(Qt::ApplicationModal); @@ -186,7 +399,7 @@ void ImportCsvDialog::accept() objectMap objects = pdb->getBrowsableObjects(); for(auto it=objects.constBegin();it!=objects.constEnd();++it) { - if((*it)->type() == sqlb::Object::Types::Table && (*it)->name() == ui->editName->text()) + if((*it)->type() == sqlb::Object::Types::Table && (*it)->name() == tableName) { if((size_t)(*it).dynamicCast()->fields().size() != csv.columns()) { @@ -194,6 +407,13 @@ void ImportCsvDialog::accept() tr("There is already a table of that name and an import into an existing table is only possible if the number of columns match.")); return; } else { + // If we are importing multiple files, we can skip the warning and perform the inserts + // To omit further warnings there is a filter button to select all conforming sql + if (!ui->checkBoxSeparateTables->isChecked()) { + importToExistingTable = true; + break; + } + if(QMessageBox::question(this, QApplication::applicationName(), tr("There is already a table of that name. Do you want to import the data into it?"), QMessageBox::Yes, QMessageBox::No) == QMessageBox::Yes) { importToExistingTable = true; @@ -214,16 +434,17 @@ void ImportCsvDialog::accept() // Create table if(!importToExistingTable) { - if(!pdb->createTable(ui->editName->text(), fieldList)) + if(!pdb->createTable(tableName, fieldList)) return rollback(this, pdb, progress, restorepointName, 0, tr("Creating the table failed: %1").arg(pdb->lastError())); } // now lets import all data, one row at a time + CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); for(CSVParser::TCSVResult::const_iterator it = itBegin; it != csv.csv().end(); ++it) { - QString sql = QString("INSERT INTO %1 VALUES(").arg(sqlb::escapeIdentifier(ui->editName->text())); + QString sql = QString("INSERT INTO %1 VALUES(").arg(sqlb::escapeIdentifier(tableName)); QStringList insertlist; for(QStringList::const_iterator jt = it->begin(); jt != it->end(); ++jt) @@ -255,76 +476,6 @@ void ImportCsvDialog::accept() if(progress.wasCanceled()) return rollback(this, pdb, progress, restorepointName, std::distance(itBegin, it) + 1, ""); } - - QApplication::restoreOverrideCursor(); // restore original cursor - QDialog::accept(); -} - -void ImportCsvDialog::updatePreview() -{ - // Show/hide custom quote/separator input fields - ui->editCustomQuote->setVisible(ui->comboQuote->currentIndex() == ui->comboQuote->count()-1); - ui->editCustomSeparator->setVisible(ui->comboSeparator->currentIndex() == ui->comboSeparator->count()-1); - ui->editCustomEncoding->setVisible(ui->comboEncoding->currentIndex() == ui->comboEncoding->count()-1); - - // Get preview data - QFile file(csvFilename); - file.open(QIODevice::ReadOnly); - - CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar()); - - QTextStream tstream(&file); - tstream.setCodec(currentEncoding().toUtf8()); - csv.parse(tstream, 20); - file.close(); - - // Reset preview widget - ui->tablePreview->clear(); - ui->tablePreview->setColumnCount(csv.columns()); - - // Exit if there are no lines to preview at all - if(csv.columns() == 0) - return; - - // Use first row as header if necessary - CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin(); - if(ui->checkboxHeader->isChecked()) - { - ui->tablePreview->setHorizontalHeaderLabels(*itBegin); - ++itBegin; - } - - // Fill data section - ui->tablePreview->setRowCount(std::distance(itBegin, csv.csv().end())); - - for(CSVParser::TCSVResult::const_iterator ct = itBegin; - ct != csv.csv().end(); - ++ct) - { - for(QStringList::const_iterator it = ct->begin(); it != ct->end(); ++it) - { - int rowNum = std::distance(itBegin, ct); - if(it == ct->begin()) - { - ui->tablePreview->setVerticalHeaderItem( - rowNum, - new QTableWidgetItem(QString::number(rowNum + 1))); - } - ui->tablePreview->setItem( - rowNum, - std::distance(ct->begin(), it), - new QTableWidgetItem(*it)); - } - } -} - -void ImportCsvDialog::checkInput() -{ - bool valid = true; - if(ui->editName->text().isEmpty()) - valid = false; - - ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(valid); } void ImportCsvDialog::setQuoteChar(const QChar& c) diff --git a/src/ImportCsvDialog.h b/src/ImportCsvDialog.h index 6797d10a70..b4e355fe10 100644 --- a/src/ImportCsvDialog.h +++ b/src/ImportCsvDialog.h @@ -1,10 +1,13 @@ #ifndef IMPORTCSVDIALOG_H #define IMPORTCSVDIALOG_H +#include "sqlitetypes.h" #include class DBBrowserDB; class QCompleter; +class CSVParser; +class QListWidgetItem; namespace Ui { class ImportCsvDialog; @@ -15,20 +18,30 @@ class ImportCsvDialog : public QDialog Q_OBJECT public: - explicit ImportCsvDialog(const QString& filename, DBBrowserDB* db, QWidget* parent = 0); + explicit ImportCsvDialog(const QStringList& filenames, DBBrowserDB* db, QWidget* parent = 0); ~ImportCsvDialog(); private slots: virtual void accept(); void updatePreview(); void checkInput(); + void selectFiles(); + void updateSelectedFilePreview(QListWidgetItem*); + void updateSelection(bool); + void matchSimilar(); private: Ui::ImportCsvDialog* ui; - QString csvFilename; + QStringList csvFilenames; + QString selectedFile; DBBrowserDB* pdb; QCompleter* encodingCompleter; + CSVParser parseCSV(const QString &f, qint64 count = -1); + sqlb::FieldVector generateFieldList(const CSVParser& parser); + + void importCsv(const QString& f, const QString &n = QString()); + void setQuoteChar(const QChar& c); char currentQuoteChar() const; diff --git a/src/ImportCsvDialog.ui b/src/ImportCsvDialog.ui index 7cd61be29d..60e464188a 100644 --- a/src/ImportCsvDialog.ui +++ b/src/ImportCsvDialog.ui @@ -13,7 +13,7 @@ Import CSV file - + @@ -241,8 +241,98 @@ + + + + + + + Separate tables + + + + + + + + + + + + + + true + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + + + + 7 + + + QLayout::SetDefaultConstraint + + + 0 + + + + + Deselect All + + + true + + + true + + + + + + + false + + + Match Similar + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + @@ -268,179 +358,231 @@ comboEncoding editCustomEncoding checkBoxTrimFields + checkBoxSeparateTables + filePicker + toggleSelected + matchSimilar tablePreview - buttonBox - accepted() + comboSeparator + currentIndexChanged(int) ImportCsvDialog - accept() + updatePreview() - 261 - 480 + 232 + 99 - 157 - 274 + 445 + 64 - buttonBox - rejected() + editName + textChanged(QString) ImportCsvDialog - reject() + checkInput() - 329 - 480 + 609 + 13 - 286 - 274 + 631 + 45 - checkboxHeader - toggled(bool) + editCustomSeparator + textChanged(QString) ImportCsvDialog updatePreview() - 187 - 46 + 478 + 99 - 354 - 45 + 577 + 76 - comboSeparator - currentIndexChanged(int) + editCustomQuote + textChanged(QString) ImportCsvDialog updatePreview() - 216 - 71 + 478 + 132 - 445 - 64 + 530 + 90 - comboQuote - currentIndexChanged(int) + editCustomEncoding + textChanged(QString) ImportCsvDialog updatePreview() - 224 - 100 + 495 + 165 - 350 - 88 + 540 + 133 - editName - textChanged(QString) + checkBoxTrimFields + toggled(bool) ImportCsvDialog - checkInput() + updatePreview() - 609 - 13 + 184 + 191 - 631 - 45 + 368 + 244 - editCustomSeparator - textChanged(QString) + comboEncoding + currentIndexChanged(int) ImportCsvDialog updatePreview() - 325 - 70 + 266 + 165 - 577 - 76 + 572 + 121 - editCustomQuote - textChanged(QString) + checkboxHeader + toggled(bool) ImportCsvDialog updatePreview() - 467 - 101 + 184 + 60 - 530 - 90 + 354 + 45 - comboEncoding - currentIndexChanged(int) + toggleSelected + toggled(bool) ImportCsvDialog - updatePreview() + updateSelection(bool) - 198 - 131 + 674 + 258 - 572 - 121 + 368 + 244 - editCustomEncoding - textChanged(QString) + comboQuote + currentIndexChanged(int) ImportCsvDialog updatePreview() - 378 + 232 132 - 540 - 133 + 350 + 88 - checkBoxTrimFields + buttonBox + accepted() + ImportCsvDialog + accept() + + + 272 + 478 + + + 157 + 274 + + + + + buttonBox + rejected() + ImportCsvDialog + reject() + + + 340 + 478 + + + 286 + 274 + + + + + checkBoxSeparateTables toggled(bool) ImportCsvDialog - updatePreview() + checkInput() + + + 176 + 216 + + + 368 + 244 + + + + + matchSimilar + pressed() + ImportCsvDialog + matchSimilar() - 191 - 189 + 682 + 279 368 @@ -452,5 +594,7 @@ updatePreview() checkInput() + updateSelection(bool) + matchSimilar() diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index 83f21bd879..286a059300 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -1094,16 +1094,20 @@ void MainWindow::mainTabSelected(int tabindex) void MainWindow::importTableFromCSV() { - QString wFile = FileDialog::getOpenFileName( - this, - tr("Choose a text file"), - tr("Text files(*.csv *.txt);;All files(*)")); + QStringList wFiles = FileDialog::getOpenFileNames( + this, + tr("Choose text files"), + tr("Text files(*.csv *.txt);;All files(*)")); + + QStringList validFiles; + foreach(auto file, wFiles) { + if (QFile::exists(file)) + validFiles.append(file); + } - if (QFile::exists(wFile) ) - { - ImportCsvDialog dialog(wFile, &db, this); - if(dialog.exec()) - { + if (!validFiles.isEmpty()) { + ImportCsvDialog dialog(validFiles, &db, this); + if (dialog.exec()) { populateTable(); QMessageBox::information(this, QApplication::applicationName(), tr("Import completed")); }