Skip to content

Commit f124ff7

Browse files
MauricioFauthMauricioFauth
committed
Merge PR #19954 Refactor csv import
Closes #19954 Signed-off-by: Maurício Meneghini Fauth <mauricio@mfauth.net>
2 parents de85bcb + d671070 commit f124ff7

4 files changed

Lines changed: 241 additions & 254 deletions

File tree

‎phpstan-baseline.neon‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9549,24 +9549,12 @@ parameters:
95499549
count: 2
95509550
path: src/Plugins/Import/ImportCsv.php
95519551

9552-
-
9553-
message: '#^Foreach overwrites \$i with its key variable\.$#'
9554-
identifier: foreach.keyOverwrite
9555-
count: 1
9556-
path: src/Plugins/Import/ImportCsv.php
9557-
95589552
-
95599553
message: '#^Loose comparison via "\!\=" is not allowed\.$#'
95609554
identifier: notEqual.notAllowed
95619555
count: 3
95629556
path: src/Plugins/Import/ImportCsv.php
95639557

9564-
-
9565-
message: '#^Loose comparison via "\=\=" is not allowed\.$#'
9566-
identifier: equal.notAllowed
9567-
count: 16
9568-
path: src/Plugins/Import/ImportCsv.php
9569-
95709558
-
95719559
message: '#^Only booleans are allowed in &&, string given on the right side\.$#'
95729560
identifier: booleanAnd.rightNotBoolean

‎src/Import/Import.php‎

Lines changed: 25 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ class Import
6666
public static string $importText = '';
6767
public static ResultInterface|false $result = false;
6868
public static string $errorUrl = '';
69+
private bool $forceExecute = false;
6970

7071
public function __construct()
7172
{
@@ -198,7 +199,7 @@ public function runQuery(string $sql, array &$sqlData): void
198199

199200
ImportSettings::$executedQueries++;
200201

201-
if (ImportSettings::$runQuery && ImportSettings::$executedQueries < 50) {
202+
if (ImportSettings::$runQuery && ImportSettings::$executedQueries < 50 && ! $this->forceExecute) {
202203
ImportSettings::$goSql = true;
203204

204205
if (! ImportSettings::$sqlQueryDisabled) {
@@ -609,7 +610,7 @@ private function detectSize(
609610
return -1;
610611
}
611612

612-
public function detectType(ColumnType|null $lastCumulativeType, string|null $cell): ColumnType
613+
public function detectType(ColumnType|null $lastCumulativeType, string $cell): ColumnType
613614
{
614615
/**
615616
* If numeric, determine if decimal, int or bigint
@@ -652,57 +653,43 @@ public function detectType(ColumnType|null $lastCumulativeType, string|null $cel
652653
*/
653654
public function analyzeTable(ImportTable $table): array
654655
{
655-
/* Get number of rows in table */
656-
/* Get number of columns */
657656
$numberOfColumns = count($table->columns);
658657

659658
$columns = [];
660659
for ($i = 0; $i < $numberOfColumns; ++$i) {
661660
$columns[] = new AnalysedColumn(ColumnType::None, 0);
662-
}
663661

664-
/* Analyze each column */
665-
for ($i = 0; $i < $numberOfColumns; ++$i) {
666-
/* Analyze the column in each row */
667662
foreach ($table->rows as $row) {
668663
$cellValue = $row[$i];
669-
/* Determine type of the current cell */
670-
$currType = $this->detectType($columns[$i]->type, $cellValue === null ? null : (string) $cellValue);
671-
/* Determine size of the current cell */
664+
$detectedType = $this->detectType($columns[$i]->type, (string) $cellValue);
672665
$columns[$i]->size = $this->detectSize(
673666
$columns[$i]->size,
674667
$columns[$i]->type,
675-
$currType,
668+
$detectedType,
676669
(string) $cellValue,
677670
);
678671

679672
/**
680673
* If a type for this column has already been declared,
681674
* only alter it if it was a number and a varchar was found
682675
*/
683-
if ($currType === ColumnType::None) {
676+
if ($detectedType === ColumnType::None) {
684677
continue;
685678
}
686679

687-
if ($currType === ColumnType::Varchar) {
688-
$columns[$i]->type = ColumnType::Varchar;
689-
} elseif ($currType === ColumnType::Decimal) {
690-
if ($columns[$i]->type !== ColumnType::Varchar) {
691-
$columns[$i]->type = ColumnType::Decimal;
692-
}
693-
} elseif ($currType === ColumnType::BigInt) {
694-
if ($columns[$i]->type !== ColumnType::Varchar && $columns[$i]->type !== ColumnType::Decimal) {
695-
$columns[$i]->type = ColumnType::BigInt;
696-
}
697-
} elseif ($currType === ColumnType::Int) {
698-
if (
699-
$columns[$i]->type !== ColumnType::Varchar
700-
&& $columns[$i]->type !== ColumnType::Decimal
701-
&& $columns[$i]->type !== ColumnType::BigInt
702-
) {
703-
$columns[$i]->type = ColumnType::Int;
704-
}
705-
}
680+
$columns[$i]->type = match ($columns[$i]->type) {
681+
default => 0,
682+
ColumnType::Int => 1,
683+
ColumnType::BigInt => 2,
684+
ColumnType::Decimal => 3,
685+
ColumnType::Varchar => 4,
686+
} >= match ($detectedType) {
687+
default => 0,
688+
ColumnType::Int => 1,
689+
ColumnType::BigInt => 2,
690+
ColumnType::Decimal => 3,
691+
ColumnType::Varchar => 4,
692+
} ? $columns[$i]->type : $detectedType;
706693
}
707694
}
708695

@@ -838,6 +825,12 @@ public function buildSql(
838825

839826
$tempSQLStr .= ') VALUES ';
840827

828+
// This line is added to trick the runQuery method into thinking it executed more than one query
829+
// Without this, large imports will be redirected to the /sql page and suffer performance issues
830+
if (count($table->rows) > 50) {
831+
$this->forceExecute = true;
832+
}
833+
841834
$lastRowKey = array_key_last($table->rows);
842835
foreach ($table->rows as $rowIndex => $row) {
843836
$tempSQLStr .= '(';

0 commit comments

Comments
 (0)