From 9dd32c67f4b750613a8e73158538308f360b3539 Mon Sep 17 00:00:00 2001 From: Fred Robinson Date: Thu, 8 Aug 2019 21:42:19 -0700 Subject: [PATCH 01/14] apply patch --- mysql2sqlite | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mysql2sqlite b/mysql2sqlite index deaa469..435cd19 100755 --- a/mysql2sqlite +++ b/mysql2sqlite @@ -195,6 +195,10 @@ aInc == 1 && /PRIMARY KEY|primary key/ { next } if( match($0, ere_bit_field) ){ sub( ere_bit_field, bit_to_int( substr( $0, RSTART +2, RLENGTH -2 -1 ) ) ) } + + #remove USING BTREE and other suffixes for USING, for example: "UNIQUE KEY `hostname_domain` (`hostname`,`domain`) USING BTREE," + gsub( / USING [^, ]+/, "" ) + # field comments are not supported gsub( / (COMMENT|comment).+$/, "" ) # Get commas off end of line From ff802601b171cd6f304a5306b11d4679e8838cfe Mon Sep 17 00:00:00 2001 From: Fred Robinson Date: Sat, 10 Aug 2019 11:58:17 -0700 Subject: [PATCH 02/14] add test case --- unit_tests.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/unit_tests.sh b/unit_tests.sh index 46d1304..6bfb6dd 100755 --- a/unit_tests.sh +++ b/unit_tests.sh @@ -121,5 +121,10 @@ SQLout cat <<\SQL +CREATE TABLE `scimag` ( + `ID` int(15) unsigned NOT NULL AUTO_INCREMENT, + `DOI` varchar(200) NOT NULL, + PRIMARY KEY (`ID`) USING BTREE, + UNIQUE KEY `DOIUNIQUE` (`DOI`) USING BTREE, +); SQL - From 1fda3fed330ede7db6ff68f9bc0cc51f7d34cbb9 Mon Sep 17 00:00:00 2001 From: Fred Robinson Date: Sat, 10 Aug 2019 12:00:46 -0700 Subject: [PATCH 03/14] wrap comment --- mysql2sqlite | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql2sqlite b/mysql2sqlite index 435cd19..782883d 100755 --- a/mysql2sqlite +++ b/mysql2sqlite @@ -196,7 +196,8 @@ aInc == 1 && /PRIMARY KEY|primary key/ { next } sub( ere_bit_field, bit_to_int( substr( $0, RSTART +2, RLENGTH -2 -1 ) ) ) } - #remove USING BTREE and other suffixes for USING, for example: "UNIQUE KEY `hostname_domain` (`hostname`,`domain`) USING BTREE," + # remove USING BTREE and other suffixes for USING, for example: "UNIQUE KEY + # `hostname_domain` (`hostname`,`domain`) USING BTREE," gsub( / USING [^, ]+/, "" ) # field comments are not supported From a6526b548fc7c682edaa23a64602e0442d05e18d Mon Sep 17 00:00:00 2001 From: Jordan Hoff Date: Thu, 31 Oct 2019 10:55:36 -0500 Subject: [PATCH 04/14] Fix fulltext matching If the index name on a fulltext key contains the string `key`, then the `.+` pattern greedily matches too far into the string and produces an invalid line. This fix will correctly replace ` FULLTEXT KEY` with ` KEY` regardless of the contents of the index name. --- mysql2sqlite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql2sqlite b/mysql2sqlite index 782883d..3394d0a 100755 --- a/mysql2sqlite +++ b/mysql2sqlite @@ -157,7 +157,7 @@ inView != 0 { next } } # Replace `FULLTEXT KEY` (probably other `XXXXX KEY`) -/^ (FULLTEXT KEY|fulltext key)/ { gsub( /.+(KEY|key)/, " KEY" ) } +/^ (FULLTEXT KEY|fulltext key)/ { gsub( /[A-Za-z ]+(KEY|key)/, " KEY" ) } # Get rid of field lengths in KEY lines / (PRIMARY |primary )?(KEY|key)/ { gsub( /\([0-9]+\)/, "" ) } From 81ba8cd8512f702ef9d98acb7aba31b943d2aad1 Mon Sep 17 00:00:00 2001 From: Johnny English Date: Fri, 10 Jan 2020 21:03:29 +0100 Subject: [PATCH 05/14] minor: silence GNU AWK 5.0.1 (incorrectly complaining about regex syntax) --- mysql2sqlite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql2sqlite b/mysql2sqlite index 3394d0a..9c5030f 100755 --- a/mysql2sqlite +++ b/mysql2sqlite @@ -216,7 +216,7 @@ aInc == 1 && /PRIMARY KEY|primary key/ { next } else { # FIXME check if this is correct in all cases if( match( $1, - /(CONSTRAINT|constraint) \".*\" (FOREIGN KEY|foreign key)/ ) ){ + /(CONSTRAINT|constraint) ["].*["] (FOREIGN KEY|foreign key)/ ) ){ print "," } } From 8b925f6e24c61afc7d2bf0bbdd492348d3648831 Mon Sep 17 00:00:00 2001 From: Johnny English Date: Tue, 14 Jan 2020 16:43:04 +0100 Subject: [PATCH 06/14] minor: add note about potential stdout/stderr messages coming from sqlite3 and not just mysql2sqlite --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fadf352..65adf15 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ Converts MySQL dump to SQLite3 compatible dump (including MySQL `KEY xxxxx` stat ./mysql2sqlite dump_mysql.sql | sqlite3 mysqlite3.db ~~~~ +(both `mysql2sqlite` and `sqlite3` might write something to stdout and stderr - e.g. `memory` coming from `PRAGMA journal_mode = MEMORY;` is not harmful) + ## Development The script is written in *awk* (tested with gawk, but should work with original awk, and the lightning fast mawk) and shall be fully POSIX compliant. From 8fcb20b75ef68c3d5045860bbf6375bffb0a78fb Mon Sep 17 00:00:00 2001 From: Yogev Swisa Date: Wed, 12 Feb 2020 13:55:47 +0200 Subject: [PATCH 07/14] problem fixed --- mysql2sqlite | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql2sqlite b/mysql2sqlite index 9c5030f..7deefb6 100755 --- a/mysql2sqlite +++ b/mysql2sqlite @@ -189,6 +189,7 @@ aInc == 1 && /PRIMARY KEY|primary key/ { next } gsub( /(ENUM|enum)[^)]+\)/, "text " ) gsub( /(SET|set)\([^)]+\)/, "text " ) gsub( /UNSIGNED|unsigned/, "" ) + gsub( /_utf8mb3/, "" ) gsub( /` [^ ]*(INT|int|BIT|bit)[^ ]*/, "` integer" ) gsub( /" [^ ]*(INT|int|BIT|bit)[^ ]*/, "\" integer" ) ere_bit_field = "[bB]'[10]+'" From 13cd90a2c65e13df22a972917a4d71d3edbcea9b Mon Sep 17 00:00:00 2001 From: Yogev Swisa Date: Wed, 12 Feb 2020 19:22:43 +0200 Subject: [PATCH 08/14] unit test added --- unit_tests.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/unit_tests.sh b/unit_tests.sh index 6bfb6dd..ff2e665 100755 --- a/unit_tests.sh +++ b/unit_tests.sh @@ -128,3 +128,13 @@ CREATE TABLE `scimag` ( UNIQUE KEY `DOIUNIQUE` (`DOI`) USING BTREE, ); SQL + +cat <<\SQL +CREATE TABLE `scimag` ( + `ID` int(15) unsigned NOT NULL AUTO_INCREMENT, + `DOI` varchar(200) NOT NULL, + `TEXTFIELD` TEXT DEFAULT ('text_value'), + PRIMARY KEY (`ID`) USING BTREE, + UNIQUE KEY `DOIUNIQUE` (`DOI`) USING BTREE, +); +SQL From 839685360f57ff5a0cb1065ca5a34af305f92d56 Mon Sep 17 00:00:00 2001 From: Yogev Swisa Date: Thu, 13 Feb 2020 10:43:11 +0200 Subject: [PATCH 09/14] unit_tests.sh - MySQL query updated to be after the mysqldump. SQL to SQLin --- unit_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_tests.sh b/unit_tests.sh index ff2e665..9a206ed 100755 --- a/unit_tests.sh +++ b/unit_tests.sh @@ -129,11 +129,11 @@ CREATE TABLE `scimag` ( ); SQL -cat <<\SQL +cat <<\SQLin CREATE TABLE `scimag` ( `ID` int(15) unsigned NOT NULL AUTO_INCREMENT, `DOI` varchar(200) NOT NULL, - `TEXTFIELD` TEXT DEFAULT ('text_value'), + `TEXTFIELD` text DEFAULT (_utf8mb3'text_value'), PRIMARY KEY (`ID`) USING BTREE, UNIQUE KEY `DOIUNIQUE` (`DOI`) USING BTREE, ); From dbe7e0158531a999f8d3f44850eef1740d6d8523 Mon Sep 17 00:00:00 2001 From: Yogev Swisa Date: Thu, 13 Feb 2020 10:50:47 +0200 Subject: [PATCH 10/14] added SQLout - forgot to add in last commit --- unit_tests.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/unit_tests.sh b/unit_tests.sh index 9a206ed..4948b55 100755 --- a/unit_tests.sh +++ b/unit_tests.sh @@ -137,4 +137,17 @@ CREATE TABLE `scimag` ( PRIMARY KEY (`ID`) USING BTREE, UNIQUE KEY `DOIUNIQUE` (`DOI`) USING BTREE, ); -SQL +SQLin + +cat <<\SQLout +PRAGMA synchronous = OFF; +PRAGMA journal_mode = MEMORY; +BEGIN TRANSACTION; +CREATE TABLE `scimag` ( + `ID` integer NOT NULL PRIMARY KEY AUTOINCREMENT +, `DOI` varchar(200) NOT NULL +, `TEXTFIELD` text DEFAULT ('text_value') +, UNIQUE (`DOI`) +); +END TRANSACTION; +SQLout From f1242a0a88ebc958928fb5908727323771ac9b6a Mon Sep 17 00:00:00 2001 From: Yogev Swisa Date: Thu, 13 Feb 2020 10:53:06 +0200 Subject: [PATCH 11/14] made shorted query. --- unit_tests.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/unit_tests.sh b/unit_tests.sh index 4948b55..8ad82b0 100755 --- a/unit_tests.sh +++ b/unit_tests.sh @@ -131,11 +131,7 @@ SQL cat <<\SQLin CREATE TABLE `scimag` ( - `ID` int(15) unsigned NOT NULL AUTO_INCREMENT, - `DOI` varchar(200) NOT NULL, `TEXTFIELD` text DEFAULT (_utf8mb3'text_value'), - PRIMARY KEY (`ID`) USING BTREE, - UNIQUE KEY `DOIUNIQUE` (`DOI`) USING BTREE, ); SQLin @@ -144,10 +140,7 @@ PRAGMA synchronous = OFF; PRAGMA journal_mode = MEMORY; BEGIN TRANSACTION; CREATE TABLE `scimag` ( - `ID` integer NOT NULL PRIMARY KEY AUTOINCREMENT -, `DOI` varchar(200) NOT NULL -, `TEXTFIELD` text DEFAULT ('text_value') -, UNIQUE (`DOI`) + `TEXTFIELD` text DEFAULT ('text_value') ); END TRANSACTION; SQLout From 4acf080c456f565ca173fb5efb70ea92fa79d018 Mon Sep 17 00:00:00 2001 From: corey Date: Mon, 24 Aug 2020 14:25:50 -0400 Subject: [PATCH 12/14] Improve instructions on how to use this script. Instructions now reflect the "direct pipe" method as noted in the README. Direct piping to sqlite3 is typically how this script will be used since you get a sqlite database file as a result. --- mysql2sqlite | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mysql2sqlite b/mysql2sqlite index 7deefb6..af72a01 100755 --- a/mysql2sqlite +++ b/mysql2sqlite @@ -9,8 +9,13 @@ function printerr( s ){ print s | "cat >&2" } BEGIN { if( ARGC != 2 ){ printerr( \ - "USAGE: mysql2sqlite dump_mysql.sql > dump_sqlite3.sql\n" \ - " file name - (dash) is not supported, because - means stdin") + "USAGE:\n"\ + " - mysql2sqlite dump_mysql.sql > dump_sqlite3.sql\n" \ + " OR\n" \ + " - mysql2sqlite dump_mysql.sql | sqlite3 sqlite.db\n" \ + "\n" \ + "NOTES:\n" \ + " - Dash in filename is not supported, because dash (-) means stdin.") no_END = 1 exit 1 } From ddcbfee93d09d287fc19c786cce4f778b784bcb6 Mon Sep 17 00:00:00 2001 From: corey Date: Mon, 24 Aug 2020 14:47:37 -0400 Subject: [PATCH 13/14] Remove dashes so lines are more easily copy/pasted. --- mysql2sqlite | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql2sqlite b/mysql2sqlite index af72a01..86f11c0 100755 --- a/mysql2sqlite +++ b/mysql2sqlite @@ -10,12 +10,12 @@ BEGIN { if( ARGC != 2 ){ printerr( \ "USAGE:\n"\ - " - mysql2sqlite dump_mysql.sql > dump_sqlite3.sql\n" \ + " mysql2sqlite dump_mysql.sql > dump_sqlite3.sql\n" \ " OR\n" \ - " - mysql2sqlite dump_mysql.sql | sqlite3 sqlite.db\n" \ + " mysql2sqlite dump_mysql.sql | sqlite3 sqlite.db\n" \ "\n" \ "NOTES:\n" \ - " - Dash in filename is not supported, because dash (-) means stdin.") + " Dash in filename is not supported, because dash (-) means stdin." ) no_END = 1 exit 1 } From c6ab632eb3ad5798c85f643bd7ecf76ea2d3c63e Mon Sep 17 00:00:00 2001 From: Yogev Swisa Date: Tue, 1 Sep 2020 16:28:01 +0300 Subject: [PATCH 14/14] skip only if the word CREATE is followed by the word DATABASE to prevent skipping names with the word DATABASE in them --- mysql2sqlite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql2sqlite b/mysql2sqlite index 86f11c0..2a3e374 100755 --- a/mysql2sqlite +++ b/mysql2sqlite @@ -141,7 +141,7 @@ inView != 0 { next } } # CREATE DATABASE is not supported -/^(CREATE.*DATABASE|create.*database)/ { next } +/^(CREATE DATABASE|create database)/ { next } # print the CREATE line as is and capture the table name /^(CREATE|create)/ {