From 2696dbc12cd066dd0aeefa82cc022f0c9c34aed4 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 11 Dec 2013 15:37:10 +0100 Subject: [PATCH 001/216] Added the licence --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 8441533..0c8f763 100644 --- a/README.md +++ b/README.md @@ -146,3 +146,8 @@ What it this IGNORE NULLS I have to change in kettle.properties ? Because Kettle behaves by default the way Oracle behaves: for Oracle, a NULL and an empty string are the same. You don't want this here, because neither SQL Server nor PostgreSQL do this. + +================================ +Licence: GPL v3 + +http://www.gnu.org/licenses/gpl.html From 849054b78e469f5779b44c47e8951c06f95c039e Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 5 Mar 2014 11:11:49 +0100 Subject: [PATCH 002/216] small doc corrections --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0c8f763..c843686 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ As Kettle is a Java program, you'll also need a recent JVM (Java 6 or 7 should d ========================== -Ok, I have installed Kettle and Java, I have mssql2pg.pl, what do I do now ? +Ok, I have installed Kettle and Java, I have sqlserver2pgsql.pl, what do I do now ? You'll need several things: @@ -64,11 +64,11 @@ To get this SQL Dump, follow this procedure, in SQL Server's management interfac * Finish You'll get a file containing a SQL script. Get it on the server you'll want to -run mssql2pg.pl from. +run sqlserver2pgsql.pl from. If you just want to convert this schema, run: -> mssql2pg -f my_sqlserver_script.txt -b name_of_before_script -a name_of_after_script -u name_of_unsure_script +> sqlserver2pgsql.pl -f my_sqlserver_script.txt -b name_of_before_script -a name_of_after_script -u name_of_unsure_script The before script contains what is needed to import data (types, tables and columns). The after script contains the rest (indexes, constraints). It should be run @@ -142,7 +142,7 @@ variable to a higher value (4096) for 4GB for instance. -What it this IGNORE NULLS I have to change in kettle.properties ? +What is this IGNORE NULLS I have to change in kettle.properties ? Because Kettle behaves by default the way Oracle behaves: for Oracle, a NULL and an empty string are the same. You don't want this here, because neither SQL Server nor PostgreSQL do this. From e4fb09447668754482cdc245a4036afdbce493c9 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 7 Mar 2014 09:34:04 +0100 Subject: [PATCH 003/216] Add missing word in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c843686..3c1c9cb 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ Why didn't you do everything in the Perl script ? I don't want to use Kettle. Because I have once installed a Perl DBD driver for MS SQL's server, and I don't want to ever do it again... nor force you to do it :) You can either do it using ODBC, or using Sybase's driver. In both case, it will take you -quite a while, and you'll suffer with large objects. With Kettle, on the other, you're using the JBDC driver, +quite a while, and you'll suffer with large objects. With Kettle, on the other hand, you're using the JBDC driver, which is already one of Kettle's default drivers, along with PostgreSQL. So all the heavy lifting (converting LOBs and IMAGES and whatever to bytea) is directly done by both JDBC drivers, and should work with no efforts (except maybe adjust Java's memory parameters). If you get a memory error, try setting a JAVAMAXMEM environment From 8098b5827650eda184f209979c46fe34e4f4e671 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 14 Mar 2014 16:52:53 +0100 Subject: [PATCH 004/216] Improve parsing with a new provided file --- sqlserver2pgsql.pl | 68 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 96053db..d99bd94 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -119,20 +119,24 @@ sub convert_numeric_to_int my %types=('int'=>'int', 'nvarchar'=>'varchar', 'nchar'=>'char', + 'char'=>'char', 'varchar'=>'varchar', + 'text'=>'text', 'char'=>'char', 'smallint'=>'smallint', 'tinyint'=>'smallint', + 'bigint'=>'bigint', + 'decimal'=>'numeric', + 'float'=>'double precision', + 'real'=>'real', + 'date'=>'date', 'datetime'=>'timestamp', 'smalldatetime'=>'timestamp', - 'char'=>'char', - 'image'=>'bytea', - 'text'=>'text', - 'bigint'=>'bigint', 'timestamp'=>'timestamp', - 'decimal'=>'numeric', + 'image'=>'bytea', 'binary'=>'bytea', 'varbinary'=>'bytea', + 'money'=>'numeric' ); # Types with no qualifier, and no point in putting one @@ -715,7 +719,7 @@ sub parse_dump # containing only a single quote (end of the dbo.sp_executesql) # The problem is that SQL Server seems to be spitting the original query used to create the view, not a normalized version # of it, as PostgreSQL does. So we capture the query, and hope it works for now. - elsif ($line =~ /^\s*(create\s*view)\s*(?:\[(\S+)\])?\.\[(.*?)\]\s*(.*)$/i) + elsif ($line =~ /^\s*(create\s*view)\s*(?:\[(\S+)\]\.)?\[(.*?)\]\s*(.*)$/i) { my $viewname=$3; my $schemaname; @@ -727,14 +731,17 @@ sub parse_dump { $schemaname='dbo'; } - my $sql=$1 . ' ' . $3 . ' ' . $4 . "\n"; + $schemaname=dboreplace($schemaname); + + my $sql=$1 . ' ' . $schemaname . '.' . $3 . ' ' . $4 . "\n"; while (my $line_cont=read_and_clean($file)) { - if ($line_cont =~ /^\s*'\s*$/) + if ($line_cont =~ /^\s*'\s*|^GO$/) # We may have a quote if the view is 'quoted', or a real sql query { # The view definition is complete. # We get rid of dbo. schemas - $sql =~ s/dbo\.//g; # We put this in the current schema + $sql =~ s/(dbo)\./dboreplace($1) . '.'/eg; # We put this in the replacement schema + # Views will be stored without the full schema in them. We will # have to generate the schema in the output file $objects->{$schemaname}->{'VIEWS'}->{$viewname}->{SQL}=$sql; @@ -757,7 +764,7 @@ sub parse_dump # but they point to themselves, with the schema corrected: we want them substituted by themselves $types{$schema . '.' . $type}=dboreplace($schema) . '.' . $type; # We store the schema with it } - elsif ($line =~ /^CREATE (UNIQUE )?NONCLUSTERED INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/) + elsif ($line =~ /^CREATE (UNIQUE )?(NONCLUSTERED|CLUSTERED) INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/) { # Index creation. Index are namespaced per table in SQL Server, not in PostgreSQL # In PostgreSQL they are in the same namespace as the tables, and in the same @@ -765,9 +772,10 @@ sub parse_dump # So we store them in $objects, attached to the table # Conflicts will be sorted by resolve_name_conflicts() later my $isunique=$1; - my $idxname=$2; - my $schemaname=$3; - my $tablename=$4; + my $isclustered=$2; + my $idxname=$3; + my $schemaname=$4; + my $tablename=$5; if ($isunique) { $objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES}->{$idxname}->{UNIQUE}=1; @@ -778,12 +786,12 @@ sub parse_dump } while (my $idx=read_and_clean($file)) { - # Exit when read a line beginning with ). The index is complete - if ($idx =~ /^\)/) + # Exit when read a line with a GO. The index is complete + if ($idx =~ /^GO/) { next MAIN; } - next if ($idx =~ /^\(/); # Begin of the columns declaration + next if ($idx =~ /^\(|^\)/); # Begin/end of the columns declaration if ($idx =~ /\t\[(.*)\] (ASC|DESC)(,)?/) { if (defined $2) @@ -795,6 +803,11 @@ sub parse_dump push @{$objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES}->{$idxname}->{COLS}}, ("$1"); } } + if ($idx =~ /^INCLUDE \(/) + { + next; # Nothing equivalent in PG. Maybe if the index isn't unique, these columns should be added? + # Or we should print a warning + } } } # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. @@ -873,6 +886,10 @@ sub parse_dump { $constraint->{ON_DEL_CASC}=1; } + elsif ($fk =~ /^ON UPDATE CASCADE\s*$/) + { + $constraint->{ON_UPD_CASC}=1; + } else { die "Cannot parse $fk $., in a FK. This is a bug"; @@ -951,7 +968,7 @@ sub parse_dump } } # Ignore USE, GO, and things that have no meaning for postgresql - elsif ($line =~ /^USE\s|^GO\s*$|\/\*\*\*\*|^SET ANSI_NULLS ON|^SET QUOTED_IDENTIFIER|^SET ANSI_PADDING|CHECK CONSTRAINT|^BEGIN|^END/) + elsif ($line =~ /^USE\s|^GO\s*$|\/\*\*\*\*|^SET ANSI_NULLS (ON|OFF)|^SET QUOTED_IDENTIFIER|^SET ANSI_PADDING|CHECK CONSTRAINT|^BEGIN|^END/) { next; } @@ -974,6 +991,19 @@ sub parse_dump { next; } + # Ignore CREATE DATABASE: we hope that we are given a single database as an option. It is multiline. + # Ignore everything until next GO + # Ignore ALTER DATABASE for the same reason. The given parameters have no meaning in PG anyway + # Same for tests about full text search. + elsif ($line =~ /^(CREATE|ALTER) DATABASE|^IF \(1 = FULLTEXTSERVICEPROPERTY/) + { + while ($line !~ /^GO$/) + { + $line=read_and_clean($file); + } + # We read everything in the CREATE DATABASE. Back to work ! + next; + } # Ignore EXEC dbo.sp_executesql, for now only seen for a create view. Views sql command aren't executed directly, don't know why elsif ($line =~ /^EXEC dbo.sp_executesql/) { @@ -1151,6 +1181,10 @@ sub generate_schema { $consdef.= " ON DELETE CASCADE"; } + if (defined $constraint->{ON_UPD_CASC} and $constraint->{ON_UPD_CASC}) + { + $consdef.= " ON UPDATE CASCADE"; + } $consdef.= ";\n"; print AFTER $consdef; } From a5e4776526553fd9d9934e14ed22a8a67062271a Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 17 Mar 2014 09:03:42 +0100 Subject: [PATCH 005/216] Add new corrections, coming from yet another dump file --- sqlserver2pgsql.pl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index d99bd94..b1a07fc 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -648,7 +648,8 @@ sub parse_dump } elsif ($line =~ /^\s*(?:CONSTRAINT \[(.*)\] )?UNIQUE/) { - # This is not forbidden by SQL, of course. I just never saw this in a sql server dump, + # This (having the constraint inside a create table )is not forbidden by SQL, + # of course. I just never saw this in a sql server dump, # so it should be an error for now (it will be syntaxically different if outside a table anyhow) die "Unique key defined outside a table\n: $line" unless ($create_table); @@ -830,10 +831,10 @@ sub parse_dump } # Default values. numeric, then text, then bit - elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(((?:-)?\d+)\)\) FOR \[(.*)\]/) + elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?((?:-)?\d+(?:\.\d+)?)\)?\) FOR \[(.*)\]/) { $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}=$3; - # Default value, for a numeric (yes sql server puts it in another pair of parenthesis, don't know why) + # Default value, for a numeric (yes sql server often puts it in another pair of parenthesis, don't know why) } elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \('(.*)'\) FOR \[(.*)\]/) { @@ -857,6 +858,13 @@ sub parse_dump die "not expected for a boolean: $line $. This is a bug"; # Get an error if the true/false hypothesis is wrong } } + # Yes, we also get default NULL (what for ? :) ) + elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((NULL)\) FOR \[(.*)\]/) + { + # NULL WITHOUT quotes around it ! + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}='NULL'; + + } # FK constraint. It's multi line, we have to look for references, and what to do on update, detele, etc (I have only seen delete cascade for now) elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD\s+CONSTRAINT \[(.*)\] FOREIGN KEY\((.*?)\)/) { From 7dbea7340faaeaec98603301058e22776540d1cd Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 17 Mar 2014 09:11:35 +0100 Subject: [PATCH 006/216] Add a warning when an index with an INCLUDE clause is found --- sqlserver2pgsql.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index b1a07fc..fa412fb 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -806,8 +806,9 @@ sub parse_dump } if ($idx =~ /^INCLUDE \(/) { + print STDERR "This index ($schemaname.$tablename.$idxname) has some include columns. This isn't supported in PostgreSQL.\n"; + print STDERR "The columns in the INCLUDE clause have been ignored.\n"; next; # Nothing equivalent in PG. Maybe if the index isn't unique, these columns should be added? - # Or we should print a warning } } } From 3a95170e2623b7168f966c18eb54a177e448ed12 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 17 Mar 2014 10:54:12 +0100 Subject: [PATCH 007/216] New improvements for default values --- sqlserver2pgsql.pl | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index fa412fb..f32d388 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -584,8 +584,11 @@ sub parse_dump my $startseq=$1; my $stepseq=$2; my $seqname= lc("${tablename}_${colname}_seq"); - $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS}->{$colname}->{DEFAULT}= + # We get a sure default value. + $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS}->{$colname}->{DEFAULT}->{VALUE}= "nextval('" . dboreplace(${schemaname}) . '.' . ${seqname}. "')"; + $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS}->{$colname}->{DEFAULT}->{UNSURE}=0; + $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{START}=$startseq; $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP}=$stepseq; $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{OWNERTABLE}=$tablename . "." . $colname; @@ -831,16 +834,18 @@ sub parse_dump } } - # Default values. numeric, then text, then bit + # Default values. numeric, then text, then bit. These are 100% sure, they will parse in PG elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?((?:-)?\d+(?:\.\d+)?)\)?\) FOR \[(.*)\]/) { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}=$3; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}=$3; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; # Default value, for a numeric (yes sql server often puts it in another pair of parenthesis, don't know why) } elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \('(.*)'\) FOR \[(.*)\]/) { # Default text value, text, between commas - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}="'$3'"; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}="'$3'"; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; } elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((\d)\) FOR \[(.*)\]/) { @@ -848,24 +853,33 @@ sub parse_dump # convert to true/false if ($3 eq '0') { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}='false'; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}='false'; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; } elsif ($3 eq '1') { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}='true'; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}='true'; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; } else { die "not expected for a boolean: $line $. This is a bug"; # Get an error if the true/false hypothesis is wrong } } - # Yes, we also get default NULL (what for ? :) ) - elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((NULL)\) FOR \[(.*)\]/) + # Yes, we also get default NULL (what for ? :) ), and sometimes with a different case + elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(((?i)NULL)\) FOR \[(.*)\]/) { # NULL WITHOUT quotes around it ! - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}='NULL'; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}='NULL'; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; } + # And there are also constraints with functions and other strange code in them. Put them as unsure + elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/) + { + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}=$3; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=1; + } # FK constraint. It's multi line, we have to look for references, and what to do on update, detele, etc (I have only seen delete cascade for now) elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD\s+CONSTRAINT \[(.*)\] FOREIGN KEY\((.*?)\)/) { @@ -1248,7 +1262,15 @@ sub generate_schema { my $colref=$refschema->{TABLES}->{$table}->{COLS}->{$col}; next unless (defined $colref->{DEFAULT}); - print AFTER "ALTER TABLE $schema.$table ALTER COLUMN $col SET DEFAULT " . $colref->{DEFAULT} . ";\n"; + my $definition= "ALTER TABLE $schema.$table ALTER COLUMN $col SET DEFAULT " . $colref->{DEFAULT}->{VALUE} . ";\n"; + if ($colref->{DEFAULT}->{UNSURE}) + { + print UNSURE $definition; + } + else + { + print AFTER $definition; + } } } } From f25b9e0179d74de5ec2c925beb4ae748e672fcef Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 19 Mar 2014 17:42:00 +0100 Subject: [PATCH 008/216] Add support (warning) for calculated columns --- sqlserver2pgsql.pl | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index f32d388..d7baa1a 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -619,6 +619,23 @@ sub parse_dump die "I don't understand $line. This is a bug"; } } + # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) + # So just put it as a varchar, and issue a warning is STDOUT + elsif ($line =~ /^\t\[(.*)\]\s+AS\s+/) + { + # We just get the column name + $colnumber++; + my $colname=$1; + my $coltype='varchar'; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{POS}=$colnumber; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{TYPE}=$coltype; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{NOT_NULL}=0; + # Big fat warning + print STDERR "Warning: There is a calculated column: $schemaname.$tablename.$colname. This isn't done the same way in PG at all\n"; + print STDERR "\tFor now it has been declared as a varchar in PG, so that the values can be copied\n"; + print STDERR "\tYou should change its type manually in the dump (sorry for that),\n"; + print STDERR "\tand write a trigger to maintain this value\n"; + } elsif ($line =~ /^(?: CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED/) { # This is not forbidden by SQL, of course. I just never saw this in a sql server dump, @@ -694,7 +711,7 @@ sub parse_dump } elsif ($line =~ /CREATE\s+PROC(?:EDURE)?\s+\[.*\]\.\[(.*)\]/i) { - print STDERR "Procedure $1 ignored\n"; + print STDERR "Warning: Procedure $1 ignored\n"; # We have to find next GO to know we are out of the procedure while (my $contline=read_and_clean($file)) { @@ -703,7 +720,7 @@ sub parse_dump } elsif ($line =~ /CREATE\s+FUNCTION\s+\[.*\]\.\[(.*)\]/i) { - print STDERR "Function $1 ignored\n"; + print STDERR "Warning: Function $1 ignored\n"; # We have to find next GO to know we are out of the procedure while (my $contline=read_and_clean($file)) { @@ -712,7 +729,7 @@ sub parse_dump } elsif ($line =~ /CREATE\s+TRIGGER\s+\[(.*)\]/i) { - print STDERR "Trigger $1 ignored\n"; + print STDERR "Warning: Trigger $1 ignored\n"; # We have to find next GO to know we are out of the procedure while (my $contline=read_and_clean($file)) { @@ -809,8 +826,8 @@ sub parse_dump } if ($idx =~ /^INCLUDE \(/) { - print STDERR "This index ($schemaname.$tablename.$idxname) has some include columns. This isn't supported in PostgreSQL.\n"; - print STDERR "The columns in the INCLUDE clause have been ignored.\n"; + print STDERR "Warning: This index ($schemaname.$tablename.$idxname) has some include columns. This isn't supported in PostgreSQL.\n"; + print STDERR "\tThe columns in the INCLUDE clause have been ignored.\n"; next; # Nothing equivalent in PG. Maybe if the index isn't unique, these columns should be added? } } From 8b929c56d0bce7b513d70c6b423eafb9c4d9c99e Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 20 Mar 2014 16:40:48 +0100 Subject: [PATCH 009/216] Create a trigger for calculated column --- sqlserver2pgsql.pl | 3002 ++++++++++++++++++++++++-------------------- 1 file changed, 1668 insertions(+), 1334 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index d7baa1a..2a42bf3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -18,9 +18,7 @@ use strict; - - -# Global objects definition structure: we need it to store all seen tables, detect which ones have LOBS +# Global objects definition structure: we need it to store all seen tables, detect which ones have LOBS # and if their PK is a simple integer (so we can parallelize readings on these tables in a special # kettle transformation) @@ -30,186 +28,208 @@ my $objects; # These are global variables, from configuration file or command line arguments -our ($sd,$sh,$sp,$su,$sw,$pd,$ph,$pp,$pu,$pw);# Connection args +our ($sd, $sh, $sp, $su, $sw, $pd, $ph, $pp, $pu, $pw); # Connection args our $conf_file; -our $filename;# Filename passed as arg -our $case_insensitive=0; # Passed as arg: was SQL Server installation case insensitive ? PostgreSQL can't ignore accents anyway - # If yes, we will generate citext with CHECK constraints, that's the best we can do -our $norelabel_dbo=0; # Passed as arg: should we convert DBO to public ? -our $convert_numeric_to_int=0; # Should we convert numerics to int when possible ? (numeric (4,0) could be converted an int, for instance) +our $filename; # Filename passed as arg +our $case_insensitive = 0 + ; # Passed as arg: was SQL Server installation case insensitive ? PostgreSQL can't ignore accents anyway + # If yes, we will generate citext with CHECK constraints, that's the best we can do +our $norelabel_dbo = 0; # Passed as arg: should we convert DBO to public ? +our $convert_numeric_to_int = 0 + ; # Should we convert numerics to int when possible ? (numeric (4,0) could be converted an int, for instance) our $kettle; our $before_file; our $after_file; our $unsure_file; -my $template; # These two variables are loaded in the BEGIN block at the end of this file (they are very big -my $template_lob; # putting them there won't pollute the code as much) -my ($job_header,$job_middle,$job_footer); # These are used to create the static parts of the job -my ($job_entry,$job_hop); # These are used to create the dynamic parts of the job (XML file) - +my $template + ; # These two variables are loaded in the BEGIN block at the end of this file (they are very big +my $template_lob; # putting them there won't pollute the code as much) +my ($job_header, $job_middle, $job_footer) + ; # These are used to create the static parts of the job +my ($job_entry, $job_hop) + ; # These are used to create the dynamic parts of the job (XML file) # Opens the configuration file # Sets $sd $sh $sp $su $sw $pd $ph $pp $pu $pw when they are not set in the command line already # Also gets kettle parameters... sub parse_conf_file { - # Correspondance between conf_file parameter and program variable - # This is also used as the list of accepted parameters in the configuration file - my %parameters=( 'sql server database' => 'sd', - 'sql server host' => 'sh', - 'sql server port' => 'sp', - 'sql server username' => 'su', - 'sql server password' => 'sw', - 'postgresql database' => 'pd', - 'postgresql host' => 'ph', - 'postgresql port' => 'pp', - 'postgresql username' => 'pu', - 'postgresql password' => 'pw', - 'kettle directory' => 'kettle', - 'before file' => 'before_file', - 'after file' => 'after_file', - 'unsure file' => 'unsure_file', - 'sql server dump filename' => 'filename', - 'case insensitive' => 'case_insensitive', - 'no relabel dbo' => 'norelabel_dbo', - 'convert numeric to int' => 'convert_numeric_to_int', - ); - # Open the conf file or die - open CONF,$conf_file or die "Cannot open $conf_file"; - while (my $line = ) - { - $line =~ s/#.*//; # Remove comments - $line =~ s/\s+=\s+//; # Remove whitespaces around the = - $line =~ s/\s+$//; # Remove trailing whitespaces - next if ($line =~ /^$/); # Empty line after comments have been removed - $line =~ /^(.*)=(.*)$/ or die "Cannot parse $line from $conf_file"; - my ($param,$value)=($1,$2); - no strict 'refs'; # Using references by name, temporarily - unless (defined $parameters{$param}) - { - die "Cannot understand parameter $param in $conf_file"; - } - my $param_name=$parameters{$param}; - if (defined $$param_name) - { - next; # Parameter overriden in command line - } - $$param_name=$value; - use strict 'refs'; - } - close CONF; + + # Correspondance between conf_file parameter and program variable + # This is also used as the list of accepted parameters in the configuration file + my %parameters = ('sql server database' => 'sd', + 'sql server host' => 'sh', + 'sql server port' => 'sp', + 'sql server username' => 'su', + 'sql server password' => 'sw', + 'postgresql database' => 'pd', + 'postgresql host' => 'ph', + 'postgresql port' => 'pp', + 'postgresql username' => 'pu', + 'postgresql password' => 'pw', + 'kettle directory' => 'kettle', + 'before file' => 'before_file', + 'after file' => 'after_file', + 'unsure file' => 'unsure_file', + 'sql server dump filename' => 'filename', + 'case insensitive' => 'case_insensitive', + 'no relabel dbo' => 'norelabel_dbo', + 'convert numeric to int' => 'convert_numeric_to_int', + ); + + # Open the conf file or die + open CONF, $conf_file or die "Cannot open $conf_file"; + while (my $line = ) + { + $line =~ s/#.*//; # Remove comments + $line =~ s/\s+=\s+//; # Remove whitespaces around the = + $line =~ s/\s+$//; # Remove trailing whitespaces + next + if ($line =~ /^$/); # Empty line after comments have been removed + $line =~ /^(.*)=(.*)$/ or die "Cannot parse $line from $conf_file"; + my ($param, $value) = ($1, $2); + no strict 'refs'; # Using references by name, temporarily + unless (defined $parameters{$param}) + { + die "Cannot understand parameter $param in $conf_file"; + } + my $param_name = $parameters{$param}; + if (defined $$param_name) + { + next; # Parameter overriden in command line + } + $$param_name = $value; + use strict 'refs'; + } + close CONF; } # Converts numeric(4,0) and similar to int, bigint, smallint sub convert_numeric_to_int { - my ($qual)=@_; - croak "not a good qualifier $qual" unless ($qual =~ /^(\d+),\s*(\d+)$/); - my $precision=$1; - my $scale=$2; - croak "scale should be 0\n" unless ($scale eq '0'); - return 'smallint' if ($precision<=4); - return 'integer' if ($precision<=9); - return 'bigint' if ($precision<=18); - return 'numeric($qual)'; + my ($qual) = @_; + croak "not a good qualifier $qual" unless ($qual =~ /^(\d+),\s*(\d+)$/); + my $precision = $1; + my $scale = $2; + croak "scale should be 0\n" unless ($scale eq '0'); + return 'smallint' if ($precision <= 4); + return 'integer' if ($precision <= 9); + return 'bigint' if ($precision <= 18); + return 'numeric($qual)'; } # These are the no-brainer conversions # There is still a special case for text types and case insensitivity (see convert_type) though -my %types=('int'=>'int', - 'nvarchar'=>'varchar', - 'nchar'=>'char', - 'char'=>'char', - 'varchar'=>'varchar', - 'text'=>'text', - 'char'=>'char', - 'smallint'=>'smallint', - 'tinyint'=>'smallint', - 'bigint'=>'bigint', - 'decimal'=>'numeric', - 'float'=>'double precision', - 'real'=>'real', - 'date'=>'date', - 'datetime'=>'timestamp', - 'smalldatetime'=>'timestamp', - 'timestamp'=>'timestamp', - 'image'=>'bytea', - 'binary'=>'bytea', - 'varbinary'=>'bytea', - 'money'=>'numeric' - ); +my %types = ('int' => 'int', + 'nvarchar' => 'varchar', + 'nchar' => 'char', + 'char' => 'char', + 'varchar' => 'varchar', + 'text' => 'text', + 'char' => 'char', + 'smallint' => 'smallint', + 'tinyint' => 'smallint', + 'bigint' => 'bigint', + 'decimal' => 'numeric', + 'float' => 'double precision', + 'real' => 'real', + 'date' => 'date', + 'datetime' => 'timestamp', + 'smalldatetime' => 'timestamp', + 'timestamp' => 'timestamp', + 'image' => 'bytea', + 'binary' => 'bytea', + 'varbinary' => 'bytea', + 'money' => 'numeric'); # Types with no qualifier, and no point in putting one -my %unqual=('bytea'=>1); +my %unqual = ('bytea' => 1); # This function uses the two static lists above, plus domains and citext types that # may have been created during parsing, to convert mssql's types to pgsql's sub convert_type { - my ($sqlstype,$sqlqual,$colname,$tablename,$typname,$schemaname)=@_; - my $rettype; - if (defined $types{$sqlstype}) - { - if ((defined $sqlqual and defined ($unqual{$types{$sqlstype}})) or not defined $sqlqual) - { - # This is one of the few types that have to be unqualified (binary type) - $rettype= $types{$sqlstype}; - } - elsif (defined $sqlqual) - { - $rettype= ($types{$sqlstype}."($sqlqual)"); - } - } - # A few special cases - elsif ($sqlstype eq 'bit' and not defined $sqlqual) - { - $rettype= "boolean"; - } - elsif ($sqlstype eq 'ntext' and not defined $sqlqual) - { - $rettype= "text"; - } - elsif ($sqlstype eq 'numeric') - { - # Numeric is a special case: - # No qualifier. We have to use numeric - return 'numeric' unless ($sqlqual); - return "numeric($sqlqual)" unless ($sqlqual =~ /\d+,\s*0/); # If the qualifier is not x,0 - return "numeric($sqlqual)" unless ($convert_numeric_to_int); # If we have not activated conversion - return convert_numeric_to_int($sqlqual); # We got there: convert ! - } - else - { - print "Types: " , Dumper(\%types); - croak "Cannot determine the PostgreSQL's datatype corresponding to $sqlstype. This is a bug\n"; - } - # We special case when type is varchar, to be case insensitive - if ($sqlstype =~ /text|varchar/ and $case_insensitive) - { - $rettype="citext"; - # Do we have a SQL qualifier ? (we'll have to do check constraints then) - if ($sqlqual) - { - # Check we have a table name and a colname, or a typname - if (defined $colname and defined $tablename) # We are called from a CREATE TABLE, we have to add a check constraint - { - my $constraint; - $constraint->{TYPE}='CHECK_CITEXT'; - $constraint->{TABLE}=$tablename; - $constraint->{TEXT}="char_length($colname) <= $sqlqual"; - push @{$objects->{$schemaname}->{TABLES}->{$tablename}->{CONSTRAINTS}},($constraint); - } - elsif (defined $typname) # We are called from a CREATE TYPE, which will be converted to a CREATE DOMAIN - { - $rettype="citext CHECK(char_length(value)<=$sqlqual)"; - } - else - { - die "Called in a case sensitive, trying to generate a check constraint, failed. This is a bug!"; - } - } - } - return $rettype; + my ($sqlstype, $sqlqual, $colname, $tablename, $typname, $schemaname) = + @_; + my $rettype; + if (defined $types{$sqlstype}) + { + if ((defined $sqlqual and defined($unqual{$types{$sqlstype}})) + or not defined $sqlqual) + { + # This is one of the few types that have to be unqualified (binary type) + $rettype = $types{$sqlstype}; + } + elsif (defined $sqlqual) + { + $rettype = ($types{$sqlstype} . "($sqlqual)"); + } + } + + # A few special cases + elsif ($sqlstype eq 'bit' and not defined $sqlqual) + { + $rettype = "boolean"; + } + elsif ($sqlstype eq 'ntext' and not defined $sqlqual) + { + $rettype = "text"; + } + elsif ($sqlstype eq 'numeric') + { + + # Numeric is a special case: + # No qualifier. We have to use numeric + return 'numeric' unless ($sqlqual); + return "numeric($sqlqual)" + unless ($sqlqual =~ /\d+,\s*0/); # If the qualifier is not x,0 + return "numeric($sqlqual)" + unless ($convert_numeric_to_int) + ; # If we have not activated conversion + return convert_numeric_to_int($sqlqual); # We got there: convert ! + } + else + { + print "Types: ", Dumper(\%types); + croak + "Cannot determine the PostgreSQL's datatype corresponding to $sqlstype. This is a bug\n"; + } + + # We special case when type is varchar, to be case insensitive + if ($sqlstype =~ /text|varchar/ and $case_insensitive) + { + $rettype = "citext"; + + # Do we have a SQL qualifier ? (we'll have to do check constraints then) + if ($sqlqual) + { + + # Check we have a table name and a colname, or a typname + if ( defined $colname + and defined $tablename + ) # We are called from a CREATE TABLE, we have to add a check constraint + { + my $constraint; + $constraint->{TYPE} = 'CHECK_CITEXT'; + $constraint->{TABLE} = $tablename; + $constraint->{TEXT} = "char_length($colname) <= $sqlqual"; + push @{$objects->{$schemaname}->{TABLES}->{$tablename} + ->{CONSTRAINTS}}, ($constraint); + } + elsif (defined $typname + ) # We are called from a CREATE TYPE, which will be converted to a CREATE DOMAIN + { + $rettype = "citext CHECK(char_length(value)<=$sqlqual)"; + } + else + { + die + "Called in a case sensitive, trying to generate a check constraint, failed. This is a bug!"; + } + } + } + return $rettype; } # This gives the next column position for a table @@ -217,241 +237,275 @@ sub convert_type # These tables are added at the end of the table, in %objects sub next_col_pos { - my ($schema,$table)=@_; - if (defined $objects->{$schema}->{TABlES}->{$table}->{COLS}) - { - my $max=0; - foreach my $col(values (%{$objects->{$schema}->{TABlES}->{$table}->{COLS}})) - { - if ($col->{POS} > $max) - { - $max=$col->{POS}; - } - } - return $max+1; - } - else - { - die "We tried to add a column to an unknown table"; - } + my ($schema, $table) = @_; + if (defined $objects->{$schema}->{TABlES}->{$table}->{COLS}) + { + my $max = 0; + foreach my $col ( + values(%{$objects->{$schema}->{TABlES}->{$table}->{COLS}})) + { + if ($col->{POS} > $max) + { + $max = $col->{POS}; + } + } + return $max + 1; + } + else + { + die "We tried to add a column to an unknown table"; + } } # This relabels the string if it is dbo and we want to relabel it to public sub dboreplace { - my ($schema)=@_; - return $schema if ($schema ne 'dbo'); - return 'public' unless ($norelabel_dbo); - return 'dbo'; + my ($schema) = @_; + return $schema if ($schema ne 'dbo'); + return 'public' unless ($norelabel_dbo); + return 'dbo'; } # Test if we are on windows. We will have to convert / to \ in the XML files sub is_windows { - if ($^O =~ /win/i) - { - return 1; - } - return 0; + if ($^O =~ /win/i) + { + return 1; + } + return 0; } # Die if kettle is not set up correctly sub kettle_die { - my ($file)=@_; - die "You have to set up KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in $file.\nIf this file doesn't exist yet, start spoon from the kettle directory once."; + my ($file) = @_; + die + "You have to set up KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in $file.\nIf this file doesn't exist yet, start spoon from the kettle directory once."; } - # This sub checks ~/.kettle/kettle.properties to be sure # KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y is in place # We die if not sub check_kettle_properties { - my $ok=0; - my $file; - if (!is_windows()) - { - $file= $ENV{'HOME'}.'/.kettle/kettle.properties'; - } - else - { - $file= $ENV{'USERPROFILE'}.'/.kettle/kettle.properties'; - } - open FILE, $file or kettle_die($file); - while () - { - next unless (/KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL\s*=\s*Y/); - $ok=1; - } - close FILE; - if (not $ok) - { - kettle_die($file); - } - return 0; + my $ok = 0; + my $file; + if (!is_windows()) + { + $file = $ENV{'HOME'} . '/.kettle/kettle.properties'; + } + else + { + $file = $ENV{'USERPROFILE'} . '/.kettle/kettle.properties'; + } + open FILE, $file or kettle_die($file); + while () + { + next unless (/KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL\s*=\s*Y/); + $ok = 1; + } + close FILE; + if (not $ok) + { + kettle_die($file); + } + return 0; } # Usage, obviously. Has to be kept in sync with new command line options sub usage { - print "$0 [-k kettle_output_directory] -b before_file -a after_file -u unsure_file -f sql_server_schema_file[-h] [-i]\n"; - print "\nExpects a SQL Server SQL structure dump as -f (preferably unicode)\n"; - print "-i tells $0 to create a case-insensitive PostgreSQL schema\n"; - print "-nr tells $0 not to convert the dbo schema to public. dbo will stay dbo\n"; - print "-num tells $0 to convert numeric xxx,0 to int, bigint, etc. Will not keep numeric scale and precision for the converted\n"; - print "before_file contains the structure\n"; - print "after_file contains index, constraints\n"; - print "unsure_file contains things we cannot guarantee will work, such as views\n"; - print "\n"; - print "If you are generating for kettle, you'll need to provide connection information\n"; - print "for connecting to both databases:\n"; - print "-sd: sqlserver database\n"; - print "-sh: sqlserver host\n"; - print "-sp: sqlserver port\n"; - print "-su: sqlserver username\n"; - print "-sw: sqlserver password\n"; - print "-pd: postgresql database\n"; - print "-ph: postgresql host\n"; - print "-pp: postgresql port\n"; - print "-pu: postgresql username\n"; - print "-pw: postgresql password\n"; + print + "$0 [-k kettle_output_directory] -b before_file -a after_file -u unsure_file -f sql_server_schema_file[-h] [-i]\n"; + print + "\nExpects a SQL Server SQL structure dump as -f (preferably unicode)\n"; + print "-i tells $0 to create a case-insensitive PostgreSQL schema\n"; + print + "-nr tells $0 not to convert the dbo schema to public. dbo will stay dbo\n"; + print + "-num tells $0 to convert numeric xxx,0 to int, bigint, etc. Will not keep numeric scale and precision for the converted\n"; + print "before_file contains the structure\n"; + print "after_file contains index, constraints\n"; + print + "unsure_file contains things we cannot guarantee will work, such as views\n"; + print "\n"; + print + "If you are generating for kettle, you'll need to provide connection information\n"; + print "for connecting to both databases:\n"; + print "-sd: sqlserver database\n"; + print "-sh: sqlserver host\n"; + print "-sp: sqlserver port\n"; + print "-su: sqlserver username\n"; + print "-sw: sqlserver password\n"; + print "-pd: postgresql database\n"; + print "-ph: postgresql host\n"; + print "-pp: postgresql port\n"; + print "-pu: postgresql username\n"; + print "-pw: postgresql password\n"; } # This function generates kettle transformations, and a kettle job running all these # transformations sequentially, for all the tables, in all the schemas, in sql server's dump sub generate_kettle { - my ($dir)=@_; - # first, create the kettle directory - unless (-d $dir) - { - mkdir ($dir) or die "Cannot create $dir"; - } - # For each table in each schema in $objects, we generate a kettle file in the directory - - foreach my $schema ( sort keys %{$objects}) - { - my $refschema=$objects->{$schema}; - my $targetschema=dboreplace($schema); - - foreach my $table (sort keys %{$refschema->{TABLES}}) - { - # First, does this table have LOBs ? The template depends on this - my $newtemplate; - if ($refschema->{TABLES}->{$table}->{haslobs}) - { - $newtemplate=$template_lob; - # Is the PK int and on only one column ? - # If yes, we can use several threads in kettle to read this table to - # improve performance - if (defined ($refschema->{TABLES}->{$table}->{PK}->{COLS}) and scalar(@{$refschema->{TABLES}->{$table}->{PK}->{COLS}})==1 - and - ($refschema->{TABLES}->{$table}->{COLS}->{($refschema->{TABLES}->{$table}->{PK}->{COLS}->[0])}->{TYPE} =~ /int$/) - ) - { - my $wherefilter='WHERE ' . $refschema->{TABLES}->{$table}->{PK}->{COLS}->[0] - . '% ${Internal.Step.Unique.Count} = ${Internal.Step.Unique.Number}'; - $newtemplate =~ s/__sqlserver_where_filter__/$wherefilter/; - $newtemplate =~ s/__sqlserver_copies__/4/g; - } - else - # No way to do this optimization. Use standard template - { - $newtemplate =~ s/__sqlserver_where_filter__//; - $newtemplate =~ s/__sqlserver_copies__/1/g - } - } - else - { - $newtemplate=$template; - } - # Substitute every connection placeholder with the real value - $newtemplate =~ s/__sqlserver_database__/$sd/g; - $newtemplate =~ s/__sqlserver_host__/$sh/g; - $newtemplate =~ s/__sqlserver_port__/$sp/g; - $newtemplate =~ s/__sqlserver_username__/$su/g; - $newtemplate =~ s/__sqlserver_password__/$sw/g; - $newtemplate =~ s/__postgres_database__/$pd/g; - $newtemplate =~ s/__postgres_host__/$ph/g; - $newtemplate =~ s/__postgres_port__/$pp/g; - $newtemplate =~ s/__postgres_username__/$pu/g; - $newtemplate =~ s/__postgres_password__/$pw/g; - $newtemplate =~ s/__sqlserver_table_name__/$schema.$table/g; - $newtemplate =~ s/__postgres_table_name__/$table/g; - $newtemplate =~ s/__postgres_schema_name__/$targetschema/g; - # Store this new transformation into its file - open FILE, ">$dir/$schema-$table.ktr" or die "Cannot write to $dir/$schema-$table.ktr"; - print FILE $newtemplate; - close FILE; - } - } - # All transformations are done - # We have to create a job to launch everything in one go - open FILE, ">$dir/migration.kjb" or die "Cannot write to $dir/migration.kjb"; - my $real_dir=getcwd; - my $entries=''; - my $hops=''; - my $prev_node='START'; - my $cur_vert_pos=100; # Not that useful, it's just not to be ugly if someone wanted to open - # the job with spoon (kettle's gui) and work on it graphically - # We sort only so that it will be easier to find a transformation in the job if one needed to - # edit it. It's also easier to track progress if tables are sorted alphabetically - foreach my $schema(sort keys %{$objects}) - { - my $refschema=$objects->{$schema}; - foreach my $table (sort {lc($a) cmp lc($b)}keys %{$refschema->{TABLES}}) - { - my $tmp_entry=$job_entry; - # We build the entries with regexp substitutions. The tablename contains the schema - $tmp_entry =~ s/__table_name__/${schema}_${table}/; - # Filename to use. We need the full path to the transformations - my $filename; - if ( $dir =~ /^(\\|\/)/) # Absolute path - { - $filename = $dir . '/' . $schema . '-' . $table . '.ktr'; - } - else - { - $filename = $real_dir . '/' . $dir . '/' . $schema . '-' . $table . '.ktr'; - } - # Different for windows and linux, obviously: we change / to \ for windows - unless (is_windows()) - { - $filename =~ s/\////g; - } - else - { - $filename =~ s/\//\\/g; - } - $tmp_entry =~ s/__file_name__/$filename/; - $tmp_entry =~ s/__y_loc__/$cur_vert_pos/; - $entries.=$tmp_entry; - - # We build the hop with the regexp too - my $tmp_hop=$job_hop; - $tmp_hop =~ s/__table_1__/$prev_node/; - $tmp_hop =~ s/__table_2__/${schema}_${table}/; - if ($prev_node eq 'START') - { - # Specific to the start node. It has to be unconditional - $tmp_hop =~ s/N<\/unconditional>/Y<\/unconditional>/; - } - $hops.=$tmp_hop; - - # We increment everything for next loop - $prev_node="${schema}_${table}"; # For the next hop - $cur_vert_pos+=80; # To be pretty in spoon - } - } - - print FILE $job_header; - print FILE $entries; - print FILE $job_middle; - print FILE $hops; - print FILE $job_footer; - close FILE; - + my ($dir) = @_; + + # first, create the kettle directory + unless (-d $dir) + { + mkdir($dir) or die "Cannot create $dir"; + } + + # For each table in each schema in $objects, we generate a kettle file in the directory + + foreach my $schema (sort keys %{$objects}) + { + my $refschema = $objects->{$schema}; + my $targetschema = dboreplace($schema); + + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + + # First, does this table have LOBs ? The template depends on this + my $newtemplate; + if ($refschema->{TABLES}->{$table}->{haslobs}) + { + $newtemplate = $template_lob; + + # Is the PK int and on only one column ? + # If yes, we can use several threads in kettle to read this table to + # improve performance + if (defined($refschema->{TABLES}->{$table}->{PK}->{COLS}) + and + scalar(@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}) + == 1 + and ($refschema->{TABLES}->{$table}->{COLS} + ->{($refschema->{TABLES}->{$table}->{PK}->{COLS}->[0]) + }->{TYPE} =~ /int$/)) + { + my $wherefilter = + 'WHERE ' + . $refschema->{TABLES}->{$table}->{PK}->{COLS}->[0] + . '% ${Internal.Step.Unique.Count} = ${Internal.Step.Unique.Number}'; + $newtemplate =~ + s/__sqlserver_where_filter__/$wherefilter/; + $newtemplate =~ s/__sqlserver_copies__/4/g; + } + else + + # No way to do this optimization. Use standard template + { + $newtemplate =~ s/__sqlserver_where_filter__//; + $newtemplate =~ s/__sqlserver_copies__/1/g; + } + } + else + { + $newtemplate = $template; + } + + # Substitute every connection placeholder with the real value + $newtemplate =~ s/__sqlserver_database__/$sd/g; + $newtemplate =~ s/__sqlserver_host__/$sh/g; + $newtemplate =~ s/__sqlserver_port__/$sp/g; + $newtemplate =~ s/__sqlserver_username__/$su/g; + $newtemplate =~ s/__sqlserver_password__/$sw/g; + $newtemplate =~ s/__postgres_database__/$pd/g; + $newtemplate =~ s/__postgres_host__/$ph/g; + $newtemplate =~ s/__postgres_port__/$pp/g; + $newtemplate =~ s/__postgres_username__/$pu/g; + $newtemplate =~ s/__postgres_password__/$pw/g; + $newtemplate =~ s/__sqlserver_table_name__/$schema.$table/g; + $newtemplate =~ s/__postgres_table_name__/$table/g; + $newtemplate =~ s/__postgres_schema_name__/$targetschema/g; + + # Store this new transformation into its file + open FILE, ">$dir/$schema-$table.ktr" + or die "Cannot write to $dir/$schema-$table.ktr"; + print FILE $newtemplate; + close FILE; + } + } + + # All transformations are done + # We have to create a job to launch everything in one go + open FILE, ">$dir/migration.kjb" + or die "Cannot write to $dir/migration.kjb"; + my $real_dir = getcwd; + my $entries = ''; + my $hops = ''; + my $prev_node = 'START'; + my $cur_vert_pos = 100 + ; # Not that useful, it's just not to be ugly if someone wanted to open + # the job with spoon (kettle's gui) and work on it graphically + # We sort only so that it will be easier to find a transformation in the job if one needed to + # edit it. It's also easier to track progress if tables are sorted alphabetically + + foreach my $schema (sort keys %{$objects}) + { + my $refschema = $objects->{$schema}; + foreach my $table (sort { lc($a) cmp lc($b) } + keys %{$refschema->{TABLES}}) + { + my $tmp_entry = $job_entry; + + # We build the entries with regexp substitutions. The tablename contains the schema + $tmp_entry =~ s/__table_name__/${schema}_${table}/; + + # Filename to use. We need the full path to the transformations + my $filename; + if ($dir =~ /^(\\|\/)/) # Absolute path + { + $filename = $dir . '/' . $schema . '-' . $table . '.ktr'; + } + else + { + $filename = + $real_dir . '/' + . $dir . '/' + . $schema . '-' + . $table . '.ktr'; + } + + # Different for windows and linux, obviously: we change / to \ for windows + unless (is_windows()) + { + $filename =~ s/\////g; + } + else + { + $filename =~ s/\//\\/g; + } + $tmp_entry =~ s/__file_name__/$filename/; + $tmp_entry =~ s/__y_loc__/$cur_vert_pos/; + $entries .= $tmp_entry; + + # We build the hop with the regexp too + my $tmp_hop = $job_hop; + $tmp_hop =~ s/__table_1__/$prev_node/; + $tmp_hop =~ s/__table_2__/${schema}_${table}/; + if ($prev_node eq 'START') + { + + # Specific to the start node. It has to be unconditional + $tmp_hop =~ + s/N<\/unconditional>/Y<\/unconditional>/; + } + $hops .= $tmp_hop; + + # We increment everything for next loop + $prev_node = "${schema}_${table}"; # For the next hop + $cur_vert_pos += 80; # To be pretty in spoon + } + } + + print FILE $job_header; + print FILE $entries; + print FILE $job_middle; + print FILE $hops; + print FILE $job_footer; + close FILE; } @@ -460,883 +514,1140 @@ sub generate_kettle # It takes into account the status (in or out of comment) of the previous line, hence # the scoped $in_comment { - my $in_comment=0; - - sub read_and_clean - { - my ($fd)=@_; - my $line=<$fd>; - return undef if (not defined $line); - $line =~ s/\r//g; # Remove \r from windows output - $line =~ s/EXEC(ute)?\s*(dbo|sys)\.sp_executesql( \@statement =)? N'//i; # Remove executesql… it's a bit weird in the SQL Server's dump - # If we are not in comment, we look for /* - # If we are in comment, we look for */, and we remove everything until */ - if (not $in_comment) - { - # We first remove all one-line only comments (there may be several on this line) - $line =~ s/\/\*.*?\*\///g; - # Is there a comment left ? - if ($line =~ /\/\*/) - { - $in_comment=1; - $line =~ s/\/\*.*//; # Remove everything after the comment - } - } - else - { - # We do the reverse: keep only what is not commented - $line =~ s/\*\/(.*?)\/\*/$1/g; - # Is there an uncomment left ? - if ($line =~ /\*\//) - { - $in_comment=0; - $line =~ s/.*\*\///; # Remove everything before the uncomment - } - else - { - # There is no uncomment. The line should be empty - $line = "\n"; - } - } - return $line; - } + my $in_comment = 0; + + sub read_and_clean + { + my ($fd) = @_; + my $line = <$fd>; + return undef if (not defined $line); + $line =~ s/\r//g; # Remove \r from windows output + $line =~ + s/EXEC(ute)?\s*(dbo|sys)\.sp_executesql( \@statement =)? N'//i + ; # Remove executesql… it's a bit weird in the SQL Server's dump + # If we are not in comment, we look for /* + # If we are in comment, we look for */, and we remove everything until */ + if (not $in_comment) + { + + # We first remove all one-line only comments (there may be several on this line) + $line =~ s/\/\*.*?\*\///g; + + # Is there a comment left ? + if ($line =~ /\/\*/) + { + $in_comment = 1; + $line =~ s/\/\*.*//; # Remove everything after the comment + } + } + else + { + # We do the reverse: keep only what is not commented + $line =~ s/\*\/(.*?)\/\*/$1/g; + + # Is there an uncomment left ? + if ($line =~ /\*\//) + { + $in_comment = 0; + $line =~ s/.*\*\///; # Remove everything before the uncomment + } + else + { + # There is no uncomment. The line should be empty + $line = "\n"; + } + } + return $line; + } } + # Reads the dump passed as -f # Generates the $object structure # That's THE MAIN FUNCTION sub parse_dump { - # Open the input file or die. This first pass is to detect encoding, and open it correctly afterwards - my $data; - my $file; - open $file,"<$filename" or die "Cannot open $filename"; - while (my $line=<$file>) - { - $data.=$line; - } - close $file; - - # We now ask guess... - my $decoder=guess_encoding($data, qw/iso8859-15/); - die $decoder unless ref($decoder); - - # If we got to here, it means we have found the right decoder - # or at least, perl thinks it has :) - open $file,"<:encoding(".$decoder->name.")",$filename or die "Cannot open $filename"; - - # Parsing loop variables - my $create_table=0; # Are we in a create table statement ? - my $tablename=''; # If yes, what's the table name ? - my $schemaname=''; # If yes, what's the schema name ? - my $colnumber=0; # Column number (just to put the commas in the right places) ? - - # Tagged because sql statements are often multi-line, so there are inner loops in some conditions - MAIN: while (my $line=read_and_clean($file)) - { - # Create table, obviously. There will be other lines below for the rest of the table definition - if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\(/) - { - $create_table=1; # We are now inside a create table - $schemaname=$1; - $tablename=$2; - $colnumber=0; - $objects->{$schemaname}->{TABLES}->{$tablename}->{haslobs}=0; - } - # Here is a col definition. We should be inside a create table - elsif ($line =~ /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))? (NOT NULL|NULL)(,)?/) - { - if ($create_table) # We are inside a create table, this is a column definition - { - $colnumber++; - my $colname=$1; - my $coltypeschema=$2; - my $coltype=$3; - if (defined $coltypeschema) - { - # The datatype is a user defined datatype - # It has already been declared before. We just need to find it - $coltype=$coltypeschema . '.' . $coltype; - } - my $colqual=$4; - my $isidentity=$5; - my $colisnull=$6; - if ($colqual) - { - if ($colqual eq '(max)') - { - $colqual=undef; # max in sql server is the same as putting no colqual in pg - } - else - { - # We need the number (or 2 numbers) in this qual - $colqual=~ /\((\d+(?:,\s*\d+)?)\)/ or die "Cannot parse colqual <$colqual>"; - $colqual= "$1"; - } - } - my $newtype=convert_type($coltype,$colqual,$colname,$tablename,undef,$schemaname); - # If it is an identity, we'll map to serial/bigserial (create a sequence, then link it - # to the column) - if ($isidentity) - { - # We have an identity field. We remember the default value and - # initialize the sequence correctly in the after script - $isidentity=~ /IDENTITY\((\d+),\s*(\d+)\)/ or die "Cannot understand <$isidentity>"; - my $startseq=$1; - my $stepseq=$2; - my $seqname= lc("${tablename}_${colname}_seq"); - # We get a sure default value. - $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS}->{$colname}->{DEFAULT}->{VALUE}= - "nextval('" . dboreplace(${schemaname}) . '.' . ${seqname}. "')"; - $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS}->{$colname}->{DEFAULT}->{UNSURE}=0; - - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{START}=$startseq; - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP}=$stepseq; - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{OWNERTABLE}=$tablename . "." . $colname; - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{OWNERSCHEMA}=$schemaname; - } - # If there is a bytea generated, this table will contain a blob: - # use a special kettle transformation for it if generating kettle - # (see generate_kettle() ) - if ($newtype eq 'bytea' or $coltype eq 'ntext') # Ntext is very slow, stored out of page - { - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{haslobs}=1; - } - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{POS}=$colnumber; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{TYPE}=$newtype; - - - if ($colisnull eq 'NOT NULL') - { - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{NOT_NULL}=1; - } - else - { - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{NOT_NULL}=0; - } - } - else - { - die "I don't understand $line. This is a bug"; - } - } - # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) - # So just put it as a varchar, and issue a warning is STDOUT - elsif ($line =~ /^\t\[(.*)\]\s+AS\s+/) - { - # We just get the column name - $colnumber++; - my $colname=$1; - my $coltype='varchar'; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{POS}=$colnumber; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{TYPE}=$coltype; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS}->{$colname}->{NOT_NULL}=0; - # Big fat warning - print STDERR "Warning: There is a calculated column: $schemaname.$tablename.$colname. This isn't done the same way in PG at all\n"; - print STDERR "\tFor now it has been declared as a varchar in PG, so that the values can be copied\n"; - print STDERR "\tYou should change its type manually in the dump (sorry for that),\n"; - print STDERR "\tand write a trigger to maintain this value\n"; - } - elsif ($line =~ /^(?: CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED/) - { - # This is not forbidden by SQL, of course. I just never saw this in a sql server dump, - # so it should be an error for now (it will be syntaxically different if outside a table anyhow) - die "PK defined outside a table\n: $line" unless ($create_table); - - my $constraint; # We put everything inside this hashref, we'll push it into the constraint list later - $constraint->{TYPE}='PK'; - if (defined $1) - { - $constraint->{NAME}=$1; - } - - # Here is the PK. We read the following lines until the end of the constraint - while (my $pk=read_and_clean($file)) - { - # Exit when read a line beginning with ). The constraint is complete. We store it and go back to main loop - if ($pk =~ /^\)/) - { - push @{$objects->{$schemaname}->{TABLES}->{$tablename}->{CONSTRAINTS}},($constraint); - # We also directly put the constraint reference in a direct path (for ease of use in generate_kettle) - $objects->{$schemaname}->{TABLES}->{$tablename}->{PK}=$constraint; - next MAIN; - } - if ($pk =~ /^\t\[(.*)\] (ASC|DESC)(,?)/) - { - push @{$constraint->{COLS}},($1); - } - } - } - elsif ($line =~ /^\s*(?:CONSTRAINT \[(.*)\] )?UNIQUE/) - { - # This (having the constraint inside a create table )is not forbidden by SQL, - # of course. I just never saw this in a sql server dump, - # so it should be an error for now (it will be syntaxically different if outside a table anyhow) - die "Unique key defined outside a table\n: $line" unless ($create_table); - - my $constraint; # We put everything inside this hashref, we'll push it into the constraint list later - $constraint->{TYPE}='UNIQUE'; - if (defined $1) - { - $constraint->{NAME}=$1; - } - # Unique key definition. We read following lines until the end of the constraint - while (my $uk=read_and_clean($file)) - { - # Exit when read a line beginning with ). The constraint is complete - if ($uk =~ /^\)/) - { - push @{$objects->{$schemaname}->{'TABLES'}->{$tablename}->{CONSTRAINTS}},($constraint); - next MAIN; - } - if ($uk =~ /^\t\[(.*)\] (ASC|DESC)(,?)/) - { - push @{$constraint->{COLS}},($1); - } - } - - } - elsif ($line =~ /^\) ON \[PRIMARY\]/) - { - # End of the table - $create_table=0; - $tablename=''; - } - ################################################################ - # From HERE, these SQL commands are not linked to a create table - ################################################################ - - elsif ($line =~ /CREATE SCHEMA \[(.*)\] AUTHORIZATION \[.*\]/) - { - $objects->{$1}=undef; # Nothing to add here, we create the schema, and put undef in it for now - } - elsif ($line =~ /CREATE\s+PROC(?:EDURE)?\s+\[.*\]\.\[(.*)\]/i) - { - print STDERR "Warning: Procedure $1 ignored\n"; - # We have to find next GO to know we are out of the procedure - while (my $contline=read_and_clean($file)) - { - next MAIN if ($contline =~ /^GO$/); - } - } - elsif ($line =~ /CREATE\s+FUNCTION\s+\[.*\]\.\[(.*)\]/i) - { - print STDERR "Warning: Function $1 ignored\n"; - # We have to find next GO to know we are out of the procedure - while (my $contline=read_and_clean($file)) - { - next MAIN if ($contline =~ /^GO$/); - } - } - elsif ($line =~ /CREATE\s+TRIGGER\s+\[(.*)\]/i) - { - print STDERR "Warning: Trigger $1 ignored\n"; - # We have to find next GO to know we are out of the procedure - while (my $contline=read_and_clean($file)) - { - next MAIN if ($contline =~ /^GO$/); - } - } - # Now we parse the create view. It is multi-line, so the code looks like like create table: we parse everything until a line - # containing only a single quote (end of the dbo.sp_executesql) - # The problem is that SQL Server seems to be spitting the original query used to create the view, not a normalized version - # of it, as PostgreSQL does. So we capture the query, and hope it works for now. - elsif ($line =~ /^\s*(create\s*view)\s*(?:\[(\S+)\]\.)?\[(.*?)\]\s*(.*)$/i) - { - my $viewname=$3; - my $schemaname; - if (defined $2) - { - $schemaname=$2; - } - else - { - $schemaname='dbo'; - } - $schemaname=dboreplace($schemaname); - - my $sql=$1 . ' ' . $schemaname . '.' . $3 . ' ' . $4 . "\n"; - while (my $line_cont=read_and_clean($file)) - { - if ($line_cont =~ /^\s*'\s*|^GO$/) # We may have a quote if the view is 'quoted', or a real sql query - { - # The view definition is complete. - # We get rid of dbo. schemas - $sql =~ s/(dbo)\./dboreplace($1) . '.'/eg; # We put this in the replacement schema - - # Views will be stored without the full schema in them. We will - # have to generate the schema in the output file - $objects->{$schemaname}->{'VIEWS'}->{$viewname}->{SQL}=$sql; - next MAIN; - } - $sql.=$line_cont; - } - } - - # I only have seen types with added constraints ( create type foo varchar(50)) for now - # These are domains with PostgreSQL - elsif ($line =~ /^CREATE TYPE \[(.*?)\]\.\[(.*?)\] FROM \[(.*?)](?:\((\d+(?:,\s*\d+)?)?\))?/) - { - # Dependency between types is not done for now. If the problem arises, it should be added - my ($schema,$type,$origtype,$quals)=($1,$2,$3,$4); - my $newtype=convert_type($origtype,$quals,undef,undef,$type,$schema); - $objects->{$schema}->{DOMAINS}->{$type}=$newtype; - - # We add them to known data types, as they probably will be used in table definitions - # but they point to themselves, with the schema corrected: we want them substituted by themselves - $types{$schema . '.' . $type}=dboreplace($schema) . '.' . $type; # We store the schema with it - } - elsif ($line =~ /^CREATE (UNIQUE )?(NONCLUSTERED|CLUSTERED) INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/) - { - # Index creation. Index are namespaced per table in SQL Server, not in PostgreSQL - # In PostgreSQL they are in the same namespace as the tables, and in the same - # schema as the table they are attached to - # So we store them in $objects, attached to the table - # Conflicts will be sorted by resolve_name_conflicts() later - my $isunique=$1; - my $isclustered=$2; - my $idxname=$3; - my $schemaname=$4; - my $tablename=$5; - if ($isunique) - { - $objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES}->{$idxname}->{UNIQUE}=1; - } - else - { - $objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES}->{$idxname}->{UNIQUE}=0; - } - while (my $idx=read_and_clean($file)) - { - # Exit when read a line with a GO. The index is complete - if ($idx =~ /^GO/) - { - next MAIN; - } - next if ($idx =~ /^\(|^\)/); # Begin/end of the columns declaration - if ($idx =~ /\t\[(.*)\] (ASC|DESC)(,)?/) - { - if (defined $2) - { - push @{$objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES}->{$idxname}->{COLS}}, ("$1 $2"); - } - else - { - push @{$objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES}->{$idxname}->{COLS}}, ("$1"); - } - } - if ($idx =~ /^INCLUDE \(/) - { - print STDERR "Warning: This index ($schemaname.$tablename.$idxname) has some include columns. This isn't supported in PostgreSQL.\n"; - print STDERR "\tThe columns in the INCLUDE clause have been ignored.\n"; - next; # Nothing equivalent in PG. Maybe if the index isn't unique, these columns should be added? - } - } - } - # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. - # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table - elsif ($line =~ /^ALTER TABLE \[.*\]\.\[(.*)\] ADD \[(.*)\] (?:\[.*\]\.)?\[(.*)\](\(.+?\))?( .*\(\d+,\s*\d+\))? (NOT NULL|NULL)$/) - { - # For now I don't know what to do with them. So die - die "$line: not understood. This is a bug"; - } - # Table constraints - # Primary key. Multiline - elsif ($line =~ /^ALTER TABLE \[.*\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED)?/) - { - # Never seen one for now. The code is there though, in case, with a die for now - die "$line: not understood. This is a bug"; - while (my $contline=read_and_clean($file)) - { - next MAIN if ($contline =~ /^GO/); - } - } - - # Default values. numeric, then text, then bit. These are 100% sure, they will parse in PG - elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?((?:-)?\d+(?:\.\d+)?)\)?\) FOR \[(.*)\]/) - { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}=$3; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; - # Default value, for a numeric (yes sql server often puts it in another pair of parenthesis, don't know why) - } - elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \('(.*)'\) FOR \[(.*)\]/) - { - # Default text value, text, between commas - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}="'$3'"; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; - } - elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((\d)\) FOR \[(.*)\]/) - { - # Weird one: for bit type, there is no supplementary parenthesis... for now, let's say this is a bit type, and - # convert to true/false - if ($3 eq '0') - { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}='false'; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; - } - elsif ($3 eq '1') - { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}='true'; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; - } - else - { - die "not expected for a boolean: $line $. This is a bug"; # Get an error if the true/false hypothesis is wrong - } - } - # Yes, we also get default NULL (what for ? :) ), and sometimes with a different case - elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(((?i)NULL)\) FOR \[(.*)\]/) - { - # NULL WITHOUT quotes around it ! - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}='NULL'; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=0; - - } - # And there are also constraints with functions and other strange code in them. Put them as unsure - elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/) - { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE}=$3; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE}=1; - } - # FK constraint. It's multi line, we have to look for references, and what to do on update, detele, etc (I have only seen delete cascade for now) - elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD\s+CONSTRAINT \[(.*)\] FOREIGN KEY\((.*?)\)/) - { - # This is a FK definition. We have the foreign table definition in next line. - my $constraint; - my $table=$2; - my $schema=$1; - $constraint->{TYPE}='FK'; - $constraint->{LOCAL_COLS}=$4; - $constraint->{LOCAL_TABLE}=$2; - $constraint->{LOCAL_COLS} =~ s/\[|\]//g; # Remove brackets - while (my $fk = read_and_clean($file)) - { - if ($fk =~ /^GO/) - { - push @{$objects->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}},($constraint); - next MAIN; - } - elsif ($fk =~ /^REFERENCES \[(.*)\]\.\[(.*)\] \((.*?)\)/) - { - $constraint->{REMOTE_COLS}=$3; - $constraint->{REMOTE_TABLE}=$2; - $constraint->{REMOTE_SCHEMA}=$1; - $constraint->{REMOTE_COLS} =~ s/\[|\]//g; # Get rid of square brackets - } - elsif ($fk =~ /^ON DELETE CASCADE\s*$/) - { - $constraint->{ON_DEL_CASC}=1; - } - elsif ($fk =~ /^ON UPDATE CASCADE\s*$/) - { - $constraint->{ON_UPD_CASC}=1; - } - else - { - die "Cannot parse $fk $., in a FK. This is a bug"; - } - } - } - # Check constraint. As it can be arbitrary code, we just get this code, and hope it will work on PG (it will be stored in a special script file) - elsif ($line =~ /ALTER TABLE \[(.*)\]\.\[(.*)\] WITH (?:NO)?CHECK ADD CONSTRAINT \[(.*)\] CHECK \(\((.*)\)\)/ ) - { - # Check constraint. We'll do what we can, syntax may be different. - my $constraint; - my $table=$2; - my $constxt=$4; - my $schema=$1; - $constraint->{TABLE}=$table; - $constraint->{NAME}=$3; - $constraint->{TYPE}='CHECK'; - $constxt =~ s/\[(\S+)\]/$1/g; # We remove the []. And hope this will parse - $constraint->{TEXT}=$constxt; - push @{$objects->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}},($constraint); - } - # These are comments or extended attributes on objets. They can be multiline, so aggregate everything - # Until next GO command - # If fact in can be a lot of things. So we have to ignore things like MS_DiagramPaneCount - elsif ($line =~ /^EXEC sys.sp_addextendedproperty/) - { - my $sqlproperty=$line; - while (my $inline=read_and_clean($file)) - { - last if ($inline =~ /^GO/); - $sqlproperty.=$inline - } - - - # We have all the extended property. Let's parse it. - - # First step: what kind is it ? we are only interested in comments for now - $sqlproperty =~ /\@name=N'(.*?)'/ or die "Cannot find a name for this extended property: $sqlproperty"; - my $propertyname=$1; - if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1)$/) - { - # We don't dump these. They are graphical descriptions of the GUI - next; - } - - elsif ($propertyname eq 'MS_Description') - { - # This is a comment. We parse it. - # Spaces are mostly random it seems, in SQL Server's dump code. So \s* everywhere :( - # There can be quotes inside a string. So (?{$schema}->{TABLES}->{$objname}->{COMMENT}=$comment; - } - elsif ($obj eq 'VIEW' and not defined $subobj) - { - $objects->{$schema}->{VIEWS}->{$objname}->{COMMENT}=$comment; - } - elsif ($obj eq 'TABLE' and $subobj eq 'COLUMN') - { - $objects->{$schema}->{TABLES}->{$objname}->{COLS}->{$subobjname}->{COMMENT}=$comment; - } - else - { - die "Cannot understand this comment: $sqlproperty"; - } - } - else - { - die "Don't know what to do with this extendedproperty: $sqlproperty"; - } - } - # Ignore USE, GO, and things that have no meaning for postgresql - elsif ($line =~ /^USE\s|^GO\s*$|\/\*\*\*\*|^SET ANSI_NULLS (ON|OFF)|^SET QUOTED_IDENTIFIER|^SET ANSI_PADDING|CHECK CONSTRAINT|^BEGIN|^END/) - { - next; - } - elsif ($line =~ /^--/) # Comment - { - next; - } - # Don't know what it is. If you know, and it is worth converting, tell me :) - elsif ($line =~ /^EXEC .*bindrule/) - { - next; - } - # Ignore users and roles. Security models will probably be very different between the two databases - elsif ($line =~ /^CREATE (ROLE|USER)/) - { - next; - } - # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views - elsif ($line =~ /^IF NOT EXISTS/) - { - next; - } - # Ignore CREATE DATABASE: we hope that we are given a single database as an option. It is multiline. - # Ignore everything until next GO - # Ignore ALTER DATABASE for the same reason. The given parameters have no meaning in PG anyway - # Same for tests about full text search. - elsif ($line =~ /^(CREATE|ALTER) DATABASE|^IF \(1 = FULLTEXTSERVICEPROPERTY/) - { - while ($line !~ /^GO$/) - { - $line=read_and_clean($file); - } - # We read everything in the CREATE DATABASE. Back to work ! - next; - } - # Ignore EXEC dbo.sp_executesql, for now only seen for a create view. Views sql command aren't executed directly, don't know why - elsif ($line =~ /^EXEC dbo.sp_executesql/) - { - next; - } - # Still on views: there are empty lines, and C-style comments - elsif ($line =~ /^\s*$/) - { - next; - } - else - { - die "Line <$line> ($.) not understood. This is a bug"; - } - } - close $file; + + # Open the input file or die. This first pass is to detect encoding, and open it correctly afterwards + my $data; + my $file; + open $file, "<$filename" or die "Cannot open $filename"; + while (my $line = <$file>) + { + $data .= $line; + } + close $file; + + # We now ask guess... + my $decoder = guess_encoding($data, qw/iso8859-15/); + die $decoder unless ref($decoder); + + # If we got to here, it means we have found the right decoder + # or at least, perl thinks it has :) + open $file, "<:encoding(" . $decoder->name . ")", $filename + or die "Cannot open $filename"; + + # Parsing loop variables + my $create_table = 0; # Are we in a create table statement ? + my $tablename = ''; # If yes, what's the table name ? + my $schemaname = ''; # If yes, what's the schema name ? + my $colnumber = + 0; # Column number (just to put the commas in the right places) ? + + # Tagged because sql statements are often multi-line, so there are inner loops in some conditions + MAIN: while (my $line = read_and_clean($file)) + { + + # Create table, obviously. There will be other lines below for the rest of the table definition + if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\(/) + { + $create_table = 1; # We are now inside a create table + $schemaname = $1; + $tablename = $2; + $colnumber = 0; + $objects->{$schemaname}->{TABLES}->{$tablename}->{haslobs} = 0; + } + + # Here is a col definition. We should be inside a create table + elsif ($line =~ + /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))? (NOT NULL|NULL)(,)?/ + ) + { + if ($create_table + ) # We are inside a create table, this is a column definition + { + $colnumber++; + my $colname = $1; + my $coltypeschema = $2; + my $coltype = $3; + if (defined $coltypeschema) + { + + # The datatype is a user defined datatype + # It has already been declared before. We just need to find it + $coltype = $coltypeschema . '.' . $coltype; + } + my $colqual = $4; + my $isidentity = $5; + my $colisnull = $6; + if ($colqual) + { + if ($colqual eq '(max)') + { + $colqual = undef + ; # max in sql server is the same as putting no colqual in pg + } + else + { + # We need the number (or 2 numbers) in this qual + $colqual =~ /\((\d+(?:,\s*\d+)?)\)/ + or die "Cannot parse colqual <$colqual>"; + $colqual = "$1"; + } + } + my $newtype = + convert_type($coltype, $colqual, $colname, + $tablename, undef, $schemaname); + + # If it is an identity, we'll map to serial/bigserial (create a sequence, then link it + # to the column) + if ($isidentity) + { + + # We have an identity field. We remember the default value and + # initialize the sequence correctly in the after script + $isidentity =~ /IDENTITY\((\d+),\s*(\d+)\)/ + or die "Cannot understand <$isidentity>"; + my $startseq = $1; + my $stepseq = $2; + my $seqname = lc("${tablename}_${colname}_seq"); + + # We get a sure default value. + $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} + ->{$colname}->{DEFAULT}->{VALUE} = + "nextval('" + . dboreplace(${schemaname}) . '.' + . ${seqname} . "')"; + $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} + ->{$colname}->{DEFAULT}->{UNSURE} = 0; + + $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{START} + = $startseq; + $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} + = $stepseq; + $objects->{$schemaname}->{SEQUENCES}->{$seqname} + ->{OWNERTABLE} = $tablename . "." . $colname; + $objects->{$schemaname}->{SEQUENCES}->{$seqname} + ->{OWNERSCHEMA} = $schemaname; + } + + # If there is a bytea generated, this table will contain a blob: + # use a special kettle transformation for it if generating kettle + # (see generate_kettle() ) + if ( $newtype eq 'bytea' + or $coltype eq + 'ntext') # Ntext is very slow, stored out of page + { + $objects->{$schemaname}->{'TABLES'}->{$tablename} + ->{haslobs} = 1; + } + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{POS} = $colnumber; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{TYPE} = $newtype; + + if ($colisnull eq 'NOT NULL') + { + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 1; + } + else + { + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 0; + } + } + else + { + die "I don't understand $line. This is a bug"; + } + } + + # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) + # So just put it as a varchar, and issue a warning is STDOUT + elsif ($line =~ /^\t\[(.*)\]\s+AS\s+\((.*)\)/) + { + + # We just get the column name + $colnumber++; + my $colname = $1; + my $code = $2; + my $coltype = 'varchar'; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{POS} = $colnumber; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{TYPE} = $coltype; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 0; + + # Big fat warning + print STDERR + "Warning: There is a calculated column: $schemaname.$tablename.$colname. This isn't done the same way in PG at all\n"; + print STDERR + "\tFor now it has been declared as a varchar in PG, so that the values can be copied\n"; + print STDERR + "\tYou should change its type manually in the dump (sorry for that),\n"; + print STDERR "\tA trigger has been written in the unsure file. It probably won't work as is.\n"; + print STDERR "\tPlease review it.\n"; + + # Try to correct what can be corrected from the AS : replace [COL] with NEW.COL + $code =~ s/\[(.*?)\]/NEW.$1/g; + my $triggerfunc = <{$schemaname}->{'TRIG_FUNCTIONS'} + ->{'trig_func_ins_or_upd' || $tablename}->{DEF} = + $triggerfunc; + $objects->{$schemaname}->{'TRIG_FUNCTIONS'} + ->{'trig_func_ins_or_upd' || $tablename}->{LANG} = + 'plpgsql'; + my %trigger; + $trigger{EVENTS} = 'before insert or update'; + $trigger{WHEN} = 'for each row'; + $trigger{FUNCTION} = + 'trig_func_ins_or_upd' || $tablename; # In the same schema + $trigger{NAME} = 'trig_ins_or_upd' || $tablename; + push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} + ->{TRIGGERS}}, (\%trigger); + + } + elsif ($line =~ + /^(?: CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED/) + { + # This is not forbidden by SQL, of course. I just never saw this in a sql server dump, + # so it should be an error for now (it will be syntaxically different if outside a table anyhow) + die "PK defined outside a table\n: $line" unless ($create_table); + + my $constraint + ; # We put everything inside this hashref, we'll push it into the constraint list later + $constraint->{TYPE} = 'PK'; + if (defined $1) + { + $constraint->{NAME} = $1; + } + + # Here is the PK. We read the following lines until the end of the constraint + while (my $pk = read_and_clean($file)) + { + + # Exit when read a line beginning with ). The constraint is complete. We store it and go back to main loop + if ($pk =~ /^\)/) + { + push @{$objects->{$schemaname}->{TABLES}->{$tablename} + ->{CONSTRAINTS}}, ($constraint); + + # We also directly put the constraint reference in a direct path (for ease of use in generate_kettle) + $objects->{$schemaname}->{TABLES}->{$tablename}->{PK} = + $constraint; + next MAIN; + } + if ($pk =~ /^\t\[(.*)\] (ASC|DESC)(,?)/) + { + push @{$constraint->{COLS}}, ($1); + } + } + } + elsif ($line =~ /^\s*(?:CONSTRAINT \[(.*)\] )?UNIQUE/) + { + + # This (having the constraint inside a create table )is not forbidden by SQL, + # of course. I just never saw this in a sql server dump, + # so it should be an error for now (it will be syntaxically different if outside a table anyhow) + die "Unique key defined outside a table\n: $line" + unless ($create_table); + + my $constraint + ; # We put everything inside this hashref, we'll push it into the constraint list later + $constraint->{TYPE} = 'UNIQUE'; + if (defined $1) + { + $constraint->{NAME} = $1; + } + + # Unique key definition. We read following lines until the end of the constraint + while (my $uk = read_and_clean($file)) + { + + # Exit when read a line beginning with ). The constraint is complete + if ($uk =~ /^\)/) + { + push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} + ->{CONSTRAINTS}}, ($constraint); + next MAIN; + } + if ($uk =~ /^\t\[(.*)\] (ASC|DESC)(,?)/) + { + push @{$constraint->{COLS}}, ($1); + } + } + + } + elsif ($line =~ /^\) ON \[PRIMARY\]/) + { + + # End of the table + $create_table = 0; + $tablename = ''; + } + ################################################################ + # From HERE, these SQL commands are not linked to a create table + ################################################################ + + elsif ($line =~ /CREATE SCHEMA \[(.*)\] AUTHORIZATION \[.*\]/) + { + $objects->{$1} = undef + ; # Nothing to add here, we create the schema, and put undef in it for now + } + elsif ($line =~ /CREATE\s+PROC(?:EDURE)?\s+\[.*\]\.\[(.*)\]/i) + { + print STDERR "Warning: Procedure $1 ignored\n"; + + # We have to find next GO to know we are out of the procedure + while (my $contline = read_and_clean($file)) + { + next MAIN if ($contline =~ /^GO$/); + } + } + elsif ($line =~ /CREATE\s+FUNCTION\s+\[.*\]\.\[(.*)\]/i) + { + print STDERR "Warning: Function $1 ignored\n"; + + # We have to find next GO to know we are out of the procedure + while (my $contline = read_and_clean($file)) + { + next MAIN if ($contline =~ /^GO$/); + } + } + elsif ($line =~ /CREATE\s+TRIGGER\s+\[(.*)\]/i) + { + print STDERR "Warning: Trigger $1 ignored\n"; + + # We have to find next GO to know we are out of the procedure + while (my $contline = read_and_clean($file)) + { + next MAIN if ($contline =~ /^GO$/); + } + } + + # Now we parse the create view. It is multi-line, so the code looks like like create table: we parse everything until a line + # containing only a single quote (end of the dbo.sp_executesql) + # The problem is that SQL Server seems to be spitting the original query used to create the view, not a normalized version + # of it, as PostgreSQL does. So we capture the query, and hope it works for now. + elsif ($line =~ + /^\s*(create\s*view)\s*(?:\[(\S+)\]\.)?\[(.*?)\]\s*(.*)$/i) + { + my $viewname = $3; + my $schemaname; + if (defined $2) + { + $schemaname = $2; + } + else + { + $schemaname = 'dbo'; + } + $schemaname = dboreplace($schemaname); + + my $sql = $1 . ' ' . $schemaname . '.' . $3 . ' ' . $4 . "\n"; + while (my $line_cont = read_and_clean($file)) + { + if ($line_cont =~ /^\s*'\s*|^GO$/ + ) # We may have a quote if the view is 'quoted', or a real sql query + { + # The view definition is complete. + # We get rid of dbo. schemas + $sql =~ s/(dbo)\./dboreplace($1) . '.'/eg + ; # We put this in the replacement schema + + # Views will be stored without the full schema in them. We will + # have to generate the schema in the output file + $objects->{$schemaname}->{'VIEWS'}->{$viewname}->{SQL} = + $sql; + next MAIN; + } + $sql .= $line_cont; + } + } + + # I only have seen types with added constraints ( create type foo varchar(50)) for now + # These are domains with PostgreSQL + elsif ($line =~ + /^CREATE TYPE \[(.*?)\]\.\[(.*?)\] FROM \[(.*?)](?:\((\d+(?:,\s*\d+)?)?\))?/ + ) + { + # Dependency between types is not done for now. If the problem arises, it should be added + my ($schema, $type, $origtype, $quals) = ($1, $2, $3, $4); + my $newtype = + convert_type($origtype, $quals, undef, undef, $type, $schema); + $objects->{$schema}->{DOMAINS}->{$type} = $newtype; + + # We add them to known data types, as they probably will be used in table definitions + # but they point to themselves, with the schema corrected: we want them substituted by themselves + $types{$schema . '.' . $type} = dboreplace($schema) . '.' + . $type; # We store the schema with it + } + elsif ($line =~ + /^CREATE (UNIQUE )?(NONCLUSTERED|CLUSTERED) INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/ + ) + { + # Index creation. Index are namespaced per table in SQL Server, not in PostgreSQL + # In PostgreSQL they are in the same namespace as the tables, and in the same + # schema as the table they are attached to + # So we store them in $objects, attached to the table + # Conflicts will be sorted by resolve_name_conflicts() later + my $isunique = $1; + my $isclustered = $2; + my $idxname = $3; + my $schemaname = $4; + my $tablename = $5; + if ($isunique) + { + $objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} + ->{$idxname}->{UNIQUE} = 1; + } + else + { + $objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} + ->{$idxname}->{UNIQUE} = 0; + } + while (my $idx = read_and_clean($file)) + { + + # Exit when read a line with a GO. The index is complete + if ($idx =~ /^GO/) + { + next MAIN; + } + next + if ($idx =~ /^\(|^\)/) + ; # Begin/end of the columns declaration + if ($idx =~ /\t\[(.*)\] (ASC|DESC)(,)?/) + { + if (defined $2) + { + push @{$objects->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{COLS}}, ("$1 $2"); + } + else + { + push @{$objects->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{COLS}}, ("$1"); + } + } + if ($idx =~ /^INCLUDE \(/) + { + print STDERR + "Warning: This index ($schemaname.$tablename.$idxname) has some include columns. This isn't supported in PostgreSQL.\n"; + print STDERR + "\tThe columns in the INCLUDE clause have been ignored.\n"; + next + ; # Nothing equivalent in PG. Maybe if the index isn't unique, these columns should be added? + } + } + } + + # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. + # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table + elsif ($line =~ + /^ALTER TABLE \[.*\]\.\[(.*)\] ADD \[(.*)\] (?:\[.*\]\.)?\[(.*)\](\(.+?\))?( .*\(\d+,\s*\d+\))? (NOT NULL|NULL)$/ + ) + { + # For now I don't know what to do with them. So die + die "$line: not understood. This is a bug"; + } + + # Table constraints + # Primary key. Multiline + elsif ($line =~ + /^ALTER TABLE \[.*\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED)?/ + ) + { + # Never seen one for now. The code is there though, in case, with a die for now + die "$line: not understood. This is a bug"; + while (my $contline = read_and_clean($file)) + { + next MAIN if ($contline =~ /^GO/); + } + } + + # Default values. numeric, then text, then bit. These are 100% sure, they will parse in PG + elsif ($line =~ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?((?:-)?\d+(?:\.\d+)?)\)?\) FOR \[(.*)\]/ + ) + { + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + = $3; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + = 0; + + # Default value, for a numeric (yes sql server often puts it in another pair of parenthesis, don't know why) + } + elsif ($line =~ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \('(.*)'\) FOR \[(.*)\]/ + ) + { + # Default text value, text, between commas + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + = "'$3'"; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + = 0; + } + elsif ($line =~ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((\d)\) FOR \[(.*)\]/ + ) + { + # Weird one: for bit type, there is no supplementary parenthesis... for now, let's say this is a bit type, and + # convert to true/false + if ($3 eq '0') + { + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} + ->{VALUE} = 'false'; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} + ->{UNSURE} = 0; + } + elsif ($3 eq '1') + { + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} + ->{VALUE} = 'true'; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} + ->{UNSURE} = 0; + } + else + { + die "not expected for a boolean: $line $. This is a bug" + ; # Get an error if the true/false hypothesis is wrong + } + } + + # Yes, we also get default NULL (what for ? :) ), and sometimes with a different case + elsif ($line =~ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(((?i)NULL)\) FOR \[(.*)\]/ + ) + { + # NULL WITHOUT quotes around it ! + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + = 'NULL'; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + = 0; + + } + + # And there are also constraints with functions and other strange code in them. Put them as unsure + elsif ($line =~ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/ + ) + { + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + = $3; + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + = 1; + } + + # FK constraint. It's multi line, we have to look for references, and what to do on update, delete, etc (I have only seen delete cascade for now) + elsif ($line =~ + /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD\s+CONSTRAINT \[(.*)\] FOREIGN KEY\((.*?)\)/ + ) + { + # This is a FK definition. We have the foreign table definition in next line. + my $constraint; + my $table = $2; + my $schema = $1; + $constraint->{TYPE} = 'FK'; + $constraint->{LOCAL_COLS} = $4; + $constraint->{LOCAL_TABLE} = $2; + $constraint->{LOCAL_COLS} =~ s/\[|\]//g; # Remove brackets + + while (my $fk = read_and_clean($file)) + { + if ($fk =~ /^GO/) + { + push @{$objects->{$schema}->{'TABLES'}->{$table} + ->{CONSTRAINTS}}, ($constraint); + next MAIN; + } + elsif ($fk =~ /^REFERENCES \[(.*)\]\.\[(.*)\] \((.*?)\)/) + { + $constraint->{REMOTE_COLS} = $3; + $constraint->{REMOTE_TABLE} = $2; + $constraint->{REMOTE_SCHEMA} = $1; + $constraint->{REMOTE_COLS} =~ + s/\[|\]//g; # Get rid of square brackets + } + elsif ($fk =~ /^ON DELETE CASCADE\s*$/) + { + $constraint->{ON_DEL_CASC} = 1; + } + elsif ($fk =~ /^ON UPDATE CASCADE\s*$/) + { + $constraint->{ON_UPD_CASC} = 1; + } + else + { + die "Cannot parse $fk $., in a FK. This is a bug"; + } + } + } + + # Check constraint. As it can be arbitrary code, we just get this code, and hope it will work on PG (it will be stored in a special script file) + elsif ($line =~ + /ALTER TABLE \[(.*)\]\.\[(.*)\] WITH (?:NO)?CHECK ADD CONSTRAINT \[(.*)\] CHECK \(\((.*)\)\)/ + ) + { + # Check constraint. We'll do what we can, syntax may be different. + my $constraint; + my $table = $2; + my $constxt = $4; + my $schema = $1; + $constraint->{TABLE} = $table; + $constraint->{NAME} = $3; + $constraint->{TYPE} = 'CHECK'; + $constxt =~ + s/\[(\S+)\]/$1/g; # We remove the []. And hope this will parse + $constraint->{TEXT} = $constxt; + push @{$objects->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}}, + ($constraint); + } + + # These are comments or extended attributes on objets. They can be multiline, so aggregate everything + # Until next GO command + # If fact in can be a lot of things. So we have to ignore things like MS_DiagramPaneCount + elsif ($line =~ /^EXEC sys.sp_addextendedproperty/) + { + my $sqlproperty = $line; + while (my $inline = read_and_clean($file)) + { + last if ($inline =~ /^GO/); + $sqlproperty .= $inline; + } + + # We have all the extended property. Let's parse it. + + # First step: what kind is it ? we are only interested in comments for now + $sqlproperty =~ /\@name=N'(.*?)'/ + or die + "Cannot find a name for this extended property: $sqlproperty"; + my $propertyname = $1; + if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1)$/) + { + # We don't dump these. They are graphical descriptions of the GUI + next; + } + + elsif ($propertyname eq 'MS_Description') + { + + # This is a comment. We parse it. + # Spaces are mostly random it seems, in SQL Server's dump code. So \s* everywhere :( + # There can be quotes inside a string. So (?{$schema}->{TABLES}->{$objname}->{COMMENT} = + $comment; + } + elsif ($obj eq 'VIEW' and not defined $subobj) + { + $objects->{$schema}->{VIEWS}->{$objname}->{COMMENT} = + $comment; + } + elsif ($obj eq 'TABLE' and $subobj eq 'COLUMN') + { + $objects->{$schema}->{TABLES}->{$objname}->{COLS} + ->{$subobjname}->{COMMENT} = $comment; + } + else + { + die "Cannot understand this comment: $sqlproperty"; + } + } + else + { + die + "Don't know what to do with this extendedproperty: $sqlproperty"; + } + } + + # Ignore USE, GO, and things that have no meaning for postgresql + elsif ($line =~ + /^USE\s|^GO\s*$|\/\*\*\*\*|^SET ANSI_NULLS (ON|OFF)|^SET QUOTED_IDENTIFIER|^SET ANSI_PADDING|CHECK CONSTRAINT|^BEGIN|^END/ + ) + { + next; + } + elsif ($line =~ /^--/) # Comment + { + next; + } + + # Don't know what it is. If you know, and it is worth converting, tell me :) + elsif ($line =~ /^EXEC .*bindrule/) + { + next; + } + + # Ignore users and roles. Security models will probably be very different between the two databases + elsif ($line =~ /^CREATE (ROLE|USER)/) + { + next; + } + + # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views + elsif ($line =~ /^IF NOT EXISTS/) + { + next; + } + + # Ignore CREATE DATABASE: we hope that we are given a single database as an option. It is multiline. + # Ignore everything until next GO + # Ignore ALTER DATABASE for the same reason. The given parameters have no meaning in PG anyway + # Same for tests about full text search. + elsif ($line =~ + /^(CREATE|ALTER) DATABASE|^IF \(1 = FULLTEXTSERVICEPROPERTY/) + { + while ($line !~ /^GO$/) + { + $line = read_and_clean($file); + } + + # We read everything in the CREATE DATABASE. Back to work ! + next; + } + + # Ignore EXEC dbo.sp_executesql, for now only seen for a create view. Views sql command aren't executed directly, don't know why + elsif ($line =~ /^EXEC dbo.sp_executesql/) + { + next; + } + + # Still on views: there are empty lines, and C-style comments + elsif ($line =~ /^\s*$/) + { + next; + } + else + { + die "Line <$line> ($.) not understood. This is a bug"; + } + } + close $file; } # Creates the SQL scripts from $object # We generate alphabetically, to make things less random (this data comes from a hash) sub generate_schema { - my ($before_file,$after_file,$unsure_file)=@_; - # Open the output files (except kettle, we'll do that at the end) - open BEFORE,">:utf8",$before_file or die "Cannot open $before_file, $!"; - open AFTER,">:utf8",$after_file or die "Cannot open $after_file, $!"; - open UNSURE,">:utf8",$unsure_file or die "Cannot open $unsure_file, $!"; - print BEFORE "\\set ON_ERROR_STOP\n"; - print BEFORE "\\set ECHO all\n"; - print BEFORE "BEGIN;\n"; - print AFTER "\\set ON_ERROR_STOP\n"; - print AFTER "\\set ECHO all\n"; - print AFTER "BEGIN;\n"; - print UNSURE "\\set ON_ERROR_STOP\n"; - print AFTER "\\set ECHO all\n"; - print UNSURE "BEGIN;\n"; - - # Are we case insensitive ? We have to install citext then - # Won't work on pre-9.1 database. But as this is a migration tool - # if someone wants to start with an older version, it's their problem :) - if ($case_insensitive) - { - print BEFORE "CREATE EXTENSION citext;\n"; - } - - # Ok, we have parsed everything, and definitions are in $objects - # We will put in the BEFORE file only table and columns definitions. - # The rest will go in the AFTER script (check constraints, put default values, etc...) - - # The schemas. don't create empty schema, sql server creates a schema per user, even if it ends empty - foreach my $schema (sort keys %{$objects}) - { - unless (dboreplace($schema) eq 'public' or not defined $objects->{$schema}) - { - print BEFORE "CREATE SCHEMA $schema;\n"; - } - } - - # For the rest, we iterate over schemas - # The tables, columns, etc... will be created in the before script, so there is no dependancy - # problem with constraints, that will be in the after script - # We have to do all domains before all tables - while (my ($schema,$refschema)=each %{$objects}) - { - $schema=dboreplace($schema); # If dbo, put into public, unless asked otherwise - # The user-defined types (domains, etc) - foreach my $domain (sort keys %{$refschema->{DOMAINS}}) - { - print BEFORE "CREATE DOMAIN $schema.$domain " . $refschema->{DOMAINS}->{$domain} . ";\n"; - } - - print BEFORE "\n"; # We change sections in the dump file - } - while (my ($schema,$refschema)=each %{$objects}) - { - $schema=dboreplace($schema); # If dbo, put into public, unless asked otherwise - - # The tables - foreach my $table (sort keys %{$refschema->{TABLES}}) - { - my @colsdef; - foreach my $col (sort { $refschema->{TABLES}->{$table}->{COLS}->{$a}->{POS} - <=> - $refschema->{TABLES}->{$table}->{COLS}->{$b}->{POS} - } (keys %{$refschema->{TABLES}->{$table}->{COLS}}) ) - - { - my $colref=$refschema->{TABLES}->{$table}->{COLS}->{$col}; - my $coldef=$col . " " . $colref->{TYPE}; - if ($colref->{NOT_NULL}) - { - $coldef .= ' NOT NULL'; - } - push @colsdef,($coldef); - } - print BEFORE "CREATE TABLE $schema.$table ( \n\t" . join (",\n\t",@colsdef) . ");\n\n"; - } - } - - while (my ($schema,$refschema)=each %{$objects}) - { - $schema=dboreplace($schema); # If dbo, put into public, unless asked otherwise - # We now add all "AFTER" objects - # We start with SEQUENCES, PKs and INDEXES (will be needed for FK) - - foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) - { - my $seqref=$refschema->{SEQUENCES}->{$sequence}; - print AFTER "CREATE SEQUENCE $schema.$sequence INCREMENT BY " . $seqref->{STEP} - . " START WITH " . $seqref->{START} - . " OWNED BY " . dboreplace($seqref->{OWNERSCHEMA}) . '.' . $seqref->{OWNERTABLE} . ";\n"; - } - - # Now PK. We have to go through all tables - foreach my $table (sort keys %{$refschema->{TABLES}}) - { - my $refpk= $refschema->{TABLES}->{$table}->{PK}; - # Warn if no PK! - if (not defined $refpk) - { - # Don't know if it should be displayed - #print STDERR "Warning: $table has no primary key.\n"; - next; - } - my $pkdef= "ALTER TABLE $schema.$table ADD"; - if (defined $refpk->{NAME}) - { - $pkdef .= " CONSTRAINT " . $refpk->{NAME}; - } - $pkdef .= " PRIMARY KEY (" . join (',',@{$refpk->{COLS}}) . ");\n"; - print AFTER $pkdef; - } - } - while (my ($schema,$refschema)=each %{$objects}) - { - $schema=dboreplace($schema); # If dbo, put into public, unless asked otherwise - - # Now The UNIQUE constraints. They may be used for FK (if columns are not null) - foreach my $table (sort keys %{$refschema->{TABLES}}) - { - foreach my $constraint (@{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) - { - next unless ($constraint->{TYPE} eq 'UNIQUE'); - my $consdef= "ALTER TABLE $schema.$table ADD"; - if (defined $constraint->{NAME}) - { - $consdef.= " CONSTRAINT " . $constraint->{NAME}; - } - $consdef.= " UNIQUE (" . join (",",@{$constraint->{COLS}}) . ");\n"; - print AFTER $consdef; - } - } - } - while (my ($schema,$refschema)=each %{$objects}) - { - $schema=dboreplace($schema); # If dbo, put into public, unless asked otherwise - - # We have all we need for FKs now. We can put all other constraints (except PK of course) - foreach my $table (sort keys %{$refschema->{TABLES}}) - { - foreach my $constraint (@{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) - { - next if ($constraint->{TYPE} =~ /^UNIQUE|PK$/); - my $consdef= "ALTER TABLE $schema.$table ADD"; - if (defined $constraint->{NAME}) - { - $consdef.= " CONSTRAINT " . $constraint->{NAME}; - } - if ($constraint->{TYPE} eq 'FK') # COLS are already a comma separated list - { - $consdef.= " FOREIGN KEY (" . $constraint->{LOCAL_COLS} . ")" . - " REFERENCES " . dboreplace($constraint->{REMOTE_SCHEMA}) . '.' . $constraint->{REMOTE_TABLE} . - " ( " . $constraint->{REMOTE_COLS} . ")"; - if (defined $constraint->{ON_DEL_CASC} and $constraint->{ON_DEL_CASC}) - { - $consdef.= " ON DELETE CASCADE"; - } - if (defined $constraint->{ON_UPD_CASC} and $constraint->{ON_UPD_CASC}) - { - $consdef.= " ON UPDATE CASCADE"; - } - $consdef.= ";\n"; - print AFTER $consdef; - } - elsif ($constraint->{TYPE} eq 'CHECK') - { - $consdef.= " CHECK (" . $constraint->{TEXT} . ");\n"; - print UNSURE $consdef; # Check constraints are SQL, so cannot be sure - } - elsif ($constraint->{TYPE} eq 'CHECK_CITEXT') - { - # These have been generated here, for citext mostly. So we know their syntax is ok - $consdef.= " CHECK (" . $constraint->{TEXT} . ");\n"; - print BEFORE $consdef; # These are for citext. So they should be checked asap - } - else - { - # Shouldn't get there. it would mean I have forgotten a type of constraint - die "I couldn't translate a constraint. This is a bug"; - } - } - } - } - while (my ($schema,$refschema)=each %{$objects}) - { - $schema=dboreplace($schema); # If dbo, put into public, unless asked otherwise - # Indexes - # They don't have a schema qualifier. But their table has, and they are in the same schema as their table - foreach my $table (sort keys %{$refschema->{TABLES}}) - { - foreach my $index (sort keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) - { - my $idxref=$refschema->{TABLES}->{$table}->{INDEXES}->{$index}; - my $idxdef="CREATE"; - if ($idxref->{UNIQUE}) - { - $idxdef.=" UNIQUE"; - } - $idxdef.= " INDEX $index ON $schema.$table (" . - join(",",@{$idxref->{COLS}}) . - ");\n"; - print AFTER $idxdef; - } - } - } - while (my ($schema,$refschema)=each %{$objects}) - { - $schema=dboreplace($schema); # If dbo, put into public, unless asked otherwise - # Default values - foreach my $table (sort keys %{$refschema->{TABLES}}) - { - foreach my $col (sort keys %{$refschema->{TABLES}->{$table}->{COLS}}) - { - my $colref=$refschema->{TABLES}->{$table}->{COLS}->{$col}; - next unless (defined $colref->{DEFAULT}); - my $definition= "ALTER TABLE $schema.$table ALTER COLUMN $col SET DEFAULT " . $colref->{DEFAULT}->{VALUE} . ";\n"; - if ($colref->{DEFAULT}->{UNSURE}) - { - print UNSURE $definition; - } - else - { - print AFTER $definition; - } - } - } - } - while (my ($schema,$refschema)=each %{$objects}) - { - $schema=dboreplace($schema); # If dbo, put into public, unless asked otherwise - # Comments on tables - foreach my $table (sort keys %{$refschema->{TABLES}}) - { - if (defined ($refschema->{TABLES}->{$table}->{COMMENT})) - { - print AFTER "COMMENT ON TABLE $schema.$table IS '" . - $refschema->{TABLES}->{$table}->{COMMENT} . - "';\n"; - } - foreach my $col (sort keys %{$refschema->{TABLES}->{$table}->{COLS}}) - { - my $colref=$refschema->{TABLES}->{$table}->{COLS}->{$col}; - if (defined ($colref->{COMMENT})) - { - print AFTER "COMMENT ON COLUMN $schema.$table.$col IS '" . - $colref->{COMMENT} . "';\n"; - } - } - } - } - - while (my ($schema,$refschema)=each %{$objects}) - { - $schema=dboreplace($schema); # If dbo, put into public, unless asked otherwise - # The views, and comments - foreach my $view (sort keys %{$refschema->{VIEWS}}) - { - print UNSURE $refschema->{VIEWS}->{$view}->{SQL},";\n"; - if (defined $refschema->{VIEWS}->{$view}->{COMMENT}) - { - print UNSURE "COMMENT ON VIEW $schema.$view IS '" . - $refschema->{VIEWS}->{$view}->{COMMENT} . - "';\n"; - } - } - } - - print BEFORE "COMMIT;\n"; - print AFTER "COMMIT;\n"; - print UNSURE "COMMIT;\n"; - close BEFORE; - close AFTER; - close UNSURE; + my ($before_file, $after_file, $unsure_file) = @_; + + # Open the output files (except kettle, we'll do that at the end) + open BEFORE, ">:utf8", $before_file or die "Cannot open $before_file, $!"; + open AFTER, ">:utf8", $after_file or die "Cannot open $after_file, $!"; + open UNSURE, ">:utf8", $unsure_file or die "Cannot open $unsure_file, $!"; + print BEFORE "\\set ON_ERROR_STOP\n"; + print BEFORE "\\set ECHO all\n"; + print BEFORE "BEGIN;\n"; + print AFTER "\\set ON_ERROR_STOP\n"; + print AFTER "\\set ECHO all\n"; + print AFTER "BEGIN;\n"; + print UNSURE "\\set ON_ERROR_STOP\n"; + print AFTER "\\set ECHO all\n"; + print UNSURE "BEGIN;\n"; + + # Are we case insensitive ? We have to install citext then + # Won't work on pre-9.1 database. But as this is a migration tool + # if someone wants to start with an older version, it's their problem :) + if ($case_insensitive) + { + print BEFORE "CREATE EXTENSION citext;\n"; + } + + # Ok, we have parsed everything, and definitions are in $objects + # We will put in the BEFORE file only table and columns definitions. + # The rest will go in the AFTER script (check constraints, put default values, etc...) + + # The schemas. don't create empty schema, sql server creates a schema per user, even if it ends empty + foreach my $schema (sort keys %{$objects}) + { + unless (dboreplace($schema) eq 'public' + or not defined $objects->{$schema}) + { + print BEFORE "CREATE SCHEMA $schema;\n"; + } + } + + # For the rest, we iterate over schemas + # The tables, columns, etc... will be created in the before script, so there is no dependancy + # problem with constraints, that will be in the after script + # We have to do all domains before all tables + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + # The user-defined types (domains, etc) + foreach my $domain (sort keys %{$refschema->{DOMAINS}}) + { + print BEFORE "CREATE DOMAIN $schema.$domain " + . $refschema->{DOMAINS}->{$domain} . ";\n"; + } + + print BEFORE "\n"; # We change sections in the dump file + } + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + + # The tables + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + my @colsdef; + foreach my $col ( + sort { + $refschema->{TABLES}->{$table}->{COLS}->{$a}->{POS} + <=> $refschema->{TABLES}->{$table}->{COLS}->{$b} + ->{POS} + } (keys %{$refschema->{TABLES}->{$table}->{COLS}})) + + { + my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; + my $coldef = $col . " " . $colref->{TYPE}; + if ($colref->{NOT_NULL}) + { + $coldef .= ' NOT NULL'; + } + push @colsdef, ($coldef); + } + print BEFORE "CREATE TABLE $schema.$table ( \n\t" + . join(",\n\t", @colsdef) + . ");\n\n"; + } + } + + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + # We now add all "AFTER" objects + # We start with SEQUENCES, PKs and INDEXES (will be needed for FK) + + foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) + { + my $seqref = $refschema->{SEQUENCES}->{$sequence}; + print AFTER "CREATE SEQUENCE $schema.$sequence INCREMENT BY " + . $seqref->{STEP} + . " START WITH " + . $seqref->{START} + . " OWNED BY " + . dboreplace($seqref->{OWNERSCHEMA}) . '.' + . $seqref->{OWNERTABLE} . ";\n"; + } + + # Now PK. We have to go through all tables + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + my $refpk = $refschema->{TABLES}->{$table}->{PK}; + + # Warn if no PK! + if (not defined $refpk) + { + + # Don't know if it should be displayed + #print STDERR "Warning: $table has no primary key.\n"; + next; + } + my $pkdef = "ALTER TABLE $schema.$table ADD"; + if (defined $refpk->{NAME}) + { + $pkdef .= " CONSTRAINT " . $refpk->{NAME}; + } + $pkdef .= + " PRIMARY KEY (" . join(',', @{$refpk->{COLS}}) . ");\n"; + print AFTER $pkdef; + } + } + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + + # Now The UNIQUE constraints. They may be used for FK (if columns are not null) + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + foreach my $constraint ( + @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) + { + next unless ($constraint->{TYPE} eq 'UNIQUE'); + my $consdef = "ALTER TABLE $schema.$table ADD"; + if (defined $constraint->{NAME}) + { + $consdef .= " CONSTRAINT " . $constraint->{NAME}; + } + $consdef .= + " UNIQUE (" . join(",", @{$constraint->{COLS}}) . ");\n"; + print AFTER $consdef; + } + } + } + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + + # We have all we need for FKs now. We can put all other constraints (except PK of course) + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + foreach my $constraint ( + @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) + { + next if ($constraint->{TYPE} =~ /^UNIQUE|PK$/); + my $consdef = "ALTER TABLE $schema.$table ADD"; + if (defined $constraint->{NAME}) + { + $consdef .= " CONSTRAINT " . $constraint->{NAME}; + } + if ($constraint->{TYPE} eq + 'FK') # COLS are already a comma separated list + { + $consdef .= + " FOREIGN KEY (" + . $constraint->{LOCAL_COLS} . ")" + . " REFERENCES " + . dboreplace($constraint->{REMOTE_SCHEMA}) . '.' + . $constraint->{REMOTE_TABLE} . " ( " + . $constraint->{REMOTE_COLS} . ")"; + if (defined $constraint->{ON_DEL_CASC} + and $constraint->{ON_DEL_CASC}) + { + $consdef .= " ON DELETE CASCADE"; + } + if (defined $constraint->{ON_UPD_CASC} + and $constraint->{ON_UPD_CASC}) + { + $consdef .= " ON UPDATE CASCADE"; + } + $consdef .= ";\n"; + print AFTER $consdef; + } + elsif ($constraint->{TYPE} eq 'CHECK') + { + $consdef .= " CHECK (" . $constraint->{TEXT} . ");\n"; + print UNSURE $consdef + ; # Check constraints are SQL, so cannot be sure + } + elsif ($constraint->{TYPE} eq 'CHECK_CITEXT') + { + + # These have been generated here, for citext mostly. So we know their syntax is ok + $consdef .= " CHECK (" . $constraint->{TEXT} . ");\n"; + print BEFORE $consdef + ; # These are for citext. So they should be checked asap + } + else + { + # Shouldn't get there. it would mean I have forgotten a type of constraint + die "I couldn't translate a constraint. This is a bug"; + } + } + } + } + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + # Indexes + # They don't have a schema qualifier. But their table has, and they are in the same schema as their table + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + foreach my $index ( + sort keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) + { + my $idxref = + $refschema->{TABLES}->{$table}->{INDEXES}->{$index}; + my $idxdef = "CREATE"; + if ($idxref->{UNIQUE}) + { + $idxdef .= " UNIQUE"; + } + $idxdef .= " INDEX $index ON $schema.$table (" + . join(",", @{$idxref->{COLS}}) . ");\n"; + print AFTER $idxdef; + } + } + } + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + # Default values + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + foreach + my $col (sort keys %{$refschema->{TABLES}->{$table}->{COLS}}) + { + my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; + next unless (defined $colref->{DEFAULT}); + my $definition = + "ALTER TABLE $schema.$table ALTER COLUMN $col SET DEFAULT " + . $colref->{DEFAULT}->{VALUE} . ";\n"; + if ($colref->{DEFAULT}->{UNSURE}) + { + print UNSURE $definition; + } + else + { + print AFTER $definition; + } + } + } + } + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + # Comments on tables + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + if (defined($refschema->{TABLES}->{$table}->{COMMENT})) + { + print AFTER "COMMENT ON TABLE $schema.$table IS '" + . $refschema->{TABLES}->{$table}->{COMMENT} . "';\n"; + } + foreach + my $col (sort keys %{$refschema->{TABLES}->{$table}->{COLS}}) + { + my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; + if (defined($colref->{COMMENT})) + { + print AFTER "COMMENT ON COLUMN $schema.$table.$col IS '" + . $colref->{COMMENT} . "';\n"; + } + } + } + } + + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + # The views, and comments + foreach my $view (sort keys %{$refschema->{VIEWS}}) + { + print UNSURE $refschema->{VIEWS}->{$view}->{SQL}, ";\n"; + if (defined $refschema->{VIEWS}->{$view}->{COMMENT}) + { + print UNSURE "COMMENT ON VIEW $schema.$view IS '" + . $refschema->{VIEWS}->{$view}->{COMMENT} . "';\n"; + } + } + } + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + # The trigger functions + foreach my $triggerfunc (sort keys %{$refschema->{TRIG_FUNCTIONS}}) + { + my $code = $refschema->{TRIG_FUNCTIONS}->{$triggerfunc}->{DEF}; + my $language = $refschema->{TRIG_FUNCTIONS}->{$triggerfunc}->{LANG}; + print UNSURE + "CREATE FUNCTION $schema.$triggerfunc() RETURNS trigger LANGUAGE $language AS \$def\$\n"; + print UNSURE $code; + print UNSURE "\$def\$;\n"; + } + } + + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = dboreplace($schema) + ; # If dbo, put into public, unless asked otherwise + # triggers on tables, as these functions are declared now + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + foreach + my $reftrigger (@{$refschema->{TABLES}->{$table}->{TRIGGERS}}) + { + print UNSURE "CREATE TRIGGER "; + print UNSURE $reftrigger->{NAME}; + print UNSURE ' '; + print UNSURE $reftrigger->{EVENTS}; + print UNSURE ' ON '; + print UNSURE$schema; + print UNSURE '.'; + print UNSURE $table; + print UNSURE ' '; + print UNSURE $reftrigger->{WHEN}; + print UNSURE ' execute procedure '; + print UNSURE $schema; + print UNSURE '.'; + print UNSURE $reftrigger->{FUNCTION}; + print UNSURE "();\n"; + } + } + } + + print BEFORE "COMMIT;\n"; + print AFTER "COMMIT;\n"; + print UNSURE "COMMIT;\n"; + close BEFORE; + close AFTER; + close UNSURE; } @@ -1347,126 +1658,153 @@ sub generate_schema # We do this schema per schema sub resolve_name_conflicts { - while (my ($schema,$refschema)=each %{$objects}) - { - my %known_names; - # Store all known names - foreach my $table (keys %{$refschema->{TABLES}}) - { - $known_names{$table}=1; - } - - # We scan all types. For now, this tool only generates domains, so we scan domains - foreach my $domain (keys %{$refschema->{DOMAINS}}) - { - if (not defined ($known_names{$domain})) - { - # Great. Just skip to the next and remember this name - $known_names{$domain}=1; - } - else - { - # We rename - $refschema->{DOMAINS}->{$domain."2pgd"} = $refschema->{DOMAINS}->{$domain}; - delete $refschema->{DOMAINS}->{$domain}; - print STDERR "Warning: I had to rename domain $domain to ${domain}2pgd because of naming conflicts between a table and a domain, in source schema $schema\n"; - # I also have to check all cols type to rename this - foreach my $table (values %{$refschema->{TABLES}}) - { - foreach my $col (values %{$table->{COLS}}) - { - # If a column has a custom type, it will be prefixed by schema - # The schema will be the destination schema: dbo may have been replaced by public - if ($col->{TYPE} eq (dboreplace($schema) . '.' . $domain) ) - { - $col->{TYPE} = dboreplace($schema) . '.' . $domain . "2pgd"; - } - } - } - } - - } - - # Then we scan all indexes - foreach my $table (keys %{$refschema->{TABLES}}) - { - foreach my $idx (keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) - { - if (not defined ($known_names{$idx})) - { - # Great. Just skip to the next and remember this name - $known_names{$idx}=1; - } - else - { - # We have to rename :/ - # Postfix with a 2pg - # We have to update the name in the $refschema hash - $refschema->{TABLES}->{$table}->{INDEXES}->{"$idx"."2pgi"}=$refschema->{TABLES}->{$table}->{INDEXES}->{$idx}; - delete $refschema->{TABLES}->{$table}->{INDEXES}->{$idx}; - print STDERR "Warning: I had to rename index $table.$idx to ${idx}2pgi because of naming conflicts in source schema $schema\n"; - $known_names{$idx . "2pgi"}=1; - } - } - } - } + while (my ($schema, $refschema) = each %{$objects}) + { + my %known_names; + + # Store all known names + foreach my $table (keys %{$refschema->{TABLES}}) + { + $known_names{$table} = 1; + } + + # We scan all types. For now, this tool only generates domains, so we scan domains + foreach my $domain (keys %{$refschema->{DOMAINS}}) + { + if (not defined($known_names{$domain})) + { + + # Great. Just skip to the next and remember this name + $known_names{$domain} = 1; + } + else + { + # We rename + $refschema->{DOMAINS}->{$domain . "2pgd"} = + $refschema->{DOMAINS}->{$domain}; + delete $refschema->{DOMAINS}->{$domain}; + print STDERR + "Warning: I had to rename domain $domain to ${domain}2pgd because of naming conflicts between a table and a domain, in source schema $schema\n"; + + # I also have to check all cols type to rename this + foreach my $table (values %{$refschema->{TABLES}}) + { + foreach my $col (values %{$table->{COLS}}) + { + + # If a column has a custom type, it will be prefixed by schema + # The schema will be the destination schema: dbo may have been replaced by public + if ($col->{TYPE} eq + (dboreplace($schema) . '.' . $domain)) + { + $col->{TYPE} = + dboreplace($schema) . '.' . $domain . "2pgd"; + } + } + } + } + + } + + # Then we scan all indexes + foreach my $table (keys %{$refschema->{TABLES}}) + { + foreach + my $idx (keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) + { + if (not defined($known_names{$idx})) + { + + # Great. Just skip to the next and remember this name + $known_names{$idx} = 1; + } + else + { + # We have to rename :/ + # Postfix with a 2pg + # We have to update the name in the $refschema hash + $refschema->{TABLES}->{$table}->{INDEXES} + ->{"$idx" . "2pgi"} = + $refschema->{TABLES}->{$table}->{INDEXES}->{$idx}; + delete $refschema->{TABLES}->{$table}->{INDEXES}->{$idx}; + print STDERR + "Warning: I had to rename index $table.$idx to ${idx}2pgi because of naming conflicts in source schema $schema\n"; + $known_names{$idx . "2pgi"} = 1; + } + } + } + } } # Main # Parse command line -my $help=0; - -my $options = GetOptions ( "k=s" => \$kettle, - "b=s" => \$before_file, - "a=s" => \$after_file, - "u=s" => \$unsure_file, - "h" => \$help, - "conf=s" => \$conf_file, - "sd=s" => \$sd, - "sh=s" => \$sh, - "sp=s" => \$sp, - "su=s" => \$su, - "sw=s" => \$sw, - "pd=s" => \$pd, - "ph=s" => \$ph, - "pp=s" => \$pp, - "pu=s" => \$pu, - "pw=s" => \$pw, - "f=s" => \$filename, - "i" => \$case_insensitive, - "nr" => \$norelabel_dbo, - "num" => \$convert_numeric_to_int, - ); +my $help = 0; + +my $options = GetOptions("k=s" => \$kettle, + "b=s" => \$before_file, + "a=s" => \$after_file, + "u=s" => \$unsure_file, + "h" => \$help, + "conf=s" => \$conf_file, + "sd=s" => \$sd, + "sh=s" => \$sh, + "sp=s" => \$sp, + "su=s" => \$su, + "sw=s" => \$sw, + "pd=s" => \$pd, + "ph=s" => \$ph, + "pp=s" => \$pp, + "pu=s" => \$pu, + "pw=s" => \$pw, + "f=s" => \$filename, + "i" => \$case_insensitive, + "nr" => \$norelabel_dbo, + "num" => \$convert_numeric_to_int,); # We don't understand command line or have been asked for usage if (not $options or $help) { - usage(); - exit 1; + usage(); + exit 1; } -# We have a configuration file. We load it, and set + +# We have a configuration file. We load it, and set # all we can find in it -if ($conf_file) +if ($conf_file) { - parse_conf_file(); + parse_conf_file(); } # We have no before, after, or unsure -if (not $before_file or not $after_file or not $unsure_file or not $filename) +if ( not $before_file + or not $after_file + or not $unsure_file + or not $filename) { - usage(); - exit 1; + usage(); + exit 1; } + # We have been asked for kettle, but the compulsory parameters are not there -if ($kettle and (not $sd or not $sh or not $sp or not $su or not $sw or not $pd or not $ph or not $pp or not $pu or not $pw)) +if ($kettle + and ( not $sd + or not $sh + or not $sp + or not $su + or not $sw + or not $pd + or not $ph + or not $pp + or not $pu + or not $pw)) { - usage(); - print "You have to provide all connection information, if using -k or kettle directory set in configuration file\n"; - exit 1; + usage(); + print + "You have to provide all connection information, if using -k or kettle directory set in configuration file\n"; + exit 1; } - # Read SQL Server's dump file parse_dump(); @@ -1480,23 +1818,19 @@ sub resolve_name_conflicts generate_schema($before_file, $after_file, $unsure_file); # If asked, create the kettle job -if ( $kettle and (defined $ENV{'HOME'} or defined $ENV{'USERPROFILE'} ) ) +if ($kettle and (defined $ENV{'HOME'} or defined $ENV{'USERPROFILE'})) { - check_kettle_properties(); + check_kettle_properties(); } generate_kettle($kettle) if ($kettle); - - - ##################################################################################################################################### - # Begin block to load ugly template variables BEGIN { - $template= < __sqlserver_table_name__ @@ -1711,7 +2045,7 @@ BEGIN EOF #################################################################################### - $template_lob= < __sqlserver_table_name__ @@ -1927,7 +2261,7 @@ BEGIN EOF #################################################################################### - $job_header= < Migration @@ -1986,13 +2320,13 @@ BEGIN EOF #################################################################################### - $job_middle= < EOF #################################################################################### - $job_footer= < @@ -2000,7 +2334,7 @@ BEGIN EOF #################################################################################### - $job_entry= < __table_name__ @@ -2036,7 +2370,7 @@ BEGIN EOF #################################################################################### - $job_hop= < __table_1__ __table_2__ From 6e9554164dbfcf4334a541e1bfada6239b33a1b5 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 20 Mar 2014 16:45:44 +0100 Subject: [PATCH 010/216] try some basic conversions on transact sql code --- sqlserver2pgsql.pl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2a42bf3..dfde1d6 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -232,6 +232,16 @@ sub convert_type return $rettype; } +# This one will try to convert what can obviously be converted from transact to PG +# Things such as getdate() which can become now() +sub convert_transactsql_code +{ + my ($code)=@_; + $code =~ s/getdate\s*\(\)/now()/g; + return $code; +} + + # This gives the next column position for a table # It is used when we receive alter tables in the sql server dump # These tables are added at the end of the table, in %objects @@ -1090,7 +1100,7 @@ sub parse_dump ) { $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} - = $3; + = convert_transactsql_code($3); $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 1; } From 18e7e039d9246ce17e695be651e9c248df26fe22 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 21 Mar 2014 14:45:44 +0100 Subject: [PATCH 011/216] Improve handling of bit datatype --- sqlserver2pgsql.pl | 70 ++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index dfde1d6..0dcd2dc 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1032,17 +1032,40 @@ sub parse_dump } } - # Default values. numeric, then text, then bit. These are 100% sure, they will parse in PG + # Default values. numeric, then text. These are 100% sure, they will parse in PG + # Sometimes there is a second pair of parenthesis. I don't even want to know why... + # Bit just need a little bit of work to be converted to 'true'/'false' elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?((?:-)?\d+(?:\.\d+)?)\)?\) FOR \[(.*)\]/ ) { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} - = $3; + if ($objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{TYPE} eq 'boolean') + { + # Ok, it IS a boolean, and we have received a number + if ($3 eq '0') + { + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} + ->{VALUE} = 'false'; + } + elsif ($3 eq '1') + { + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} + ->{VALUE} = 'true'; + } + else + { + # We should not get here: we have a numeric which isn't 0 or 1, and is supposed to be a boolean + die "Got an unexpected boolean : $3, for line $line\n"; + } + } + else + { + $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + = $3; + } $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 0; - # Default value, for a numeric (yes sql server often puts it in another pair of parenthesis, don't know why) } elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \('(.*)'\) FOR \[(.*)\]/ @@ -1054,32 +1077,6 @@ sub parse_dump $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 0; } - elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((\d)\) FOR \[(.*)\]/ - ) - { - # Weird one: for bit type, there is no supplementary parenthesis... for now, let's say this is a bit type, and - # convert to true/false - if ($3 eq '0') - { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} - ->{VALUE} = 'false'; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} - ->{UNSURE} = 0; - } - elsif ($3 eq '1') - { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} - ->{VALUE} = 'true'; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} - ->{UNSURE} = 0; - } - else - { - die "not expected for a boolean: $line $. This is a bug" - ; # Get an error if the true/false hypothesis is wrong - } - } # Yes, we also get default NULL (what for ? :) ), and sometimes with a different case elsif ($line =~ @@ -1359,6 +1356,7 @@ sub generate_schema print BEFORE "\n"; # We change sections in the dump file } + # Tables and columns while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) @@ -1389,7 +1387,7 @@ sub generate_schema . ");\n\n"; } } - + # Sequences, PKs, Indexes while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) @@ -1432,6 +1430,7 @@ sub generate_schema print AFTER $pkdef; } } + # Unique while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) @@ -1455,6 +1454,7 @@ sub generate_schema } } } + # Other constraints while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) @@ -1517,6 +1517,7 @@ sub generate_schema } } } + # Indexes. while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) @@ -1541,6 +1542,7 @@ sub generate_schema } } } + # Default values while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) @@ -1567,6 +1569,7 @@ sub generate_schema } } } + # Comments on tables and columns while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) @@ -1591,7 +1594,7 @@ sub generate_schema } } } - + # Views, and their comments while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) @@ -1607,6 +1610,7 @@ sub generate_schema } } } + # Trigger functions while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) @@ -1622,7 +1626,7 @@ sub generate_schema print UNSURE "\$def\$;\n"; } } - + # Triggers while (my ($schema, $refschema) = each %{$objects}) { $schema = dboreplace($schema) From 6700ef7eec6ef52238919b5e1fc1cf7787890449 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 27 Mar 2014 19:48:51 +0100 Subject: [PATCH 012/216] Add support for some more comments --- sqlserver2pgsql.pl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 0dcd2dc..9381439 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1225,6 +1225,26 @@ sub parse_dump die "Cannot understand this comment: $sqlproperty"; } } + elsif ($propertyname eq 'Dictionary') + { + # It seems to be another way to declare table comments. I hope this is right + $sqlproperty =~ + /^EXEC sys.sp_addextendedproperty \@name=N'(.*?)'\s*,\s*\@value=N'(.*?)(?{$schema}->{TABLES}->{$objname}->{COMMENT} = + $comment; + } + elsif ($obj eq 'SCHEMA') + { + # There is no schema comments in PG. Just print a warning and ignore them + print STDERR "Schema comment : <$comment> ignored (no schema comment in PG)\n"; + } + + } else { die From 5b8317cb4e9386b8fd5cf0061be189ba0d5ae4b0 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 27 Mar 2014 20:32:47 +0100 Subject: [PATCH 013/216] Added a remapping functionnality --- README.md | 3 ++ example_conf_file | 4 +- sqlserver2pgsql.pl | 103 ++++++++++++++++++++++++++++++--------------- 3 files changed, 74 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 3c1c9cb..bb465f1 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,9 @@ You can also use the -i, -num and/or -nr options: -nr : Don't convert the dbo schema to public. By default, this conversion is done, as it converts MSSQL's default schema to PostgreSQL's default schema +-relabel_schemas is a list of schemas to remap. The syntax is : 'source1=>dest1;source2=>dest2'. Don't forget to quote this option or the shell might alter it +there is a default dbo=>public remapping, that can be cancelled with -nr + -num : Converts numeric (xxx,0) to the appropriate smallint, integer or bigint. It won't keep the constraint on the size of the scale of the numeric diff --git a/example_conf_file b/example_conf_file index fad6761..54be17a 100644 --- a/example_conf_file +++ b/example_conf_file @@ -21,5 +21,7 @@ postgresql password=bar_password # Optional behaviour case insensitive=1 # set it to 0 to generate a dump with citext and check constraints all over the place -no relabel dbo=0 # set it to 0 to convert the dbo schema to public +no relabel dbo=1 # set it to 0 to convert the dbo schema to public convert numeric to int=1 # set it to 0 to keep numeric(xx,0) as numeric(xx,0). Will be converted to smallint, int or bigint by default +relabel schemas=dbo=>foo;schema1=>bar + diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 9381439..6eca2da 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -31,12 +31,11 @@ our ($sd, $sh, $sp, $su, $sw, $pd, $ph, $pp, $pu, $pw); # Connection args our $conf_file; our $filename; # Filename passed as arg -our $case_insensitive = 0 - ; # Passed as arg: was SQL Server installation case insensitive ? PostgreSQL can't ignore accents anyway +our $case_insensitive; # Passed as arg: was SQL Server installation case insensitive ? PostgreSQL can't ignore accents anyway # If yes, we will generate citext with CHECK constraints, that's the best we can do -our $norelabel_dbo = 0; # Passed as arg: should we convert DBO to public ? -our $convert_numeric_to_int = 0 - ; # Should we convert numerics to int when possible ? (numeric (4,0) could be converted an int, for instance) +our $norelabel_dbo; # Passed as arg: should we convert DBO to public ? +our $relabel_schemas; +our $convert_numeric_to_int; # Should we convert numerics to int when possible ? (numeric (4,0) could be converted an int, for instance) our $kettle; our $before_file; our $after_file; @@ -76,6 +75,7 @@ sub parse_conf_file 'case insensitive' => 'case_insensitive', 'no relabel dbo' => 'norelabel_dbo', 'convert numeric to int' => 'convert_numeric_to_int', + 'relabel schemas' => 'relabel_schemas', ); # Open the conf file or die @@ -87,7 +87,7 @@ sub parse_conf_file $line =~ s/\s+$//; # Remove trailing whitespaces next if ($line =~ /^$/); # Empty line after comments have been removed - $line =~ /^(.*)=(.*)$/ or die "Cannot parse $line from $conf_file"; + $line =~ /^(.*?)=(.*)$/ or die "Cannot parse $line from $conf_file"; my ($param, $value) = ($1, $2); no strict 'refs'; # Using references by name, temporarily unless (defined $parameters{$param}) @@ -102,6 +102,10 @@ sub parse_conf_file $$param_name = $value; use strict 'refs'; } + # Hard coded default values + $case_insensitive=0 unless (defined ($case_insensitive)); + $norelabel_dbo=0 unless (defined ($norelabel_dbo)); + $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); close CONF; } @@ -267,15 +271,36 @@ sub next_col_pos } } -# This relabels the string if it is dbo and we want to relabel it to public -sub dboreplace { - my ($schema) = @_; - return $schema if ($schema ne 'dbo'); - return 'public' unless ($norelabel_dbo); - return 'dbo'; -} + # This builds %relabel_schemas for use in the next function. Both are scoped so that %relabel_schemas is not visible from outside + my %relabel_schemas; + sub build_relabel_schemas + { + foreach my $pair (split (';',$relabel_schemas)) + { + my @pair=split('=>',$pair); + unless (scalar(@pair)==2) + { + die "Cannot parse the schema list given as argument: <$relabel_schemas>\n"; + } + $relabel_schemas{$pair[0]}=$pair[1]; + } + # Don't forget dbo -> public if it was asked + unless ($norelabel_dbo) + { + $relabel_schemas{'dbo'}='public'; + } + } + + # This relabels the schemas + sub relabel_schemas + { + my ($schema) = @_; + return $schema unless (defined $relabel_schemas{$schema}); + return $relabel_schemas{$schema}; + } +} # Test if we are on windows. We will have to convert / to \ in the XML files sub is_windows { @@ -335,6 +360,9 @@ sub usage "-nr tells $0 not to convert the dbo schema to public. dbo will stay dbo\n"; print "-num tells $0 to convert numeric xxx,0 to int, bigint, etc. Will not keep numeric scale and precision for the converted\n"; + print + "-relabel_schemas gives a list of schemas to rename. For instance -relabel_schemas 'source1=>dest1;source2=>dest2'\n"; + print " -nr simply cancels the default dbo=>public remapping. Don't forget to put the remapping between quotes\n"; print "before_file contains the structure\n"; print "after_file contains index, constraints\n"; print @@ -372,7 +400,7 @@ sub generate_kettle foreach my $schema (sort keys %{$objects}) { my $refschema = $objects->{$schema}; - my $targetschema = dboreplace($schema); + my $targetschema = relabel_schemas($schema); foreach my $table (sort keys %{$refschema->{TABLES}}) { @@ -675,7 +703,7 @@ sub parse_dump $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} ->{$colname}->{DEFAULT}->{VALUE} = "nextval('" - . dboreplace(${schemaname}) . '.' + . relabel_schemas(${schemaname}) . '.' . ${seqname} . "')"; $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} ->{$colname}->{DEFAULT}->{UNSURE} = 0; @@ -908,7 +936,7 @@ sub parse_dump { $schemaname = 'dbo'; } - $schemaname = dboreplace($schemaname); + $schemaname = relabel_schemas($schemaname); my $sql = $1 . ' ' . $schemaname . '.' . $3 . ' ' . $4 . "\n"; while (my $line_cont = read_and_clean($file)) @@ -918,7 +946,7 @@ sub parse_dump { # The view definition is complete. # We get rid of dbo. schemas - $sql =~ s/(dbo)\./dboreplace($1) . '.'/eg + $sql =~ s/(dbo)\./relabel_schemas($1) . '.'/eg ; # We put this in the replacement schema # Views will be stored without the full schema in them. We will @@ -945,7 +973,7 @@ sub parse_dump # We add them to known data types, as they probably will be used in table definitions # but they point to themselves, with the schema corrected: we want them substituted by themselves - $types{$schema . '.' . $type} = dboreplace($schema) . '.' + $types{$schema . '.' . $type} = relabel_schemas($schema) . '.' . $type; # We store the schema with it } elsif ($line =~ @@ -1352,7 +1380,7 @@ sub generate_schema # The schemas. don't create empty schema, sql server creates a schema per user, even if it ends empty foreach my $schema (sort keys %{$objects}) { - unless (dboreplace($schema) eq 'public' + unless (relabel_schemas($schema) eq 'public' or not defined $objects->{$schema}) { print BEFORE "CREATE SCHEMA $schema;\n"; @@ -1365,7 +1393,7 @@ sub generate_schema # We have to do all domains before all tables while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # The user-defined types (domains, etc) foreach my $domain (sort keys %{$refschema->{DOMAINS}}) @@ -1379,7 +1407,7 @@ sub generate_schema # Tables and columns while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # The tables @@ -1410,7 +1438,7 @@ sub generate_schema # Sequences, PKs, Indexes while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # We now add all "AFTER" objects # We start with SEQUENCES, PKs and INDEXES (will be needed for FK) @@ -1423,7 +1451,7 @@ sub generate_schema . " START WITH " . $seqref->{START} . " OWNED BY " - . dboreplace($seqref->{OWNERSCHEMA}) . '.' + . relabel_schemas($seqref->{OWNERSCHEMA}) . '.' . $seqref->{OWNERTABLE} . ";\n"; } @@ -1453,7 +1481,7 @@ sub generate_schema # Unique while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # Now The UNIQUE constraints. They may be used for FK (if columns are not null) @@ -1477,7 +1505,7 @@ sub generate_schema # Other constraints while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # We have all we need for FKs now. We can put all other constraints (except PK of course) @@ -1499,7 +1527,7 @@ sub generate_schema " FOREIGN KEY (" . $constraint->{LOCAL_COLS} . ")" . " REFERENCES " - . dboreplace($constraint->{REMOTE_SCHEMA}) . '.' + . relabel_schemas($constraint->{REMOTE_SCHEMA}) . '.' . $constraint->{REMOTE_TABLE} . " ( " . $constraint->{REMOTE_COLS} . ")"; if (defined $constraint->{ON_DEL_CASC} @@ -1540,7 +1568,7 @@ sub generate_schema # Indexes. while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # Indexes # They don't have a schema qualifier. But their table has, and they are in the same schema as their table @@ -1565,7 +1593,7 @@ sub generate_schema # Default values while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # Default values foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -1592,7 +1620,7 @@ sub generate_schema # Comments on tables and columns while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # Comments on tables foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -1617,7 +1645,7 @@ sub generate_schema # Views, and their comments while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # The views, and comments foreach my $view (sort keys %{$refschema->{VIEWS}}) @@ -1633,7 +1661,7 @@ sub generate_schema # Trigger functions while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # The trigger functions foreach my $triggerfunc (sort keys %{$refschema->{TRIG_FUNCTIONS}}) @@ -1649,7 +1677,7 @@ sub generate_schema # Triggers while (my ($schema, $refschema) = each %{$objects}) { - $schema = dboreplace($schema) + $schema = relabel_schemas($schema) ; # If dbo, put into public, unless asked otherwise # triggers on tables, as these functions are declared now foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -1729,10 +1757,10 @@ sub resolve_name_conflicts # If a column has a custom type, it will be prefixed by schema # The schema will be the destination schema: dbo may have been replaced by public if ($col->{TYPE} eq - (dboreplace($schema) . '.' . $domain)) + (relabel_schemas($schema) . '.' . $domain)) { $col->{TYPE} = - dboreplace($schema) . '.' . $domain . "2pgd"; + relabel_schemas($schema) . '.' . $domain . "2pgd"; } } } @@ -1794,7 +1822,8 @@ sub resolve_name_conflicts "f=s" => \$filename, "i" => \$case_insensitive, "nr" => \$norelabel_dbo, - "num" => \$convert_numeric_to_int,); + "num" => \$convert_numeric_to_int, + "relabel_schemas=s" => \$relabel_schemas,); # We don't understand command line or have been asked for usage if (not $options or $help) @@ -1839,6 +1868,10 @@ sub resolve_name_conflicts exit 1; } +# We need to build %relabel_schemas from $relabel_schemas +build_relabel_schemas(); + + # Read SQL Server's dump file parse_dump(); From 687d12bbb2510af300dceb47382798b98764d119 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 27 Mar 2014 20:37:39 +0100 Subject: [PATCH 014/216] Corrected a bug with schema generation when relabel_schema is used --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 6eca2da..18de737 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1383,7 +1383,7 @@ sub generate_schema unless (relabel_schemas($schema) eq 'public' or not defined $objects->{$schema}) { - print BEFORE "CREATE SCHEMA $schema;\n"; + print BEFORE "CREATE SCHEMA ",relabel_schemas($schema),";\n"; } } From 11c9fdaea9b00fb9b069273d0fa47bfae23d9499 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 27 Mar 2014 20:43:05 +0100 Subject: [PATCH 015/216] Correct a sequence bug: minvalue has to be set to a value lower or equal to the start value --- sqlserver2pgsql.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 18de737..4265253 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1448,6 +1448,8 @@ sub generate_schema my $seqref = $refschema->{SEQUENCES}->{$sequence}; print AFTER "CREATE SEQUENCE $schema.$sequence INCREMENT BY " . $seqref->{STEP} + . " MINVALUE " + . $seqref->{START} . " START WITH " . $seqref->{START} . " OWNED BY " From e6dcb4942fcbdee754fe46b7d0672b6bd0f34fb3 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 28 Mar 2014 17:01:51 +0100 Subject: [PATCH 016/216] Now it can keep the identifiers (column, table, etc names)'s case, or not. And reserved keywords are accepted, as every identifier is double-quoted --- example_conf_file | 2 +- regression/reg.pl | 6 +- sqlserver2pgsql.pl | 152 ++++++++++++++++++++++++++++++--------------- 3 files changed, 108 insertions(+), 52 deletions(-) diff --git a/example_conf_file b/example_conf_file index 54be17a..ae34f55 100644 --- a/example_conf_file +++ b/example_conf_file @@ -24,4 +24,4 @@ case insensitive=1 # set it to 0 to generate a dump with citext and check constr no relabel dbo=1 # set it to 0 to convert the dbo schema to public convert numeric to int=1 # set it to 0 to keep numeric(xx,0) as numeric(xx,0). Will be converted to smallint, int or bigint by default relabel schemas=dbo=>foo;schema1=>bar - +keep identifier case=1 # keep case of database objects diff --git a/regression/reg.pl b/regression/reg.pl index 8cbf973..9f6290e 100755 --- a/regression/reg.pl +++ b/regression/reg.pl @@ -7,7 +7,7 @@ foreach my $file (<*.sql>) { - my @options_to_try=('-i','-nr','-num'); + my @options_to_try=('-i','-nr','-num', '-keep_identifier_case'); my @all_combinations=(''); foreach my $option (@options_to_try) { @@ -26,7 +26,9 @@ print "======================================\n"; system("dropdb reg"); system("createdb reg"); - system(".././sqlserver2pgsql.pl -f $file -b /tmp/before -a /tmp/after -u /tmp/unsure -k /tmp/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 $options"); + my $command=".././sqlserver2pgsql.pl -f $file -b /tmp/before -a /tmp/after -u /tmp/unsure -k /tmp/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 $options"; + print $command,"\n"; + system($command); if ($? >>8) { die "Could not generate files for $file\n"; diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 4265253..2c4e602 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -40,6 +40,7 @@ our $before_file; our $after_file; our $unsure_file; +our $keep_identifier_case; my $template ; # These two variables are loaded in the BEGIN block at the end of this file (they are very big @@ -76,6 +77,7 @@ sub parse_conf_file 'no relabel dbo' => 'norelabel_dbo', 'convert numeric to int' => 'convert_numeric_to_int', 'relabel schemas' => 'relabel_schemas', + 'keep identifier case' => 'keep_identifier_case', ); # Open the conf file or die @@ -106,6 +108,7 @@ sub parse_conf_file $case_insensitive=0 unless (defined ($case_insensitive)); $norelabel_dbo=0 unless (defined ($norelabel_dbo)); $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); + $keep_identifier_case=0 unless (defined ($keep_identifier_case)); close CONF; } @@ -217,7 +220,7 @@ sub convert_type my $constraint; $constraint->{TYPE} = 'CHECK_CITEXT'; $constraint->{TABLE} = $tablename; - $constraint->{TEXT} = "char_length($colname) <= $sqlqual"; + $constraint->{TEXT} = "char_length(" . format_identifier($colname) . ") <= $sqlqual"; push @{$objects->{$schemaname}->{TABLES}->{$tablename} ->{CONSTRAINTS}}, ($constraint); } @@ -236,6 +239,34 @@ sub convert_type return $rettype; } +# This function formats the identifiers (object name), putting double quotes around it +# It also converts case if asked +sub format_identifier +{ + my ($identifier)=@_; + croak "identifier not defined in format_identifier" unless (defined $identifier); + unless ($keep_identifier_case) + { + $identifier=lc($identifier); + } + # Now, we protect the identifier (similar to quote_ident in PG) + $identifier=~ s/"/""/g; + $identifier='"'.$identifier.'"'; + return $identifier; +} + +# This is a bit of a ugly hack: for indexes, in the column definition, there may be ASC/DESC at the end +# Instead of changing the whole structure of the code, just detect this asc/desc and split it before calling format_identifier +sub format_identifier_cols_index +{ + my ($idx_identifier)=@_; + $idx_identifier =~ /^(.*?)(?: (ASC|DESC))?$/; + my ($col,$order)=($1,$2); + my $formatted=format_identifier($col); + return $formatted unless defined ($order); + return $formatted . ' ' . $order; +} + # This one will try to convert what can obviously be converted from transact to PG # Things such as getdate() which can become now() sub convert_transactsql_code @@ -276,14 +307,17 @@ sub next_col_pos my %relabel_schemas; sub build_relabel_schemas { - foreach my $pair (split (';',$relabel_schemas)) + if (defined $relabel_schemas) { - my @pair=split('=>',$pair); - unless (scalar(@pair)==2) + foreach my $pair (split (';',$relabel_schemas)) { - die "Cannot parse the schema list given as argument: <$relabel_schemas>\n"; + my @pair=split('=>',$pair); + unless (scalar(@pair)==2) + { + die "Cannot parse the schema list given as argument: <$relabel_schemas>\n"; + } + $relabel_schemas{$pair[0]}=$pair[1]; } - $relabel_schemas{$pair[0]}=$pair[1]; } # Don't forget dbo -> public if it was asked unless ($norelabel_dbo) @@ -363,6 +397,8 @@ sub usage print "-relabel_schemas gives a list of schemas to rename. For instance -relabel_schemas 'source1=>dest1;source2=>dest2'\n"; print " -nr simply cancels the default dbo=>public remapping. Don't forget to put the remapping between quotes\n"; + print + "-keep_identifier_case tells $0 to keep the case of sql server database objects (not advised). Default is to lowercase everything.\n"; print "before_file contains the structure\n"; print "after_file contains index, constraints\n"; print @@ -455,8 +491,10 @@ sub generate_kettle $newtemplate =~ s/__postgres_username__/$pu/g; $newtemplate =~ s/__postgres_password__/$pw/g; $newtemplate =~ s/__sqlserver_table_name__/$schema.$table/g; - $newtemplate =~ s/__postgres_table_name__/$table/g; - $newtemplate =~ s/__postgres_schema_name__/$targetschema/g; + my $pgtable=format_identifier($table); + my $pgschema=format_identifier($targetschema); + $newtemplate =~ s/__postgres_table_name__/$pgtable/g; + $newtemplate =~ s/__postgres_schema_name__/$pgschema/g; # Store this new transformation into its file open FILE, ">$dir/$schema-$table.ktr" @@ -703,7 +741,7 @@ sub parse_dump $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} ->{$colname}->{DEFAULT}->{VALUE} = "nextval('" - . relabel_schemas(${schemaname}) . '.' + . format_identifier(relabel_schemas(${schemaname})) . '.' . ${seqname} . "')"; $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} ->{$colname}->{DEFAULT}->{UNSURE} = 0; @@ -713,7 +751,9 @@ sub parse_dump $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} = $stepseq; $objects->{$schemaname}->{SEQUENCES}->{$seqname} - ->{OWNERTABLE} = $tablename . "." . $colname; + ->{OWNERTABLE} = $tablename; + $objects->{$schemaname}->{SEQUENCES}->{$seqname} + ->{OWNERCOL} = $colname; $objects->{$schemaname}->{SEQUENCES}->{$seqname} ->{OWNERSCHEMA} = $schemaname; } @@ -973,8 +1013,8 @@ sub parse_dump # We add them to known data types, as they probably will be used in table definitions # but they point to themselves, with the schema corrected: we want them substituted by themselves - $types{$schema . '.' . $type} = relabel_schemas($schema) . '.' - . $type; # We store the schema with it + $types{$schema . '.' . $type} = format_identifier(relabel_schemas($schema)) . '.' + . format_identifier($type); # We store the schema with it. And we do the case conversion, the quoting, etc right now } elsif ($line =~ /^CREATE (UNIQUE )?(NONCLUSTERED|CLUSTERED) INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/ @@ -1140,9 +1180,10 @@ sub parse_dump my $table = $2; my $schema = $1; $constraint->{TYPE} = 'FK'; - $constraint->{LOCAL_COLS} = $4; + my @local_cols = split (/\s*,\s*/,$4); # Split around the comma. There may be whitespaces + @local_cols=map{s/^\[//;s/]$//;$_;} @local_cols; # Remove the brackets around the columns + $constraint->{LOCAL_COLS}=\@local_cols; $constraint->{LOCAL_TABLE} = $2; - $constraint->{LOCAL_COLS} =~ s/\[|\]//g; # Remove brackets while (my $fk = read_and_clean($file)) { @@ -1154,7 +1195,10 @@ sub parse_dump } elsif ($fk =~ /^REFERENCES \[(.*)\]\.\[(.*)\] \((.*?)\)/) { - $constraint->{REMOTE_COLS} = $3; + + my @remote_cols = split (/\s*,\s*/,$3); # Split around the comma. There may be whitespaces + @remote_cols=map{s/^\[//;s/]$//;$_;} @remote_cols; # Remove the brackets around the columns + $constraint->{REMOTE_COLS}=\@remote_cols; $constraint->{REMOTE_TABLE} = $2; $constraint->{REMOTE_SCHEMA} = $1; $constraint->{REMOTE_COLS} =~ @@ -1383,7 +1427,7 @@ sub generate_schema unless (relabel_schemas($schema) eq 'public' or not defined $objects->{$schema}) { - print BEFORE "CREATE SCHEMA ",relabel_schemas($schema),";\n"; + print BEFORE "CREATE SCHEMA ",format_identifier(relabel_schemas($schema)),";\n"; } } @@ -1398,7 +1442,7 @@ sub generate_schema # The user-defined types (domains, etc) foreach my $domain (sort keys %{$refschema->{DOMAINS}}) { - print BEFORE "CREATE DOMAIN $schema.$domain " + print BEFORE "CREATE DOMAIN " . format_identifier($schema) . '.' . format_identifier($domain) . ' ' . $refschema->{DOMAINS}->{$domain} . ";\n"; } @@ -1423,14 +1467,14 @@ sub generate_schema { my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; - my $coldef = $col . " " . $colref->{TYPE}; + my $coldef = format_identifier($col) . " " . $colref->{TYPE}; if ($colref->{NOT_NULL}) { $coldef .= ' NOT NULL'; } push @colsdef, ($coldef); } - print BEFORE "CREATE TABLE $schema.$table ( \n\t" + print BEFORE "CREATE TABLE " . format_identifier($schema) . '.' . format_identifier($table) . "( \n\t" . join(",\n\t", @colsdef) . ");\n\n"; } @@ -1446,15 +1490,16 @@ sub generate_schema foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) { my $seqref = $refschema->{SEQUENCES}->{$sequence}; - print AFTER "CREATE SEQUENCE $schema.$sequence INCREMENT BY " + print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence) . " INCREMENT BY " . $seqref->{STEP} . " MINVALUE " . $seqref->{START} . " START WITH " . $seqref->{START} . " OWNED BY " - . relabel_schemas($seqref->{OWNERSCHEMA}) . '.' - . $seqref->{OWNERTABLE} . ";\n"; + . format_identifier(relabel_schemas($seqref->{OWNERSCHEMA})) . '.' + . format_identifier($seqref->{OWNERTABLE}) . '.' + . format_identifier($seqref->{OWNERCOL}) . ";\n"; } # Now PK. We have to go through all tables @@ -1470,13 +1515,15 @@ sub generate_schema #print STDERR "Warning: $table has no primary key.\n"; next; } - my $pkdef = "ALTER TABLE $schema.$table ADD"; + my $pkdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; if (defined $refpk->{NAME}) { - $pkdef .= " CONSTRAINT " . $refpk->{NAME}; + $pkdef .= " CONSTRAINT " . format_identifier($refpk->{NAME}); } + # Create a list of formatted columns + my @collist=map{format_identifier($_)} @{$refpk->{COLS}}; $pkdef .= - " PRIMARY KEY (" . join(',', @{$refpk->{COLS}}) . ");\n"; + " PRIMARY KEY (" . join(',', @collist) . ");\n"; print AFTER $pkdef; } } @@ -1493,13 +1540,14 @@ sub generate_schema @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) { next unless ($constraint->{TYPE} eq 'UNIQUE'); - my $consdef = "ALTER TABLE $schema.$table ADD"; + my $consdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; if (defined $constraint->{NAME}) { - $consdef .= " CONSTRAINT " . $constraint->{NAME}; + $consdef .= " CONSTRAINT " . format_identifier($constraint->{NAME}); } + my @collist=map{format_identifier($_)} @{$constraint->{COLS}}; $consdef .= - " UNIQUE (" . join(",", @{$constraint->{COLS}}) . ");\n"; + " UNIQUE (" . join(",", @collist) . ");\n"; print AFTER $consdef; } } @@ -1517,21 +1565,24 @@ sub generate_schema @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) { next if ($constraint->{TYPE} =~ /^UNIQUE|PK$/); - my $consdef = "ALTER TABLE $schema.$table ADD"; + my $consdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; if (defined $constraint->{NAME}) { - $consdef .= " CONSTRAINT " . $constraint->{NAME}; + $consdef .= " CONSTRAINT " . format_identifier($constraint->{NAME}); } if ($constraint->{TYPE} eq 'FK') # COLS are already a comma separated list { + # We need to convert the column list to protected names + my @localcollist=map{format_identifier($_)} @{$constraint->{LOCAL_COLS}}; + my @remotecollist=map{format_identifier($_)} @{$constraint->{REMOTE_COLS}}; $consdef .= " FOREIGN KEY (" - . $constraint->{LOCAL_COLS} . ")" + . join(',',@localcollist) . ")" . " REFERENCES " - . relabel_schemas($constraint->{REMOTE_SCHEMA}) . '.' - . $constraint->{REMOTE_TABLE} . " ( " - . $constraint->{REMOTE_COLS} . ")"; + . format_identifier(relabel_schemas($constraint->{REMOTE_SCHEMA})) . '.' + . format_identifier($constraint->{REMOTE_TABLE}) . " ( " + . join(',',@remotecollist) . ")"; if (defined $constraint->{ON_DEL_CASC} and $constraint->{ON_DEL_CASC}) { @@ -1586,8 +1637,8 @@ sub generate_schema { $idxdef .= " UNIQUE"; } - $idxdef .= " INDEX $index ON $schema.$table (" - . join(",", @{$idxref->{COLS}}) . ");\n"; + $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" + . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ");\n"; print AFTER $idxdef; } } @@ -1606,7 +1657,7 @@ sub generate_schema my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; next unless (defined $colref->{DEFAULT}); my $definition = - "ALTER TABLE $schema.$table ALTER COLUMN $col SET DEFAULT " + "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ALTER COLUMN " . format_identifier($col) . " SET DEFAULT " . $colref->{DEFAULT}->{VALUE} . ";\n"; if ($colref->{DEFAULT}->{UNSURE}) { @@ -1629,7 +1680,7 @@ sub generate_schema { if (defined($refschema->{TABLES}->{$table}->{COMMENT})) { - print AFTER "COMMENT ON TABLE $schema.$table IS '" + print AFTER "COMMENT ON TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " IS '" . $refschema->{TABLES}->{$table}->{COMMENT} . "';\n"; } foreach @@ -1638,7 +1689,7 @@ sub generate_schema my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; if (defined($colref->{COMMENT})) { - print AFTER "COMMENT ON COLUMN $schema.$table.$col IS '" + print AFTER "COMMENT ON COLUMN " . format_identifier($schema) . '.' . format_identifier($table) . '.' . format_identifier($col) . " IS '" . $colref->{COMMENT} . "';\n"; } } @@ -1671,7 +1722,7 @@ sub generate_schema my $code = $refschema->{TRIG_FUNCTIONS}->{$triggerfunc}->{DEF}; my $language = $refschema->{TRIG_FUNCTIONS}->{$triggerfunc}->{LANG}; print UNSURE - "CREATE FUNCTION $schema.$triggerfunc() RETURNS trigger LANGUAGE $language AS \$def\$\n"; + "CREATE FUNCTION " . format_identifier($schema) . '.' . $triggerfunc . "() RETURNS trigger LANGUAGE $language AS \$def\$\n"; print UNSURE $code; print UNSURE "\$def\$;\n"; } @@ -1729,17 +1780,17 @@ sub resolve_name_conflicts # Store all known names foreach my $table (keys %{$refschema->{TABLES}}) { - $known_names{$table} = 1; + $known_names{format_identifier($table)} = 1; } # We scan all types. For now, this tool only generates domains, so we scan domains foreach my $domain (keys %{$refschema->{DOMAINS}}) { - if (not defined($known_names{$domain})) + if (not defined($known_names{format_identifier($domain)})) { # Great. Just skip to the next and remember this name - $known_names{$domain} = 1; + $known_names{format_identifier($domain)} = 1; } else { @@ -1758,14 +1809,16 @@ sub resolve_name_conflicts # If a column has a custom type, it will be prefixed by schema # The schema will be the destination schema: dbo may have been replaced by public + # Be careful that they are stored formatted through format_identifier if ($col->{TYPE} eq - (relabel_schemas($schema) . '.' . $domain)) + (format_identifier(relabel_schemas($schema)) . '.' . format_identifier($domain))) { $col->{TYPE} = - relabel_schemas($schema) . '.' . $domain . "2pgd"; + format_identifier(relabel_schemas($schema)) . '.' . format_identifier($domain . "2pgd"); } } } + $known_names{format_identifier($domain."2pgd")}=1; } } @@ -1776,11 +1829,11 @@ sub resolve_name_conflicts foreach my $idx (keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) { - if (not defined($known_names{$idx})) + if (not defined($known_names{format_identifier($idx)})) { # Great. Just skip to the next and remember this name - $known_names{$idx} = 1; + $known_names{format_identifier($idx)} = 1; } else { @@ -1793,7 +1846,7 @@ sub resolve_name_conflicts delete $refschema->{TABLES}->{$table}->{INDEXES}->{$idx}; print STDERR "Warning: I had to rename index $table.$idx to ${idx}2pgi because of naming conflicts in source schema $schema\n"; - $known_names{$idx . "2pgi"} = 1; + $known_names{format_identifier($idx . "2pgi")} = 1; } } } @@ -1825,7 +1878,8 @@ sub resolve_name_conflicts "i" => \$case_insensitive, "nr" => \$norelabel_dbo, "num" => \$convert_numeric_to_int, - "relabel_schemas=s" => \$relabel_schemas,); + "relabel_schemas=s" => \$relabel_schemas, + "keep_identifier_case" =>\$keep_identifier_case,); # We don't understand command line or have been asked for usage if (not $options or $help) From b2d609f12c070b14cc2bf4bb23e4061941ab475b Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Sun, 30 Mar 2014 18:11:31 +0200 Subject: [PATCH 017/216] Correct order between indexes and foreign keys. Some foreign keys depend on unique indexes or constraints instead of primary keys --- sqlserver2pgsql.pl | 53 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2c4e602..ef80ab4 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1433,7 +1433,7 @@ sub generate_schema # For the rest, we iterate over schemas # The tables, columns, etc... will be created in the before script, so there is no dependancy - # problem with constraints, that will be in the after script + # problem with constraints, that will be in the after script, except foreign keys which depend on unique indexes # We have to do all domains before all tables while (my ($schema, $refschema) = each %{$objects}) { @@ -1552,6 +1552,31 @@ sub generate_schema } } } + # Indexes. Unique indexes are needed before foreign key constraints + while (my ($schema, $refschema) = each %{$objects}) + { + $schema = relabel_schemas($schema) + ; # If dbo, put into public, unless asked otherwise + # Indexes + # They don't have a schema qualifier. But their table has, and they are in the same schema as their table + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + foreach my $index ( + sort keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) + { + my $idxref = + $refschema->{TABLES}->{$table}->{INDEXES}->{$index}; + my $idxdef = "CREATE"; + if ($idxref->{UNIQUE}) + { + $idxdef .= " UNIQUE"; + } + $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" + . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ");\n"; + print AFTER $idxdef; + } + } + } # Other constraints while (my ($schema, $refschema) = each %{$objects}) { @@ -1618,31 +1643,7 @@ sub generate_schema } } } - # Indexes. - while (my ($schema, $refschema) = each %{$objects}) - { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # Indexes - # They don't have a schema qualifier. But their table has, and they are in the same schema as their table - foreach my $table (sort keys %{$refschema->{TABLES}}) - { - foreach my $index ( - sort keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) - { - my $idxref = - $refschema->{TABLES}->{$table}->{INDEXES}->{$index}; - my $idxdef = "CREATE"; - if ($idxref->{UNIQUE}) - { - $idxdef .= " UNIQUE"; - } - $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" - . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ");\n"; - print AFTER $idxdef; - } - } - } + # Default values while (my ($schema, $refschema) = each %{$objects}) { From 2f60cc4ce30e135b31b57ce49e9770fb5b9ecd7d Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 1 Apr 2014 16:49:24 +0200 Subject: [PATCH 018/216] Add a first try at making kettle work with case conversions --- sqlserver2pgsql.pl | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index ef80ab4..9c193b4 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -450,6 +450,7 @@ sub generate_kettle # Is the PK int and on only one column ? # If yes, we can use several threads in kettle to read this table to # improve performance + my $wherefilter; if (defined($refschema->{TABLES}->{$table}->{PK}->{COLS}) and scalar(@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}) @@ -458,7 +459,7 @@ sub generate_kettle ->{($refschema->{TABLES}->{$table}->{PK}->{COLS}->[0]) }->{TYPE} =~ /int$/)) { - my $wherefilter = + $wherefilter = 'WHERE ' . $refschema->{TABLES}->{$table}->{PK}->{COLS}->[0] . '% ${Internal.Step.Unique.Count} = ${Internal.Step.Unique.Number}'; @@ -470,15 +471,32 @@ sub generate_kettle # No way to do this optimization. Use standard template { + $wherefilter=''; $newtemplate =~ s/__sqlserver_where_filter__//; $newtemplate =~ s/__sqlserver_copies__/1/g; } + $newtemplate =~ s/__sqlserver_where_filter__/$wherefilter/; } else { $newtemplate = $template; } + # Build the column list of the table to put into the SQL Server query + my @colsdef; + foreach my $col ( + sort { + $refschema->{TABLES}->{$table}->{COLS}->{$a}->{POS} + <=> $refschema->{TABLES}->{$table}->{COLS}->{$b} + ->{POS} + } (keys %{$refschema->{TABLES}->{$table}->{COLS}})) + + { + my $coldef = "[$col] AS " . format_identifier($col); + push @colsdef,($coldef); + } + my $colsdef=join(',',@colsdef); + # Substitute every connection placeholder with the real value $newtemplate =~ s/__sqlserver_database__/$sd/g; $newtemplate =~ s/__sqlserver_host__/$sh/g; @@ -491,6 +509,7 @@ sub generate_kettle $newtemplate =~ s/__postgres_username__/$pu/g; $newtemplate =~ s/__postgres_password__/$pw/g; $newtemplate =~ s/__sqlserver_table_name__/$schema.$table/g; + $newtemplate =~ s/__sqlserver_table_cols__/$colsdef/g; my $pgtable=format_identifier($table); my $pgschema=format_identifier($targetschema); $newtemplate =~ s/__postgres_table_name__/$pgtable/g; @@ -499,6 +518,7 @@ sub generate_kettle # Store this new transformation into its file open FILE, ">$dir/$schema-$table.ktr" or die "Cannot write to $dir/$schema-$table.ktr"; + binmode(FILE,":utf8"); print FILE $newtemplate; close FILE; } @@ -2110,7 +2130,7 @@ BEGIN __sqlserver_db__ - SELECT * FROM __sqlserver_table_name__ WITH(NOLOCK) + SELECT __sqlserver_table_cols__ FROM __sqlserver_table_name__ WITH(NOLOCK) 0 N From 78c2d22ec8376f83fde57aeb0fa9ac302edfcec9 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 1 Apr 2014 16:51:15 +0200 Subject: [PATCH 019/216] fix obvious oversight (forgot LOB kettle jobs) --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 9c193b4..2bd6fc5 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2345,7 +2345,7 @@ BEGIN __sqlserver_db__ - SELECT * FROM __sqlserver_table_name__ WITH(NOLOCK) __sqlserver_where_filter__ + SELECT __sqlserver_table_cols__ FROM __sqlserver_table_name__ WITH(NOLOCK) __sqlserver_where_filter__ 0 N From e752cdef9798891ad611e8e965fa28d44fa93b11 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 1 Apr 2014 17:26:33 +0200 Subject: [PATCH 020/216] Ask kettle to quote all identifiers to avoid problems with some identifiers (starting with a number for instance) --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2bd6fc5..1ac13f2 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2277,7 +2277,7 @@ BEGIN IS_CLUSTEREDN MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ - QUOTE_ALL_FIELDSN + QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN USE_POOLINGN @@ -2299,7 +2299,7 @@ BEGIN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN PORT_NUMBER__postgres_port__ - QUOTE_ALL_FIELDSN + QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN From bf10d22cdfc0c42a2a18709660a86c514e830e4c Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 1 Apr 2014 18:14:09 +0200 Subject: [PATCH 021/216] Forgot some quote identifiers in the kettle xml --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 1ac13f2..25277ad 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2062,7 +2062,7 @@ BEGIN IS_CLUSTEREDN MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ - QUOTE_ALL_FIELDSN + QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN USE_POOLINGN @@ -2084,7 +2084,7 @@ BEGIN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN PORT_NUMBER__postgres_port__ - QUOTE_ALL_FIELDSN + QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN From 7f9107aaf69c91067fb52222cfa58f3ddb487b7a Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 8 Apr 2014 16:35:25 +0200 Subject: [PATCH 022/216] Add if not exists in the create schema and create extension --- sqlserver2pgsql.pl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 25277ad..070b567 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1434,7 +1434,7 @@ sub generate_schema # if someone wants to start with an older version, it's their problem :) if ($case_insensitive) { - print BEFORE "CREATE EXTENSION citext;\n"; + print BEFORE "CREATE EXTENSION IF NOT EXISTS citext;\n"; } # Ok, we have parsed everything, and definitions are in $objects @@ -1447,7 +1447,8 @@ sub generate_schema unless (relabel_schemas($schema) eq 'public' or not defined $objects->{$schema}) { - print BEFORE "CREATE SCHEMA ",format_identifier(relabel_schemas($schema)),";\n"; + # Not compatible before 9.3. This is the logical target for this tool anyway + print BEFORE "CREATE SCHEMA IF NOT EXISTS ",format_identifier(relabel_schemas($schema)),";\n"; } } From e89081ace908ebbbc2e6905a0ce5690b7e361b07 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 8 Apr 2014 16:37:08 +0200 Subject: [PATCH 023/216] correct documentation for quoting under windows --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb465f1..eb19f2c 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ You can also use the -i, -num and/or -nr options: schema to PostgreSQL's default schema -relabel_schemas is a list of schemas to remap. The syntax is : 'source1=>dest1;source2=>dest2'. Don't forget to quote this option or the shell might alter it -there is a default dbo=>public remapping, that can be cancelled with -nr +there is a default dbo=>public remapping, that can be cancelled with -nr. Use double quotes instead of simple quotes on Windows. -num : Converts numeric (xxx,0) to the appropriate smallint, integer or bigint. It won't keep the constraint on the size of the scale of the numeric From 5929d0716cbf77a0368494ede6154d4848442d83 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 8 Apr 2014 16:39:46 +0200 Subject: [PATCH 024/216] overwrite relabel_dbo with the relabel_schema, if both are given --- sqlserver2pgsql.pl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 070b567..28600bb 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -307,6 +307,13 @@ sub next_col_pos my %relabel_schemas; sub build_relabel_schemas { + # Don't forget dbo -> public if it was asked (default) + unless ($norelabel_dbo) + { + $relabel_schemas{'dbo'}='public'; + } + # dbo can be overwritten in relabel_schema (the user will probably forget to deactivate the relabel). + # so we do the real relabeling after the norelabel_dbo, to overwrite if (defined $relabel_schemas) { foreach my $pair (split (';',$relabel_schemas)) @@ -319,11 +326,7 @@ sub next_col_pos $relabel_schemas{$pair[0]}=$pair[1]; } } - # Don't forget dbo -> public if it was asked - unless ($norelabel_dbo) - { - $relabel_schemas{'dbo'}='public'; - } + } From e70da085a872428fb23e67c9acc11d5a1689efdd Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 8 Apr 2014 16:42:35 +0200 Subject: [PATCH 025/216] document the configuration file's existence --- README.md | 3 +++ sqlserver2pgsql.pl | 1 + 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index eb19f2c..f3a0145 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,9 @@ The after script contains the rest (indexes, constraints). It should be run after data is imported. The unsure script contains objects where we attempt to migrate, but cannot guarantee, such as views. +-conf uses a conf file. All options below can also be set there. Command line options will overwrite conf options. +There is an example of such a conf file (example_conf_file) + You can also use the -i, -num and/or -nr options: -i : Generate an "ignore case" schema, using citext, to emulate MSSQL's case insensitive collation. diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 28600bb..10bc6f3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -392,6 +392,7 @@ sub usage "$0 [-k kettle_output_directory] -b before_file -a after_file -u unsure_file -f sql_server_schema_file[-h] [-i]\n"; print "\nExpects a SQL Server SQL structure dump as -f (preferably unicode)\n"; + print "-conf uses a conf file. All options below can also be set there. Command line options will overwrite conf options\n"; print "-i tells $0 to create a case-insensitive PostgreSQL schema\n"; print "-nr tells $0 not to convert the dbo schema to public. dbo will stay dbo\n"; From 419365bf1a082f394796c3ed817bc7ab98a190fc Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 8 Apr 2014 16:43:38 +0200 Subject: [PATCH 026/216] correct case insensivity in the example conf file --- example_conf_file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_conf_file b/example_conf_file index ae34f55..ba8538c 100644 --- a/example_conf_file +++ b/example_conf_file @@ -20,7 +20,7 @@ postgresql username=bar_user postgresql password=bar_password # Optional behaviour -case insensitive=1 # set it to 0 to generate a dump with citext and check constraints all over the place +case insensitive=0 # set it to 1 to generate a dump with citext and check constraints all over the place no relabel dbo=1 # set it to 0 to convert the dbo schema to public convert numeric to int=1 # set it to 0 to keep numeric(xx,0) as numeric(xx,0). Will be converted to smallint, int or bigint by default relabel schemas=dbo=>foo;schema1=>bar From 4b1a382419e489ef1b775d5e3e233de25f30e22e Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 8 Apr 2014 16:50:56 +0200 Subject: [PATCH 027/216] Die on schema comment: never met one for now, ask for an example --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 10bc6f3..5b16624 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1336,8 +1336,8 @@ sub parse_dump } elsif ($obj eq 'SCHEMA') { - # There is no schema comments in PG. Just print a warning and ignore them - print STDERR "Schema comment : <$comment> ignored (no schema comment in PG)\n"; + # Never met one for now. Die and ask to send me an example + die "Schema comment : <$comment> not understood. Please send a bug report\n"; } } From df44b1190babe3cb8ed01bc4b1438ebcf42043e6 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 8 Apr 2014 16:52:23 +0200 Subject: [PATCH 028/216] replace now() with CURRENT_TIMESTAMP, which is the SQL standard --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 5b16624..2183e66 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -268,11 +268,11 @@ sub format_identifier_cols_index } # This one will try to convert what can obviously be converted from transact to PG -# Things such as getdate() which can become now() +# Things such as getdate() which can become CURRENT_TIMESTAMP sub convert_transactsql_code { my ($code)=@_; - $code =~ s/getdate\s*\(\)/now()/g; + $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/g; return $code; } From 5840a1966fa9d05b720f3cd44dae601ab9922874 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 9 Apr 2014 11:27:51 +0200 Subject: [PATCH 029/216] Refactored the table parsing to make it work like all other parsings --- sqlserver2pgsql.pl | 398 +++++++++++++++++++++------------------------ 1 file changed, 186 insertions(+), 212 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2183e66..8b06b7b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -686,13 +686,6 @@ sub parse_dump open $file, "<:encoding(" . $decoder->name . ")", $filename or die "Cannot open $filename"; - # Parsing loop variables - my $create_table = 0; # Are we in a create table statement ? - my $tablename = ''; # If yes, what's the table name ? - my $schemaname = ''; # If yes, what's the schema name ? - my $colnumber = - 0; # Column number (just to put the commas in the right places) ? - # Tagged because sql statements are often multi-line, so there are inner loops in some conditions MAIN: while (my $line = read_and_clean($file)) { @@ -700,248 +693,229 @@ sub parse_dump # Create table, obviously. There will be other lines below for the rest of the table definition if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\(/) { - $create_table = 1; # We are now inside a create table - $schemaname = $1; - $tablename = $2; - $colnumber = 0; + my $schemaname = $1; + my $tablename = $2; + my $colnumber = 0; $objects->{$schemaname}->{TABLES}->{$tablename}->{haslobs} = 0; - } - - # Here is a col definition. We should be inside a create table - elsif ($line =~ - /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))? (NOT NULL|NULL)(,)?/ - ) - { - if ($create_table - ) # We are inside a create table, this is a column definition + # We are in a create table. Read everything until its end... + TABLE: while (my $line = read_and_clean($file)) { - $colnumber++; - my $colname = $1; - my $coltypeschema = $2; - my $coltype = $3; - if (defined $coltypeschema) + # Here is a col definition. + if ($line =~ + /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))? (NOT NULL|NULL)(,)?/ + ) { + $colnumber++; + my $colname = $1; + my $coltypeschema = $2; + my $coltype = $3; + if (defined $coltypeschema) + { - # The datatype is a user defined datatype - # It has already been declared before. We just need to find it - $coltype = $coltypeschema . '.' . $coltype; - } - my $colqual = $4; - my $isidentity = $5; - my $colisnull = $6; - if ($colqual) - { - if ($colqual eq '(max)') + # The datatype is a user defined datatype + # It has already been declared before. We just need to find it + $coltype = $coltypeschema . '.' . $coltype; + } + my $colqual = $4; + my $isidentity = $5; + my $colisnull = $6; + if ($colqual) { - $colqual = undef - ; # max in sql server is the same as putting no colqual in pg + if ($colqual eq '(max)') + { + $colqual = undef + ; # max in sql server is the same as putting no colqual in pg + } + else + { + # We need the number (or 2 numbers) in this qual + $colqual =~ /\((\d+(?:,\s*\d+)?)\)/ + or die "Cannot parse colqual <$colqual>"; + $colqual = "$1"; + } } - else + my $newtype = + convert_type($coltype, $colqual, $colname, + $tablename, undef, $schemaname); + + # If it is an identity, we'll map to serial/bigserial (create a sequence, then link it + # to the column) + if ($isidentity) { - # We need the number (or 2 numbers) in this qual - $colqual =~ /\((\d+(?:,\s*\d+)?)\)/ - or die "Cannot parse colqual <$colqual>"; - $colqual = "$1"; + + # We have an identity field. We remember the default value and + # initialize the sequence correctly in the after script + $isidentity =~ /IDENTITY\((\d+),\s*(\d+)\)/ + or die "Cannot understand <$isidentity>"; + my $startseq = $1; + my $stepseq = $2; + my $seqname = lc("${tablename}_${colname}_seq"); + + # We get a sure default value. + $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} + ->{$colname}->{DEFAULT}->{VALUE} = + "nextval('" + . format_identifier(relabel_schemas(${schemaname})) . '.' + . ${seqname} . "')"; + $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} + ->{$colname}->{DEFAULT}->{UNSURE} = 0; + + $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{START} + = $startseq; + $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} + = $stepseq; + $objects->{$schemaname}->{SEQUENCES}->{$seqname} + ->{OWNERTABLE} = $tablename; + $objects->{$schemaname}->{SEQUENCES}->{$seqname} + ->{OWNERCOL} = $colname; + $objects->{$schemaname}->{SEQUENCES}->{$seqname} + ->{OWNERSCHEMA} = $schemaname; } - } - my $newtype = - convert_type($coltype, $colqual, $colname, - $tablename, undef, $schemaname); - # If it is an identity, we'll map to serial/bigserial (create a sequence, then link it - # to the column) - if ($isidentity) - { + # If there is a bytea generated, this table will contain a blob: + # use a special kettle transformation for it if generating kettle + # (see generate_kettle() ) + if ( $newtype eq 'bytea' + or $coltype eq + 'ntext') # Ntext is very slow, stored out of page + { + $objects->{$schemaname}->{'TABLES'}->{$tablename} + ->{haslobs} = 1; + } + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{POS} = $colnumber; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{TYPE} = $newtype; - # We have an identity field. We remember the default value and - # initialize the sequence correctly in the after script - $isidentity =~ /IDENTITY\((\d+),\s*(\d+)\)/ - or die "Cannot understand <$isidentity>"; - my $startseq = $1; - my $stepseq = $2; - my $seqname = lc("${tablename}_${colname}_seq"); - - # We get a sure default value. - $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} - ->{$colname}->{DEFAULT}->{VALUE} = - "nextval('" - . format_identifier(relabel_schemas(${schemaname})) . '.' - . ${seqname} . "')"; - $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} - ->{$colname}->{DEFAULT}->{UNSURE} = 0; - - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{START} - = $startseq; - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} - = $stepseq; - $objects->{$schemaname}->{SEQUENCES}->{$seqname} - ->{OWNERTABLE} = $tablename; - $objects->{$schemaname}->{SEQUENCES}->{$seqname} - ->{OWNERCOL} = $colname; - $objects->{$schemaname}->{SEQUENCES}->{$seqname} - ->{OWNERSCHEMA} = $schemaname; + if ($colisnull eq 'NOT NULL') + { + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 1; + } + else + { + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 0; + } } - # If there is a bytea generated, this table will contain a blob: - # use a special kettle transformation for it if generating kettle - # (see generate_kettle() ) - if ( $newtype eq 'bytea' - or $coltype eq - 'ntext') # Ntext is very slow, stored out of page + # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) + # So just put it as a varchar, and issue a warning is STDOUT + elsif ($line =~ /^\t\[(.*)\]\s+AS\s+\((.*)\)/) { - $objects->{$schemaname}->{'TABLES'}->{$tablename} - ->{haslobs} = 1; - } - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{POS} = $colnumber; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{TYPE} = $newtype; - if ($colisnull eq 'NOT NULL') - { + # We just get the column name + $colnumber++; + my $colname = $1; + my $code = $2; + my $coltype = 'varchar'; $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{NOT_NULL} = 1; - } - else - { + ->{$colname}->{POS} = $colnumber; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{TYPE} = $coltype; $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{NOT_NULL} = 0; - } - } - else - { - die "I don't understand $line. This is a bug"; - } - } - # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) - # So just put it as a varchar, and issue a warning is STDOUT - elsif ($line =~ /^\t\[(.*)\]\s+AS\s+\((.*)\)/) - { + # Big fat warning + print STDERR + "Warning: There is a calculated column: $schemaname.$tablename.$colname. This isn't done the same way in PG at all\n"; + print STDERR + "\tFor now it has been declared as a varchar in PG, so that the values can be copied\n"; + print STDERR + "\tYou should change its type manually in the dump (sorry for that),\n"; + print STDERR "\tA trigger has been written in the unsure file. It probably won't work as is.\n"; + print STDERR "\tPlease review it.\n"; - # We just get the column name - $colnumber++; - my $colname = $1; - my $code = $2; - my $coltype = 'varchar'; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{POS} = $colnumber; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{TYPE} = $coltype; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{NOT_NULL} = 0; - - # Big fat warning - print STDERR - "Warning: There is a calculated column: $schemaname.$tablename.$colname. This isn't done the same way in PG at all\n"; - print STDERR - "\tFor now it has been declared as a varchar in PG, so that the values can be copied\n"; - print STDERR - "\tYou should change its type manually in the dump (sorry for that),\n"; - print STDERR "\tA trigger has been written in the unsure file. It probably won't work as is.\n"; - print STDERR "\tPlease review it.\n"; - - # Try to correct what can be corrected from the AS : replace [COL] with NEW.COL - $code =~ s/\[(.*?)\]/NEW.$1/g; - my $triggerfunc = <{$schemaname}->{'TRIG_FUNCTIONS'} - ->{'trig_func_ins_or_upd' || $tablename}->{DEF} = - $triggerfunc; - $objects->{$schemaname}->{'TRIG_FUNCTIONS'} - ->{'trig_func_ins_or_upd' || $tablename}->{LANG} = - 'plpgsql'; - my %trigger; - $trigger{EVENTS} = 'before insert or update'; - $trigger{WHEN} = 'for each row'; - $trigger{FUNCTION} = - 'trig_func_ins_or_upd' || $tablename; # In the same schema - $trigger{NAME} = 'trig_ins_or_upd' || $tablename; - push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} - ->{TRIGGERS}}, (\%trigger); + $objects->{$schemaname}->{'TRIG_FUNCTIONS'} + ->{'trig_func_ins_or_upd' || $tablename}->{DEF} = + $triggerfunc; + $objects->{$schemaname}->{'TRIG_FUNCTIONS'} + ->{'trig_func_ins_or_upd' || $tablename}->{LANG} = + 'plpgsql'; + my %trigger; + $trigger{EVENTS} = 'before insert or update'; + $trigger{WHEN} = 'for each row'; + $trigger{FUNCTION} = + 'trig_func_ins_or_upd' || $tablename; # In the same schema + $trigger{NAME} = 'trig_ins_or_upd' || $tablename; + push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} + ->{TRIGGERS}}, (\%trigger); - } - elsif ($line =~ - /^(?: CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED/) - { - # This is not forbidden by SQL, of course. I just never saw this in a sql server dump, - # so it should be an error for now (it will be syntaxically different if outside a table anyhow) - die "PK defined outside a table\n: $line" unless ($create_table); - - my $constraint - ; # We put everything inside this hashref, we'll push it into the constraint list later - $constraint->{TYPE} = 'PK'; - if (defined $1) - { - $constraint->{NAME} = $1; - } + } + elsif ($line =~ + /^(?: CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED/) + { + my $constraint + ; # We put everything inside this hashref, we'll push it into the constraint list later + $constraint->{TYPE} = 'PK'; + if (defined $1) + { + $constraint->{NAME} = $1; + } - # Here is the PK. We read the following lines until the end of the constraint - while (my $pk = read_and_clean($file)) - { + # Here is the PK. We read the following lines until the end of the constraint + while (my $pk = read_and_clean($file)) + { - # Exit when read a line beginning with ). The constraint is complete. We store it and go back to main loop - if ($pk =~ /^\)/) - { - push @{$objects->{$schemaname}->{TABLES}->{$tablename} - ->{CONSTRAINTS}}, ($constraint); + # Exit when read a line beginning with ). The constraint is complete. We store it and go back to main loop + if ($pk =~ /^\)/) + { + push @{$objects->{$schemaname}->{TABLES}->{$tablename} + ->{CONSTRAINTS}}, ($constraint); - # We also directly put the constraint reference in a direct path (for ease of use in generate_kettle) - $objects->{$schemaname}->{TABLES}->{$tablename}->{PK} = - $constraint; - next MAIN; + # We also directly put the constraint reference in a direct path (for ease of use in generate_kettle) + $objects->{$schemaname}->{TABLES}->{$tablename}->{PK} = + $constraint; + next TABLE; + } + if ($pk =~ /^\t\[(.*)\] (ASC|DESC)(,?)/) + { + push @{$constraint->{COLS}}, ($1); + } + } } - if ($pk =~ /^\t\[(.*)\] (ASC|DESC)(,?)/) + elsif ($line =~ /^\s*(?:CONSTRAINT \[(.*)\] )?UNIQUE/) { - push @{$constraint->{COLS}}, ($1); - } - } - } - elsif ($line =~ /^\s*(?:CONSTRAINT \[(.*)\] )?UNIQUE/) - { - - # This (having the constraint inside a create table )is not forbidden by SQL, - # of course. I just never saw this in a sql server dump, - # so it should be an error for now (it will be syntaxically different if outside a table anyhow) - die "Unique key defined outside a table\n: $line" - unless ($create_table); + my $constraint + ; # We put everything inside this hashref, we'll push it into the constraint list later + $constraint->{TYPE} = 'UNIQUE'; + if (defined $1) + { + $constraint->{NAME} = $1; + } - my $constraint - ; # We put everything inside this hashref, we'll push it into the constraint list later - $constraint->{TYPE} = 'UNIQUE'; - if (defined $1) - { - $constraint->{NAME} = $1; - } + # Unique key definition. We read following lines until the end of the constraint + while (my $uk = read_and_clean($file)) + { - # Unique key definition. We read following lines until the end of the constraint - while (my $uk = read_and_clean($file)) - { + # Exit when read a line beginning with ). The constraint is complete + if ($uk =~ /^\)/) + { + push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} + ->{CONSTRAINTS}}, ($constraint); + next TABLE; + } + if ($uk =~ /^\t\[(.*)\] (ASC|DESC)(,?)/) + { + push @{$constraint->{COLS}}, ($1); + } + } - # Exit when read a line beginning with ). The constraint is complete - if ($uk =~ /^\)/) - { - push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} - ->{CONSTRAINTS}}, ($constraint); - next MAIN; } - if ($uk =~ /^\t\[(.*)\] (ASC|DESC)(,?)/) + elsif ($line =~ /^\) ON \[PRIMARY\]/) { - push @{$constraint->{COLS}}, ($1); + # End of the table + next MAIN; } } - - } - elsif ($line =~ /^\) ON \[PRIMARY\]/) - { - - # End of the table - $create_table = 0; - $tablename = ''; } ################################################################ # From HERE, these SQL commands are not linked to a create table From fbc51fff3b3bbb429a190516acc51cfeb39348ab Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 9 Apr 2014 12:37:08 +0200 Subject: [PATCH 030/216] Parse foreign keys without names. The name wasn't retrieved at all, so correct this bug at the same time --- sqlserver2pgsql.pl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 8b06b7b..652f420 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -996,8 +996,6 @@ sub parse_dump $sql .= $line_cont; } } - - # I only have seen types with added constraints ( create type foo varchar(50)) for now # These are domains with PostgreSQL elsif ($line =~ /^CREATE TYPE \[(.*?)\]\.\[(.*?)\] FROM \[(.*?)](?:\((\d+(?:,\s*\d+)?)?\))?/ @@ -1092,10 +1090,6 @@ sub parse_dump { # Never seen one for now. The code is there though, in case, with a die for now die "$line: not understood. This is a bug"; - while (my $contline = read_and_clean($file)) - { - next MAIN if ($contline =~ /^GO/); - } } # Default values. numeric, then text. These are 100% sure, they will parse in PG @@ -1169,19 +1163,25 @@ sub parse_dump } # FK constraint. It's multi line, we have to look for references, and what to do on update, delete, etc (I have only seen delete cascade for now) + # Constraint name is optionnal elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD\s+CONSTRAINT \[(.*)\] FOREIGN KEY\((.*?)\)/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? FOREIGN KEY\((.*?)\)/ ) { # This is a FK definition. We have the foreign table definition in next line. my $constraint; my $table = $2; my $schema = $1; + my $consname= $3; $constraint->{TYPE} = 'FK'; my @local_cols = split (/\s*,\s*/,$4); # Split around the comma. There may be whitespaces @local_cols=map{s/^\[//;s/]$//;$_;} @local_cols; # Remove the brackets around the columns $constraint->{LOCAL_COLS}=\@local_cols; $constraint->{LOCAL_TABLE} = $2; + if (defined $consname) + { + $constraint->{NAME}=$consname; + } while (my $fk = read_and_clean($file)) { From 497fcbd36b5683304dd9a642794bd4b549c8457a Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 9 Apr 2014 18:03:17 +0200 Subject: [PATCH 031/216] big refactoring, and initial support for create type as table from sql server --- sqlserver2pgsql.pl | 183 ++++++++++++++++++++++++++++++--------------- 1 file changed, 122 insertions(+), 61 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 652f420..b740fd1 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -236,6 +236,16 @@ sub convert_type } } } + # We special case SQL Server's TABLE types: they should be converted into an array + if ( $sqlstype =~ /^(\S+)\.(\S+)$/) + { + # This is a namespaced type. So we check if this is a special array + my ($schema,$type)=($1,$2); + if (defined($objects->{$schema}->{TABLE_TYPES}->{$type})) + { + $rettype=$rettype.'[]'; + } + } return $rettype; } @@ -693,7 +703,7 @@ sub parse_dump # Create table, obviously. There will be other lines below for the rest of the table definition if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\(/) { - my $schemaname = $1; + my $schemaname = relabel_schemas($1); my $tablename = $2; my $colnumber = 0; $objects->{$schemaname}->{TABLES}->{$tablename}->{haslobs} = 0; @@ -714,7 +724,7 @@ sub parse_dump # The datatype is a user defined datatype # It has already been declared before. We just need to find it - $coltype = $coltypeschema . '.' . $coltype; + $coltype = relabel_schemas($coltypeschema) . '.' . $coltype; } my $colqual = $4; my $isidentity = $5; @@ -915,15 +925,18 @@ sub parse_dump # End of the table next MAIN; } + else + { + die "Cannot understand $line\n"; + } } } ################################################################ # From HERE, these SQL commands are not linked to a create table ################################################################ - elsif ($line =~ /CREATE SCHEMA \[(.*)\] AUTHORIZATION \[.*\]/) { - $objects->{$1} = undef + $objects->{relabel_schemas($1)} = undef ; # Nothing to add here, we create the schema, and put undef in it for now } elsif ($line =~ /CREATE\s+PROC(?:EDURE)?\s+\[.*\]\.\[(.*)\]/i) @@ -1001,17 +1014,74 @@ sub parse_dump /^CREATE TYPE \[(.*?)\]\.\[(.*?)\] FROM \[(.*?)](?:\((\d+(?:,\s*\d+)?)?\))?/ ) { - # Dependency between types is not done for now. If the problem arises, it should be added + # Dependency between types is not done for now. If the problem arises, it may be added my ($schema, $type, $origtype, $quals) = ($1, $2, $3, $4); + $schema=relabel_schemas($schema); my $newtype = convert_type($origtype, $quals, undef, undef, $type, $schema); $objects->{$schema}->{DOMAINS}->{$type} = $newtype; # We add them to known data types, as they probably will be used in table definitions # but they point to themselves, with the schema corrected: we want them substituted by themselves - $types{$schema . '.' . $type} = format_identifier(relabel_schemas($schema)) . '.' + $types{$schema . '.' . $type} = format_identifier($schema) . '.' . format_identifier($type); # We store the schema with it. And we do the case conversion, the quoting, etc right now } + # These are like arrays of composite (with added functionnality, but we'll skip these + # This will look like a table, but we'll ignore anything that isn't a column definition + # If any of the type isn't a base type, this will die. But anyway, we wouldn't be able to convert properly + elsif ($line =~ /^CREATE TYPE \[(.*)\]\.\[(.*)\] AS TABLE\(/) + { + my $schema=relabel_schemas($1); + my $typename=$2; + my $newbasetype=''; + my @cols_newbasetype; + my $colname; + my $type; + my $typequal; + my $newtype; + TYPE: while (my $typeline= read_and_clean($file)) + { + if ($typeline =~ /^\t\[(.*)\] \[(.*)\](?: \((\d+(?:,\d+)?)\))?(?: (?:NOT )?NULL),?$/) + { + # This is another column for this type + $colname=$1; + $type=$2; + $typequal=$3; + $newtype = + convert_type($type, $typequal, undef, undef, undef, undef); + push @cols_newbasetype,(format_identifier($colname) . ' ' . $newtype); + } + elsif ( $typeline =~ /PRIMARY KEY/) + { + print STDERR "Warning: TABLE type in SQL Server, input line $., ignored a primary key constraint\n"; + # Let's skip everything till next parenthesis (probably the end) + while (my $to_skip= read_and_clean($file)) + { + next TYPE if ($to_skip =~ /\)/); + } + } + elsif ($typeline =~ /^\)$/) # We reached the end of the type def + { + next; + } + elsif ($typeline =~ /^GO$/) + { + # We reached the end. We add this new type + # create the new type declaration + $newbasetype=join(",\n",@cols_newbasetype); + $objects->{$schema}->{TABLE_TYPES}->{$typename}=$newbasetype; + # We add this to known data types, it will be used in table definitions + $types{$schema . '.' . $typename} = format_identifier($schema) . '.' + . format_identifier($typename); # We store the schema with it. And we do the case conversion, the quoting, etc right now + next MAIN; + } + else + { + die "Cannot understand $type. This is a bug"; + } + } + } + elsif ($line =~ /^CREATE (UNIQUE )?(NONCLUSTERED|CLUSTERED) INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/ ) @@ -1024,7 +1094,7 @@ sub parse_dump my $isunique = $1; my $isclustered = $2; my $idxname = $3; - my $schemaname = $4; + my $schemaname = relabel_schemas($4); my $tablename = $5; if ($isunique) { @@ -1099,17 +1169,17 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?((?:-)?\d+(?:\.\d+)?)\)?\) FOR \[(.*)\]/ ) { - if ($objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{TYPE} eq 'boolean') + if ($objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{TYPE} eq 'boolean') { # Ok, it IS a boolean, and we have received a number if ($3 eq '0') { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} ->{VALUE} = 'false'; } elsif ($3 eq '1') { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} ->{VALUE} = 'true'; } else @@ -1120,10 +1190,10 @@ sub parse_dump } else { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = $3; } - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 0; } @@ -1132,9 +1202,9 @@ sub parse_dump ) { # Default text value, text, between commas - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = "'$3'"; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 0; } @@ -1144,9 +1214,9 @@ sub parse_dump ) { # NULL WITHOUT quotes around it ! - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'NULL'; - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 0; } @@ -1156,9 +1226,9 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/ ) { - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = convert_transactsql_code($3); - $objects->{$1}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 1; } @@ -1171,7 +1241,7 @@ sub parse_dump # This is a FK definition. We have the foreign table definition in next line. my $constraint; my $table = $2; - my $schema = $1; + my $schema = relabel_schemas($1); my $consname= $3; $constraint->{TYPE} = 'FK'; my @local_cols = split (/\s*,\s*/,$4); # Split around the comma. There may be whitespaces @@ -1198,7 +1268,7 @@ sub parse_dump @remote_cols=map{s/^\[//;s/]$//;$_;} @remote_cols; # Remove the brackets around the columns $constraint->{REMOTE_COLS}=\@remote_cols; $constraint->{REMOTE_TABLE} = $2; - $constraint->{REMOTE_SCHEMA} = $1; + $constraint->{REMOTE_SCHEMA} = relabel_schemas($1); $constraint->{REMOTE_COLS} =~ s/\[|\]//g; # Get rid of square brackets } @@ -1226,7 +1296,7 @@ sub parse_dump my $constraint; my $table = $2; my $constxt = $4; - my $schema = $1; + my $schema = relabel_schemas($1); $constraint->{TABLE} = $table; $constraint->{NAME} = $3; $constraint->{TYPE} = 'CHECK'; @@ -1275,6 +1345,7 @@ sub parse_dump or die "Could not parse $sqlproperty. This is a bug."; my ($comment, $schema, $obj, $objname, $subobj, $subobjname) = ($2, $4, $5, $6, $7, $8); + $schema=relabel_schemas($schema); if ($obj eq 'TABLE' and not defined $subobj) { $objects->{$schema}->{TABLES}->{$objname}->{COMMENT} = @@ -1303,6 +1374,7 @@ sub parse_dump or die "Could not parse $sqlproperty. This is a bug."; my ($comment, $schema, $obj, $objname) = ($2, $4, $5, $6); + $schema=relabel_schemas($schema); if ($obj eq 'TABLE') { $objects->{$schema}->{TABLES}->{$objname}->{COMMENT} = @@ -1422,23 +1494,33 @@ sub generate_schema # The schemas. don't create empty schema, sql server creates a schema per user, even if it ends empty foreach my $schema (sort keys %{$objects}) { - unless (relabel_schemas($schema) eq 'public' + unless ($schema eq 'public' or not defined $objects->{$schema}) { # Not compatible before 9.3. This is the logical target for this tool anyway - print BEFORE "CREATE SCHEMA IF NOT EXISTS ",format_identifier(relabel_schemas($schema)),";\n"; + print BEFORE "CREATE SCHEMA IF NOT EXISTS ",format_identifier($schema),";\n"; } } - # For the rest, we iterate over schemas + # For the rest, we iterate over schemas, except for array types (no point in complicating this) # The tables, columns, etc... will be created in the before script, so there is no dependancy # problem with constraints, that will be in the after script, except foreign keys which depend on unique indexes - # We have to do all domains before all tables + # We have to do all domains and types before all tables + # Don't care for dependancy while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # The user-defined types (domains, etc) + # The user-defined types (domains, etc) + foreach my $tabletype (sort keys %{$refschema->{TABLE_TYPES}}) + { + print BEFORE "CREATE TYPE " . format_identifier($schema) . '.' . format_identifier($tabletype) . " AS (\n" + . $refschema->{TABLE_TYPES}->{$tabletype} . "\n);\n"; + } + + print BEFORE "\n"; # We change sections in the dump file + } + while (my ($schema, $refschema) = each %{$objects}) + { + # The user-defined types (domains, etc) foreach my $domain (sort keys %{$refschema->{DOMAINS}}) { print BEFORE "CREATE DOMAIN " . format_identifier($schema) . '.' . format_identifier($domain) . ' ' @@ -1450,8 +1532,6 @@ sub generate_schema # Tables and columns while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise # The tables foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -1481,9 +1561,7 @@ sub generate_schema # Sequences, PKs, Indexes while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # We now add all "AFTER" objects + # We now add all "AFTER" objects # We start with SEQUENCES, PKs and INDEXES (will be needed for FK) foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) @@ -1496,7 +1574,7 @@ sub generate_schema . " START WITH " . $seqref->{START} . " OWNED BY " - . format_identifier(relabel_schemas($seqref->{OWNERSCHEMA})) . '.' + . format_identifier($seqref->{OWNERSCHEMA}) . '.' . format_identifier($seqref->{OWNERTABLE}) . '.' . format_identifier($seqref->{OWNERCOL}) . ";\n"; } @@ -1529,9 +1607,6 @@ sub generate_schema # Unique while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # Now The UNIQUE constraints. They may be used for FK (if columns are not null) foreach my $table (sort keys %{$refschema->{TABLES}}) { @@ -1554,10 +1629,8 @@ sub generate_schema # Indexes. Unique indexes are needed before foreign key constraints while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # Indexes - # They don't have a schema qualifier. But their table has, and they are in the same schema as their table + # Indexes + # They don't have a schema qualifier. But their table has, and they are in the same schema as their table foreach my $table (sort keys %{$refschema->{TABLES}}) { foreach my $index ( @@ -1579,8 +1652,6 @@ sub generate_schema # Other constraints while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise # We have all we need for FKs now. We can put all other constraints (except PK of course) foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -1604,7 +1675,7 @@ sub generate_schema " FOREIGN KEY (" . join(',',@localcollist) . ")" . " REFERENCES " - . format_identifier(relabel_schemas($constraint->{REMOTE_SCHEMA})) . '.' + . format_identifier($constraint->{REMOTE_SCHEMA}) . '.' . format_identifier($constraint->{REMOTE_TABLE}) . " ( " . join(',',@remotecollist) . ")"; if (defined $constraint->{ON_DEL_CASC} @@ -1646,9 +1717,7 @@ sub generate_schema # Default values while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # Default values + # Default values foreach my $table (sort keys %{$refschema->{TABLES}}) { foreach @@ -1673,9 +1742,7 @@ sub generate_schema # Comments on tables and columns while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # Comments on tables + # Comments on tables foreach my $table (sort keys %{$refschema->{TABLES}}) { if (defined($refschema->{TABLES}->{$table}->{COMMENT})) @@ -1698,9 +1765,7 @@ sub generate_schema # Views, and their comments while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # The views, and comments + # The views, and comments foreach my $view (sort keys %{$refschema->{VIEWS}}) { print UNSURE $refschema->{VIEWS}->{$view}->{SQL}, ";\n"; @@ -1714,9 +1779,7 @@ sub generate_schema # Trigger functions while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # The trigger functions + # The trigger functions foreach my $triggerfunc (sort keys %{$refschema->{TRIG_FUNCTIONS}}) { my $code = $refschema->{TRIG_FUNCTIONS}->{$triggerfunc}->{DEF}; @@ -1730,9 +1793,7 @@ sub generate_schema # Triggers while (my ($schema, $refschema) = each %{$objects}) { - $schema = relabel_schemas($schema) - ; # If dbo, put into public, unless asked otherwise - # triggers on tables, as these functions are declared now + # triggers on tables, as these functions are declared now foreach my $table (sort keys %{$refschema->{TABLES}}) { foreach @@ -1811,10 +1872,10 @@ sub resolve_name_conflicts # The schema will be the destination schema: dbo may have been replaced by public # Be careful that they are stored formatted through format_identifier if ($col->{TYPE} eq - (format_identifier(relabel_schemas($schema)) . '.' . format_identifier($domain))) + (format_identifier($schema) . '.' . format_identifier($domain))) { $col->{TYPE} = - format_identifier(relabel_schemas($schema)) . '.' . format_identifier($domain . "2pgd"); + format_identifier($schema) . '.' . format_identifier($domain . "2pgd"); } } } From 0374c97287d1a15e85df7b0a63e46fe5f90dca76 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 9 Apr 2014 18:15:16 +0200 Subject: [PATCH 032/216] Still more refactoring on relabel of schemas, for kettle --- sqlserver2pgsql.pl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index b740fd1..33ea019 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -450,11 +450,11 @@ sub generate_kettle foreach my $schema (sort keys %{$objects}) { my $refschema = $objects->{$schema}; - my $targetschema = relabel_schemas($schema); + my $targetschema = $schema; foreach my $table (sort keys %{$refschema->{TABLES}}) { - + my $origschema=$refschema->{TABLES}->{$table}->{origschema}; # First, does this table have LOBs ? The template depends on this my $newtemplate; if ($refschema->{TABLES}->{$table}->{haslobs}) @@ -522,7 +522,7 @@ sub generate_kettle $newtemplate =~ s/__postgres_port__/$pp/g; $newtemplate =~ s/__postgres_username__/$pu/g; $newtemplate =~ s/__postgres_password__/$pw/g; - $newtemplate =~ s/__sqlserver_table_name__/$schema.$table/g; + $newtemplate =~ s/__sqlserver_table_name__/$origschema.$table/g; $newtemplate =~ s/__sqlserver_table_cols__/$colsdef/g; my $pgtable=format_identifier($table); my $pgschema=format_identifier($targetschema); @@ -704,9 +704,11 @@ sub parse_dump if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\(/) { my $schemaname = relabel_schemas($1); + my $orig_schema = $1; my $tablename = $2; my $colnumber = 0; $objects->{$schemaname}->{TABLES}->{$tablename}->{haslobs} = 0; + $objects->{$schemaname}->{TABLES}->{$tablename}->{origschema} = $orig_schema; # We are in a create table. Read everything until its end... TABLE: while (my $line = read_and_clean($file)) { From 4436be2d7bf6d026d05552531b73526bc0d948fa Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 14 Apr 2014 12:24:29 +0200 Subject: [PATCH 033/216] Big rework for new dumps --- sqlserver2pgsql.pl | 320 +++++++++++++++++++++++++++++---------------- 1 file changed, 206 insertions(+), 114 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 33ea019..ee25ed3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -123,7 +123,7 @@ sub convert_numeric_to_int return 'smallint' if ($precision <= 4); return 'integer' if ($precision <= 9); return 'bigint' if ($precision <= 18); - return 'numeric($qual)'; + return "numeric($qual)"; } # These are the no-brainer conversions @@ -148,7 +148,8 @@ sub convert_numeric_to_int 'image' => 'bytea', 'binary' => 'bytea', 'varbinary' => 'bytea', - 'money' => 'numeric'); + 'money' => 'numeric', + 'uniqueidentifier' => 'uuid',); # Types with no qualifier, and no point in putting one my %unqual = ('bytea' => 1); @@ -288,16 +289,16 @@ sub convert_transactsql_code # This gives the next column position for a table -# It is used when we receive alter tables in the sql server dump +# It is used when we add a new column # These tables are added at the end of the table, in %objects sub next_col_pos { my ($schema, $table) = @_; - if (defined $objects->{$schema}->{TABlES}->{$table}->{COLS}) + if (defined $objects->{$schema}->{TABLES}->{$table}->{COLS}) { my $max = 0; foreach my $col ( - values(%{$objects->{$schema}->{TABlES}->{$table}->{COLS}})) + values(%{$objects->{$schema}->{TABLES}->{$table}->{COLS}})) { if ($col->{POS} > $max) { @@ -306,6 +307,11 @@ sub next_col_pos } return $max + 1; } + elsif (defined $objects->{$schema}->{TABLES}->{$table}) + { + # First column + return 1; + } else { die "We tried to add a column to an unknown table"; @@ -671,6 +677,98 @@ sub generate_kettle } } +# This adds a column we just read to a table +sub add_column_to_table +{ + my ($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull)=@_; + my $colnumber=next_col_pos($schemaname,$tablename); + if (defined $coltypeschema) + { + + # The datatype is a user defined datatype + # It has already been declared before. We just need to find it + $coltype = relabel_schemas($coltypeschema) . '.' . $coltype; + } + if ($colqual) + { + if ($colqual eq '(max)') + { + $colqual = undef + ; # max in sql server is the same as putting no colqual in pg + } + else + { + # We need the number (or 2 numbers) in this qual + $colqual =~ /\((\d+(?:,\s*\d+)?)\)/ + or die "Cannot parse colqual <$colqual>"; + $colqual = "$1"; + } + } + my $newtype = + convert_type($coltype, $colqual, $colname, + $tablename, undef, $schemaname); + + # If it is an identity, we'll map to serial/bigserial (create a sequence, then link it + # to the column) + if ($isidentity) + { + + # We have an identity field. We remember the default value and + # initialize the sequence correctly in the after script + $isidentity =~ /IDENTITY\((\d+),\s*(\d+)\)/ + or die "Cannot understand <$isidentity>"; + my $startseq = $1; + my $stepseq = $2; + my $seqname = lc("${tablename}_${colname}_seq"); + + # We get a sure default value. + $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} + ->{$colname}->{DEFAULT}->{VALUE} = + "nextval('" + . format_identifier(relabel_schemas(${schemaname})) . '.' + . ${seqname} . "')"; + $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} + ->{$colname}->{DEFAULT}->{UNSURE} = 0; + + $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{START} + = $startseq; + $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} + = $stepseq; + $objects->{$schemaname}->{SEQUENCES}->{$seqname} + ->{OWNERTABLE} = $tablename; + $objects->{$schemaname}->{SEQUENCES}->{$seqname} + ->{OWNERCOL} = $colname; + $objects->{$schemaname}->{SEQUENCES}->{$seqname} + ->{OWNERSCHEMA} = $schemaname; + } + + # If there is a bytea generated, this table will contain a blob: + # use a special kettle transformation for it if generating kettle + # (see generate_kettle() ) + if ( $newtype eq 'bytea' + or $coltype eq + 'ntext') # Ntext is very slow, stored out of page + { + $objects->{$schemaname}->{'TABLES'}->{$tablename} + ->{haslobs} = 1; + } + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{POS} = $colnumber; + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{TYPE} = $newtype; + + if ($colisnull eq 'NOT NULL') + { + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 1; + } + else + { + $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 0; + } +} + # Reads the dump passed as -f # Generates the $object structure # That's THE MAIN FUNCTION @@ -706,7 +804,6 @@ sub parse_dump my $schemaname = relabel_schemas($1); my $orig_schema = $1; my $tablename = $2; - my $colnumber = 0; $objects->{$schemaname}->{TABLES}->{$tablename}->{haslobs} = 0; $objects->{$schemaname}->{TABLES}->{$tablename}->{origschema} = $orig_schema; # We are in a create table. Read everything until its end... @@ -717,107 +814,24 @@ sub parse_dump /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))? (NOT NULL|NULL)(,)?/ ) { - $colnumber++; + #Deported into a function because we can also meet alter table add columns on their own my $colname = $1; my $coltypeschema = $2; my $coltype = $3; - if (defined $coltypeschema) - { - - # The datatype is a user defined datatype - # It has already been declared before. We just need to find it - $coltype = relabel_schemas($coltypeschema) . '.' . $coltype; - } my $colqual = $4; my $isidentity = $5; my $colisnull = $6; - if ($colqual) - { - if ($colqual eq '(max)') - { - $colqual = undef - ; # max in sql server is the same as putting no colqual in pg - } - else - { - # We need the number (or 2 numbers) in this qual - $colqual =~ /\((\d+(?:,\s*\d+)?)\)/ - or die "Cannot parse colqual <$colqual>"; - $colqual = "$1"; - } - } - my $newtype = - convert_type($coltype, $colqual, $colname, - $tablename, undef, $schemaname); - - # If it is an identity, we'll map to serial/bigserial (create a sequence, then link it - # to the column) - if ($isidentity) - { - - # We have an identity field. We remember the default value and - # initialize the sequence correctly in the after script - $isidentity =~ /IDENTITY\((\d+),\s*(\d+)\)/ - or die "Cannot understand <$isidentity>"; - my $startseq = $1; - my $stepseq = $2; - my $seqname = lc("${tablename}_${colname}_seq"); - - # We get a sure default value. - $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} - ->{$colname}->{DEFAULT}->{VALUE} = - "nextval('" - . format_identifier(relabel_schemas(${schemaname})) . '.' - . ${seqname} . "')"; - $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} - ->{$colname}->{DEFAULT}->{UNSURE} = 0; - - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{START} - = $startseq; - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} - = $stepseq; - $objects->{$schemaname}->{SEQUENCES}->{$seqname} - ->{OWNERTABLE} = $tablename; - $objects->{$schemaname}->{SEQUENCES}->{$seqname} - ->{OWNERCOL} = $colname; - $objects->{$schemaname}->{SEQUENCES}->{$seqname} - ->{OWNERSCHEMA} = $schemaname; - } - - # If there is a bytea generated, this table will contain a blob: - # use a special kettle transformation for it if generating kettle - # (see generate_kettle() ) - if ( $newtype eq 'bytea' - or $coltype eq - 'ntext') # Ntext is very slow, stored out of page - { - $objects->{$schemaname}->{'TABLES'}->{$tablename} - ->{haslobs} = 1; - } - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{POS} = $colnumber; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{TYPE} = $newtype; - - if ($colisnull eq 'NOT NULL') - { - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{NOT_NULL} = 1; - } - else - { - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{NOT_NULL} = 0; - } + add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); } + # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) # So just put it as a varchar, and issue a warning is STDOUT elsif ($line =~ /^\t\[(.*)\]\s+AS\s+\((.*)\)/) { # We just get the column name - $colnumber++; + my $colnumber=next_col_pos($schemaname,$tablename); my $colname = $1; my $code = $2; my $coltype = 'varchar'; @@ -1147,21 +1161,87 @@ sub parse_dump # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table elsif ($line =~ - /^ALTER TABLE \[.*\]\.\[(.*)\] ADD \[(.*)\] (?:\[.*\]\.)?\[(.*)\](\(.+?\))?( .*\(\d+,\s*\d+\))? (NOT NULL|NULL)$/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD \[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))? (NOT NULL|NULL)$/ ) { - # For now I don't know what to do with them. So die - die "$line: not understood. This is a bug"; + my $schemaname=relabel_schemas($1); + my $tablename=$2; + my $colname=$3; + my $coltypeschema=$4; + my $coltype=$5; + my $colqual=$6; + my $isidentity=$7; + my $colisnull=$8; + add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); } # Table constraints # Primary key. Multiline elsif ($line =~ - /^ALTER TABLE \[.*\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED)?/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED|NONCLUSTERED)?/ ) { - # Never seen one for now. The code is there though, in case, with a die for now - die "$line: not understood. This is a bug"; + my $schemaname=relabel_schemas($1); + my $tablename=$2; + my $constraint; + $constraint->{TYPE}='PK'; + if (defined $3) + { + $constraint->{NAME} = $3; + } + + + CONS: while (my $consline= read_and_clean($file)) + { + next if ($consline =~ /^\($/); + if ($consline =~ /^\t\[(.*)\] ASC,?$/) + { + push @{$constraint->{COLS}}, ($1); + } + elsif ($consline =~ /^\).*$/) + { + push @{$objects->{$schemaname}->{TABLES}->{$tablename} + ->{CONSTRAINTS}}, ($constraint); + + # We also directly put the constraint reference in a direct path (for ease of use in generate_kettle) + $objects->{$schemaname}->{TABLES}->{$tablename}->{PK} = $constraint; + # We are done here + next MAIN; + } + else + { + die "Cannot understand $consline."; + } + } + } + elsif ($line =~ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[(.*)\])? UNIQUE (?:CLUSTERED|NONCLUSTERED)?/ + ) + { + my $schemaname=relabel_schemas($1); + my $tablename=$2; + my $constraint; + $constraint->{TYPE}='UNIQUE'; + if (defined $3) + { + $constraint->{NAME}=$3; + } + while (my $uk = read_and_clean($file)) + { + + # Exit when read a line beginning with ). The constraint is complete + if ($uk =~ /^\)/) + { + push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} + ->{CONSTRAINTS}}, ($constraint); + next MAIN; + } + if ($uk =~ /^\t\[(.*)\] (ASC|DESC)(,?)/) + { + push @{$constraint->{COLS}}, ($1); + } + } + } # Default values. numeric, then text. These are 100% sure, they will parse in PG @@ -1176,13 +1256,11 @@ sub parse_dump # Ok, it IS a boolean, and we have received a number if ($3 eq '0') { - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} - ->{VALUE} = 'false'; + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'false'; } elsif ($3 eq '1') { - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT} - ->{VALUE} = 'true'; + $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'true'; } else { @@ -1291,7 +1369,7 @@ sub parse_dump # Check constraint. As it can be arbitrary code, we just get this code, and hope it will work on PG (it will be stored in a special script file) elsif ($line =~ - /ALTER TABLE \[(.*)\]\.\[(.*)\] WITH (?:NO)?CHECK ADD CONSTRAINT \[(.*)\] CHECK \(\((.*)\)\)/ + /ALTER TABLE \[(.*)\]\.\[(.*)\] WITH (?:NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? CHECK \(\((.*)\)\)/ ) { # Check constraint. We'll do what we can, syntax may be different. @@ -1300,7 +1378,10 @@ sub parse_dump my $constxt = $4; my $schema = relabel_schemas($1); $constraint->{TABLE} = $table; - $constraint->{NAME} = $3; + if (defined $3) + { + $constraint->{NAME} = $3; + } $constraint->{TYPE} = 'CHECK'; $constxt =~ s/\[(\S+)\]/$1/g; # We remove the []. And hope this will parse @@ -1310,15 +1391,22 @@ sub parse_dump } # These are comments or extended attributes on objets. They can be multiline, so aggregate everything - # Until next GO command + # Until a line that ends with a quote (but not two of them). We remove pair of quotes to make it simpler # If fact in can be a lot of things. So we have to ignore things like MS_DiagramPaneCount elsif ($line =~ /^EXEC sys.sp_addextendedproperty/) { + $line =~ s/''//g; my $sqlproperty = $line; - while (my $inline = read_and_clean($file)) + # If it ends with a single quote, and it is not the start (some people start their comments with a linefeed) + unless ($line =~ /'$/ and $line !~ /=N'$/) { - last if ($inline =~ /^GO/); - $sqlproperty .= $inline; + while (my $inline = read_and_clean($file)) + { + $inline =~ s/''//g; + $sqlproperty .= $inline; + # If it ends with a single quote, and it is not the start (some people start their comments with a linefeed) + last if ($inline =~ /'$/ and $inline !~ /=N'$/); + } } # We have all the extended property. Let's parse it. @@ -1342,9 +1430,13 @@ sub parse_dump # There can be quotes inside a string. So (?. Ignored.\n"; + next MAIN; + } my ($comment, $schema, $obj, $objname, $subobj, $subobjname) = ($2, $4, $5, $6, $7, $8); $schema=relabel_schemas($schema); @@ -1865,9 +1957,9 @@ sub resolve_name_conflicts "Warning: I had to rename domain $domain to ${domain}2pgd because of naming conflicts between a table and a domain, in source schema $schema\n"; # I also have to check all cols type to rename this - foreach my $table (values %{$refschema->{TABLES}}) + while ( my ($tablename,$table) = each %{$refschema->{TABLES}}) { - foreach my $col (values %{$table->{COLS}}) + while (my ($colname,$col) =each %{$table->{COLS}}) { # If a column has a custom type, it will be prefixed by schema From 8a3deda5bf475c0de16745db2ad6fb1510e4a09d Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 14 Apr 2014 12:39:11 +0200 Subject: [PATCH 034/216] Correction for kettle job when SQL Server's table name have to be protected with carrets --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index ee25ed3..951d9a1 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -528,7 +528,7 @@ sub generate_kettle $newtemplate =~ s/__postgres_port__/$pp/g; $newtemplate =~ s/__postgres_username__/$pu/g; $newtemplate =~ s/__postgres_password__/$pw/g; - $newtemplate =~ s/__sqlserver_table_name__/$origschema.$table/g; + $newtemplate =~ s/__sqlserver_table_name__/[$origschema].[$table]/g; $newtemplate =~ s/__sqlserver_table_cols__/$colsdef/g; my $pgtable=format_identifier($table); my $pgschema=format_identifier($targetschema); From 9c4b7261f4cf4480e3a5eb7acd926b68755d76d2 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 16 Apr 2014 15:14:04 +0200 Subject: [PATCH 035/216] Reduce the size of kettle's rowsets to more reasonable values --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 951d9a1..f5d27ad 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2152,7 +2152,7 @@ BEGIN 0.0 0.0 - 2000 + 200 50 50 N @@ -2367,7 +2367,7 @@ BEGIN 0.0 0.0 - 100 + 10 50 50 N From 2b89e2bc145c6d8964582f82b5fbe6d82a2d819b Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 16 Apr 2014 18:14:59 +0200 Subject: [PATCH 036/216] Add support for casts. They are used for some special types, such as UUID, so that the kettle job doesn't complain (they are seen as varchars in kettle) --- sqlserver2pgsql.pl | 207 +++++++++++++++++++++++++++------------------ 1 file changed, 124 insertions(+), 83 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index f5d27ad..9e37736 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -126,6 +126,23 @@ sub convert_numeric_to_int return "numeric($qual)"; } +# This is a list of the types that require a cast to be imported in kettle +my %types_to_cast = ('uuid' => '1',); + +# This sub adds a cast (if not defined already) if +# - we generate for kettle +# - the passed type is in %types_to_cast +sub add_cast +{ + my ($type)=@_; + if (defined $types_to_cast{$type}) + { + $objects->{CASTS}->{uuid}=1; + } +} + + + # These are the no-brainer conversions # There is still a special case for text types and case insensitivity (see convert_type) though my %types = ('int' => 'int', @@ -189,13 +206,22 @@ sub convert_type # Numeric is a special case: # No qualifier. We have to use numeric - return 'numeric' unless ($sqlqual); - return "numeric($sqlqual)" - unless ($sqlqual =~ /\d+,\s*0/); # If the qualifier is not x,0 - return "numeric($sqlqual)" - unless ($convert_numeric_to_int) - ; # If we have not activated conversion - return convert_numeric_to_int($sqlqual); # We got there: convert ! + if (not $sqlqual) + { + $rettype='numeric'; + } + elsif ($sqlqual !~ /\d+,\s*0/) + { + $rettype="numeric($sqlqual)"; + } + elsif ( my $tmprettype=convert_numeric_to_int($sqlqual)) + { + $rettype=$tmprettype; + } + else + { + $rettype="numeric($sqlqual)"; + } } else { @@ -222,7 +248,7 @@ sub convert_type $constraint->{TYPE} = 'CHECK_CITEXT'; $constraint->{TABLE} = $tablename; $constraint->{TEXT} = "char_length(" . format_identifier($colname) . ") <= $sqlqual"; - push @{$objects->{$schemaname}->{TABLES}->{$tablename} + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} ->{CONSTRAINTS}}, ($constraint); } elsif (defined $typname @@ -242,11 +268,14 @@ sub convert_type { # This is a namespaced type. So we check if this is a special array my ($schema,$type)=($1,$2); - if (defined($objects->{$schema}->{TABLE_TYPES}->{$type})) + if (defined($objects->{SCHEMAS}->{$schema}->{TABLE_TYPES}->{$type})) { $rettype=$rettype.'[]'; } } + # Add this type to casts to perform if necessary + add_cast($rettype); + return $rettype; } @@ -294,11 +323,11 @@ sub convert_transactsql_code sub next_col_pos { my ($schema, $table) = @_; - if (defined $objects->{$schema}->{TABLES}->{$table}->{COLS}) + if (defined $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}) { my $max = 0; foreach my $col ( - values(%{$objects->{$schema}->{TABLES}->{$table}->{COLS}})) + values(%{$objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}})) { if ($col->{POS} > $max) { @@ -307,7 +336,7 @@ sub next_col_pos } return $max + 1; } - elsif (defined $objects->{$schema}->{TABLES}->{$table}) + elsif (defined $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}) { # First column return 1; @@ -453,9 +482,9 @@ sub generate_kettle # For each table in each schema in $objects, we generate a kettle file in the directory - foreach my $schema (sort keys %{$objects}) + foreach my $schema (sort keys %{$objects->{SCHEMAS}}) { - my $refschema = $objects->{$schema}; + my $refschema = $objects->{SCHEMAS}->{$schema}; my $targetschema = $schema; foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -558,9 +587,9 @@ sub generate_kettle # We sort only so that it will be easier to find a transformation in the job if one needed to # edit it. It's also easier to track progress if tables are sorted alphabetically - foreach my $schema (sort keys %{$objects}) + foreach my $schema (sort keys %{$objects->{SCHEMAS}}) { - my $refschema = $objects->{$schema}; + my $refschema = $objects->{SCHEMAS}->{$schema}; foreach my $table (sort { lc($a) cmp lc($b) } keys %{$refschema->{TABLES}}) { @@ -722,23 +751,23 @@ sub add_column_to_table my $seqname = lc("${tablename}_${colname}_seq"); # We get a sure default value. - $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{COLS} ->{$colname}->{DEFAULT}->{VALUE} = "nextval('" . format_identifier(relabel_schemas(${schemaname})) . '.' . ${seqname} . "')"; - $objects->{$schemaname}->{TABLES}->{$tablename}->{COLS} + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{COLS} ->{$colname}->{DEFAULT}->{UNSURE} = 0; - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{START} + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{START} = $startseq; - $objects->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} = $stepseq; - $objects->{$schemaname}->{SEQUENCES}->{$seqname} + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname} ->{OWNERTABLE} = $tablename; - $objects->{$schemaname}->{SEQUENCES}->{$seqname} + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname} ->{OWNERCOL} = $colname; - $objects->{$schemaname}->{SEQUENCES}->{$seqname} + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname} ->{OWNERSCHEMA} = $schemaname; } @@ -749,22 +778,22 @@ sub add_column_to_table or $coltype eq 'ntext') # Ntext is very slow, stored out of page { - $objects->{$schemaname}->{'TABLES'}->{$tablename} + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename} ->{haslobs} = 1; } - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{POS} = $colnumber; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{TYPE} = $newtype; if ($colisnull eq 'NOT NULL') { - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{NOT_NULL} = 1; } else { - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{NOT_NULL} = 0; } } @@ -804,8 +833,8 @@ sub parse_dump my $schemaname = relabel_schemas($1); my $orig_schema = $1; my $tablename = $2; - $objects->{$schemaname}->{TABLES}->{$tablename}->{haslobs} = 0; - $objects->{$schemaname}->{TABLES}->{$tablename}->{origschema} = $orig_schema; + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{haslobs} = 0; + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{origschema} = $orig_schema; # We are in a create table. Read everything until its end... TABLE: while (my $line = read_and_clean($file)) { @@ -835,11 +864,11 @@ sub parse_dump my $colname = $1; my $code = $2; my $coltype = 'varchar'; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{POS} = $colnumber; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{TYPE} = $coltype; - $objects->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{NOT_NULL} = 0; # Big fat warning @@ -860,10 +889,10 @@ sub parse_dump RETURN NEW; end; EOF - $objects->{$schemaname}->{'TRIG_FUNCTIONS'} + $objects->{SCHEMAS}->{$schemaname}->{'TRIG_FUNCTIONS'} ->{'trig_func_ins_or_upd' || $tablename}->{DEF} = $triggerfunc; - $objects->{$schemaname}->{'TRIG_FUNCTIONS'} + $objects->{SCHEMAS}->{$schemaname}->{'TRIG_FUNCTIONS'} ->{'trig_func_ins_or_upd' || $tablename}->{LANG} = 'plpgsql'; my %trigger; @@ -872,7 +901,7 @@ sub parse_dump $trigger{FUNCTION} = 'trig_func_ins_or_upd' || $tablename; # In the same schema $trigger{NAME} = 'trig_ins_or_upd' || $tablename; - push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} + push @{$objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename} ->{TRIGGERS}}, (\%trigger); } @@ -894,11 +923,11 @@ sub parse_dump # Exit when read a line beginning with ). The constraint is complete. We store it and go back to main loop if ($pk =~ /^\)/) { - push @{$objects->{$schemaname}->{TABLES}->{$tablename} + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} ->{CONSTRAINTS}}, ($constraint); # We also directly put the constraint reference in a direct path (for ease of use in generate_kettle) - $objects->{$schemaname}->{TABLES}->{$tablename}->{PK} = + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{PK} = $constraint; next TABLE; } @@ -925,7 +954,7 @@ sub parse_dump # Exit when read a line beginning with ). The constraint is complete if ($uk =~ /^\)/) { - push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} + push @{$objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename} ->{CONSTRAINTS}}, ($constraint); next TABLE; } @@ -952,7 +981,7 @@ sub parse_dump ################################################################ elsif ($line =~ /CREATE SCHEMA \[(.*)\] AUTHORIZATION \[.*\]/) { - $objects->{relabel_schemas($1)} = undef + $objects->{SCHEMAS}->{relabel_schemas($1)} = undef ; # Nothing to add here, we create the schema, and put undef in it for now } elsif ($line =~ /CREATE\s+PROC(?:EDURE)?\s+\[.*\]\.\[(.*)\]/i) @@ -1018,7 +1047,7 @@ sub parse_dump # Views will be stored without the full schema in them. We will # have to generate the schema in the output file - $objects->{$schemaname}->{'VIEWS'}->{$viewname}->{SQL} = + $objects->{SCHEMAS}->{$schemaname}->{'VIEWS'}->{$viewname}->{SQL} = $sql; next MAIN; } @@ -1035,7 +1064,7 @@ sub parse_dump $schema=relabel_schemas($schema); my $newtype = convert_type($origtype, $quals, undef, undef, $type, $schema); - $objects->{$schema}->{DOMAINS}->{$type} = $newtype; + $objects->{SCHEMAS}->{$schema}->{DOMAINS}->{$type} = $newtype; # We add them to known data types, as they probably will be used in table definitions # but they point to themselves, with the schema corrected: we want them substituted by themselves @@ -1085,7 +1114,7 @@ sub parse_dump # We reached the end. We add this new type # create the new type declaration $newbasetype=join(",\n",@cols_newbasetype); - $objects->{$schema}->{TABLE_TYPES}->{$typename}=$newbasetype; + $objects->{SCHEMAS}->{$schema}->{TABLE_TYPES}->{$typename}=$newbasetype; # We add this to known data types, it will be used in table definitions $types{$schema . '.' . $typename} = format_identifier($schema) . '.' . format_identifier($typename); # We store the schema with it. And we do the case conversion, the quoting, etc right now @@ -1114,12 +1143,12 @@ sub parse_dump my $tablename = $5; if ($isunique) { - $objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} ->{$idxname}->{UNIQUE} = 1; } else { - $objects->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} ->{$idxname}->{UNIQUE} = 0; } while (my $idx = read_and_clean($file)) @@ -1137,12 +1166,12 @@ sub parse_dump { if (defined $2) { - push @{$objects->{$schemaname}->{TABLES}->{$tablename} + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} ->{INDEXES}->{$idxname}->{COLS}}, ("$1 $2"); } else { - push @{$objects->{$schemaname}->{TABLES}->{$tablename} + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} ->{INDEXES}->{$idxname}->{COLS}}, ("$1"); } } @@ -1200,11 +1229,11 @@ sub parse_dump } elsif ($consline =~ /^\).*$/) { - push @{$objects->{$schemaname}->{TABLES}->{$tablename} + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} ->{CONSTRAINTS}}, ($constraint); # We also directly put the constraint reference in a direct path (for ease of use in generate_kettle) - $objects->{$schemaname}->{TABLES}->{$tablename}->{PK} = $constraint; + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{PK} = $constraint; # We are done here next MAIN; } @@ -1232,7 +1261,7 @@ sub parse_dump # Exit when read a line beginning with ). The constraint is complete if ($uk =~ /^\)/) { - push @{$objects->{$schemaname}->{'TABLES'}->{$tablename} + push @{$objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename} ->{CONSTRAINTS}}, ($constraint); next MAIN; } @@ -1251,16 +1280,16 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?((?:-)?\d+(?:\.\d+)?)\)?\) FOR \[(.*)\]/ ) { - if ($objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{TYPE} eq 'boolean') + if ($objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{TYPE} eq 'boolean') { # Ok, it IS a boolean, and we have received a number if ($3 eq '0') { - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'false'; + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'false'; } elsif ($3 eq '1') { - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'true'; + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'true'; } else { @@ -1270,10 +1299,10 @@ sub parse_dump } else { - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = $3; } - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 0; } @@ -1282,9 +1311,9 @@ sub parse_dump ) { # Default text value, text, between commas - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = "'$3'"; - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 0; } @@ -1294,9 +1323,9 @@ sub parse_dump ) { # NULL WITHOUT quotes around it ! - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'NULL'; - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 0; } @@ -1306,9 +1335,9 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/ ) { - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = convert_transactsql_code($3); - $objects->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} + $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} = 1; } @@ -1337,7 +1366,7 @@ sub parse_dump { if ($fk =~ /^GO/) { - push @{$objects->{$schema}->{'TABLES'}->{$table} + push @{$objects->{SCHEMAS}->{$schema}->{'TABLES'}->{$table} ->{CONSTRAINTS}}, ($constraint); next MAIN; } @@ -1386,7 +1415,7 @@ sub parse_dump $constxt =~ s/\[(\S+)\]/$1/g; # We remove the []. And hope this will parse $constraint->{TEXT} = $constxt; - push @{$objects->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}}, + push @{$objects->{SCHEMAS}->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}}, ($constraint); } @@ -1442,17 +1471,17 @@ sub parse_dump $schema=relabel_schemas($schema); if ($obj eq 'TABLE' and not defined $subobj) { - $objects->{$schema}->{TABLES}->{$objname}->{COMMENT} = + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{COMMENT} = $comment; } elsif ($obj eq 'VIEW' and not defined $subobj) { - $objects->{$schema}->{VIEWS}->{$objname}->{COMMENT} = + $objects->{SCHEMAS}->{$schema}->{VIEWS}->{$objname}->{COMMENT} = $comment; } elsif ($obj eq 'TABLE' and $subobj eq 'COLUMN') { - $objects->{$schema}->{TABLES}->{$objname}->{COLS} + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{COLS} ->{$subobjname}->{COMMENT} = $comment; } else @@ -1471,7 +1500,7 @@ sub parse_dump $schema=relabel_schemas($schema); if ($obj eq 'TABLE') { - $objects->{$schema}->{TABLES}->{$objname}->{COMMENT} = + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{COMMENT} = $comment; } elsif ($obj eq 'SCHEMA') @@ -1580,16 +1609,28 @@ sub generate_schema { print BEFORE "CREATE EXTENSION IF NOT EXISTS citext;\n"; } + if (defined ($objects->{CASTS})) + { + foreach my $cast (keys %{$objects->{CASTS}}) + { + # We create the cast in the BEFORE. Add a comment, as this is not obvious + print BEFORE "-- Create a cast. Used for the kettle job\n"; + print BEFORE "CREATE CAST (varchar as $cast) with inout as implicit\n"; + # We drop the cast in the AFTER + print AFTER "-- drop a cast. Used for the kettle job\n"; + print AFTER "DROP CAST (varchar as $cast)\n"; + } + } # Ok, we have parsed everything, and definitions are in $objects # We will put in the BEFORE file only table and columns definitions. # The rest will go in the AFTER script (check constraints, put default values, etc...) # The schemas. don't create empty schema, sql server creates a schema per user, even if it ends empty - foreach my $schema (sort keys %{$objects}) + foreach my $schema (sort keys %{$objects->{SCHEMAS}}) { unless ($schema eq 'public' - or not defined $objects->{$schema}) + or not defined $objects->{SCHEMAS}->{$schema}) { # Not compatible before 9.3. This is the logical target for this tool anyway print BEFORE "CREATE SCHEMA IF NOT EXISTS ",format_identifier($schema),";\n"; @@ -1601,7 +1642,7 @@ sub generate_schema # problem with constraints, that will be in the after script, except foreign keys which depend on unique indexes # We have to do all domains and types before all tables # Don't care for dependancy - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # The user-defined types (domains, etc) foreach my $tabletype (sort keys %{$refschema->{TABLE_TYPES}}) @@ -1612,7 +1653,7 @@ sub generate_schema print BEFORE "\n"; # We change sections in the dump file } - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # The user-defined types (domains, etc) foreach my $domain (sort keys %{$refschema->{DOMAINS}}) @@ -1624,7 +1665,7 @@ sub generate_schema print BEFORE "\n"; # We change sections in the dump file } # Tables and columns - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # The tables @@ -1653,7 +1694,7 @@ sub generate_schema } } # Sequences, PKs, Indexes - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # We now add all "AFTER" objects # We start with SEQUENCES, PKs and INDEXES (will be needed for FK) @@ -1699,7 +1740,7 @@ sub generate_schema } } # Unique - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # Now The UNIQUE constraints. They may be used for FK (if columns are not null) foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -1721,7 +1762,7 @@ sub generate_schema } } # Indexes. Unique indexes are needed before foreign key constraints - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # Indexes # They don't have a schema qualifier. But their table has, and they are in the same schema as their table @@ -1744,7 +1785,7 @@ sub generate_schema } } # Other constraints - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # We have all we need for FKs now. We can put all other constraints (except PK of course) @@ -1809,7 +1850,7 @@ sub generate_schema } # Default values - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # Default values foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -1834,7 +1875,7 @@ sub generate_schema } } # Comments on tables and columns - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # Comments on tables foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -1857,7 +1898,7 @@ sub generate_schema } } # Views, and their comments - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # The views, and comments foreach my $view (sort keys %{$refschema->{VIEWS}}) @@ -1871,7 +1912,7 @@ sub generate_schema } } # Trigger functions - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # The trigger functions foreach my $triggerfunc (sort keys %{$refschema->{TRIG_FUNCTIONS}}) @@ -1885,7 +1926,7 @@ sub generate_schema } } # Triggers - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # triggers on tables, as these functions are declared now foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -1928,7 +1969,7 @@ sub generate_schema # We do this schema per schema sub resolve_name_conflicts { - while (my ($schema, $refschema) = each %{$objects}) + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { my %known_names; @@ -2087,7 +2128,7 @@ sub resolve_name_conflicts parse_dump(); # Debug, uncomment: -#print Dumper($objects); +print Dumper($objects); # Rename indexes if they conflict resolve_name_conflicts(); From 36819e382151d774cf9b98436da3647a28df9583 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 16 Apr 2014 18:16:10 +0200 Subject: [PATCH 037/216] Remove the Dumper output --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 9e37736..61cbcb7 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2128,7 +2128,7 @@ sub resolve_name_conflicts parse_dump(); # Debug, uncomment: -print Dumper($objects); +#print Dumper($objects); # Rename indexes if they conflict resolve_name_conflicts(); From 64d16523122226b71383fe8728161ca4f0e1aed2 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 16 Apr 2014 18:18:09 +0200 Subject: [PATCH 038/216] Add the semi colons for create casts. --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 61cbcb7..266dbb2 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1615,10 +1615,10 @@ sub generate_schema { # We create the cast in the BEFORE. Add a comment, as this is not obvious print BEFORE "-- Create a cast. Used for the kettle job\n"; - print BEFORE "CREATE CAST (varchar as $cast) with inout as implicit\n"; + print BEFORE "CREATE CAST (varchar as $cast) with inout as implicit;\n"; # We drop the cast in the AFTER print AFTER "-- drop a cast. Used for the kettle job\n"; - print AFTER "DROP CAST (varchar as $cast)\n"; + print AFTER "DROP CAST (varchar as $cast);\n"; } } From a2d0ed62b656b4f09d694fc959e1a91bbdfd2cf2 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 16 Apr 2014 19:13:56 +0200 Subject: [PATCH 039/216] Added the ability to defer foreign keys validation to the unsure script: they can now be created as not valid, and validated later --- README.md | 4 ++++ regression/reg.pl | 2 +- sqlserver2pgsql.pl | 22 ++++++++++++++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f3a0145..feeb937 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,10 @@ there is a default dbo=>public remapping, that can be cancelled with -nr. Use do -num : Converts numeric (xxx,0) to the appropriate smallint, integer or bigint. It won't keep the constraint on the size of the scale of the numeric +-validate_constraints=yes/after/no: for foreign keys, if yes: foreign keys are created as valid in the after script (default) + if no: they are created as not valid (enforced only for new rows) + if after: they are created as not valid, but the statements to validate them are put in the unsure file + If you want to also import data: > ./sqlserver2pgsql.pl -b before.sql -a after.sql -u unsure.sql -k kettledir \ diff --git a/regression/reg.pl b/regression/reg.pl index 9f6290e..f76e24d 100755 --- a/regression/reg.pl +++ b/regression/reg.pl @@ -7,7 +7,7 @@ foreach my $file (<*.sql>) { - my @options_to_try=('-i','-nr','-num', '-keep_identifier_case'); + my @options_to_try=('-i','-nr','-num', '-keep_identifier_case', '-validate_constraints=after'); my @all_combinations=(''); foreach my $option (@options_to_try) { diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 266dbb2..9a559e0 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -41,6 +41,7 @@ our $after_file; our $unsure_file; our $keep_identifier_case; +our $validate_constraints='yes'; my $template ; # These two variables are loaded in the BEGIN block at the end of this file (they are very big @@ -77,7 +78,8 @@ sub parse_conf_file 'no relabel dbo' => 'norelabel_dbo', 'convert numeric to int' => 'convert_numeric_to_int', 'relabel schemas' => 'relabel_schemas', - 'keep identifier case' => 'keep_identifier_case', + 'keep identifier case' => 'keep_identifier_case', + 'validate constraints' => 'validate_constraints', ); # Open the conf file or die @@ -1823,8 +1825,18 @@ sub generate_schema { $consdef .= " ON UPDATE CASCADE"; } + # We need a name on the constraint to be able to validate it later. Maybe it would be better to generate one + # FIXME: we'll see later if a generator is needed (probably) + if ($constraint->{TYPE} eq 'FK' and ($validate_constraints =~ /^after|no$/) and defined($constraint->{NAME})) + { + $consdef .= " NOT VALID"; + } $consdef .= ";\n"; print AFTER $consdef; + if ($constraint->{TYPE} eq 'FK' and $validate_constraints eq 'after' and defined $constraint->{NAME}) + { + print UNSURE "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " VALIDATE CONSTRAINT " . format_identifier($constraint->{NAME}) . ";\n"; + } } elsif ($constraint->{TYPE} eq 'CHECK') { @@ -2075,7 +2087,8 @@ sub resolve_name_conflicts "nr" => \$norelabel_dbo, "num" => \$convert_numeric_to_int, "relabel_schemas=s" => \$relabel_schemas, - "keep_identifier_case" =>\$keep_identifier_case,); + "keep_identifier_case" =>\$keep_identifier_case, + "validate_constraints=s" =>\$validate_constraints,); # We don't understand command line or have been asked for usage if (not $options or $help) @@ -2101,6 +2114,11 @@ sub resolve_name_conflicts exit 1; } +if ($validate_constraints !~ '^(yes|after|no)$') +{ + die "validate_constraints should be yes, after or no (default yes)\n"; +} + # We have been asked for kettle, but the compulsory parameters are not there if ($kettle and ( not $sd From 525794050126b3bfacebd9f69bee278ba97c04f8 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 17 Apr 2014 15:20:12 +0200 Subject: [PATCH 040/216] Now finds the correct current value for sequences --- sqlserver2pgsql.pl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 9a559e0..5e87d96 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1886,6 +1886,18 @@ sub generate_schema } } } + + # Current values for sequences: autodetect the current max in the table, now that we probably have the indexes + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) + { + foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) + { + my $seqref = $refschema->{SEQUENCES}->{$sequence}; + print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select max(". format_identifier($seqref->{OWNERCOL}) .") from " . format_identifier($seqref->{OWNERSCHEMA}) . '.'. format_identifier($seqref->{OWNERTABLE}) . "::bigint));\n"; + } + } + + # Comments on tables and columns while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { From ff9450f3f9e31530fad6abdb778c8d1b4f9ee834 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 17 Apr 2014 16:31:58 +0200 Subject: [PATCH 041/216] Correction on generation of nextval --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 5e87d96..508a869 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1893,7 +1893,7 @@ sub generate_schema foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) { my $seqref = $refschema->{SEQUENCES}->{$sequence}; - print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select max(". format_identifier($seqref->{OWNERCOL}) .") from " . format_identifier($seqref->{OWNERSCHEMA}) . '.'. format_identifier($seqref->{OWNERTABLE}) . "::bigint));\n"; + print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select max(". format_identifier($seqref->{OWNERCOL}) .") from " . format_identifier($seqref->{OWNERSCHEMA}) . '.'. format_identifier($seqref->{OWNERTABLE}) . ")::bigint);\n"; } } From b97519f342e32508e19d854b9d0da71168f3ad55 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 23 Apr 2014 18:08:03 +0200 Subject: [PATCH 042/216] Speed improvements by rewriting the step removing \0 from javascript to java --- sqlserver2pgsql.pl | 185 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 151 insertions(+), 34 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 508a869..e0eb15a 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2295,10 +2295,10 @@ BEGIN - Table inputModified Java Script ValueY Modified Java Script ValueTable outputY + Table inputUser Defined Java ClassY User Defined Java ClassTable outputY - Modified Java Script Value - ScriptValueMod + User Defined Java Class + UserDefinedJavaClass Y 4 @@ -2306,20 +2306,79 @@ BEGIN none - N - 9 - 0 - Script 1 - for (var i=0;i<getInputRowMeta().size();i++) { - var valueMeta = getInputRowMeta().getValueMeta(i); - if (valueMeta.getTypeDesc().equals("String")) { - row[i]=replace(row[i],"\\00",''); - } -} - + + + + TRANSFORM_CLASS + + Processor + + + + + + N + - 243 - 159 + 280 + 332 Y @@ -2510,35 +2569,93 @@ BEGIN - Table inputModified Java Script ValueY Modified Java Script ValueTable outputY + Table inputUser Defined Java ClassY User Defined Java ClassTable outputY - Modified Java Script Value - ScriptValueMod + User Defined Java Class + UserDefinedJavaClass Y - __sqlserver_copies__ + 4 none - N - 9 - 0 - Script 1 - for (var i=0;i<getInputRowMeta().size();i++) { - var valueMeta = getInputRowMeta().getValueMeta(i); - if (valueMeta.getTypeDesc().equals("String")) { - row[i]=replace(row[i],"\\00",''); - } -} - + + + + TRANSFORM_CLASS + + Processor + + + + + + N + - 243 - 159 + 280 + 332 Y - Table input TableInput From cb892901718bd23249c3d21a764e4063cebeaa04 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 23 Apr 2014 18:19:28 +0200 Subject: [PATCH 043/216] protect \ in \00 for new java generation --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index e0eb15a..8b42de8 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2319,7 +2319,7 @@ BEGIN String[] fieldNames; long numFields; -Pattern pattern = Pattern.compile("\00"); +Pattern pattern = Pattern.compile("\\00"); RowMetaInterface inputRowMeta; public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException @@ -2593,7 +2593,7 @@ BEGIN String[] fieldNames; long numFields; -Pattern pattern = Pattern.compile("\00"); +Pattern pattern = Pattern.compile("\\00"); RowMetaInterface inputRowMeta; public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException From 8ff2afe3ba9802a457fb8bd90b3f33b4dc7a451a Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 23 Apr 2014 18:27:05 +0200 Subject: [PATCH 044/216] Put parallelism level as a global variable, instead of constants all over --- sqlserver2pgsql.pl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 8b42de8..ad3c78d 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -42,6 +42,7 @@ our $unsure_file; our $keep_identifier_case; our $validate_constraints='yes'; +our $parallelism=8; my $template ; # These two variables are loaded in the BEGIN block at the end of this file (they are very big @@ -516,7 +517,7 @@ sub generate_kettle . '% ${Internal.Step.Unique.Count} = ${Internal.Step.Unique.Number}'; $newtemplate =~ s/__sqlserver_where_filter__/$wherefilter/; - $newtemplate =~ s/__sqlserver_copies__/4/g; + $newtemplate =~ s/__PARALLELISM__/$parallelism/g; } else @@ -2301,7 +2302,7 @@ BEGIN UserDefinedJavaClass Y - 4 + __PARALLELISM__ none @@ -2413,7 +2414,7 @@ BEGIN TableOutput Y - 4 + __PARALLELISM__ none @@ -2575,7 +2576,7 @@ BEGIN UserDefinedJavaClass Y - 4 + __PARALLELISM__ none @@ -2661,7 +2662,7 @@ BEGIN TableInput Y - __sqlserver_copies__ + __PARALLELISM__ none @@ -2686,7 +2687,7 @@ BEGIN TableOutput Y - 4 + __PARALLELISM__ none From 7132b2747d5c26bc9470e4ec8498bd722fea71e4 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 24 Apr 2014 08:02:45 +0200 Subject: [PATCH 045/216] Small correction to make the new java step work with ketle 5 --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index ad3c78d..3df33df 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2345,7 +2345,7 @@ BEGIN numFields = fieldNames.length; int fieldnum; for (fieldnum = 0; fieldnum < numFields; fieldnum++) { - if(inputRowMeta.getValueMeta(fieldnum).getType()!= ValueMeta.TYPE_STRING) { + if(inputRowMeta.getValueMeta(fieldnum).getType()!= ValueMetaInterface.TYPE_STRING) { fieldNames[fieldnum]="-1"; } } @@ -2619,7 +2619,7 @@ BEGIN numFields = fieldNames.length; int fieldnum; for (fieldnum = 0; fieldnum < numFields; fieldnum++) { - if(inputRowMeta.getValueMeta(fieldnum).getType()!= ValueMeta.TYPE_STRING) { + if(inputRowMeta.getValueMeta(fieldnum).getType()!= ValueMetaInterface.TYPE_STRING) { fieldNames[fieldnum]="-1"; } } From ee905cdbd71b583d8c7732a1b740745bef2bf566 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 24 Apr 2014 09:19:30 +0200 Subject: [PATCH 046/216] correct a bug with parallelism factorisation --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3df33df..377fc56 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -517,7 +517,6 @@ sub generate_kettle . '% ${Internal.Step.Unique.Count} = ${Internal.Step.Unique.Number}'; $newtemplate =~ s/__sqlserver_where_filter__/$wherefilter/; - $newtemplate =~ s/__PARALLELISM__/$parallelism/g; } else @@ -566,6 +565,7 @@ sub generate_kettle my $pgschema=format_identifier($targetschema); $newtemplate =~ s/__postgres_table_name__/$pgtable/g; $newtemplate =~ s/__postgres_schema_name__/$pgschema/g; + $newtemplate =~ s/__PARALLELISM__/$parallelism/g; # Store this new transformation into its file open FILE, ">$dir/$schema-$table.ktr" From 4ecbe35b3b764a2aff2de5a67b9da798156097b8 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 24 Apr 2014 14:19:15 +0200 Subject: [PATCH 047/216] fix silly oversight in trigger generation for calculated column --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 377fc56..fa7baa6 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -888,7 +888,7 @@ sub parse_dump $code =~ s/\[(.*?)\]/NEW.$1/g; my $triggerfunc = < Date: Wed, 30 Apr 2014 16:23:44 +0200 Subject: [PATCH 048/216] Small corrections on alter table --- sqlserver2pgsql.pl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index fa7baa6..85ed51b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -885,6 +885,7 @@ sub parse_dump print STDERR "\tPlease review it.\n"; # Try to correct what can be corrected from the AS : replace [COL] with NEW.COL + # It is obviously not going to work for anything a bit complicated $code =~ s/\[(.*?)\]/NEW.$1/g; my $triggerfunc = <{SCHEMAS}->{relabel_schemas($schemaname)}->{TABLES}->{$tablename}->{COLS}->{$colname}->{DEFAULT}->{VALUE} = $default; + } } # Table constraints From 188a7e53ae780c3f9b71e9311d33688c1319020d Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 16 May 2014 11:59:07 +0200 Subject: [PATCH 049/216] Change size of commits to avoid deadlocks and do a set synchronous_commit to off in all postgresql sessions --- sqlserver2pgsql.pl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 85ed51b..f331939 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2299,6 +2299,7 @@ BEGIN QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN + SQL_CONNECTset synchronous_commit to off; @@ -2428,7 +2429,7 @@ BEGIN __postgres_db__ __postgres_schema_name__ __postgres_table_name__
- 500 + 100 Y N Y @@ -2573,6 +2574,7 @@ BEGIN QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN + SQL_CONNECTset synchronous_commit to off; From 4e5d109c6bf3d750ea02f4a7c107f1426e56b198 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 19 May 2014 16:43:49 +0200 Subject: [PATCH 050/216] preserve view ordering --- sqlserver2pgsql.pl | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index f331939..e191665 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -51,6 +51,7 @@ ; # These are used to create the static parts of the job my ($job_entry, $job_hop) ; # These are used to create the dynamic parts of the job (XML file) +my @view_list; # array to keep view ordering from sql server's dump (a view may depend on another view) # Opens the configuration file # Sets $sd $sh $sp $su $sw $pd $ph $pp $pu $pw when they are not set in the command line already @@ -1053,6 +1054,8 @@ sub parse_dump # have to generate the schema in the output file $objects->{SCHEMAS}->{$schemaname}->{'VIEWS'}->{$viewname}->{SQL} = $sql; + my @view_array=($schemaname,$viewname); + push @view_list,(\@view_array); # adds another schema/view to the list next MAIN; } $sql .= $line_cont; @@ -1929,17 +1932,16 @@ sub generate_schema } } # Views, and their comments - while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) + # This is different from other objets: we keep the views ordering + foreach my $viewref(@view_list) { - # The views, and comments - foreach my $view (sort keys %{$refschema->{VIEWS}}) + my ($schema,$view)=@$viewref; + my $refschema=$objects->{SCHEMAS}->{$schema}; + print UNSURE $refschema->{VIEWS}->{$view}->{SQL}, ";\n"; + if (defined $refschema->{VIEWS}->{$view}->{COMMENT}) { - print UNSURE $refschema->{VIEWS}->{$view}->{SQL}, ";\n"; - if (defined $refschema->{VIEWS}->{$view}->{COMMENT}) - { - print UNSURE "COMMENT ON VIEW $schema.$view IS '" - . $refschema->{VIEWS}->{$view}->{COMMENT} . "';\n"; - } + print UNSURE "COMMENT ON VIEW $schema.$view IS '" + . $refschema->{VIEWS}->{$view}->{COMMENT} . "';\n"; } } # Trigger functions @@ -2165,7 +2167,7 @@ sub resolve_name_conflicts parse_dump(); # Debug, uncomment: -#print Dumper($objects); +print Dumper($objects); # Rename indexes if they conflict resolve_name_conflicts(); From ca2ffe447fb13f6a6526277bf7a1c07500c52e12 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 17 Jun 2014 15:43:25 +0200 Subject: [PATCH 051/216] Split the doc into 3 files --- FAQ.md | 19 +++++++++++++++++++ INSTALL.md | 13 +++++++++++++ README.md | 37 +++---------------------------------- 3 files changed, 35 insertions(+), 34 deletions(-) create mode 100644 FAQ.md create mode 100644 INSTALL.md diff --git a/FAQ.md b/FAQ.md new file mode 100644 index 0000000..5a95254 --- /dev/null +++ b/FAQ.md @@ -0,0 +1,19 @@ +================================ +FAQ: + +Why didn't you do everything in the Perl script ? I don't want to use Kettle. + +Because I have once installed a Perl DBD driver for MS SQL's server, and I don't want to ever do it again... nor +force you to do it :) You can either do it using ODBC, or using Sybase's driver. In both case, it will take you +quite a while, and you'll suffer with large objects. With Kettle, on the other hand, you're using the JBDC driver, +which is already one of Kettle's default drivers, along with PostgreSQL. So all the heavy lifting (converting +LOBs and IMAGES and whatever to bytea) is directly done by both JDBC drivers, and should work with no efforts +(except maybe adjust Java's memory parameters). If you get a memory error, try setting a JAVAMAXMEM environment +variable to a higher value (4096) for 4GB for instance. + + + +What is this IGNORE NULLS I have to change in kettle.properties ? + +Because Kettle behaves by default the way Oracle behaves: for Oracle, a NULL and an empty string are the same. +You don't want this here, because neither SQL Server nor PostgreSQL do this. diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..eb79b36 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,13 @@ +Installation: +========================== +Nothing to do on this part, as long as you don't want to use the data migration part: the script +has no dependancy on fancy Perl modules, it only uses modules from the base perl distribution. + +Just run the script with your perl interpreter, providing it with the requested options (--help +will tell you what to do). + +If you want to migrate the data, you'll need "Kettle", an Open Source ETL. Get the latest version +from here: http://kettle.pentaho.com/ . +You'll also need a SQL Server account with the permission to SELECT from the tables you want to migrate. + +As Kettle is a Java program, you'll also need a recent JVM (Java 6 or 7 should do the trick). diff --git a/README.md b/README.md index feeb937..6bf9591 100644 --- a/README.md +++ b/README.md @@ -26,22 +26,11 @@ It won't migrate PL procedures, the languages are too different. I usually only test this script under Linux. It should work on Windows, as I had to do it once with Windows, and on any Unix system. -Installation: -========================== -Nothing to do on this part, as long as you don't want to use the data migration part: the script -has no dependancy on fancy Perl modules, it only uses modules from the base perl distribution. - -Just run the script with your perl interpreter, providing it with the requested options (--help -will tell you what to do). -If you want to migrate the data, you'll need "Kettle", an Open Source ETL. Get the latest version -from here: http://kettle.pentaho.com/ . -You'll also need a SQL Server account with the permission to SELECT from the tables you want to migrate. - -As Kettle is a Java program, you'll also need a recent JVM (Java 6 or 7 should do the trick). - -========================== +You'll need to install a few things to make it work. See INSTALL.md +Usage +============================= Ok, I have installed Kettle and Java, I have sqlserver2pgsql.pl, what do I do now ? You'll need several things: @@ -137,26 +126,6 @@ You can also use a configuration file if you like: There is an example configuration file provided. You can also mix the configuration file with command line options. Command line options have the priority over values set in the configuration file. -================================ -FAQ: - -Why didn't you do everything in the Perl script ? I don't want to use Kettle. - -Because I have once installed a Perl DBD driver for MS SQL's server, and I don't want to ever do it again... nor -force you to do it :) You can either do it using ODBC, or using Sybase's driver. In both case, it will take you -quite a while, and you'll suffer with large objects. With Kettle, on the other hand, you're using the JBDC driver, -which is already one of Kettle's default drivers, along with PostgreSQL. So all the heavy lifting (converting -LOBs and IMAGES and whatever to bytea) is directly done by both JDBC drivers, and should work with no efforts -(except maybe adjust Java's memory parameters). If you get a memory error, try setting a JAVAMAXMEM environment -variable to a higher value (4096) for 4GB for instance. - - - -What is this IGNORE NULLS I have to change in kettle.properties ? - -Because Kettle behaves by default the way Oracle behaves: for Oracle, a NULL and an empty string are the same. -You don't want this here, because neither SQL Server nor PostgreSQL do this. - ================================ Licence: GPL v3 From 07b4337253c5175ae1979ab4048d6207b3394a4a Mon Sep 17 00:00:00 2001 From: damien clochard Date: Tue, 17 Jun 2014 15:54:56 +0200 Subject: [PATCH 052/216] Update README.md minor format fixes --- README.md | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6bf9591..a5ebdec 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ It is written in Perl. It does two things: * convert a SQL Server schema to a PostgreSQL schema - * produce a Pentaho Data Integrator (Kettle) job to migrate all the data from SQL Server to PostgreSQL. This second part is optionnal @@ -29,6 +28,12 @@ with Windows, and on any Unix system. You'll need to install a few things to make it work. See INSTALL.md +Install +========================== + +See https://github.com/dalibo/sqlserver2pgsql/blob/master/INSTALL.md + + Usage ============================= Ok, I have installed Kettle and Java, I have sqlserver2pgsql.pl, what do I do now ? @@ -57,7 +62,7 @@ run sqlserver2pgsql.pl from. If you just want to convert this schema, run: -> sqlserver2pgsql.pl -f my_sqlserver_script.txt -b name_of_before_script -a name_of_after_script -u name_of_unsure_script + > sqlserver2pgsql.pl -f my_sqlserver_script.txt -b name_of_before_script -a name_of_after_script -u name_of_unsure_script The before script contains what is needed to import data (types, tables and columns). The after script contains the rest (indexes, constraints). It should be run @@ -87,9 +92,9 @@ the size of the scale of the numeric If you want to also import data: -> ./sqlserver2pgsql.pl -b before.sql -a after.sql -u unsure.sql -k kettledir \ - -sd source -sh 192.168.0.2 -sp 1433 -su dalibo -sw mysqlpass \ - -pd dest -ph localhost -pp 5432 -pu dalibo -pw mypgpass -f sql_server_schema.sql + > ./sqlserver2pgsql.pl -b before.sql -a after.sql -u unsure.sql -k kettledir \ + -sd source -sh 192.168.0.2 -sp 1433 -su dalibo -sw mysqlpass \ + -pd dest -ph localhost -pp 5432 -pu dalibo -pw mypgpass -f sql_server_schema.sql -k is the directory where you want to store the kettle xml files (there will be one for each table to copy, plus the one for the job) @@ -110,23 +115,31 @@ cleartext, so don't make this directory public): You've generated everything. Let's do the import: -# Run the before script (creates the tables) -> psql -U mypguser mypgdatabase -f name_of_before_script -# Run the kettle job: -> cd my_kettle_installation_directory -> ./kitchen.sh -file=full_path_to_kettle_job_dir/migration.kjb -level=detailed -# Run the after script (creates the indexes, constraints...) -> psql -U mypguser mypgdatabase -f name_of_after_script +``` + # Run the before script (creates the tables) + > psql -U mypguser mypgdatabase -f name_of_before_script + # Run the kettle job: + > cd my_kettle_installation_directory + > ./kitchen.sh -file=full_path_to_kettle_job_dir/migration.kjb -level=detailed + # Run the after script (creates the indexes, constraints...) + > psql -U mypguser mypgdatabase -f name_of_after_script +``` If you want to dig deeper into the kettle job, you can use kettle_report.pl to display the individual table's transfer performance. Then, if needed, you'll be able to modify the Kettle job to optimize it, using Spoon, Kettle's GUI You can also use a configuration file if you like: -> ./sqlserver2pgsql.pl -conf example_conf_file -f mydatabase_dump.sql + > ./sqlserver2pgsql.pl -conf example_conf_file -f mydatabase_dump.sql There is an example configuration file provided. You can also mix the configuration file with command line options. Command line options have the priority over values set in the configuration file. +FAQ +================================ + +See https://github.com/dalibo/sqlserver2pgsql/blob/master/FAQ.md + + +Licence ================================ -Licence: GPL v3 -http://www.gnu.org/licenses/gpl.html +GPL v3 : http://www.gnu.org/licenses/gpl.html From 5bd3b9729c27a6d8d10be92a3ca1eb22a2871c80 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 17 Jun 2014 15:59:32 +0200 Subject: [PATCH 053/216] Improve FAQ --- FAQ.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FAQ.md b/FAQ.md index 5a95254..fcb2f13 100644 --- a/FAQ.md +++ b/FAQ.md @@ -1,7 +1,7 @@ -================================ FAQ: Why didn't you do everything in the Perl script ? I don't want to use Kettle. +================================ Because I have once installed a Perl DBD driver for MS SQL's server, and I don't want to ever do it again... nor force you to do it :) You can either do it using ODBC, or using Sybase's driver. In both case, it will take you @@ -14,6 +14,7 @@ variable to a higher value (4096) for 4GB for instance. What is this IGNORE NULLS I have to change in kettle.properties ? +================================ Because Kettle behaves by default the way Oracle behaves: for Oracle, a NULL and an empty string are the same. You don't want this here, because neither SQL Server nor PostgreSQL do this. From 1ab0e70c6f97e33e80a9717b0f6d4ddd11adbbeb Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 17 Jun 2014 16:06:22 +0200 Subject: [PATCH 054/216] add more FAQ --- FAQ.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/FAQ.md b/FAQ.md index fcb2f13..c594996 100644 --- a/FAQ.md +++ b/FAQ.md @@ -1,7 +1,7 @@ FAQ: - +================ Why didn't you do everything in the Perl script ? I don't want to use Kettle. -================================ +---------------------------------- Because I have once installed a Perl DBD driver for MS SQL's server, and I don't want to ever do it again... nor force you to do it :) You can either do it using ODBC, or using Sybase's driver. In both case, it will take you @@ -14,7 +14,20 @@ variable to a higher value (4096) for 4GB for instance. What is this IGNORE NULLS I have to change in kettle.properties ? -================================ +---------------------------------- Because Kettle behaves by default the way Oracle behaves: for Oracle, a NULL and an empty string are the same. You don't want this here, because neither SQL Server nor PostgreSQL do this. + +What problems will I face after the migration ? +---------------------------------- +Here are those I faced: +* It seems that there is no way (at least with certain SQL Server versions) to tell if a constraint is enforced +on the whole table or only for new records. Anyway, this information isn't available in the SQL dump provided by +SQL Sever. So you may have constraints that won't validate anyway +* Case sensivity is on a per-column basis in SQL Server (you choose the collation per column, and case +sensitivity is part of the collation). In PostgreSQL, everything is case sensitive. I tried to emulate that +with citext, but if you can avoid it, do it. You'll have casting between citext and varchar/text everywhere, +bad plans, etc. +* Some columns may take into account trailing spaces, some won't (it must be ansi_padding). Anyway, this +doesn't exist either in PG. So more constraints will fail. From 4b103fc41167e4dedcb9f1906b256b7d7e64f69d Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 2 Jul 2014 08:47:27 +0200 Subject: [PATCH 055/216] Support sysname, and remove the debug dump (forgot to remove it last time) --- sqlserver2pgsql.pl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index e191665..e7e2da0 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -227,6 +227,11 @@ sub convert_type $rettype="numeric($sqlqual)"; } } + elsif ($sqlstype eq 'sysname') + { + # Special case. This is an internal type, and should seldom be used in production. Converted to varchar(128) + $rettype='varchar(128)'; + } else { print "Types: ", Dumper(\%types); @@ -2167,7 +2172,7 @@ sub resolve_name_conflicts parse_dump(); # Debug, uncomment: -print Dumper($objects); +#print Dumper($objects); # Rename indexes if they conflict resolve_name_conflicts(); From ba4dff6006d0de8d7eac1400c79c09ccafcd8bf5 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 28 Aug 2014 16:21:42 +0200 Subject: [PATCH 056/216] Add support for incremental migration --- README.md | 41 ++ example_conf_file | 3 + sqlserver2pgsql.pl | 1126 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 1122 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index a5ebdec..844f01c 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ It does two things: * convert a SQL Server schema to a PostgreSQL schema * produce a Pentaho Data Integrator (Kettle) job to migrate all the data from SQL Server to PostgreSQL. This second part is optionnal + * produce an incremental version of this job to migrate what has changed in the database from the previous run. This is created + when the migration job is also created. Notes, warnings: @@ -90,6 +92,34 @@ the size of the scale of the numeric if no: they are created as not valid (enforced only for new rows) if after: they are created as not valid, but the statements to validate them are put in the unsure file +-sort_size=100000: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). + +We don't sort in databases for two reasons: the sort order can be different between SQL Server +and PostgreSQL, and we don't want to stress the servers. But sorting a lot of data in Java can generate a Java Out of Heap Memory error. +If you get Out of Memory errors, raise the Java Heap memory (in the kitchen script) as much as you can. If you still have the problem, reduce +this sort size. You can also try reducing parallelism, having one on two sorts instead of 8 will of course consume less memory. The last problem is that +if the sort_size is small, kettle is going to generate a very large amount of temporary files, and then read them back sorted. So you may hit the +"too many open files" limit of your system (default 1024 on linux for instance). So you'll have to do some tuning here: + - First, use as much Java memory as you can: set the JAVAXMEM environment variable to 4096 (megabytes) or more if you can afford it. The more the better. + - If you still get Out Of Memory errors, put a smaller sort size, until you can do the sorts (decrease it tenfold each time for example). You'll obviously lose some performance + - If then you get the too many open files error, raise the maximum number of open files. In most Linux distributions, this is editing /etc/security/limits.conf and putting +@userName soft nofile 65535 +@userName hard nofile 65535 +(replace userName with your user name). Log in again, and verify with "ulimit -n" that you are now allowed to open 65535 files. +You may also have to raise the maximum number of open files on the system: echo the new value to /proc/sys/fs/file-max. + +You'll need a lot of temporary space on disk to do these sorts... + +You can also edit only the offending transformation with Spoon (Kettle's GUI), so that only this one is slowed down. + +When Kettle crashed on one of these problems, the temporary files aren't removed. They are usually in /tmp (or in your temp directory in Windows), and start with out_. Don't forget to remove them. + +-use_pk_if_possible=0/1/public.table1,myschema.table2: enable the generation of jobs doing sorts in the databases (order by in the select part of Kettle's table inputs). + +1 will ask to try for all tables, or you can give a list of tables (if for example, you cannot make these tables work with a reasonable sort size). Anyway, sqlserver2sql will only accept +to do sorts in the database if the primary key can be guaranteed to be sorted the same way in PostgreSQL and SQL Server. That means that it only accepts if the key is made only of numeric +and date/timestamp types. If not, the standard incremental job will be generated. + If you want to also import data: > ./sqlserver2pgsql.pl -b before.sql -a after.sql -u unsure.sql -k kettledir \ @@ -127,6 +157,17 @@ You've generated everything. Let's do the import: If you want to dig deeper into the kettle job, you can use kettle_report.pl to display the individual table's transfer performance. Then, if needed, you'll be able to modify the Kettle job to optimize it, using Spoon, Kettle's GUI +``` +You can also give a try to the incremental job: + > ./kitchen.sh -file=full_path_to_kettle_job_dir/incremental.kjb -level=detailed + +This one is highly experimental. I need your feedback ! :). You should only run an incremental job on an already loaded database. + +It may fail for a variety of reasons, mainly out of memory errors. If you have other unique constraints beyond the primary key, the series of queries generated by sqlserver2pgsql may generate conflicting updates. So test it several times before the migration day, if you really want to try this method. The "normal" method is safer, but of course, you'll start from scratch, and have those long indexes builds at the end. + +By the way, to be able to insert data into all tables, it deactivates triggers at the beginning and activates them back at the end of the job. So, if the job fails, those triggers won't be reactivated. + +``` You can also use a configuration file if you like: > ./sqlserver2pgsql.pl -conf example_conf_file -f mydatabase_dump.sql diff --git a/example_conf_file b/example_conf_file index ba8538c..a69094b 100644 --- a/example_conf_file +++ b/example_conf_file @@ -25,3 +25,6 @@ no relabel dbo=1 # set it to 0 to convert the dbo schema to public convert numeric to int=1 # set it to 0 to keep numeric(xx,0) as numeric(xx,0). Will be converted to smallint, int or bigint by default relabel schemas=dbo=>foo;schema1=>bar keep identifier case=1 # keep case of database objects +# Incremental job +sort size=10000 # drives the amount of memory and temporary files that will be created by an incremental job +use pk if possible=0 # 1/list of tables, for tables where you want to try getting already sorted records diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index e7e2da0..f6079cb 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -43,10 +43,15 @@ our $keep_identifier_case; our $validate_constraints='yes'; our $parallelism=8; +our $sort_size=10000; +our $use_pk_if_possible=0; + +# These three variables are loaded in the BEGIN block at the end of this file (they are very big +my $template; +my $template_lob; +my $incremental_template; +my $incremental_template_sortable_pk; -my $template - ; # These two variables are loaded in the BEGIN block at the end of this file (they are very big -my $template_lob; # putting them there won't pollute the code as much) my ($job_header, $job_middle, $job_footer) ; # These are used to create the static parts of the job my ($job_entry, $job_hop) @@ -82,6 +87,8 @@ sub parse_conf_file 'relabel schemas' => 'relabel_schemas', 'keep identifier case' => 'keep_identifier_case', 'validate constraints' => 'validate_constraints', + 'sort size' => 'sort_size', + 'use pk if possible' => 'use_pk_if_possible', ); # Open the conf file or die @@ -288,6 +295,53 @@ sub convert_type return $rettype; } +# This function is used for selects from SQL Server, in kettle. It adds a function call +# if there is a conversion to be done. +# uniqueidentifier is upper case in SQL Server, whereas uuid is lower case in PG +sub sql_convert_column +{ + my ($colname,$coltype)=@_; + my %functions = ( 'uuid' => 'lower'); + if (defined ($functions{$coltype})) + { + return $functions{$coltype} . '([' . $colname . '])'; + } + else + { + return "[$colname]"; + } +} + +# This function is used to determine if a PK will be sorted the same in SQL Server and PG +# It means that it doesn't depend on collation orders or other internals. +# For now, only numeric and date data types are considered OK +# Used for incremental jobs, to know if we can ask the databases to send us pre-sorted data +# We also filter on $use_pk_if_possible +sub is_pk_sort_order_safe +{ + my ($schema,$table)=@_; + my %safe_types = ('numeric' => 1, + 'int' => 1, + 'bigint' => 1, + 'smallint' => 1, + 'real' => 1, + 'double precision' => 1, + 'date' => 1, + 'timestamp' => 1, + ); + return 0 unless (defined $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{PK}); # There is no PK + return 0 unless ($use_pk_if_possible =~ /\b${schema}\.${table}\b/i or $use_pk_if_possible eq '1'); + my $pk=$objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{PK}; + my $isok=1; + # It is OK as long as all types are in %safe_type + foreach my $col (@{$pk->{COLS}}) + { + $isok=0 unless defined ($safe_types{$objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}}); + } + return $isok; +} + + # This function formats the identifiers (object name), putting double quotes around it # It also converts case if asked sub format_identifier @@ -459,6 +513,10 @@ sub usage "-keep_identifier_case tells $0 to keep the case of sql server database objects (not advised). Default is to lowercase everything.\n"; print "before_file contains the structure\n"; print "after_file contains index, constraints\n"; + print "validate_constraint validates the constraints that have been created\n"; + print "sort_size will change size of sort batch for the incremental job. Too small and it will be slow, too big and you will get Java Out of Heap Memory errors.\n"; + print "sort_size is 10000, which is very low, to try to avoid problems. First, raise java heap memory (in the kitchen script), then try higher values if you need more speed\n"; + print "use_pk_if_possible is false (0) by default. You can put it to 1 (true), or give a comma separated list of tables (with schema). Compared case insensitively\n"; print "unsure_file contains things we cannot guarantee will work, such as views\n"; print "\n"; @@ -490,6 +548,7 @@ sub generate_kettle } # For each table in each schema in $objects, we generate a kettle file in the directory + # We also create an incremental transformation foreach my $schema (sort keys %{$objects->{SCHEMAS}}) { @@ -538,6 +597,21 @@ sub generate_kettle { $newtemplate = $template; } + # We have a similar question for incremental jobs: can we use the primary key ? + # We obviously need a primary key, and we need the sort order to be the same in both databases + my $newincrementaltemplate; + if (is_pk_sort_order_safe($schema,$table)) + { + $newincrementaltemplate=$incremental_template_sortable_pk; + my $collist=join(',',@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}); + $newincrementaltemplate =~ s/__sqlserver_pk_condition__/$collist/g; + $newincrementaltemplate =~ s/__pg_pk_condition__/$collist/g; + } + else + { + $newincrementaltemplate=$incremental_template; + } + # Build the column list of the table to put into the SQL Server query my @colsdef; @@ -549,12 +623,16 @@ sub generate_kettle } (keys %{$refschema->{TABLES}->{$table}->{COLS}})) { - my $coldef = "[$col] AS " . format_identifier($col); + my $coldef = sql_convert_column($col,$refschema->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}) . " AS " . format_identifier($col); push @colsdef,($coldef); } my $colsdef=join(',',@colsdef); - # Substitute every connection placeholder with the real value + my $pgtable=format_identifier($table); + my $pgschema=format_identifier($targetschema); + + # Substitute every connection placeholder with the real value. We do this for both templates + $newtemplate =~ s/__sqlserver_database__/$sd/g; $newtemplate =~ s/__sqlserver_database__/$sd/g; $newtemplate =~ s/__sqlserver_host__/$sh/g; $newtemplate =~ s/__sqlserver_port__/$sp/g; @@ -567,12 +645,91 @@ sub generate_kettle $newtemplate =~ s/__postgres_password__/$pw/g; $newtemplate =~ s/__sqlserver_table_name__/[$origschema].[$table]/g; $newtemplate =~ s/__sqlserver_table_cols__/$colsdef/g; - my $pgtable=format_identifier($table); - my $pgschema=format_identifier($targetschema); $newtemplate =~ s/__postgres_table_name__/$pgtable/g; $newtemplate =~ s/__postgres_schema_name__/$pgschema/g; $newtemplate =~ s/__PARALLELISM__/$parallelism/g; + + $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; + $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; + $newincrementaltemplate =~ s/__sqlserver_host__/$sh/g; + $newincrementaltemplate =~ s/__sqlserver_port__/$sp/g; + $newincrementaltemplate =~ s/__sqlserver_username__/$su/g; + $newincrementaltemplate =~ s/__sqlserver_password__/$sw/g; + $newincrementaltemplate =~ s/__postgres_database__/$pd/g; + $newincrementaltemplate =~ s/__postgres_host__/$ph/g; + $newincrementaltemplate =~ s/__postgres_port__/$pp/g; + $newincrementaltemplate =~ s/__postgres_username__/$pu/g; + $newincrementaltemplate =~ s/__postgres_password__/$pw/g; + $newincrementaltemplate =~ s/__sqlserver_table_name__/[$origschema].[$table]/g; + $newincrementaltemplate =~ s/__sqlserver_table_cols__/$colsdef/g; + $newincrementaltemplate =~ s/__postgres_table_name__/$pgtable/g; + $newincrementaltemplate =~ s/__postgres_schema_name__/$pgschema/g; + $newincrementaltemplate =~ s/__PARALLELISM__/$parallelism/g; + $newincrementaltemplate =~ s/__sort_size__/$sort_size/g; + + # We have a bit of work to do on primary keys for the incremental template: we need them + # to compare the tables… + if (defined($refschema->{TABLES}->{$table}->{PK}->{COLS})) + { + my @pk=@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}; + my $keys; + foreach my $pk(@pk) + { + $keys.="$pk\n"; + } + $newincrementaltemplate =~ s/__KEYS_MERGE__/$keys/g; + + my $sortkeys=''; + my $synckeys=''; + foreach my $pk(@pk) + { + $sortkeys.="\n$pk\nY\nY\n\n"; + my $outcol=$pk; + unless ($keep_identifier_case) + { + $outcol=lc($outcol); + } + $synckeys.="\n$pk\n$outcol\n=\n\n\n"; + + } + $newincrementaltemplate =~ s/__SORT_KEYS_SQLSERVER__/$sortkeys/g; + $newincrementaltemplate =~ s/__SORT_KEYS_PG__/$sortkeys/g; + $newincrementaltemplate =~ s/__KEYS_SYNC__/$synckeys/g; + + # We also need to tell the merge step to compare all columns + my $valuesmerge=''; + my $valuessync=''; + foreach my $colname (keys(%{$refschema->{TABLES}->{$table}->{COLS}})) + { + # Is it a member of the PK ? If yes, no need to use it for comparison +#FIXME: unless (scalar(grep(/^${colname}$/,@pk))) # Does grep find an element in the array matching colname ? +# { + $valuesmerge.="$colname\n"; + # we need to use the correct case for postgresql output + my $outcol=$colname; + unless ($keep_identifier_case) + { + $outcol=lc($outcol); + } + $valuessync.="\n$outcol\n$colname\nY\n\n"; +# } + } + $newincrementaltemplate =~ s/__VALUES_MERGE__/$valuesmerge/g; + $newincrementaltemplate =~ s/__VALUES_SYNC__/$valuessync/g; + + + # Produce the incremental transformation + open FILE, ">$dir/incremental-$schema-$table.ktr" + or die "Cannot write to $dir/incremental-$schema-$table.ktr"; + binmode(FILE,":utf8"); + print FILE $newincrementaltemplate; + close FILE; + } + else + { + print STDERR "$schema/$table has no PK. Cannot create an incremental transformation\n"; + } # Store this new transformation into its file open FILE, ">$dir/$schema-$table.ktr" or die "Cannot write to $dir/$schema-$table.ktr"; @@ -584,17 +741,33 @@ sub generate_kettle # All transformations are done # We have to create a job to launch everything in one go - open FILE, ">$dir/migration.kjb" + # We also create an incremental job. This incremental job + # first deactivates all constraints (with triggers), then + # runs all incremental jobs, and "standard" jobs for all those + # tables where we cannot do incremental (no PK...) + open JOBFILE, ">$dir/migration.kjb" or die "Cannot write to $dir/migration.kjb"; + open INCFILE, ">$dir/incremental.kjb" + or die "Cannot write to $dir/incremental.kjb"; my $real_dir = getcwd; my $entries = ''; + my $incentries = ''; my $hops = ''; - my $prev_node = 'START'; - my $cur_vert_pos = 100 - ; # Not that useful, it's just not to be ugly if someone wanted to open - # the job with spoon (kettle's gui) and work on it graphically - # We sort only so that it will be easier to find a transformation in the job if one needed to - # edit it. It's also easier to track progress if tables are sorted alphabetically + my $prev_node = 'SQL SCRIPT START'; + # $cur_vert_pso is not that useful, it's just not to be ugly if someone wanted to open + # the job with spoon (kettle's gui) and work on it graphically + my $cur_vert_pos = 100; + + # First : the hop from START to SQL SCRIPT START + my $tmp_hop = $job_hop; + $tmp_hop =~ s/__table_1__/START/; + $tmp_hop =~ s/__table_2__/SQL SCRIPT START/; + # This is the first hop, so it is unconditionnal + $tmp_hop =~ s/N<\/unconditional>/Y<\/unconditional>/; + $hops.=$tmp_hop; + + # We sort only so that it will be easier to find a transformation in the job if one needed to + # edit it. It's also easier to track progress if tables are sorted alphabetically foreach my $schema (sort keys %{$objects->{SCHEMAS}}) { @@ -606,46 +779,58 @@ sub generate_kettle # We build the entries with regexp substitutions. The tablename contains the schema $tmp_entry =~ s/__table_name__/${schema}_${table}/; + $tmp_entry =~ s/__y_loc__/$cur_vert_pos/; - # Filename to use. We need the full path to the transformations - my $filename; + # JOBFILEname to use. We need the full path to the transformations + # The only difference between normal and incremental job is the filename of the transformation + my $JOBFILEname; + my $INCJOBFILEname; if ($dir =~ /^(\\|\/)/) # Absolute path { - $filename = $dir . '/' . $schema . '-' . $table . '.ktr'; + $JOBFILEname = $dir . '/' . $schema . '-' . $table . '.ktr'; + $INCJOBFILEname = $dir . '/' . 'incremental-' . $schema . '-' . $table . '.ktr'; } else { - $filename = + $JOBFILEname = + $real_dir . '/' + . $dir . '/' + . $schema . '-' + . $table . '.ktr'; + $INCJOBFILEname = $real_dir . '/' . $dir . '/' + . 'incremental-' . $schema . '-' . $table . '.ktr'; } + # Does the incremental transformation exist ? + unless (-e $INCJOBFILEname) + { + $INCJOBFILEname=$JOBFILEname; + } # Different for windows and linux, obviously: we change / to \ for windows unless (is_windows()) { - $filename =~ s/\////g; + $JOBFILEname =~ s/\////g; + $INCJOBFILEname =~ s/\////g; } else { - $filename =~ s/\//\\/g; + $JOBFILEname =~ s/\//\\/g; + $INCJOBFILEname =~ s/\//\\/g; } - $tmp_entry =~ s/__file_name__/$filename/; - $tmp_entry =~ s/__y_loc__/$cur_vert_pos/; + my $inctmp_entry=$tmp_entry; + $tmp_entry =~ s/__file_name__/$JOBFILEname/; + $inctmp_entry =~ s/__file_name__/$INCJOBFILEname/; $entries .= $tmp_entry; + $incentries.=$inctmp_entry; # We build the hop with the regexp too my $tmp_hop = $job_hop; $tmp_hop =~ s/__table_1__/$prev_node/; $tmp_hop =~ s/__table_2__/${schema}_${table}/; - if ($prev_node eq 'START') - { - - # Specific to the start node. It has to be unconditional - $tmp_hop =~ - s/N<\/unconditional>/Y<\/unconditional>/; - } $hops .= $tmp_hop; # We increment everything for next loop @@ -653,13 +838,59 @@ sub generate_kettle $cur_vert_pos += 80; # To be pretty in spoon } } + # Put the final hop + $tmp_hop = $job_hop; + $tmp_hop =~ s/__table_1__/$prev_node/; + $tmp_hop =~ s/__table_2__/SQL SCRIPT END/; + $hops .= $tmp_hop; - print FILE $job_header; - print FILE $entries; - print FILE $job_middle; - print FILE $hops; - print FILE $job_footer; - close FILE; + + # Build the casts in the start/stop job entries + my $beforescript; + my $afterscript; + if (defined ($objects->{CASTS})) + { + foreach my $cast (keys %{$objects->{CASTS}}) + { + $beforescript.= "DROP CAST IF EXISTS (varchar as $cast);\n"; + $beforescript.= "CREATE CAST (varchar as $cast) with inout as implicit;\n"; + $afterscript.= "DROP CAST (varchar as $cast);\n"; + } + } + # Remove/restore triggers to be able to insert without FK checks + foreach my $schema (sort keys %{$objects->{SCHEMAS}}) + { + my $refschema = $objects->{SCHEMAS}->{$schema}; + foreach my $table (sort { lc($a) cmp lc($b) } + keys %{$refschema->{TABLES}}) + { + $beforescript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " DISABLE TRIGGER ALL;\n"; + $beforescript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ENABLE TRIGGER ALL;\n"; + } + } + + + # This is for the SQL Scripts. We also need to specify the PG connection + $job_header =~ s/__SQL_SCRIPT_INIT__/$beforescript/g; + $job_header =~ s/__SQL_SCRIPT_END__/$afterscript/g; + $job_header =~ s/__postgres_database__/$pd/g; + $job_header =~ s/__postgres_host__/$ph/g; + $job_header =~ s/__postgres_port__/$pp/g; + $job_header =~ s/__postgres_username__/$pu/g; + $job_header =~ s/__postgres_password__/$pw/g; + + print JOBFILE $job_header; + print JOBFILE $entries; + print JOBFILE $job_middle; + print JOBFILE $hops; + print JOBFILE $job_footer; + close JOBFILE; + print INCFILE $job_header; + print INCFILE $incentries; + print INCFILE $job_middle; + print INCFILE $hops; + print INCFILE $job_footer; + close INCFILE; } @@ -1626,18 +1857,6 @@ sub generate_schema { print BEFORE "CREATE EXTENSION IF NOT EXISTS citext;\n"; } - if (defined ($objects->{CASTS})) - { - foreach my $cast (keys %{$objects->{CASTS}}) - { - # We create the cast in the BEFORE. Add a comment, as this is not obvious - print BEFORE "-- Create a cast. Used for the kettle job\n"; - print BEFORE "CREATE CAST (varchar as $cast) with inout as implicit;\n"; - # We drop the cast in the AFTER - print AFTER "-- drop a cast. Used for the kettle job\n"; - print AFTER "DROP CAST (varchar as $cast);\n"; - } - } # Ok, we have parsed everything, and definitions are in $objects # We will put in the BEFORE file only table and columns definitions. @@ -2114,7 +2333,9 @@ sub resolve_name_conflicts "num" => \$convert_numeric_to_int, "relabel_schemas=s" => \$relabel_schemas, "keep_identifier_case" =>\$keep_identifier_case, - "validate_constraints=s" =>\$validate_constraints,); + "validate_constraints=s" =>\$validate_constraints, + "sort_size=i" =>\$sort_size, + "use_pk_if_possible=s" =>\$use_pk_if_possible,); # We don't understand command line or have been asked for usage if (not $options or $help) @@ -2757,6 +2978,29 @@ BEGIN 2013/04/08 14:08:27.625 + + __postgres_db__ + __postgres_host__ + POSTGRESQL + Native + __postgres_database__ + __postgres_port__ + __postgres_username__ + __postgres_password__ + + + + + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER__postgres_port__ + QUOTE_ALL_FIELDSY + SUPPORTS_BOOLEAN_DATA_TYPEY + USE_POOLINGN + SQL_CONNECTset synchronous_commit to off; + + @@ -2799,6 +3043,38 @@ BEGIN 38 73 + + SQL SCRIPT START + + SQL + __SQL_SCRIPT_INIT__ + F + F + + F + __postgres_db__ + N + Y + 0 + 38 + 140 + + + SQL SCRIPT END + + SQL + __SQL_SCRIPT_END__ + F + F + + F + __postgres_db__ + N + Y + 0 + 38 + 200 + EOF #################################################################################### @@ -2863,6 +3139,760 @@ BEGIN N EOF +#################################################################################### + + $incremental_template= < + + migration__sqlserver_table_name__ + + + + Normal + 0 + / + + + + + + + + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + + +
+ + 0.0 + 0.0 + + 200 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2013/02/28 14:04:49.560 + - + 2014/08/26 15:16:59.019 + + + + + __postgres_db__ + __postgres_host__ + POSTGRESQL + Native + __postgres_database__ + __postgres_port__ + __postgres_username__ + __postgres_password__ + + + + + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER__postgres_port__ + QUOTE_ALL_FIELDSY + SQL_CONNECTset synchronous_commit to off; + SUPPORTS_BOOLEAN_DATA_TYPEY + USE_POOLINGN + + + + __sqlserver_db__ + __sqlserver_host__ + MSSQL + Native + __sqlserver_database__ + __sqlserver_port__ + __sqlserver_username__ + __sqlserver_password__ + + + + + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + MSSQL_DOUBLE_DECIMAL_SEPARATORN + PORT_NUMBER__sqlserver_port__ + QUOTE_ALL_FIELDSY + SUPPORTS_BOOLEAN_DATA_TYPEN + USE_POOLINGN + + + + + Table input 2User Defined Java ClassY User Defined Java ClassSort rows 2Y Sort rows 2Sorted Merge 2Y Table inputSort rowsY Sort rowsSorted MergeY Sorted MergeMerge Rows (diff)Y Sorted Merge 2Merge Rows (diff)Y Merge Rows (diff)Synchronize after mergeY + + Table input 2 + TableInput + + Y + 1 + + none + + + __sqlserver_db__ + SELECT __sqlserver_table_cols__ FROM __sqlserver_table_name__ WITH(NOLOCK) + 0 + + N + Y + N + + + 122 + 250 + Y + + + + + + User Defined Java Class + UserDefinedJavaClass + + Y + __PARALLELISM__ + + none + + + + + + TRANSFORM_CLASS + + Processor + + + + + + N + + + 280 + 332 + Y + + + + Table input + TableInput + + Y + 1 + + none + + + __postgres_db__ + SELECT * FROM __postgres_schema_name__.__postgres_table_name__ + 0 + + N + N + N + + + 122 + 150 + Y + + + + + Sort rows + SortRows + + Y + __PARALLELISM__ + + none + + + %%java.io.tmpdir%% + out + __sort_size__ + + N + + N + +__SORT_KEYS_PG__ + + + + 351 + 161 + Y + + + + + Sort rows 2 + SortRows + + Y + __PARALLELISM__ + + none + + + %%java.io.tmpdir%% + out + __sort_size__ + + N + + N + +__SORT_KEYS_SQLSERVER__ + + + + 352 + 253 + Y + + + + Sorted Merge + SortedMerge + + Y + + 1 + + none + + + +__SORT_KEYS_PG__ + + + + 428 + 161 + Y + + + + Sorted Merge 2 + SortedMerge + + Y + + 1 + + none + + + +__SORT_KEYS_SQLSERVER__ + + + + 429 + 245 + Y + + + + + Synchronize after merge + SynchronizeAfterMerge + + Y + __PARALLELISM__ + + none + + + __postgres_db__ + 100 + N + + N + N + __changed__ + new + changed + deleted + + __postgres_schema_name__ +
__postgres_table_name__
+ __KEYS_SYNC__ + __VALUES_SYNC__ + + + + 700 + 212 + Y + + + + + Merge Rows (diff) + MergeRows + + Y + 1 + + none + + + + __KEYS_MERGE__ + + + __VALUES_MERGE__ + +__changed__ +Sorted Merge +Sorted Merge 2 + + + + + 506 + 212 + Y + + + + + + + + N + +EOF +#################################################################################### + + $incremental_template_sortable_pk= < + + migration__sqlserver_table_name__ + + + + Normal + 0 + / + + + + + + + + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + + +
+ + 0.0 + 0.0 + + 200 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2013/02/28 14:04:49.560 + - + 2014/08/26 15:16:59.019 + + + + + __postgres_db__ + __postgres_host__ + POSTGRESQL + Native + __postgres_database__ + __postgres_port__ + __postgres_username__ + __postgres_password__ + + + + + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER__postgres_port__ + QUOTE_ALL_FIELDSY + SQL_CONNECTset synchronous_commit to off; + SUPPORTS_BOOLEAN_DATA_TYPEY + USE_POOLINGN + + + + __sqlserver_db__ + __sqlserver_host__ + MSSQL + Native + __sqlserver_database__ + __sqlserver_port__ + __sqlserver_username__ + __sqlserver_password__ + + + + + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + MSSQL_DOUBLE_DECIMAL_SEPARATORN + PORT_NUMBER__sqlserver_port__ + QUOTE_ALL_FIELDSY + SUPPORTS_BOOLEAN_DATA_TYPEN + USE_POOLINGN + + + + + Table input 2User Defined Java ClassY User Defined Java ClassMerge Rows (diff)Y Table inputMerge Rows (diff)Y Merge Rows (diff)Synchronize after mergeY + + Table input 2 + TableInput + + Y + 1 + + none + + + __sqlserver_db__ + SELECT __sqlserver_table_cols__ FROM __sqlserver_table_name__ WITH(NOLOCK) ORDER BY __sqlserver_pk_condition__ + 0 + + N + Y + N + + + 122 + 250 + Y + + + + + + User Defined Java Class + UserDefinedJavaClass + + Y + 1 + + none + + + + + + TRANSFORM_CLASS + + Processor + + + + + + N + + + 280 + 332 + Y + + + + Table input + TableInput + + Y + 1 + + none + + + __postgres_db__ + SELECT * FROM __postgres_schema_name__.__postgres_table_name__ ORDER BY __pg_pk_condition__ + 0 + + N + N + N + + + 122 + 150 + Y + + + + + Synchronize after merge + SynchronizeAfterMerge + + Y + __PARALLELISM__ + + none + + + __postgres_db__ + 100 + N + + N + N + __changed__ + new + changed + deleted + + __postgres_schema_name__ +
__postgres_table_name__
+ __KEYS_SYNC__ + __VALUES_SYNC__ + + + + 700 + 212 + Y + + + + + Merge Rows (diff) + MergeRows + + Y + 1 + + none + + + + __KEYS_MERGE__ + + + __VALUES_MERGE__ + +__changed__ +Table input +User Defined Java Class + + + + + 506 + 212 + Y + + + + + + + + N + +EOF #################################################################################### } From 53fef3a070d3503b4c2d6a6ea0773e218ada01d8 Mon Sep 17 00:00:00 2001 From: Stuart Brown Date: Fri, 10 Oct 2014 16:31:08 +0200 Subject: [PATCH 057/216] Added data type mapping between smallmoney and numeric --- sqlserver2pgsql.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index f6079cb..a0a22b2 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -177,6 +177,7 @@ sub add_cast 'binary' => 'bytea', 'varbinary' => 'bytea', 'money' => 'numeric', + 'smallmoney' => 'numeric', 'uniqueidentifier' => 'uuid',); # Types with no qualifier, and no point in putting one From ddbdfd5eacb24571f6ba1b7cc1407f776c7a9700 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 17 Nov 2014 17:55:12 +0100 Subject: [PATCH 058/216] Correct bug on job generation for lob --- sqlserver2pgsql.pl | 45 ++++++++++++++------------------------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index a0a22b2..b0ba30a 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -559,40 +559,23 @@ sub generate_kettle foreach my $table (sort keys %{$refschema->{TABLES}}) { my $origschema=$refschema->{TABLES}->{$table}->{origschema}; - # First, does this table have LOBs ? The template depends on this + # First, does this table have LOBs ? The template depends on this and is this + # table having an int PK ? my $newtemplate; - if ($refschema->{TABLES}->{$table}->{haslobs}) + if ( $refschema->{TABLES}->{$table}->{haslobs} + and defined($refschema->{TABLES}->{$table}->{PK}->{COLS}) + and scalar(@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}) == 1 + and ($refschema->{TABLES}->{$table}->{COLS}->{($refschema->{TABLES}->{$table}->{PK}->{COLS}->[0])}->{TYPE} =~ /int$/) + ) { - $newtemplate = $template_lob; - - # Is the PK int and on only one column ? - # If yes, we can use several threads in kettle to read this table to - # improve performance my $wherefilter; - if (defined($refschema->{TABLES}->{$table}->{PK}->{COLS}) - and - scalar(@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}) - == 1 - and ($refschema->{TABLES}->{$table}->{COLS} - ->{($refschema->{TABLES}->{$table}->{PK}->{COLS}->[0]) - }->{TYPE} =~ /int$/)) - { - $wherefilter = - 'WHERE ' - . $refschema->{TABLES}->{$table}->{PK}->{COLS}->[0] - . '% ${Internal.Step.Unique.Count} = ${Internal.Step.Unique.Number}'; - $newtemplate =~ - s/__sqlserver_where_filter__/$wherefilter/; - } - else - - # No way to do this optimization. Use standard template - { - $wherefilter=''; - $newtemplate =~ s/__sqlserver_where_filter__//; - $newtemplate =~ s/__sqlserver_copies__/1/g; - } - $newtemplate =~ s/__sqlserver_where_filter__/$wherefilter/; + $newtemplate = $template_lob; + $wherefilter = + 'WHERE ' + . $refschema->{TABLES}->{$table}->{PK}->{COLS}->[0] + . '% ${Internal.Step.Unique.Count} = ${Internal.Step.Unique.Number}'; + $newtemplate =~ + s/__sqlserver_where_filter__/$wherefilter/; } else { From 091fd7b726d6a97c09a153e62ba4bc205915f4f3 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 17 Nov 2014 17:59:46 +0100 Subject: [PATCH 059/216] Correct bug #8 (start scripts contains both disable and enable triggers) --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index b0ba30a..3a22801 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -849,7 +849,7 @@ sub generate_kettle keys %{$refschema->{TABLES}}) { $beforescript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " DISABLE TRIGGER ALL;\n"; - $beforescript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ENABLE TRIGGER ALL;\n"; + $afterscript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ENABLE TRIGGER ALL;\n"; } } From 4c71d51d969886aa367208e61c1cf048797cb874 Mon Sep 17 00:00:00 2001 From: marco44 Date: Mon, 17 Nov 2014 18:10:16 +0100 Subject: [PATCH 060/216] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 844f01c..dae2cba 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a migration tool to convert a Microsoft SQL Server Database into a Postg It is written in Perl. -It does two things: +It does three things: * convert a SQL Server schema to a PostgreSQL schema * produce a Pentaho Data Integrator (Kettle) job to migrate From 046e9316ee66eed766a2891eab6e4cde515b2c40 Mon Sep 17 00:00:00 2001 From: Bill Ruddock Date: Wed, 7 Jan 2015 17:49:02 +0000 Subject: [PATCH 061/216] Fix balanced close/open comment on one line Fixes a multi-line C style comment closed and reopened on same line with important SQL in between. e.g. ```sql /* comments ... */ SQL here /* ... more comments ... */ ``` --- sqlserver2pgsql.pl | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3a22801..5408429 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -896,6 +896,19 @@ sub generate_kettle ; # Remove executesql… it's a bit weird in the SQL Server's dump # If we are not in comment, we look for /* # If we are in comment, we look for */, and we remove everything until */ + if ($in_comment) + { + if ($line =~ /\*\//) + { + $in_comment = 0; + $line =~ s/.*\*\///; # Remove everything before the uncomment + } + else + { + $line = "\n"; + } + } + if (not $in_comment) { @@ -909,23 +922,6 @@ sub generate_kettle $line =~ s/\/\*.*//; # Remove everything after the comment } } - else - { - # We do the reverse: keep only what is not commented - $line =~ s/\*\/(.*?)\/\*/$1/g; - - # Is there an uncomment left ? - if ($line =~ /\*\//) - { - $in_comment = 0; - $line =~ s/.*\*\///; # Remove everything before the uncomment - } - else - { - # There is no uncomment. The line should be empty - $line = "\n"; - } - } return $line; } } From 988714552658f419594e65494db4d185179c68cf Mon Sep 17 00:00:00 2001 From: Bill Ruddock Date: Wed, 7 Jan 2015 17:55:25 +0000 Subject: [PATCH 062/216] Ignore EXEC sys.sp_db_vardecimal_storage_format Enabling vardecimal storage format was only needed in SQL Server 2005 and means nothing for PostgreSQL. See http://msdn.microsoft.com/en-us/library/bb326653.aspx --- sqlserver2pgsql.pl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3a22801..0943503 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1800,6 +1800,11 @@ sub parse_dump { next; } + # Ignore EXEC sys.sp_db_vardecimal_storage_format, enabling for vardecimal storage format was only needed in SQL Server 2005 and means nothing for PG. + elsif ($line =~ /^EXEC sys\.sp_db_vardecimal_storage_format/) + { + next; + } # Still on views: there are empty lines, and C-style comments elsif ($line =~ /^\s*$/) From 6518ef71ef87f9c9a4bce4448331d4dc2f00c6fa Mon Sep 17 00:00:00 2001 From: Bill Ruddock Date: Wed, 7 Jan 2015 18:08:35 +0000 Subject: [PATCH 063/216] Rename indices where more than 2 share same name --- sqlserver2pgsql.pl | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3a22801..0d44ef4 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2274,16 +2274,24 @@ sub resolve_name_conflicts } else { + my $i = 0; + my $postfix = "2pgi"; + while (defined($known_names{format_identifier("${idx}2pgi${i}")})) + { + $i++; + } + $postfix .= $i; + # We have to rename :/ - # Postfix with a 2pg + # Postfix with a 2pgi followed by number # We have to update the name in the $refschema hash $refschema->{TABLES}->{$table}->{INDEXES} - ->{"$idx" . "2pgi"} = + ->{"${idx}${postfix}"} = $refschema->{TABLES}->{$table}->{INDEXES}->{$idx}; delete $refschema->{TABLES}->{$table}->{INDEXES}->{$idx}; print STDERR - "Warning: I had to rename index $table.$idx to ${idx}2pgi because of naming conflicts in source schema $schema\n"; - $known_names{format_identifier($idx . "2pgi")} = 1; + "Warning: I had to rename index $table.$idx to ${idx}${postfix} because of naming conflicts in source schema $schema\n"; + $known_names{format_identifier("${idx}${postfix}")} = 1; } } } From 922169e9910ab4c4d7ec07bf6bec0e8ee5eb0861 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 20 Jan 2015 09:04:03 +0100 Subject: [PATCH 064/216] add support for the time type --- sqlserver2pgsql.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 01fc149..ca94612 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -172,6 +172,7 @@ sub add_cast 'date' => 'date', 'datetime' => 'timestamp', 'smalldatetime' => 'timestamp', + 'time' => 'time', 'timestamp' => 'timestamp', 'image' => 'bytea', 'binary' => 'bytea', From 60fe3c83765e6f11e1ebd01de68ed9f8e297c6b2 Mon Sep 17 00:00:00 2001 From: marco44 Date: Fri, 23 Jan 2015 14:45:05 +0100 Subject: [PATCH 065/216] Update FAQ.md --- FAQ.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/FAQ.md b/FAQ.md index c594996..7bfb343 100644 --- a/FAQ.md +++ b/FAQ.md @@ -5,12 +5,16 @@ Why didn't you do everything in the Perl script ? I don't want to use Kettle. Because I have once installed a Perl DBD driver for MS SQL's server, and I don't want to ever do it again... nor force you to do it :) You can either do it using ODBC, or using Sybase's driver. In both case, it will take you -quite a while, and you'll suffer with large objects. With Kettle, on the other hand, you're using the JBDC driver, +quite a while, and you'll suffer with large objects. + +With Kettle, on the other hand, you're using the JBDC driver, which is already one of Kettle's default drivers, along with PostgreSQL. So all the heavy lifting (converting LOBs and IMAGES and whatever to bytea) is directly done by both JDBC drivers, and should work with no efforts (except maybe adjust Java's memory parameters). If you get a memory error, try setting a JAVAMAXMEM environment variable to a higher value (4096) for 4GB for instance. +There is another big advantage of using Kettle: you can tailor the scripts produced by sqlserver2pgsql to your needs, such as adding some conversions, schema changes. As Kettle is an ETL, it is a good tool for doing such conversions on the fly. + What is this IGNORE NULLS I have to change in kettle.properties ? From cfe2df9ea593ce7cdbb7c0f15c26a9ad129d298e Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 6 Mar 2015 14:33:24 +0100 Subject: [PATCH 066/216] Support empty passwords --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index ca94612..604c1f0 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2364,12 +2364,12 @@ sub resolve_name_conflicts if ($kettle and ( not $sd or not $sh - or not $sp + or not defined($sp) # password can be empty, it just has to be defined) or not $su or not $sw or not $pd or not $ph - or not $pp + or not defined ($pp) # password can be empty, it just has to be defined or not $pu or not $pw)) { From 1b018ca07541b5dae42be2c14e4e4b4005f68570 Mon Sep 17 00:00:00 2001 From: sebpcspkr Date: Tue, 10 Mar 2015 11:53:20 +0100 Subject: [PATCH 067/216] p stands for Port/ w stands for Password :) --- sqlserver2pgsql.pl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 604c1f0..bcb0a42 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2364,14 +2364,16 @@ sub resolve_name_conflicts if ($kettle and ( not $sd or not $sh - or not defined($sp) # password can be empty, it just has to be defined) + or not $sp or not $su - or not $sw + or not defined($sw) # password can be empty, it just has to be defined or not $pd or not $ph - or not defined ($pp) # password can be empty, it just has to be defined + or not $pp or not $pu - or not $pw)) + or not defined($pw) # password can be empty, it just has to be defined + ) +) { usage(); print From 42de4f70042e22467627dabf207ae63ee99ed81b Mon Sep 17 00:00:00 2001 From: marco44 Date: Fri, 20 Mar 2015 12:03:33 +0100 Subject: [PATCH 068/216] Update FAQ.md --- FAQ.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FAQ.md b/FAQ.md index 7bfb343..a6aec1c 100644 --- a/FAQ.md +++ b/FAQ.md @@ -3,7 +3,7 @@ FAQ: Why didn't you do everything in the Perl script ? I don't want to use Kettle. ---------------------------------- -Because I have once installed a Perl DBD driver for MS SQL's server, and I don't want to ever do it again... nor +Because I have once installed a Perl DBD driver for Microsoft SQL server, and I don't want to ever do it again... nor force you to do it :) You can either do it using ODBC, or using Sybase's driver. In both case, it will take you quite a while, and you'll suffer with large objects. From 452556be5b89db28286afc573f6c77731e6e0da2 Mon Sep 17 00:00:00 2001 From: marco44 Date: Fri, 20 Mar 2015 12:10:38 +0100 Subject: [PATCH 069/216] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index dae2cba..6c3795a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ sqlserver2pgsql This is a migration tool to convert a Microsoft SQL Server Database into a PostgreSQL database, as automatically as possible. -It is written in Perl. +It is written in Perl and has received a fair amount of testing. It does three things: @@ -18,13 +18,13 @@ It does three things: Notes, warnings: ========================== This tool will never be completely finished. For now, it works with all the SQL Server -databases I had to migrate. If it doesn't work with yours, feel free to modify this, +databases I and anyone asking for help in an issue had to migrate. If it doesn't work with yours, feel free to modify this, send me patches, or the SQL dump from your SQL Server Database, with the -problem you are facing. I'll try to improve the code, but I need this SQL dump. +problem you are facing. I'll try to improve the code, but I need this SQL dump. Create an issue in github ! It won't migrate PL procedures, the languages are too different. -I usually only test this script under Linux. It should work on Windows, as I had to do it once +I usually only test this script under Linux. It works on Windows, as I had to do it once with Windows, and on any Unix system. From 338c5c7a3711351fc6c31947c0954194088c237a Mon Sep 17 00:00:00 2001 From: marco44 Date: Sat, 21 Mar 2015 08:23:38 +0100 Subject: [PATCH 070/216] support datetimeoffset --- sqlserver2pgsql.pl | 47 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index bcb0a42..ed85b3e 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -156,29 +156,30 @@ sub add_cast # These are the no-brainer conversions # There is still a special case for text types and case insensitivity (see convert_type) though -my %types = ('int' => 'int', - 'nvarchar' => 'varchar', - 'nchar' => 'char', - 'char' => 'char', - 'varchar' => 'varchar', - 'text' => 'text', - 'char' => 'char', - 'smallint' => 'smallint', - 'tinyint' => 'smallint', - 'bigint' => 'bigint', - 'decimal' => 'numeric', - 'float' => 'double precision', - 'real' => 'real', - 'date' => 'date', - 'datetime' => 'timestamp', - 'smalldatetime' => 'timestamp', - 'time' => 'time', - 'timestamp' => 'timestamp', - 'image' => 'bytea', - 'binary' => 'bytea', - 'varbinary' => 'bytea', - 'money' => 'numeric', - 'smallmoney' => 'numeric', +my %types = ('int' => 'int', + 'nvarchar' => 'varchar', + 'nchar' => 'char', + 'char' => 'char', + 'varchar' => 'varchar', + 'text' => 'text', + 'char' => 'char', + 'smallint' => 'smallint', + 'tinyint' => 'smallint', + 'bigint' => 'bigint', + 'decimal' => 'numeric', + 'float' => 'double precision', + 'real' => 'real', + 'date' => 'date', + 'datetime' => 'timestamp', + 'smalldatetime' => 'timestamp', + 'time' => 'time', + 'timestamp' => 'timestamp', + 'datetimeoffset' => 'timestamp with time zone', + 'image' => 'bytea', + 'binary' => 'bytea', + 'varbinary' => 'bytea', + 'money' => 'numeric', + 'smallmoney' => 'numeric', 'uniqueidentifier' => 'uuid',); # Types with no qualifier, and no point in putting one From 7392a7acc512e771e4d718e72f62ed6e19f07a37 Mon Sep 17 00:00:00 2001 From: marco44 Date: Sat, 21 Mar 2015 11:04:39 +0100 Subject: [PATCH 071/216] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6c3795a..25e3952 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ It does three things: * produce an incremental version of this job to migrate what has changed in the database from the previous run. This is created when the migration job is also created. +Please drop me a word (on github) if you use this tool, feedback is great :) Notes, warnings: ========================== From d4339178b86e3ef330fd5951ae6be9191855d31f Mon Sep 17 00:00:00 2001 From: Daniel G Date: Thu, 26 Mar 2015 21:36:46 +0100 Subject: [PATCH 072/216] Added support for ms sql instances In Ms Sql Server, instances are handled, for eample whe using sqlexpres or when using replication. Here I am adding support so the scripts output the instance property and they can run witout manually editing the outputed files. --- sqlserver2pgsql.pl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index ed85b3e..bf887a3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -28,7 +28,7 @@ my $objects; # These are global variables, from configuration file or command line arguments -our ($sd, $sh, $sp, $su, $sw, $pd, $ph, $pp, $pu, $pw); # Connection args +our ($sd, $sh, $si, $sp, $su, $sw, $pd, $ph, $pp, $pu, $pw); # Connection args our $conf_file; our $filename; # Filename passed as arg our $case_insensitive; # Passed as arg: was SQL Server installation case insensitive ? PostgreSQL can't ignore accents anyway @@ -59,7 +59,7 @@ my @view_list; # array to keep view ordering from sql server's dump (a view may depend on another view) # Opens the configuration file -# Sets $sd $sh $sp $su $sw $pd $ph $pp $pu $pw when they are not set in the command line already +# Sets $sd $sh $si $sp $su $sw $pd $ph $pp $pu $pw when they are not set in the command line already # Also gets kettle parameters... sub parse_conf_file { @@ -68,6 +68,7 @@ sub parse_conf_file # This is also used as the list of accepted parameters in the configuration file my %parameters = ('sql server database' => 'sd', 'sql server host' => 'sh', + 'sql server host instance' => 'si', 'sql server port' => 'sp', 'sql server username' => 'su', 'sql server password' => 'sw', @@ -174,7 +175,7 @@ sub add_cast 'smalldatetime' => 'timestamp', 'time' => 'time', 'timestamp' => 'timestamp', - 'datetimeoffset' => 'timestamp with time zone', + 'datetimeoffset' => 'timestamp with time zone', 'image' => 'bytea', 'binary' => 'bytea', 'varbinary' => 'bytea', @@ -617,11 +618,17 @@ sub generate_kettle my $pgtable=format_identifier($table); my $pgschema=format_identifier($targetschema); + my $sqlinstancename = ''; + if (length $si) { + $sqlinstancename = $si; + } + # Substitute every connection placeholder with the real value. We do this for both templates $newtemplate =~ s/__sqlserver_database__/$sd/g; $newtemplate =~ s/__sqlserver_database__/$sd/g; $newtemplate =~ s/__sqlserver_host__/$sh/g; $newtemplate =~ s/__sqlserver_port__/$sp/g; + $newtemplate =~ s/__sqlserver_instance__/$sqlinstancename/g; $newtemplate =~ s/__sqlserver_username__/$su/g; $newtemplate =~ s/__sqlserver_password__/$sw/g; $newtemplate =~ s/__postgres_database__/$pd/g; @@ -643,6 +650,7 @@ sub generate_kettle $newincrementaltemplate =~ s/__sqlserver_username__/$su/g; $newincrementaltemplate =~ s/__sqlserver_password__/$sw/g; $newincrementaltemplate =~ s/__postgres_database__/$pd/g; + $newincrementaltemplate =~ s/__sqlserver_instance__/$sqlinstancename/g; $newincrementaltemplate =~ s/__postgres_host__/$ph/g; $newincrementaltemplate =~ s/__postgres_port__/$pp/g; $newincrementaltemplate =~ s/__postgres_username__/$pu/g; @@ -2314,6 +2322,7 @@ sub resolve_name_conflicts "conf=s" => \$conf_file, "sd=s" => \$sd, "sh=s" => \$sh, + "si=s" => \$si, "sp=s" => \$sp, "su=s" => \$su, "sw=s" => \$sw, @@ -2494,6 +2503,7 @@ BEGIN + EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -2769,6 +2779,7 @@ BEGIN + EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3244,6 +3255,7 @@ BEGIN + EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3670,6 +3682,7 @@ BEGIN + EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN From 53bc289ab95b39d69a53339577a6623b5c2fb2bc Mon Sep 17 00:00:00 2001 From: marco44 Date: Thu, 26 Mar 2015 21:48:21 +0100 Subject: [PATCH 073/216] Add documentation for sql server instance patch --- README.md | 3 ++- example_conf_file | 1 + sqlserver2pgsql.pl | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 25e3952..b9c4f7c 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Ok, I have installed Kettle and Java, I have sqlserver2pgsql.pl, what do I do no You'll need several things: * The connection parameters to the SQL Server database: - IP address, port, username, password, database name + IP address, port, username, password, database name, instance name if not default * Access to an empty PostgreSQL database (where you want your migrated data) * A text file containing a SQL dump of the SQL Server database @@ -134,6 +134,7 @@ You'll also need to specify the connection parameters. They will be stored insid cleartext, so don't make this directory public): -sd : sql server database -sh : sql server host +-si : sql server host instance -sp : sql server port (usually 1433) -su : sql server username -sw : sql server password diff --git a/example_conf_file b/example_conf_file index a69094b..ab4a1c9 100644 --- a/example_conf_file +++ b/example_conf_file @@ -10,6 +10,7 @@ kettle directory=/tmp/kettle # Comment this line if you don't want a kettle scri # These are ignored as long as kettle is not set sql server database=foo sql server host=foo_host +sql server host instance=my_instance # You can omit this if you use the default instance sql server port=1433 sql server username=foo_user sql server password=foo_password diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index bf887a3..01ed9d5 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -529,6 +529,7 @@ sub usage print "for connecting to both databases:\n"; print "-sd: sqlserver database\n"; print "-sh: sqlserver host\n"; + print "-si: sqlserver host instance\n"; print "-sp: sqlserver port\n"; print "-su: sqlserver username\n"; print "-sw: sqlserver password\n"; From bdc99453a759f74c40dcbb94adf7d6ee8221705f Mon Sep 17 00:00:00 2001 From: Yann VERRY Date: Sun, 19 Apr 2015 22:17:33 +0200 Subject: [PATCH 074/216] datetime2 support --- sqlserver2pgsql.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 01ed9d5..b0f5c4a 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -172,6 +172,7 @@ sub add_cast 'real' => 'real', 'date' => 'date', 'datetime' => 'timestamp', + 'datetime2' => 'timestamp', 'smalldatetime' => 'timestamp', 'time' => 'time', 'timestamp' => 'timestamp', From 28222fb5b6d2be4ed181eef76d9c899ef7cf3f26 Mon Sep 17 00:00:00 2001 From: marco44 Date: Sun, 26 Apr 2015 17:28:11 +0200 Subject: [PATCH 075/216] Change the windows match regexp so that it wont catch 'darwin' anymore --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 01ed9d5..26ba0ca 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -453,7 +453,7 @@ sub next_col_pos # Test if we are on windows. We will have to convert / to \ in the XML files sub is_windows { - if ($^O =~ /win/i) + if ($^O =~ /MSWin32/) { return 1; } From 7e996663b17952caa03f5e1598beb410c32089bd Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 28 Apr 2015 11:14:26 +0200 Subject: [PATCH 076/216] Add support for preserving reserved keywords case (useful when you have a "Name" field in the target database). Only works with PDI > 5.2 though --- sqlserver2pgsql.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index a96cb60..9016202 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2537,6 +2537,7 @@ BEGIN SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN SQL_CONNECTset synchronous_commit to off; + PRESERVE_RESERVED_WORD_CASEY @@ -2813,6 +2814,7 @@ BEGIN SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN SQL_CONNECTset synchronous_commit to off; + PRESERVE_RESERVED_WORD_CASEY @@ -3009,6 +3011,7 @@ BEGIN SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN SQL_CONNECTset synchronous_commit to off; + PRESERVE_RESERVED_WORD_CASEY From aa75e4bd131072b18dfd0839208023e8b63f1395 Mon Sep 17 00:00:00 2001 From: marco44 Date: Thu, 11 Jun 2015 11:14:14 +0200 Subject: [PATCH 077/216] Correct issue 23 (not strict enough in checking kettle's setup) --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 9016202..4f1b8a7 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -487,7 +487,7 @@ sub check_kettle_properties open FILE, $file or kettle_die($file); while () { - next unless (/KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL\s*=\s*Y/); + next unless (/^KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y$/); $ok = 1; } close FILE; From b629aee74b81fe8aa7f82ee0616635dedbefa5c4 Mon Sep 17 00:00:00 2001 From: marco44 Date: Fri, 10 Jul 2015 13:04:36 +0200 Subject: [PATCH 078/216] Support for new dump provided by DavidPerezIngeniero (issue #24) --- sqlserver2pgsql.pl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 4f1b8a7..2d815f4 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1071,8 +1071,11 @@ sub parse_dump TABLE: while (my $line = read_and_clean($file)) { # Here is a col definition. + # We ignore ROWGUIDCOL as it has no meaning in PostgreSQL and cannot be emulated + # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) + # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ - /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))? (NOT NULL|NULL)(,)?/ + /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (NOT NULL|NULL)(,)?/ ) { #Deported into a function because we can also meet alter table add columns on their own @@ -1270,7 +1273,7 @@ sub parse_dump my $sql = $1 . ' ' . $schemaname . '.' . $3 . ' ' . $4 . "\n"; while (my $line_cont = read_and_clean($file)) { - if ($line_cont =~ /^\s*'\s*|^GO$/ + if ($line_cont =~ /^\s*'\s*$|^GO$/ ) # We may have a quote if the view is 'quoted', or a real sql query { # The view definition is complete. @@ -1629,6 +1632,10 @@ sub parse_dump { $constraint->{ON_UPD_CASC} = 1; } + elsif ($fk =~ /^NOT FOR REPLICATION$/) + { + next; # We don't care for this, it has no meaning for PostgreSQL + } else { die "Cannot parse $fk $., in a FK. This is a bug"; From df0948be2c325c3736e1dd2f03c7a8c6d920f455 Mon Sep 17 00:00:00 2001 From: marco44 Date: Wed, 5 Aug 2015 08:43:04 +0200 Subject: [PATCH 079/216] add support for xml type --- sqlserver2pgsql.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2d815f4..3619bd5 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -182,7 +182,8 @@ sub add_cast 'varbinary' => 'bytea', 'money' => 'numeric', 'smallmoney' => 'numeric', - 'uniqueidentifier' => 'uuid',); + 'uniqueidentifier' => 'uuid', + 'xml' => 'xml',); # Types with no qualifier, and no point in putting one my %unqual = ('bytea' => 1); From 9b8f71ce74f4bb3ab944c47ccbe252606ecf9379 Mon Sep 17 00:00:00 2001 From: marco44 Date: Fri, 14 Aug 2015 07:47:50 +0200 Subject: [PATCH 080/216] Support ARITHABORT (and warn the user). Closes #25 --- sqlserver2pgsql.pl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3619bd5..1d23a95 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1798,6 +1798,20 @@ sub parse_dump # Ignore CREATE DATABASE: we hope that we are given a single database as an option. It is multiline. # Ignore everything until next GO # Ignore ALTER DATABASE for the same reason. The given parameters have no meaning in PG anyway + # Except for SET ARITHABORT OFF, for which we print a warning because it probably means the database contents are weird (10/0 = null) + elsif ($line =~ + /^ALTER DATABASE.* SET ARITHABORT OFF/) + { + print STDERR "WARNING: the source database is set as ARITHABORT OFF.\n"; + print STDERR " It means that for SQL Server, 10/0 = NULL.\n"; + print STDERR " You'll probably have problems porting that to PostgreSQL.\n"; + while ($line !~ /^GO$/) + { + $line =read_and_clean($file); + } + # We read everything in the CREATE DATABASE. Back to work ! + next; + } # Same for tests about full text search. elsif ($line =~ /^(CREATE|ALTER) DATABASE|^IF \(1 = FULLTEXTSERVICEPROPERTY/) From d1ef8b76113b4964ca80a14a3d6de7d782975b70 Mon Sep 17 00:00:00 2001 From: marco44 Date: Sun, 23 Aug 2015 15:33:49 +0200 Subject: [PATCH 081/216] Add support for geography datatype (advising to use postgis) --- sqlserver2pgsql.pl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 1d23a95..e54f2cd 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -45,6 +45,7 @@ our $parallelism=8; our $sort_size=10000; our $use_pk_if_possible=0; +our $requires_postgis=0; # These three variables are loaded in the BEGIN block at the end of this file (they are very big my $template; @@ -245,6 +246,18 @@ sub convert_type # Special case. This is an internal type, and should seldom be used in production. Converted to varchar(128) $rettype='varchar(128)'; } + + # We special case also the geometry and geography data types + elsif ( $sqlstype =~ /^geography$|^geometry$/i ) + { + # These require that the destination database contains PostGIS + unless ($requires_postgis) + { + print STDERR "WARNING: $sqlstype detected. You will need PostGIS (http://postgis.net/).\nThe generated script will perform the CREATE EXTENSION, but please install PostGIS on this server\n"; + $requires_postgis=1; + } + $rettype=lc($sqlstype); + } else { print "Types: ", Dumper(\%types); @@ -295,6 +308,7 @@ sub convert_type $rettype=$rettype.'[]'; } } + # Add this type to casts to perform if necessary add_cast($rettype); @@ -1876,6 +1890,13 @@ sub generate_schema { print BEFORE "CREATE EXTENSION IF NOT EXISTS citext;\n"; } + + # Do we require PostGIS ? + if ($requires_postgis) + { + print BEFORE "CREATE EXTENSION IF NOT EXISTS postgis;\n"; + print BEFORE "CREATE EXTENSION IF NOT EXISTS postgis_topology;\n"; + } # Ok, we have parsed everything, and definitions are in $objects # We will put in the BEFORE file only table and columns definitions. From d8ca1fdbaabecfee2a8bd13cd0f623867f468427 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 28 Aug 2015 09:04:00 +0200 Subject: [PATCH 082/216] Added support for sql_variant. There would probably be work for data conversion, but I have no data to test on. --- sqlserver2pgsql.pl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index e54f2cd..0f2c3e3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -258,6 +258,12 @@ sub convert_type } $rettype=lc($sqlstype); } + elsif ($sqlstype eq 'sql_variant') + { + # There is no equivalent in PostgreSQL (and I think that's a good thing :) ) + print STDERR "WARNING: $sqlstype detected (in $schemaname.$tablename.$colname).\nThis is a 'not typed' field in SQL Server. There is no equivalent in PostgreSQL.\nThis is converted to text, but you'll have rework to do on your client code\n"; + $rettype='text'; + } else { print "Types: ", Dumper(\%types); From d2c3cff12f208b40158e32bb41ad1a71271b9207 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 28 Aug 2015 09:05:23 +0200 Subject: [PATCH 083/216] small fixes of warning messages --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 0f2c3e3..ce54506 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -253,7 +253,7 @@ sub convert_type # These require that the destination database contains PostGIS unless ($requires_postgis) { - print STDERR "WARNING: $sqlstype detected. You will need PostGIS (http://postgis.net/).\nThe generated script will perform the CREATE EXTENSION, but please install PostGIS on this server\n"; + print STDERR "WARNING: $sqlstype detected (in $schemaname.$tablename.$colname).\n You will need PostGIS (http://postgis.net/).\n The generated script will perform the CREATE EXTENSION, but please install PostGIS on this server\n"; $requires_postgis=1; } $rettype=lc($sqlstype); @@ -261,7 +261,7 @@ sub convert_type elsif ($sqlstype eq 'sql_variant') { # There is no equivalent in PostgreSQL (and I think that's a good thing :) ) - print STDERR "WARNING: $sqlstype detected (in $schemaname.$tablename.$colname).\nThis is a 'not typed' field in SQL Server. There is no equivalent in PostgreSQL.\nThis is converted to text, but you'll have rework to do on your client code\n"; + print STDERR "WARNING: $sqlstype detected (in $schemaname.$tablename.$colname).\n This is a 'not typed' field in SQL Server. There is no equivalent in PostgreSQL.\n This is converted to text, but you'll have rework to do on your client code\n"; $rettype='text'; } else From 675ecedd49e76bd508044ead9aa036034b664b0b Mon Sep 17 00:00:00 2001 From: marco44 Date: Wed, 16 Sep 2015 16:55:18 +0200 Subject: [PATCH 084/216] support default value as create table. as default values appear in several places now, refactor this into a function --- sqlserver2pgsql.pl | 127 ++++++++++++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 48 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index e54f2cd..df71fe8 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -399,6 +399,68 @@ sub convert_transactsql_code return $code; } +# This function does its best to convert MS's weird default values syntax into something logical +sub store_default_value +{ + my ($schema,$table,$col,$value,$line)=@_; + if ($value =~ /^\(?(\d+(\.\d+)?)\)?$/) # Value is numeric + { + $value = $1; # Get rid of parenthesis + if ($objects->{SCHEMAS}->{relabel_schemas($schema)}->{TABLES}->{$table}->{COLS}->{$col}->{TYPE} eq 'boolean') + { + # Ok, it IS a boolean, and we have received a number + if ($value eq '0') + { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'false'; + } + elsif ($value eq '1') + { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'true'; + } + else + { + # We should not get here: we have a numeric which isn't 0 or 1, and is supposed to be a boolean + die "Got an unexpected boolean : $value, for line $line\n"; + } + } + else + { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = $value; + } + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 0; + + } + elsif ($value =~ /^NULL$/) # A NULL value + { + # NULL WITHOUT quotes around it ! + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = 'NULL'; + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 0; + } + elsif ($value =~ /^N?'(.*)'$/) # There is sometimes an N before a string. + { + $value = $1; # Get rid of junk + # Default text value, text, between commas + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = "'$1'"; + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 0; + } + else + { + #This must be a function call... + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = convert_transactsql_code($value); + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 1; + + } + +} + # This gives the next column position for a table # It is used when we add a new column @@ -1002,7 +1064,7 @@ sub add_column_to_table ->{$colname}->{DEFAULT}->{VALUE} = "nextval('" . format_identifier(relabel_schemas(${schemaname})) . '.' - . ${seqname} . "')"; + . format_identifier(${seqname}) . "')"; $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{COLS} ->{$colname}->{DEFAULT}->{UNSURE} = 0; @@ -1090,17 +1152,22 @@ sub parse_dump # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ - /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (NOT NULL|NULL)(,)?/ + /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ ) { #Deported into a function because we can also meet alter table add columns on their own my $colname = $1; my $coltypeschema = $2; my $coltype = $3; - my $colqual = $4; - my $isidentity = $5; - my $colisnull = $6; + my $colqual =$4; + my $isidentity =$5; + my $colisnull =$6; + my $default =$7; add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); + if (defined $default) + { + store_default_value($schemaname,$tablename,$colname,$default,$line); + } } @@ -1458,7 +1525,7 @@ sub parse_dump add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); if (defined $default) { - $objects->{SCHEMAS}->{relabel_schemas($schemaname)}->{TABLES}->{$tablename}->{COLS}->{$colname}->{DEFAULT}->{VALUE} = $default; + store_default_value($schemaname,$tablename,$colname,$default,$line); } } @@ -1535,44 +1602,16 @@ sub parse_dump # Sometimes there is a second pair of parenthesis. I don't even want to know why... # Bit just need a little bit of work to be converted to 'true'/'false' elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?((?:-)?\d+(?:\.\d+)?)\)?\) FOR \[(.*)\]/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((\(?(?:-)?\d+(?:\.\d+)?\))?\) FOR \[(.*)\]/ ) { - if ($objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{TYPE} eq 'boolean') - { - # Ok, it IS a boolean, and we have received a number - if ($3 eq '0') - { - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'false'; - } - elsif ($3 eq '1') - { - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} = 'true'; - } - else - { - # We should not get here: we have a numeric which isn't 0 or 1, and is supposed to be a boolean - die "Got an unexpected boolean : $3, for line $line\n"; - } - } - else - { - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} - = $3; - } - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} - = 0; - + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \('(.*)'\) FOR \[(.*)\]/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(('.*')\) FOR \[(.*)\]/ ) { - # Default text value, text, between commas - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} - = "'$3'"; - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} - = 0; + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } # Yes, we also get default NULL (what for ? :) ), and sometimes with a different case @@ -1580,12 +1619,7 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(((?i)NULL)\) FOR \[(.*)\]/ ) { - # NULL WITHOUT quotes around it ! - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} - = 'NULL'; - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} - = 0; - + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } # And there are also constraints with functions and other strange code in them. Put them as unsure @@ -1593,10 +1627,7 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/ ) { - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{VALUE} - = convert_transactsql_code($3); - $objects->{SCHEMAS}->{relabel_schemas($1)}->{TABLES}->{$2}->{COLS}->{$4}->{DEFAULT}->{UNSURE} - = 1; + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } # FK constraint. It's multi line, we have to look for references, and what to do on update, delete, etc (I have only seen delete cascade for now) From 9c45c5e606ea43791fee4272231026a6b0718525 Mon Sep 17 00:00:00 2001 From: marco44 Date: Tue, 5 Apr 2016 17:11:40 +0200 Subject: [PATCH 085/216] Update FAQ.md --- FAQ.md | 1 + 1 file changed, 1 insertion(+) diff --git a/FAQ.md b/FAQ.md index a6aec1c..180c302 100644 --- a/FAQ.md +++ b/FAQ.md @@ -35,3 +35,4 @@ with citext, but if you can avoid it, do it. You'll have casting between citext bad plans, etc. * Some columns may take into account trailing spaces, some won't (it must be ansi_padding). Anyway, this doesn't exist either in PG. So more constraints will fail. +* PLpgSQL is very different from T-SQL, so you'll have trouble with stored procedures. It may be worth to give https://bitbucket.org/openscg/pgtsql a try … From dd13b2840357949266e6a609719d2c4c1d22cce0 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 7 Apr 2016 17:29:17 +0200 Subject: [PATCH 086/216] Multiple corrections to #31 --- sqlserver2pgsql.pl | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index ec0f179..061a6a2 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1303,7 +1303,7 @@ sub parse_dump ################################################################ # From HERE, these SQL commands are not linked to a create table ################################################################ - elsif ($line =~ /CREATE SCHEMA \[(.*)\] AUTHORIZATION \[.*\]/) + elsif ($line =~ /^CREATE SCHEMA \[(.*)\]/) { $objects->{SCHEMAS}->{relabel_schemas($1)} = undef ; # Nothing to add here, we create the schema, and put undef in it for now @@ -1512,6 +1512,16 @@ sub parse_dump } } } + elsif ($line =~ /^CREATE SPATIAL INDEX/) + { + my $def=$line; + while (my $idx = read_and_clean($file)) + { + $def.=$idx; + } + print STDERR "This spatial index won't be migrated:\n$def\n"; + + } # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table @@ -1744,7 +1754,7 @@ sub parse_dump or die "Cannot find a name for this extended property: $sqlproperty"; my $propertyname = $1; - if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1)$/) + if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1|Display Name|Description|Example Values|Source System|Table Description|Table Type|ETL Rules|Display Folder|SCD Type|Source Datatype)$/) { # We don't dump these. They are graphical descriptions of the GUI next; @@ -1807,7 +1817,6 @@ sub parse_dump # Never met one for now. Die and ask to send me an example die "Schema comment : <$comment> not understood. Please send a bug report\n"; } - } else { @@ -1840,6 +1849,11 @@ sub parse_dump next; } + elsif ($line =~ /^ALTER (ROLE|USER)/) + { + next; + } + # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views elsif ($line =~ /^IF NOT EXISTS/) { @@ -1863,6 +1877,29 @@ sub parse_dump # We read everything in the CREATE DATABASE. Back to work ! next; } + # Sometimes, when there is a ALTER DATABASE SET ARITHABORT OFF, there are SET ARITHABORT ON. Just ignore them + elsif ($line =~ /^SET ARITHABORT ON/) + { + next; + } + # Sometimes we meet this: SET CONCAT_NULL_YIELDS_NULL ON. That's the normal behaviour for a SQL database. Just ignore + elsif ($line =~ /^SET CONCAT_NULL_YIELDS_NULL ON/) + { + next; + } + # Same more or less + elsif ($line =~ /^SET ANSI_WARNINGS ON/) + { + next; + } + # What the hell does this do in a dump ??? + elsif ($line =~ /^SET NUMERIC_ROUNDABORT OFF/) + { + next; + } + + + # Same for tests about full text search. elsif ($line =~ /^(CREATE|ALTER) DATABASE|^IF \(1 = FULLTEXTSERVICEPROPERTY/) From 85a1f4288ca13cfc6ef952b4d3c637ab8d31a3dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=BE=D1=81=D0=BE=D0=BB=D0=BE=D0=B2=20=D0=9A=D0=BE?= =?UTF-8?q?=D0=BD=D1=81=D1=82=D0=B0=D0=BD=D1=82=D0=B8=D0=BD=20=28Mosolov?= =?UTF-8?q?=5FKN=29?= Date: Wed, 20 Apr 2016 10:57:07 +0400 Subject: [PATCH 087/216] Added "ON DELETE SET NULL" support for foreign keys. --- sqlserver2pgsql.pl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 061a6a2..795b273 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1690,6 +1690,10 @@ sub parse_dump { $constraint->{ON_DEL_CASC} = 1; } + elsif ($fk =~ /^ON DELETE SET NULL\s*$/) + { + $constraint->{ON_DEL_SET_NULL} = 1; + } elsif ($fk =~ /^ON UPDATE CASCADE\s*$/) { $constraint->{ON_UPD_CASC} = 1; @@ -2168,6 +2172,11 @@ sub generate_schema { $consdef .= " ON DELETE CASCADE"; } + if (defined $constraint->{ON_DEL_SET_NULL} + and $constraint->{ON_DEL_SET_NULL}) + { + $consdef .= " ON DELETE SET NULL"; + } if (defined $constraint->{ON_UPD_CASC} and $constraint->{ON_UPD_CASC}) { From 3672182099289aa83ab17269c85ad8374c7fd0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=BE=D1=81=D0=BE=D0=BB=D0=BE=D0=B2=20=D0=9A=D0=BE?= =?UTF-8?q?=D0=BD=D1=81=D1=82=D0=B0=D0=BD=D1=82=D0=B8=D0=BD=20=28Mosolov?= =?UTF-8?q?=5FKN=29?= Date: Wed, 4 May 2016 17:27:40 +0400 Subject: [PATCH 088/216] Added ignoring of FULLTEXT objects. --- sqlserver2pgsql.pl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 795b273..a87809c 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1902,8 +1902,6 @@ sub parse_dump next; } - - # Same for tests about full text search. elsif ($line =~ /^(CREATE|ALTER) DATABASE|^IF \(1 = FULLTEXTSERVICEPROPERTY/) @@ -1917,6 +1915,17 @@ sub parse_dump next; } + # Ignore CREATE and ALTER statements for full text search objects, such as CATALOG, INDEX or STOPLIST. + elsif ($line =~ /^(CREATE|ALTER) FULLTEXT/) + { + while ($line !~ /^GO$/) + { + $line =read_and_clean($file); + } + + next; + } + # Ignore EXEC dbo.sp_executesql, for now only seen for a create view. Views sql command aren't executed directly, don't know why elsif ($line =~ /^EXEC dbo.sp_executesql/) { From b9f353522b8c2e6114698ef9774787be714a7e20 Mon Sep 17 00:00:00 2001 From: marco44 Date: Thu, 12 May 2016 16:40:19 +0200 Subject: [PATCH 089/216] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b9c4f7c..663cbbd 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ It does three things: * produce an incremental version of this job to migrate what has changed in the database from the previous run. This is created when the migration job is also created. -Please drop me a word (on github) if you use this tool, feedback is great :) +Please drop me a word (on github) if you use this tool, feedback is great. I also like pull requests :) Notes, warnings: ========================== From df407d63505f265ca61b63cf3ac6251c1cefce88 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Fri, 8 Jul 2016 16:07:11 -0400 Subject: [PATCH 090/216] Added support for converting functions used as part of default constraints: - user_name to CURRENT_USER. - datepart to date_part. - Brakets to double quotes. Also made regex matching case insensitive. --- sqlserver2pgsql.pl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index a87809c..5fba82c 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -401,7 +401,10 @@ sub format_identifier_cols_index sub convert_transactsql_code { my ($code)=@_; - $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/g; + $code =~ s/[\[\]]/\"/gi; + $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; + $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; + $code =~ s/datepart\s*\(\s*(.*)\s*\,\s*(.*)\s*\)/date_part('$1', $2)/gi; return $code; } From a8338fb605ed15f240da13330333300c23e43809 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Mon, 11 Jul 2016 13:13:07 -0400 Subject: [PATCH 091/216] Making regex less greedy. --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 5fba82c..672aecf 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -404,7 +404,7 @@ sub convert_transactsql_code $code =~ s/[\[\]]/\"/gi; $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; - $code =~ s/datepart\s*\(\s*(.*)\s*\,\s*(.*)\s*\)/date_part('$1', $2)/gi; + $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; return $code; } From 97fd2eb8186fcb706440f5ca1423d86947eb8426 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Mon, 11 Jul 2016 15:44:03 -0400 Subject: [PATCH 092/216] Added special handling for columns of type [date] when generating kettle scripts. --- sqlserver2pgsql.pl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 672aecf..126504b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -140,7 +140,7 @@ sub convert_numeric_to_int } # This is a list of the types that require a cast to be imported in kettle -my %types_to_cast = ('uuid' => '1',); +my %types_to_cast = ('uuid' => '1','date' => '1'); # This sub adds a cast (if not defined already) if # - we generate for kettle @@ -150,7 +150,7 @@ sub add_cast my ($type)=@_; if (defined $types_to_cast{$type}) { - $objects->{CASTS}->{uuid}=1; + $objects->{CASTS}->{$type}=1; } } @@ -327,11 +327,15 @@ sub convert_type sub sql_convert_column { my ($colname,$coltype)=@_; - my %functions = ( 'uuid' => 'lower'); + my %functions = ( 'uuid' => 'lower' ); if (defined ($functions{$coltype})) { return $functions{$coltype} . '([' . $colname . '])'; } + elsif ($coltype eq 'date') + { + return 'convert(varchar, [' . $colname . '], 120)'; + } else { return "[$colname]"; From ba979a425a020d1e3bf0532f3fa297745542af7a Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Mon, 11 Jul 2016 16:03:44 -0400 Subject: [PATCH 093/216] Making sql_convert_column function a little more generic. --- sqlserver2pgsql.pl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 126504b..82ea4a0 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -327,14 +327,10 @@ sub convert_type sub sql_convert_column { my ($colname,$coltype)=@_; - my %functions = ( 'uuid' => 'lower' ); + my %functions = ( 'uuid' => 'lower({colname})', 'date' => 'convert(varchar, {colname}, 120)' ); if (defined ($functions{$coltype})) { - return $functions{$coltype} . '([' . $colname . '])'; - } - elsif ($coltype eq 'date') - { - return 'convert(varchar, [' . $colname . '], 120)'; + return $functions{$coltype} =~ s/\{colname\}/[$colname]/r; } else { From 67adae1a3ed250b3fc8fae8ee7cb37f6e952ef72 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Tue, 12 Jul 2016 15:02:32 -0400 Subject: [PATCH 094/216] Added special handling for PostgreSQL columns of type [date] when generating kettle incremental scripts. Added comments to sql_convert_column. Replaced some tabs with spaces. --- sqlserver2pgsql.pl | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 82ea4a0..f583f53 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -324,13 +324,14 @@ sub convert_type # This function is used for selects from SQL Server, in kettle. It adds a function call # if there is a conversion to be done. # uniqueidentifier is upper case in SQL Server, whereas uuid is lower case in PG +# date is converted to varchar in the YYYY-MM-DD format sub sql_convert_column { my ($colname,$coltype)=@_; my %functions = ( 'uuid' => 'lower({colname})', 'date' => 'convert(varchar, {colname}, 120)' ); if (defined ($functions{$coltype})) { - return $functions{$coltype} =~ s/\{colname\}/[$colname]/r; + return $functions{$coltype} =~ s/\{colname\}/[$colname]/r; } else { @@ -338,6 +339,24 @@ sub sql_convert_column } } +# This function is used for selects from PostgreSQL, in kettle. It adds a function call +# if there is a conversion to be done. +# uuid is converted to varchar and forced to lower case +# date is converted to varchar in the YYYY-MM-DD format +sub postgres_convert_column +{ + my ($colname,$coltype)=@_; + my %functions = ( 'uuid' => 'lower(cast({colname} as varchar))', 'date' => 'to_char({colname}, \'YYYY-MM-DD\')' ); + if (defined ($functions{$coltype})) + { + return $functions{$coltype} =~ s/\{colname\}/"$colname"/r; + } + else + { + return "\"$colname\""; + } +} + # This function is used to determine if a PK will be sorted the same in SQL Server and PG # It means that it doesn't depend on collation orders or other internals. # For now, only numeric and date data types are considered OK @@ -690,6 +709,7 @@ sub generate_kettle # Build the column list of the table to put into the SQL Server query my @colsdef; + my @pgcolsdef; foreach my $col ( sort { $refschema->{TABLES}->{$table}->{COLS}->{$a}->{POS} @@ -699,9 +719,12 @@ sub generate_kettle { my $coldef = sql_convert_column($col,$refschema->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}) . " AS " . format_identifier($col); + my $pgcoldef = postgres_convert_column($col,$refschema->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}) . " AS " . format_identifier($col); push @colsdef,($coldef); + push @pgcolsdef,($pgcoldef); } my $colsdef=join(',',@colsdef); + my $pgcolsdef=join(',',@pgcolsdef); my $pgtable=format_identifier($table); my $pgschema=format_identifier($targetschema); @@ -747,6 +770,7 @@ sub generate_kettle $newincrementaltemplate =~ s/__sqlserver_table_cols__/$colsdef/g; $newincrementaltemplate =~ s/__postgres_table_name__/$pgtable/g; $newincrementaltemplate =~ s/__postgres_schema_name__/$pgschema/g; + $newincrementaltemplate =~ s/__postgres_table_cols__/$pgcolsdef/g; $newincrementaltemplate =~ s/__PARALLELISM__/$parallelism/g; $newincrementaltemplate =~ s/__sort_size__/$sort_size/g; @@ -3535,7 +3559,7 @@ BEGIN __postgres_db__ - SELECT * FROM __postgres_schema_name__.__postgres_table_name__ + SELECT __postgres_table_cols__ FROM __postgres_schema_name__.__postgres_table_name__ 0 N @@ -3962,7 +3986,7 @@ BEGIN __postgres_db__ - SELECT * FROM __postgres_schema_name__.__postgres_table_name__ ORDER BY __pg_pk_condition__ + SELECT __postgres_table_cols__ FROM __postgres_schema_name__.__postgres_table_name__ ORDER BY __pg_pk_condition__ 0 N From 1b0e6627b6c202ddcee490371231dba0c02cdea9 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Wed, 13 Jul 2016 13:54:57 -0400 Subject: [PATCH 095/216] Ignoring xml schema collections as part of types and columns since they are not supported in pg. --- sqlserver2pgsql.pl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index f583f53..d6025b8 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1062,7 +1062,12 @@ sub add_column_to_table } if ($colqual) { - if ($colqual eq '(max)') + if ($coltype eq 'xml') + { + $colqual = undef + ; # ignoring sql server xml schema since its not supported in pg + } + elsif ($colqual eq '(max)') { $colqual = undef ; # max in sql server is the same as putting no colqual in pg @@ -1884,6 +1889,17 @@ sub parse_dump { next; } + + # Ignore xml schema collections since they are not supported in pg + elsif ($line =~ /^CREATE XML SCHEMA COLLECTION/) + { + next; + } + + elsif ($line =~ /^ALTER XML SCHEMA COLLECTION/) + { + next; + } # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views elsif ($line =~ /^IF NOT EXISTS/) From 58edecd12717f36a47cd7dcd259108b7c28372c9 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Wed, 13 Jul 2016 14:57:30 -0400 Subject: [PATCH 096/216] Fix for issue causing columns on Table types not to been parsed correctly if type and length not separated by exactly one space. --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index d6025b8..5d4ae3e 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1444,7 +1444,7 @@ sub parse_dump my $newtype; TYPE: while (my $typeline= read_and_clean($file)) { - if ($typeline =~ /^\t\[(.*)\] \[(.*)\](?: \((\d+(?:,\d+)?)\))?(?: (?:NOT )?NULL),?$/) + if ($typeline =~ /^\t\[(.*)\] \[(.*)\](?:\s*?\((\d+(?:,\d+)?)\))?(?:\s+?(?:NOT\s+?)?NULL),?$/) { # This is another column for this type $colname=$1; @@ -1480,7 +1480,7 @@ sub parse_dump } else { - die "Cannot understand $type. This is a bug"; + die "Cannot understand line: $typeline"; } } } From d541517ed829206e2dadcdab35d6b032f8fe2713 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Wed, 13 Jul 2016 15:02:03 -0400 Subject: [PATCH 097/216] Modifying error message to match style already in use on the script. --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 5d4ae3e..32cc311 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1480,7 +1480,7 @@ sub parse_dump } else { - die "Cannot understand line: $typeline"; + die "Cannot understand $typeline\n"; } } } From e92fb1b1ff104721584268107efa6c8b9288aa39 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Wed, 13 Jul 2016 15:20:46 -0400 Subject: [PATCH 098/216] Fix for an issue causing column definitions containing NOT FOR REPLICATION to fail. --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 32cc311..2ac7dc5 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1190,7 +1190,7 @@ sub parse_dump # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ - /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ + /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ ) { #Deported into a function because we can also meet alter table add columns on their own From 863b5c553b35d309c5534f6101e301cd596bfbe0 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Thu, 14 Jul 2016 14:45:03 -0400 Subject: [PATCH 099/216] Added fix to issue affecting datetimeoffset columns on kettle scripts. --- sqlserver2pgsql.pl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2ac7dc5..298129c 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -323,12 +323,16 @@ sub convert_type # This function is used for selects from SQL Server, in kettle. It adds a function call # if there is a conversion to be done. -# uniqueidentifier is upper case in SQL Server, whereas uuid is lower case in PG +# uuid is upper case in SQL Server, whereas uuid is lower case in PG # date is converted to varchar in the YYYY-MM-DD format +# timestamp with time zone is converted to varchar in the YYYY-MM-DD HH:MI:SS.MMM (24h) with time zone format sub sql_convert_column { my ($colname,$coltype)=@_; - my %functions = ( 'uuid' => 'lower({colname})', 'date' => 'convert(varchar, {colname}, 120)' ); + my %functions = ( + 'uuid' => 'lower({colname})', + 'date' => 'convert(varchar(50), {colname}, 120)', + 'timestamp with time zone(7)' => 'convert(varchar(50), {colname}, 121)'); if (defined ($functions{$coltype})) { return $functions{$coltype} =~ s/\{colname\}/[$colname]/r; @@ -343,10 +347,14 @@ sub sql_convert_column # if there is a conversion to be done. # uuid is converted to varchar and forced to lower case # date is converted to varchar in the YYYY-MM-DD format +# timestamp with time zone is converted to varchar in the YYYY-MM-DD HH:MI:SS.US+00 format (UTC) sub postgres_convert_column { my ($colname,$coltype)=@_; - my %functions = ( 'uuid' => 'lower(cast({colname} as varchar))', 'date' => 'to_char({colname}, \'YYYY-MM-DD\')' ); + my %functions = ( + 'uuid' => 'lower(cast({colname} as varchar))', + 'date' => 'to_char({colname}, \'YYYY-MM-DD\')', + 'timestamp with time zone(7)' => 'to_char({colname} AT TIME ZONE \'UTC\', \'YYYY-MM-DD HH:MI:SS.US+00\')'); if (defined ($functions{$coltype})) { return $functions{$coltype} =~ s/\{colname\}/"$colname"/r; From 4de33b1bc9cc079a9e7ca57768982ce905b4d189 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Fri, 15 Jul 2016 10:29:28 -0400 Subject: [PATCH 100/216] Corrected comment modified by mistake. --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 298129c..382048f 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -323,7 +323,7 @@ sub convert_type # This function is used for selects from SQL Server, in kettle. It adds a function call # if there is a conversion to be done. -# uuid is upper case in SQL Server, whereas uuid is lower case in PG +# uniqueidentifier is upper case in SQL Server, whereas uuid is lower case in PG # date is converted to varchar in the YYYY-MM-DD format # timestamp with time zone is converted to varchar in the YYYY-MM-DD HH:MI:SS.MMM (24h) with time zone format sub sql_convert_column From 6b31ad272d5351e648514d750897d447a53c906b Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Fri, 15 Jul 2016 14:17:53 -0400 Subject: [PATCH 101/216] Added option to update system catalog for specific casts when needed (see xml type) Added casts for "xml" and "timestamp with time zone" types. --- sqlserver2pgsql.pl | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 382048f..9feb598 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -140,7 +140,9 @@ sub convert_numeric_to_int } # This is a list of the types that require a cast to be imported in kettle -my %types_to_cast = ('uuid' => '1','date' => '1'); +# C = using CREATE CAST +# S = updating system catalog +my %types_to_cast = ('uuid' => 'C','date' => 'C','timestamp with time zone' => 'C','xml' => 'S'); # This sub adds a cast (if not defined already) if # - we generate for kettle @@ -150,7 +152,7 @@ sub add_cast my ($type)=@_; if (defined $types_to_cast{$type}) { - $objects->{CASTS}->{$type}=1; + $objects->{CASTS}->{$type}=$types_to_cast{$type}; } } @@ -184,10 +186,10 @@ sub add_cast 'money' => 'numeric', 'smallmoney' => 'numeric', 'uniqueidentifier' => 'uuid', - 'xml' => 'xml',); + 'xml' => 'xml',); # Types with no qualifier, and no point in putting one -my %unqual = ('bytea' => 1); +my %unqual = ('bytea' => 1, 'timestamp with time zone' => 1); # This function uses the two static lists above, plus domains and citext types that # may have been created during parsing, to convert mssql's types to pgsql's @@ -966,9 +968,17 @@ sub generate_kettle { foreach my $cast (keys %{$objects->{CASTS}}) { - $beforescript.= "DROP CAST IF EXISTS (varchar as $cast);\n"; - $beforescript.= "CREATE CAST (varchar as $cast) with inout as implicit;\n"; - $afterscript.= "DROP CAST (varchar as $cast);\n"; + if ($objects->{CASTS}->{$cast} eq "C") + { + $beforescript.= "DROP CAST IF EXISTS (varchar as $cast);\n"; + $beforescript.= "CREATE CAST (varchar as $cast) with inout as implicit;\n"; + $afterscript.= "DROP CAST (varchar as $cast);\n"; + } + elsif ($objects->{CASTS}->{$cast} eq "S") + { + $beforescript.= "update pg_cast set castcontext='i' where castsource='character varying'::regtype and casttarget='$cast'::regtype;\n"; + $afterscript.= "update pg_cast set castcontext='e' where castsource='character varying'::regtype and casttarget='$cast'::regtype;\n"; + } } } # Remove/restore triggers to be able to insert without FK checks From da4c7b5f4e7d43341f962b1a9a5666b5b7652d9c Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Fri, 15 Jul 2016 15:30:05 -0400 Subject: [PATCH 102/216] Added conversion expression for xml columns with empty values which will be converted to null since the empty values won't be accepted in PostgreSQL. --- sqlserver2pgsql.pl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 9feb598..0a60dd3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -328,16 +328,18 @@ sub convert_type # uniqueidentifier is upper case in SQL Server, whereas uuid is lower case in PG # date is converted to varchar in the YYYY-MM-DD format # timestamp with time zone is converted to varchar in the YYYY-MM-DD HH:MI:SS.MMM (24h) with time zone format +# xml columns with empty values will be converted to null since the empty values won't be accepted in PG (datalength of an empty xml column is 5) sub sql_convert_column { my ($colname,$coltype)=@_; my %functions = ( 'uuid' => 'lower({colname})', 'date' => 'convert(varchar(50), {colname}, 120)', - 'timestamp with time zone(7)' => 'convert(varchar(50), {colname}, 121)'); + 'timestamp with time zone(7)' => 'convert(varchar(50), {colname}, 121)', + 'xml' => 'case when datalength({colname}) > 5 then {colname} else null end'); if (defined ($functions{$coltype})) { - return $functions{$coltype} =~ s/\{colname\}/[$colname]/r; + return $functions{$coltype} =~ s/\{colname\}/[$colname]/gr; } else { @@ -976,8 +978,8 @@ sub generate_kettle } elsif ($objects->{CASTS}->{$cast} eq "S") { - $beforescript.= "update pg_cast set castcontext='i' where castsource='character varying'::regtype and casttarget='$cast'::regtype;\n"; - $afterscript.= "update pg_cast set castcontext='e' where castsource='character varying'::regtype and casttarget='$cast'::regtype;\n"; + $beforescript.= "UPDATE pg_cast SET castcontext='i' WHERE castsource='character varying'::regtype AND casttarget='$cast'::regtype;\n"; + $afterscript.= "UPDATE pg_cast SET castcontext='e' WHERE castsource='character varying'::regtype AND casttarget='$cast'::regtype;\n"; } } } From 9bea5e7bf9433581b9433df170f86eff8670e152 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Fri, 15 Jul 2016 15:42:14 -0400 Subject: [PATCH 103/216] Removed qualifier from postgres select function for "timestamp with time zone" type used on incremental kettle script. --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 0a60dd3..1b4afb6 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -358,7 +358,7 @@ sub postgres_convert_column my %functions = ( 'uuid' => 'lower(cast({colname} as varchar))', 'date' => 'to_char({colname}, \'YYYY-MM-DD\')', - 'timestamp with time zone(7)' => 'to_char({colname} AT TIME ZONE \'UTC\', \'YYYY-MM-DD HH:MI:SS.US+00\')'); + 'timestamp with time zone' => 'to_char({colname} AT TIME ZONE \'UTC\', \'YYYY-MM-DD HH:MI:SS.US+00\')'); if (defined ($functions{$coltype})) { return $functions{$coltype} =~ s/\{colname\}/"$colname"/r; From 0fd7e28cccdc53f3562f1863c7107cd3a3f7d7a6 Mon Sep 17 00:00:00 2001 From: Javier Callico Date: Thu, 21 Jul 2016 13:45:27 -0400 Subject: [PATCH 104/216] Changed target type for smallmoney from numeric to numeric(6,4) in order to fix an issue on the kettle incremental script. --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 1b4afb6..d91891d 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -184,7 +184,7 @@ sub add_cast 'binary' => 'bytea', 'varbinary' => 'bytea', 'money' => 'numeric', - 'smallmoney' => 'numeric', + 'smallmoney' => 'numeric(6,4)', 'uniqueidentifier' => 'uuid', 'xml' => 'xml',); From f4e4c6f9ec047dc7c88a75163d2510434c92c329 Mon Sep 17 00:00:00 2001 From: Joshua Rountree Date: Tue, 30 Aug 2016 09:21:44 -0400 Subject: [PATCH 105/216] Update README.md Fixed a typo with word optionnal. And just simplified the explanation to just (optional) instead of writing it out. --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 663cbbd..c26f164 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,12 @@ This is a migration tool to convert a Microsoft SQL Server Database into a Postg It is written in Perl and has received a fair amount of testing. - It does three things: * convert a SQL Server schema to a PostgreSQL schema * produce a Pentaho Data Integrator (Kettle) job to migrate - all the data from SQL Server to PostgreSQL. This second part is optionnal - * produce an incremental version of this job to migrate what has changed in the database from the previous run. This is created - when the migration job is also created. + all the data from SQL Server to PostgreSQL (optional) + * produce an incremental version of this job to migrate what has changed in the database from the previous run. This is created when the migration job is also created. Please drop me a word (on github) if you use this tool, feedback is great. I also like pull requests :) From 14ebd73f43f7af8767e4661648995804e0edd011 Mon Sep 17 00:00:00 2001 From: marco44 Date: Wed, 7 Sep 2016 12:03:16 +0200 Subject: [PATCH 106/216] Update README.md --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c26f164..62034ef 100644 --- a/README.md +++ b/README.md @@ -63,12 +63,15 @@ run sqlserver2pgsql.pl from. If you just want to convert this schema, run: - > sqlserver2pgsql.pl -f my_sqlserver_script.txt -b name_of_before_script -a name_of_after_script -u name_of_unsure_script +`sqlserver2pgsql.pl -f input_sql_dump -b output_before_script -a output_after_script -u output_unsure_script` -The before script contains what is needed to import data (types, tables and columns). -The after script contains the rest (indexes, constraints). It should be run -after data is imported. The unsure script contains objects where we attempt to migrate, but cannot guarantee, -such as views. +The sqlserver2pgsql Perl script processes your SQL raw dump "input_sql_dump" and producse these three scripts: + +- output_before_script: contains what is needed to import data (types, tables and columns) + +- output_after_script: contains the rest (indexes, constraints) + +- output_unsure_script: contains objects where we attempt to migrate, but cannot guarantee, such as views -conf uses a conf file. All options below can also be set there. Command line options will overwrite conf options. There is an example of such a conf file (example_conf_file) From 59566e27c006ac0fe8eea908e1d07b42087e66dc Mon Sep 17 00:00:00 2001 From: marco44 Date: Fri, 23 Sep 2016 15:38:35 +0200 Subject: [PATCH 107/216] Update INSTALL.md advice using strawberry perl, for windows users --- INSTALL.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index eb79b36..89b17ef 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -6,6 +6,8 @@ has no dependancy on fancy Perl modules, it only uses modules from the base perl Just run the script with your perl interpreter, providing it with the requested options (--help will tell you what to do). +If you are trying to run the script under Windows, you probably won't have a Perl interpreter. I recommand you use the latest Strawberry Perl (use either the installer or the portable version). + If you want to migrate the data, you'll need "Kettle", an Open Source ETL. Get the latest version from here: http://kettle.pentaho.com/ . You'll also need a SQL Server account with the permission to SELECT from the tables you want to migrate. From f6166e757afffb639ad8cdc5f2eb0e1ac9e12071 Mon Sep 17 00:00:00 2001 From: marco44 Date: Thu, 29 Sep 2016 09:54:21 +0200 Subject: [PATCH 108/216] Update FAQ.md --- FAQ.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/FAQ.md b/FAQ.md index 180c302..6b909aa 100644 --- a/FAQ.md +++ b/FAQ.md @@ -36,3 +36,7 @@ bad plans, etc. * Some columns may take into account trailing spaces, some won't (it must be ansi_padding). Anyway, this doesn't exist either in PG. So more constraints will fail. * PLpgSQL is very different from T-SQL, so you'll have trouble with stored procedures. It may be worth to give https://bitbucket.org/openscg/pgtsql a try … + +Can this tool migrate functions and stored procedures? +---------------------------------- +No, Transact-SQL is very different from PostgreSQL's many PL languages. You could give a look at https://bitbucket.org/openscg/pgtsql but I can't help you with this… and I don't think it is a long term viable option. From 64b99b3cbc431b9dfd8481ddbd2d94e67a1f3ee1 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 30 Sep 2016 16:54:10 +0200 Subject: [PATCH 109/216] Support reWriteBatchedInserts in PostgreSQL's jdbc driver This is not supported by the JDBC driver provided with current (6.1 PDI). When it will be, though, it can almost double the per session throughput of inserts in PostgreSQL. This probably won't have any impact on most cases, but why not take advantage of this. --- sqlserver2pgsql.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index d91891d..3659e02 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2727,6 +2727,7 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN From da5c820952c979a6d64560b6043a06b697b0931b Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 21 Nov 2016 16:05:25 +0100 Subject: [PATCH 110/216] Add support for changing the parallelism level --- README.md | 1 + example_conf_file | 1 + sqlserver2pgsql.pl | 6 +++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 62034ef..c24c504 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ cleartext, so don't make this directory public): -pu : postgresql username -pw : postgresql password -f : the SQL Server structure dump file +-p : The parallelism used in kettle jobs: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 You've generated everything. Let's do the import: diff --git a/example_conf_file b/example_conf_file index ab4a1c9..15e5a5b 100644 --- a/example_conf_file +++ b/example_conf_file @@ -19,6 +19,7 @@ postgresql host=bar_host postgresql port=5432 postgresql username=bar_user postgresql password=bar_password +parallelism=8 # Default value is 8. Number of parallel connections used by kettle to insert data into the PostgreSQL database # Optional behaviour case insensitive=0 # set it to 1 to generate a dump with citext and check constraints all over the place diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3659e02..19273e3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -42,7 +42,7 @@ our $unsure_file; our $keep_identifier_case; our $validate_constraints='yes'; -our $parallelism=8; +our $parallelism; our $sort_size=10000; our $use_pk_if_possible=0; our $requires_postgis=0; @@ -79,6 +79,7 @@ sub parse_conf_file 'postgresql username' => 'pu', 'postgresql password' => 'pw', 'kettle directory' => 'kettle', + 'parallelism' => 'parallelism', 'before file' => 'before_file', 'after file' => 'after_file', 'unsure file' => 'unsure_file', @@ -122,6 +123,7 @@ sub parse_conf_file $norelabel_dbo=0 unless (defined ($norelabel_dbo)); $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); $keep_identifier_case=0 unless (defined ($keep_identifier_case)); + $parallelism=8 unless (defined ($parallelism)); close CONF; } @@ -656,6 +658,7 @@ sub usage print "-pp: postgresql port\n"; print "-pu: postgresql username\n"; print "-pw: postgresql password\n"; + print "-p: parallelism level for the kettle job\n"; } # This function generates kettle transformations, and a kettle job running all these @@ -2515,6 +2518,7 @@ sub resolve_name_conflicts my $help = 0; my $options = GetOptions("k=s" => \$kettle, + "p=i" => \$parallelism, "b=s" => \$before_file, "a=s" => \$after_file, "u=s" => \$unsure_file, From 8accbd68f5ae24b8dbf3135c88935cc3d7e24362 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 21 Nov 2016 17:02:39 +0100 Subject: [PATCH 111/216] Set some default values correctly. And document keep_identifier_case --- README.md | 2 ++ example_conf_file | 2 ++ sqlserver2pgsql.pl | 24 ++++++++++++++++-------- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c24c504..2446a79 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ there is a default dbo=>public remapping, that can be cancelled with -nr. Use do -num : Converts numeric (xxx,0) to the appropriate smallint, integer or bigint. It won't keep the constraint on the size of the scale of the numeric +-keep_identifier_case: don't convert the dump to all lower case. This is not recommended, as you'll have to put every identifier (column, table…) in double quotes… + -validate_constraints=yes/after/no: for foreign keys, if yes: foreign keys are created as valid in the after script (default) if no: they are created as not valid (enforced only for new rows) if after: they are created as not valid, but the statements to validate them are put in the unsure file diff --git a/example_conf_file b/example_conf_file index 15e5a5b..0d06b69 100644 --- a/example_conf_file +++ b/example_conf_file @@ -27,6 +27,8 @@ no relabel dbo=1 # set it to 0 to convert the dbo schema to public convert numeric to int=1 # set it to 0 to keep numeric(xx,0) as numeric(xx,0). Will be converted to smallint, int or bigint by default relabel schemas=dbo=>foo;schema1=>bar keep identifier case=1 # keep case of database objects +validate constraints = yes # yes, after or no, should the constraints be validated by the dump ? (yes=validate during load, after after the load, no keep invalidated) + # Incremental job sort size=10000 # drives the amount of memory and temporary files that will be created by an incremental job use pk if possible=0 # 1/list of tables, for tables where you want to try getting already sorted records diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 19273e3..883e7a2 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -30,10 +30,10 @@ # These are global variables, from configuration file or command line arguments our ($sd, $sh, $si, $sp, $su, $sw, $pd, $ph, $pp, $pu, $pw); # Connection args our $conf_file; -our $filename; # Filename passed as arg -our $case_insensitive; # Passed as arg: was SQL Server installation case insensitive ? PostgreSQL can't ignore accents anyway - # If yes, we will generate citext with CHECK constraints, that's the best we can do -our $norelabel_dbo; # Passed as arg: should we convert DBO to public ? +our $filename; # Filename passed as arg +our $case_insensitive; # Passed as arg: was SQL Server installation case insensitive ? PostgreSQL can't ignore accents anyway + # If yes, we will generate citext with CHECK constraints, that's the best we can do +our $norelabel_dbo; # Passed as arg: should we convert DBO to public ? our $relabel_schemas; our $convert_numeric_to_int; # Should we convert numerics to int when possible ? (numeric (4,0) could be converted an int, for instance) our $kettle; @@ -41,10 +41,12 @@ our $after_file; our $unsure_file; our $keep_identifier_case; -our $validate_constraints='yes'; +our $validate_constraints; our $parallelism; -our $sort_size=10000; -our $use_pk_if_possible=0; +our $sort_size; +our $use_pk_if_possible; + +# Will be set if we detect GIS objects our $requires_postgis=0; # These three variables are loaded in the BEGIN block at the end of this file (they are very big @@ -124,6 +126,12 @@ sub parse_conf_file $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); $keep_identifier_case=0 unless (defined ($keep_identifier_case)); $parallelism=8 unless (defined ($parallelism)); + $sort_size=10000 unless (defined ($sort_size)); + $use_pk_if_possible unless (defined ($use_pk_if_possible)); + $validate_constraints='yes' unless (defined ($validate_constraints)); + # Default ports for PostgreSQL and SQL Server + $pp=5432 unless (defined ($pp)); + $sp=1433 unless (defined ($sp)); close CONF; } @@ -637,7 +645,7 @@ sub usage "-keep_identifier_case tells $0 to keep the case of sql server database objects (not advised). Default is to lowercase everything.\n"; print "before_file contains the structure\n"; print "after_file contains index, constraints\n"; - print "validate_constraint validates the constraints that have been created\n"; + print "validate_constraints validates the constraints that have been created\n"; print "sort_size will change size of sort batch for the incremental job. Too small and it will be slow, too big and you will get Java Out of Heap Memory errors.\n"; print "sort_size is 10000, which is very low, to try to avoid problems. First, raise java heap memory (in the kitchen script), then try higher values if you need more speed\n"; print "use_pk_if_possible is false (0) by default. You can put it to 1 (true), or give a comma separated list of tables (with schema). Compared case insensitively\n"; From d869e15bf47dbda049117bb727979457d2e68bba Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 21 Nov 2016 17:22:10 +0100 Subject: [PATCH 112/216] correction for default values --- sqlserver2pgsql.pl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 883e7a2..1794c87 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -120,19 +120,24 @@ sub parse_conf_file $$param_name = $value; use strict 'refs'; } - # Hard coded default values + + close CONF; +} + +sub set_default_conf_values +{ + # Hard coded default values, set only if not passed or found in configuration $case_insensitive=0 unless (defined ($case_insensitive)); $norelabel_dbo=0 unless (defined ($norelabel_dbo)); $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); $keep_identifier_case=0 unless (defined ($keep_identifier_case)); $parallelism=8 unless (defined ($parallelism)); $sort_size=10000 unless (defined ($sort_size)); - $use_pk_if_possible unless (defined ($use_pk_if_possible)); + $use_pk_if_possible=0 unless (defined ($use_pk_if_possible)); $validate_constraints='yes' unless (defined ($validate_constraints)); # Default ports for PostgreSQL and SQL Server $pp=5432 unless (defined ($pp)); $sp=1433 unless (defined ($sp)); - close CONF; } # Converts numeric(4,0) and similar to int, bigint, smallint @@ -2567,6 +2572,9 @@ sub resolve_name_conflicts parse_conf_file(); } +# Set default values for anything not set yet +set_default_conf_values(); + # We have no before, after, or unsure if ( not $before_file or not $after_file From 77e89f035c67db68d5f8215a8d8aef69691af892 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 25 Nov 2016 20:18:42 +0100 Subject: [PATCH 113/216] Improvements to the doc --- README.md | 94 +++++++++++++++++++++++++++------------------- sqlserver2pgsql.pl | 2 +- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 2446a79..960d36d 100644 --- a/README.md +++ b/README.md @@ -63,9 +63,14 @@ run sqlserver2pgsql.pl from. If you just want to convert this schema, run: -`sqlserver2pgsql.pl -f input_sql_dump -b output_before_script -a output_after_script -u output_unsure_script` +``` +sqlserver2pgsql.pl -f input_sql_dump \ + -b output_before_script\ + -a output_after_script\ + -u output_unsure_script +``` -The sqlserver2pgsql Perl script processes your SQL raw dump "input_sql_dump" and producse these three scripts: +The sqlserver2pgsql Perl script processes your SQL raw dump "input_sql_dump" and produces these three scripts: - output_before_script: contains what is needed to import data (types, tables and columns) @@ -79,16 +84,16 @@ There is an example of such a conf file (example_conf_file) You can also use the -i, -num and/or -nr options: -i : Generate an "ignore case" schema, using citext, to emulate MSSQL's case insensitive collation. - It will create citext fields, with check constraints. + It will create citext fields, with check constraints. This type is slower on string comparison operations. -nr : Don't convert the dbo schema to public. By default, this conversion is done, as it converts MSSQL's default -schema to PostgreSQL's default schema +schema (dbo) to PostgreSQL's default schema (public) -relabel_schemas is a list of schemas to remap. The syntax is : 'source1=>dest1;source2=>dest2'. Don't forget to quote this option or the shell might alter it there is a default dbo=>public remapping, that can be cancelled with -nr. Use double quotes instead of simple quotes on Windows. -num : Converts numeric (xxx,0) to the appropriate smallint, integer or bigint. It won't keep the constraint on -the size of the scale of the numeric +the size of the scale of the numeric. But smallint, integer and bigint types are faster than numeric. -keep_identifier_case: don't convert the dump to all lower case. This is not recommended, as you'll have to put every identifier (column, table…) in double quotes… @@ -96,39 +101,14 @@ the size of the scale of the numeric if no: they are created as not valid (enforced only for new rows) if after: they are created as not valid, but the statements to validate them are put in the unsure file --sort_size=100000: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). - -We don't sort in databases for two reasons: the sort order can be different between SQL Server -and PostgreSQL, and we don't want to stress the servers. But sorting a lot of data in Java can generate a Java Out of Heap Memory error. -If you get Out of Memory errors, raise the Java Heap memory (in the kitchen script) as much as you can. If you still have the problem, reduce -this sort size. You can also try reducing parallelism, having one on two sorts instead of 8 will of course consume less memory. The last problem is that -if the sort_size is small, kettle is going to generate a very large amount of temporary files, and then read them back sorted. So you may hit the -"too many open files" limit of your system (default 1024 on linux for instance). So you'll have to do some tuning here: - - First, use as much Java memory as you can: set the JAVAXMEM environment variable to 4096 (megabytes) or more if you can afford it. The more the better. - - If you still get Out Of Memory errors, put a smaller sort size, until you can do the sorts (decrease it tenfold each time for example). You'll obviously lose some performance - - If then you get the too many open files error, raise the maximum number of open files. In most Linux distributions, this is editing /etc/security/limits.conf and putting -@userName soft nofile 65535 -@userName hard nofile 65535 -(replace userName with your user name). Log in again, and verify with "ulimit -n" that you are now allowed to open 65535 files. -You may also have to raise the maximum number of open files on the system: echo the new value to /proc/sys/fs/file-max. - -You'll need a lot of temporary space on disk to do these sorts... - -You can also edit only the offending transformation with Spoon (Kettle's GUI), so that only this one is slowed down. - -When Kettle crashed on one of these problems, the temporary files aren't removed. They are usually in /tmp (or in your temp directory in Windows), and start with out_. Don't forget to remove them. - --use_pk_if_possible=0/1/public.table1,myschema.table2: enable the generation of jobs doing sorts in the databases (order by in the select part of Kettle's table inputs). - -1 will ask to try for all tables, or you can give a list of tables (if for example, you cannot make these tables work with a reasonable sort size). Anyway, sqlserver2sql will only accept -to do sorts in the database if the primary key can be guaranteed to be sorted the same way in PostgreSQL and SQL Server. That means that it only accepts if the key is made only of numeric -and date/timestamp types. If not, the standard incremental job will be generated. If you want to also import data: - > ./sqlserver2pgsql.pl -b before.sql -a after.sql -u unsure.sql -k kettledir \ +``` +./sqlserver2pgsql.pl -b before.sql -a after.sql -u unsure.sql -k kettledir \ -sd source -sh 192.168.0.2 -sp 1433 -su dalibo -sw mysqlpass \ -pd dest -ph localhost -pp 5432 -pu dalibo -pw mypgpass -f sql_server_schema.sql +``` -k is the directory where you want to store the kettle xml files (there will be one for each table to copy, plus the one for the job) @@ -147,9 +127,43 @@ cleartext, so don't make this directory public): -pu : postgresql username -pw : postgresql password -f : the SQL Server structure dump file + + -p : The parallelism used in kettle jobs: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 +-sort_size=100000: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). -You've generated everything. Let's do the import: +We don't sort in databases for two reasons: the sort order (collation for strings for example) can be different between SQL Server +and PostgreSQL, and we don't want to stress the servers more than needed anyway. But sorting a lot of data in Java can generate a Java Out of Heap Memory error. +If you get Out of Memory errors, raise the Java Heap memory (in the kitchen script) as much as you can. If you still have the problem, reduce +this sort size. You can also try reducing parallelism, having one or two sorts instead of 8 will of course consume less memory. The last problem is that +if the sort_size is small, kettle is going to generate a very large amount of temporary files, and then read them back sorted. So you may hit the +"too many open files" limit of your system (default 1024 on linux for instance). So you'll have to do some tuning here: + + - First, use as much Java memory as you can: set the JAVAXMEM environment variable to 4096 (megabytes) or more if you can afford it. The more the better. + - If you still get Out Of Memory errors, put a smaller sort size, until you can do the sorts (decrease it tenfold each time for example). You'll obviously lose some performance + - If then you get the too many open files error, raise the maximum number of open files. In most Linux distributions, this is editing /etc/security/limits.conf and putting +``` +@userName soft nofile 65535 +@userName hard nofile 65535 +``` + +(replace userName with your user name). Log in again, and verify with "ulimit -n" that you are now allowed to open 65535 files. +You may also have to raise the maximum number of open files on the system: echo the new value to /proc/sys/fs/file-max. + +You'll need a lot of temporary space on disk to do these sorts... + +You can also edit only the offending transformation with Spoon (Kettle's GUI), so that only this one is slowed down. + +When Kettle crashed on one of these problems, the temporary files aren't removed. They are usually in /tmp (or in your temp directory in Windows), and start with out_. Don't forget to remove them. + +-use_pk_if_possible=0/1/public.table1,myschema.table2: enable the generation of jobs doing sorts in the databases (order by in the select part of Kettle's table inputs). + +1 will ask to try for all tables, or you can give a list of tables (if for example, you cannot make these tables work with a reasonable sort size). Anyway, sqlserver2pgsql will only accept +to do sorts in the database if the primary key can be guaranteed to be sorted the same way in PostgreSQL and SQL Server. That means that it only accepts if the key is made only of numeric +and date/timestamp types. If not, the standard, kettle-sorting incremental job will be generated. + + +Now you've generated everything. Let's do the import: ``` # Run the before script (creates the tables) @@ -163,9 +177,11 @@ You've generated everything. Let's do the import: If you want to dig deeper into the kettle job, you can use kettle_report.pl to display the individual table's transfer performance. Then, if needed, you'll be able to modify the Kettle job to optimize it, using Spoon, Kettle's GUI -``` + You can also give a try to the incremental job: - > ./kitchen.sh -file=full_path_to_kettle_job_dir/incremental.kjb -level=detailed +``` +./kitchen.sh -file=full_path_to_kettle_job_dir/incremental.kjb -level=detailed +``` This one is highly experimental. I need your feedback ! :). You should only run an incremental job on an already loaded database. @@ -173,10 +189,12 @@ It may fail for a variety of reasons, mainly out of memory errors. If you have o By the way, to be able to insert data into all tables, it deactivates triggers at the beginning and activates them back at the end of the job. So, if the job fails, those triggers won't be reactivated. -``` + You can also use a configuration file if you like: - > ./sqlserver2pgsql.pl -conf example_conf_file -f mydatabase_dump.sql +``` +./sqlserver2pgsql.pl -conf example_conf_file -f mydatabase_dump.sql +``` There is an example configuration file provided. You can also mix the configuration file with command line options. Command line options have the priority over values set in the configuration file. diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 1794c87..05b46a7 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2556,7 +2556,7 @@ sub resolve_name_conflicts "keep_identifier_case" =>\$keep_identifier_case, "validate_constraints=s" =>\$validate_constraints, "sort_size=i" =>\$sort_size, - "use_pk_if_possible=s" =>\$use_pk_if_possible,); + "use_pk_if_possible=s" =>\$use_pk_if_possible,); # We don't understand command line or have been asked for usage if (not $options or $help) From cc229cd8dbd5d22ef2685da0eca98c17a0c09aa7 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 25 Nov 2016 20:31:39 +0100 Subject: [PATCH 114/216] Still more doc corrections --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 960d36d..a0cc569 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ run sqlserver2pgsql.pl from. If you just want to convert this schema, run: ``` -sqlserver2pgsql.pl -f input_sql_dump \ +./sqlserver2pgsql.pl -f input_sql_dump \ -b output_before_script\ -a output_after_script\ -u output_unsure_script @@ -167,12 +167,12 @@ Now you've generated everything. Let's do the import: ``` # Run the before script (creates the tables) - > psql -U mypguser mypgdatabase -f name_of_before_script + psql -U mypguser mypgdatabase -f name_of_before_script # Run the kettle job: - > cd my_kettle_installation_directory - > ./kitchen.sh -file=full_path_to_kettle_job_dir/migration.kjb -level=detailed + cd my_kettle_installation_directory + ./kitchen.sh -file=full_path_to_kettle_job_dir/migration.kjb -level=detailed # Run the after script (creates the indexes, constraints...) - > psql -U mypguser mypgdatabase -f name_of_after_script + psql -U mypguser mypgdatabase -f name_of_after_script ``` If you want to dig deeper into the kettle job, you can use kettle_report.pl to display the individual table's transfer performance. Then, if needed, you'll be able to modify the Kettle job to optimize it, using Spoon, Kettle's GUI From 30eae1cece2e74ab7fd02ba2047e5ffcf8a14c64 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 25 Nov 2016 20:40:19 +0100 Subject: [PATCH 115/216] README editing --- README.md | 73 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index a0cc569..e4c0c37 100644 --- a/README.md +++ b/README.md @@ -83,21 +83,22 @@ There is an example of such a conf file (example_conf_file) You can also use the -i, -num and/or -nr options: --i : Generate an "ignore case" schema, using citext, to emulate MSSQL's case insensitive collation. +`-i` : Generate an "ignore case" schema, using citext, to emulate MSSQL's case insensitive collation. It will create citext fields, with check constraints. This type is slower on string comparison operations. --nr : Don't convert the dbo schema to public. By default, this conversion is done, as it converts MSSQL's default +`-nr` : Don't convert the dbo schema to public. By default, this conversion is done, as it converts MSSQL's default schema (dbo) to PostgreSQL's default schema (public) --relabel_schemas is a list of schemas to remap. The syntax is : 'source1=>dest1;source2=>dest2'. Don't forget to quote this option or the shell might alter it -there is a default dbo=>public remapping, that can be cancelled with -nr. Use double quotes instead of simple quotes on Windows. +`-relabel_schemas` is a list of schemas to remap. The syntax is : `source1=>dest1;source2=>dest2`. Don't forget to quote this option or the shell might alter it +there is a default `dbo=>public` remapping, that can be cancelled with `-nr`. Use double quotes instead of simple quotes on Windows. --num : Converts numeric (xxx,0) to the appropriate smallint, integer or bigint. It won't keep the constraint on -the size of the scale of the numeric. But smallint, integer and bigint types are faster than numeric. +`-num` : Converts numeric (xxx,0) to the appropriate smallint, integer or bigint. It won't keep the constraint on +the size of the scale of the numeric. smallint, integer and bigint types are much faster than numeric, ano often used only as surrogate keys, +so the scale is often not important. --keep_identifier_case: don't convert the dump to all lower case. This is not recommended, as you'll have to put every identifier (column, table…) in double quotes… +`-keep_identifier_case`: don't convert the dump to all lower case. This is not recommended, as you'll have to put every identifier (column, table…) in double quotes… --validate_constraints=yes/after/no: for foreign keys, if yes: foreign keys are created as valid in the after script (default) +`-validate_constraints=yes/after/no`: for foreign keys, if yes: foreign keys are created as valid in the after script (default) if no: they are created as not valid (enforced only for new rows) if after: they are created as not valid, but the statements to validate them are put in the unsure file @@ -110,38 +111,41 @@ If you want to also import data: -pd dest -ph localhost -pp 5432 -pu dalibo -pw mypgpass -f sql_server_schema.sql ``` --k is the directory where you want to store the kettle xml files (there will be +`-k` is the directory where you want to store the kettle xml files (there will be one for each table to copy, plus the one for the job) You'll also need to specify the connection parameters. They will be stored inside the kettle files (in cleartext, so don't make this directory public): --sd : sql server database --sh : sql server host --si : sql server host instance --sp : sql server port (usually 1433) --su : sql server username --sw : sql server password --pd : postgresql database --ph : postgresql host --pp : postgresql port --pu : postgresql username --pw : postgresql password --f : the SQL Server structure dump file - - --p : The parallelism used in kettle jobs: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 --sort_size=100000: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). +`-sd` : sql server database +`-sh` : sql server host +`-si` : sql server host instance +`-sp` : sql server port (usually 1433) +`-su` : sql server username +`-sw` : sql server password +`-pd` : postgresql database +`-ph` : postgresql host +`-pp` : postgresql port +`-pu` : postgresql username +`-pw` : postgresql password +`-f` : the SQL Server structure dump file + + +`-p` : The parallelism used in kettle jobs: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 +`-sort_size=100000`: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). We don't sort in databases for two reasons: the sort order (collation for strings for example) can be different between SQL Server and PostgreSQL, and we don't want to stress the servers more than needed anyway. But sorting a lot of data in Java can generate a Java Out of Heap Memory error. + If you get Out of Memory errors, raise the Java Heap memory (in the kitchen script) as much as you can. If you still have the problem, reduce -this sort size. You can also try reducing parallelism, having one or two sorts instead of 8 will of course consume less memory. The last problem is that -if the sort_size is small, kettle is going to generate a very large amount of temporary files, and then read them back sorted. So you may hit the -"too many open files" limit of your system (default 1024 on linux for instance). So you'll have to do some tuning here: +this sort size. You can also try reducing parallelism, having one or two sorts instead of 8 will of course consume less memory. + +The last problem is that if the sort_size is small, kettle is going to generate a very large amount of temporary files, and then +read them back sorted. So you may hit the "too many open files" limit of your system (default 1024 on linux for instance). +So you'll have to do some tuning here: - - First, use as much Java memory as you can: set the JAVAXMEM environment variable to 4096 (megabytes) or more if you can afford it. The more the better. - - If you still get Out Of Memory errors, put a smaller sort size, until you can do the sorts (decrease it tenfold each time for example). You'll obviously lose some performance - - If then you get the too many open files error, raise the maximum number of open files. In most Linux distributions, this is editing /etc/security/limits.conf and putting +- First, use as much Java memory as you can: set the JAVAXMEM environment variable to 4096 (megabytes) or more if you can afford it. The more the better. +- If you still get Out Of Memory errors, put a smaller sort size, until you can do the sorts (decrease it tenfold each time for example). You'll obviously lose some performance +- If then you get the too many open files error, raise the maximum number of open files. In most Linux distributions, this is editing /etc/security/limits.conf and putting ``` @userName soft nofile 65535 @userName hard nofile 65535 @@ -156,9 +160,9 @@ You can also edit only the offending transformation with Spoon (Kettle's GUI), s When Kettle crashed on one of these problems, the temporary files aren't removed. They are usually in /tmp (or in your temp directory in Windows), and start with out_. Don't forget to remove them. --use_pk_if_possible=0/1/public.table1,myschema.table2: enable the generation of jobs doing sorts in the databases (order by in the select part of Kettle's table inputs). +`-use_pk_if_possible=0/1/public.table1,myschema.table2`: enable the generation of jobs doing sorts in the databases (order by in the select part of Kettle's table inputs). -1 will ask to try for all tables, or you can give a list of tables (if for example, you cannot make these tables work with a reasonable sort size). Anyway, sqlserver2pgsql will only accept +`1` will ask to try for all tables, or you can give a list of tables (if for example, you cannot make these tables work with a reasonable sort size). Anyway, sqlserver2pgsql will only accept to do sorts in the database if the primary key can be guaranteed to be sorted the same way in PostgreSQL and SQL Server. That means that it only accepts if the key is made only of numeric and date/timestamp types. If not, the standard, kettle-sorting incremental job will be generated. @@ -175,7 +179,8 @@ Now you've generated everything. Let's do the import: psql -U mypguser mypgdatabase -f name_of_after_script ``` -If you want to dig deeper into the kettle job, you can use kettle_report.pl to display the individual table's transfer performance. Then, if needed, you'll be able to modify the Kettle job to optimize it, using Spoon, Kettle's GUI +If you want to dig deeper into the kettle job, you can use kettle_report.pl to display the individual table's transfer performance (you'll need to redirect +kitchen's output to a file). Then, if needed, you'll be able to modify the Kettle job to optimize it, using Spoon, Kettle's GUI You can also give a try to the incremental job: From f13a015214f38704f68c3c31be7dd03f71da0fbe Mon Sep 17 00:00:00 2001 From: marco44 Date: Fri, 25 Nov 2016 20:45:48 +0100 Subject: [PATCH 116/216] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e4c0c37..dbbe2bb 100644 --- a/README.md +++ b/README.md @@ -64,13 +64,13 @@ run sqlserver2pgsql.pl from. If you just want to convert this schema, run: ``` -./sqlserver2pgsql.pl -f input_sql_dump \ +./sqlserver2pgsql.pl -f sqlserver_sql_dump \ -b output_before_script\ -a output_after_script\ -u output_unsure_script ``` -The sqlserver2pgsql Perl script processes your SQL raw dump "input_sql_dump" and produces these three scripts: +The sqlserver2pgsql Perl script processes your SQL raw dump "sqlserver_sql_dump" and produces these three scripts: - output_before_script: contains what is needed to import data (types, tables and columns) @@ -151,8 +151,8 @@ So you'll have to do some tuning here: @userName hard nofile 65535 ``` -(replace userName with your user name). Log in again, and verify with "ulimit -n" that you are now allowed to open 65535 files. -You may also have to raise the maximum number of open files on the system: echo the new value to /proc/sys/fs/file-max. +(replace userName with your user name). Log in again, and verify with `ulimit -n` that you are now allowed to open 65535 files. +You may also have to raise the maximum number of open files on the system: `echo` the new value to `/proc/sys/fs/file-max`. You'll need a lot of temporary space on disk to do these sorts... From 5eb4e00135e6326513455a275eb0df6e2867a8b1 Mon Sep 17 00:00:00 2001 From: marco44 Date: Wed, 30 Nov 2016 19:49:20 +0100 Subject: [PATCH 117/216] Support sequences --- sqlserver2pgsql.pl | 98 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 11 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 05b46a7..7bb93f7 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1144,6 +1144,8 @@ sub add_column_to_table $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{START} = $startseq; + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MIN} + = $startseq; $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} = $stepseq; $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname} @@ -1371,6 +1373,59 @@ sub parse_dump ################################################################ # From HERE, these SQL commands are not linked to a create table ################################################################ + elsif ($line =~ /^CREATE SEQUENCE \[(.*)\]\.\[(.*)\]/) + { + my $schemaname = relabel_schemas($1); + my $orig_schema = $1; + my $seqname = $2; + while (my $contline = read_and_clean($file)) + { + if ($contline =~ /^\s*AS \[.*\]\s*$/) + { + next; # We don't care, sequences are always bigint in PostgreSQL + } + elsif ($contline =~ /^\s*START WITH (\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{START}=$1; + } + elsif ($contline =~ /^\s*INCREMENT BY (\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP}=$1; + } + elsif ($contline =~ /^\s*MINVALUE (-?\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MIN}=$1; + } + elsif ($contline =~ /^\s*MAXVALUE (-?\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MAX}=$1; + } + elsif ($contline =~ /^\s*(NO)?CACHE( \d+)?\s*$/) + { + if (defined $1) + { + # It's a no cache. Equivalent to CACHE = 1 in PostgreSQL + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=1; + } + elsif (defined $2) + { + # We have a specified value + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=$2; + } + else + { + # Cache, but not specified. SQL Server isn't very clear on the size of the cache. Let's say 100 + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=100; + } + } + elsif ($contline =~ /^GO$/) + { + next MAIN; + } + } + + + } elsif ($line =~ /^CREATE SCHEMA \[(.*)\]/) { $objects->{SCHEMAS}->{relabel_schemas($1)} = undef @@ -2144,16 +2199,34 @@ sub generate_schema foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) { my $seqref = $refschema->{SEQUENCES}->{$sequence}; - print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence) . " INCREMENT BY " - . $seqref->{STEP} - . " MINVALUE " - . $seqref->{START} - . " START WITH " - . $seqref->{START} - . " OWNED BY " - . format_identifier($seqref->{OWNERSCHEMA}) . '.' - . format_identifier($seqref->{OWNERTABLE}) . '.' - . format_identifier($seqref->{OWNERCOL}) . ";\n"; + print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence); + if (defined $seqref->{STEP}) + { + print AFTER " INCREMENT BY ",$seqref->{STEP}; + } + if (defined $seqref->{MIN}) + { + print AFTER " MINVALUE ",$seqref->{MIN}; + } + if (defined $seqref->{MAX}) + { + print AFTER " MAXVALUE ",$seqref->{MAX}; + } + if (defined $seqref->{START}) + { + print AFTER " START WITH ",$seqref->{START}; + } + if (defined $seqref->{CACHE}) + { + print AFTER " CACHE ",$seqref->{CACHE}; + } + if (defined $seqref->{OWNERTABLE}) + { + print AFTER " OWNED BY ",format_identifier($seqref->{OWNERSCHEMA}), + '.',format_identifier($seqref->{OWNERTABLE}), + '.',format_identifier($seqref->{OWNERCOL}); + } + print AFTER ";\n"; } # Now PK. We have to go through all tables @@ -2337,7 +2410,10 @@ sub generate_schema { foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) { - my $seqref = $refschema->{SEQUENCES}->{$sequence}; + my $seqref = $refschema->{SEQUENCES}->{$sequence}; + # This may not be an identity. Skip it then + next unless defined ($seqref->{OWNERCOL}); + print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select max(". format_identifier($seqref->{OWNERCOL}) .") from " . format_identifier($seqref->{OWNERSCHEMA}) . '.'. format_identifier($seqref->{OWNERTABLE}) . ")::bigint);\n"; } } From 3aa123a184a2b59ad8e71501fcdecea8ef591a96 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 6 Dec 2016 14:43:35 +0100 Subject: [PATCH 118/216] Support Perl < 5.14 there was use of /r modifier in regex, which is only supported on 5.14+ --- sqlserver2pgsql.pl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 05b46a7..58f120f 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -354,7 +354,9 @@ sub sql_convert_column 'xml' => 'case when datalength({colname}) > 5 then {colname} else null end'); if (defined ($functions{$coltype})) { - return $functions{$coltype} =~ s/\{colname\}/[$colname]/gr; + my $tmpcol=$functions{$coltype}; + $tmpcol =~ s/\{colname\}/[$colname]/g; + return $tmpcol; } else { @@ -376,7 +378,9 @@ sub postgres_convert_column 'timestamp with time zone' => 'to_char({colname} AT TIME ZONE \'UTC\', \'YYYY-MM-DD HH:MI:SS.US+00\')'); if (defined ($functions{$coltype})) { - return $functions{$coltype} =~ s/\{colname\}/"$colname"/r; + my $tmpcol = $functions{$coltype}; + $tmpcol =~ s/\{colname\}/"$colname"/; + return $tmpcol; } else { From 7b2f3e285687742f0f13cec0b4680d1252bdb1e9 Mon Sep 17 00:00:00 2001 From: Julien Rouhaud Date: Tue, 6 Dec 2016 19:24:23 +0100 Subject: [PATCH 119/216] Fix parsing of conf_file --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 425fea2..f37466c 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -101,7 +101,7 @@ sub parse_conf_file while (my $line = ) { $line =~ s/#.*//; # Remove comments - $line =~ s/\s+=\s+//; # Remove whitespaces around the = + $line =~ s/\s+=\s+/=/; # Remove whitespaces around the = $line =~ s/\s+$//; # Remove trailing whitespaces next if ($line =~ /^$/); # Empty line after comments have been removed From bb8d9b50783fc226e7494d3ad0c81688c0e4cd9c Mon Sep 17 00:00:00 2001 From: Julien Rouhaud Date: Tue, 6 Dec 2016 19:36:14 +0100 Subject: [PATCH 120/216] Use non greedy modifier in constraint square bracket handling. --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index f37466c..b37cba0 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1853,7 +1853,7 @@ sub parse_dump } $constraint->{TYPE} = 'CHECK'; $constxt =~ - s/\[(\S+)\]/$1/g; # We remove the []. And hope this will parse + s/\[(\S+?)\]/$1/g; # We remove the []. And hope this will parse $constraint->{TEXT} = $constxt; push @{$objects->{SCHEMAS}->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}}, ($constraint); From b93f9909099a01a11a48f2df58f12d80f6594551 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 21 Feb 2017 13:27:07 +0100 Subject: [PATCH 121/216] Ignore SPARSE attribute. While at it, put croak instead of die, will simplify debugging --- sqlserver2pgsql.pl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index b37cba0..e81b98b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1232,7 +1232,7 @@ sub parse_dump # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ - /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ + /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE )?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ ) { #Deported into a function because we can also meet alter table add columns on their own @@ -1370,7 +1370,7 @@ sub parse_dump } else { - die "Cannot understand $line\n"; + croak "Cannot understand $line\n"; } } } @@ -1575,7 +1575,7 @@ sub parse_dump } else { - die "Cannot understand $typeline\n"; + croak "Cannot understand $typeline\n"; } } } @@ -1707,7 +1707,7 @@ sub parse_dump } else { - die "Cannot understand $consline."; + croak "Cannot understand $consline."; } } } @@ -1831,7 +1831,7 @@ sub parse_dump } else { - die "Cannot parse $fk $., in a FK. This is a bug"; + croak "Cannot parse $fk $., in a FK. This is a bug"; } } } @@ -1882,7 +1882,7 @@ sub parse_dump # First step: what kind is it ? we are only interested in comments for now $sqlproperty =~ /\@name=N'(.*?)'/ - or die + or croak "Cannot find a name for this extended property: $sqlproperty"; my $propertyname = $1; if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1|Display Name|Description|Example Values|Source System|Table Description|Table Type|ETL Rules|Display Folder|SCD Type|Source Datatype)$/) @@ -1926,7 +1926,7 @@ sub parse_dump } else { - die "Cannot understand this comment: $sqlproperty"; + croak "Cannot understand this comment: $sqlproperty"; } } elsif ($propertyname eq 'Dictionary') @@ -1934,7 +1934,7 @@ sub parse_dump # It seems to be another way to declare table comments. I hope this is right $sqlproperty =~ /^EXEC sys.sp_addextendedproperty \@name=N'(.*?)'\s*,\s*\@value=N'(.*?)(? not understood. Please send a bug report\n"; + croak "Schema comment : <$comment> not understood. Please send a bug report\n"; } } else { - die + croak "Don't know what to do with this extendedproperty: $sqlproperty"; } } @@ -2377,7 +2377,7 @@ sub generate_schema else { # Shouldn't get there. it would mean I have forgotten a type of constraint - die "I couldn't translate a constraint. This is a bug"; + croak "I couldn't translate a constraint. This is a bug"; } } } @@ -2667,7 +2667,7 @@ sub resolve_name_conflicts if ($validate_constraints !~ '^(yes|after|no)$') { - die "validate_constraints should be yes, after or no (default yes)\n"; + croak "validate_constraints should be yes, after or no (default yes)\n"; } # We have been asked for kettle, but the compulsory parameters are not there From 4019450e6c9a921a24f4ecdd832647155ce9fe06 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 21 Feb 2017 17:49:08 +0100 Subject: [PATCH 122/216] Solves Bug #55 from github (create view with view name on next line) --- sqlserver2pgsql.pl | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index e81b98b..6653512 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -101,7 +101,7 @@ sub parse_conf_file while (my $line = ) { $line =~ s/#.*//; # Remove comments - $line =~ s/\s+=\s+/=/; # Remove whitespaces around the = + $line =~ s/\s+=\s+//; # Remove whitespaces around the = $line =~ s/\s+$//; # Remove trailing whitespaces next if ($line =~ /^$/); # Empty line after comments have been removed @@ -1470,22 +1470,44 @@ sub parse_dump # containing only a single quote (end of the dbo.sp_executesql) # The problem is that SQL Server seems to be spitting the original query used to create the view, not a normalized version # of it, as PostgreSQL does. So we capture the query, and hope it works for now. - elsif ($line =~ - /^\s*(create\s*view)\s*(?:\[(\S+)\]\.)?\[(.*?)\]\s*(.*)$/i) + elsif ($line =~/^\s*(create\s*view)/i) { - my $viewname = $3; + my $viewname; my $schemaname; - if (defined $2) + my $supplement; + # Either we have the create view and the object name with it, or it is on next non-empty line + if ( $line =~ /^\s*(create\s*view)\s*(?:\[(\S+)\]\.)?\[(.*?)\]\s*(.*)$/i) { - $schemaname = $2; + $viewname = $3; + if (defined $2) + { + $schemaname = $2; + } + else + { + $schemaname = 'dbo'; + } + $supplement=$4; } else { - $schemaname = 'dbo'; + # Find the line containing the view name. For now, it's the next line. If not, well, correct your dump by yourself, sorry :) + my $line_name = read_and_clean($file); + $line_name=~ /^\s*(?:\[(\S+)\]\.)?\[(.*?)\]\s*(.*)$/ or croak "This line $line_name doesn't contain a view name"; + $viewname = $2; + if (defined $1) + { + $schemaname = $1; + } + else + { + $schemaname = 'dbo'; + } + $supplement=$3; } $schemaname = relabel_schemas($schemaname); - my $sql = $1 . ' ' . $schemaname . '.' . $3 . ' ' . $4 . "\n"; + my $sql = 'CREATE VIEW ' . $schemaname . '.' . $viewname . ' ' . $supplement . "\n"; while (my $line_cont = read_and_clean($file)) { if ($line_cont =~ /^\s*'\s*$|^GO$/ @@ -1853,7 +1875,7 @@ sub parse_dump } $constraint->{TYPE} = 'CHECK'; $constxt =~ - s/\[(\S+?)\]/$1/g; # We remove the []. And hope this will parse + s/\[(\S+)\]/$1/g; # We remove the []. And hope this will parse $constraint->{TEXT} = $constxt; push @{$objects->{SCHEMAS}->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}}, ($constraint); From 97c1e0b44bf4ce6cba9b22dfc088980386716e3a Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 22 Feb 2017 14:30:28 +0100 Subject: [PATCH 123/216] Support (as much as possible) partial indexes from sql server --- sqlserver2pgsql.pl | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 6653512..8e381fa 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -446,7 +446,7 @@ sub format_identifier_cols_index return $formatted . ' ' . $order; } -# This one will try to convert what can obviously be converted from transact to PG +# This one will try to convert what can obviously be converted from transact to (or embedded WHERE in indexes for instance) PG # Things such as getdate() which can become CURRENT_TIMESTAMP sub convert_transactsql_code { @@ -1659,6 +1659,13 @@ sub parse_dump next ; # Nothing equivalent in PG. Maybe if the index isn't unique, these columns should be added? } + if ($idx =~ /^WHERE\s*\((.*)\)$/) + { + # This is a where clause. PostgreSQL has them too. But we cannot be sure this will be exactly the same. So if an index as a WHERE clause, it has to go to unsure + my $filter=$1; + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{WHERE}=$filter; + } } } elsif ($line =~ /^CREATE SPATIAL INDEX/) @@ -2320,8 +2327,18 @@ sub generate_schema $idxdef .= " UNIQUE"; } $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" - . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ");\n"; - print AFTER $idxdef; + . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ")"; + if (not defined $idxref->{WHERE}) + { + $idxdef .= ";\n"; + print AFTER $idxdef; + } + else + { + print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; + $idxdef .= "\nWHERE (" . $idxref->{WHERE} . ");\n"; + print UNSURE $idxdef; + } } } } From b87c863a0b182ba6de190f2874bdb68e92e8cf23 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 22 Feb 2017 15:29:59 +0100 Subject: [PATCH 124/216] Remove [] instead of replacing them with ". And call convert_transactsql_code from the index generation as well --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 8e381fa..780a88b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -451,7 +451,7 @@ sub format_identifier_cols_index sub convert_transactsql_code { my ($code)=@_; - $code =~ s/[\[\]]/\"/gi; + $code =~ s/[\[\]]//gi; # Bit brutal probably $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; @@ -2336,7 +2336,7 @@ sub generate_schema else { print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; - $idxdef .= "\nWHERE (" . $idxref->{WHERE} . ");\n"; + $idxdef .= "\nWHERE (" . convert_transactsql_code($idxref->{WHERE}) . ");\n"; print UNSURE $idxdef; } } From e38cdb02f925d17a7e26fde5eae98e7f6ebddee0 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 22 Feb 2017 15:57:28 +0100 Subject: [PATCH 125/216] Improve check constraints parsing --- sqlserver2pgsql.pl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 780a88b..047520f 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1881,8 +1881,6 @@ sub parse_dump $constraint->{NAME} = $3; } $constraint->{TYPE} = 'CHECK'; - $constxt =~ - s/\[(\S+)\]/$1/g; # We remove the []. And hope this will parse $constraint->{TEXT} = $constxt; push @{$objects->{SCHEMAS}->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}}, ($constraint); @@ -2401,7 +2399,7 @@ sub generate_schema } elsif ($constraint->{TYPE} eq 'CHECK') { - $consdef .= " CHECK (" . $constraint->{TEXT} . ");\n"; + $consdef .= " CHECK (" . convert_transactsql_code($constraint->{TEXT}) . ");\n"; print UNSURE $consdef ; # Check constraints are SQL, so cannot be sure } From 774472497d7801f5bf071ebfabcc0dea12d9f897 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 24 Feb 2017 13:59:42 +0100 Subject: [PATCH 126/216] Try to be a bit better at fixing [] in transact sql and where clauses of partial indexes --- sqlserver2pgsql.pl | 151 ++++++++++++++++++++++++--------------------- 1 file changed, 79 insertions(+), 72 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 047520f..daa24f3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -50,7 +50,7 @@ our $requires_postgis=0; # These three variables are loaded in the BEGIN block at the end of this file (they are very big -my $template; +my $template; my $template_lob; my $incremental_template; my $incremental_template_sortable_pk; @@ -156,7 +156,7 @@ sub convert_numeric_to_int # This is a list of the types that require a cast to be imported in kettle # C = using CREATE CAST -# S = updating system catalog +# S = updating system catalog my %types_to_cast = ('uuid' => 'C','date' => 'C','timestamp with time zone' => 'C','xml' => 'S'); # This sub adds a cast (if not defined already) if @@ -263,7 +263,7 @@ sub convert_type # Special case. This is an internal type, and should seldom be used in production. Converted to varchar(128) $rettype='varchar(128)'; } - + # We special case also the geometry and geography data types elsif ( $sqlstype =~ /^geography$|^geometry$/i ) { @@ -447,11 +447,18 @@ sub format_identifier_cols_index } # This one will try to convert what can obviously be converted from transact to (or embedded WHERE in indexes for instance) PG -# Things such as getdate() which can become CURRENT_TIMESTAMP +# Things such as getdate() which can become CURRENT_TIMESTAMP sub convert_transactsql_code { my ($code)=@_; - $code =~ s/[\[\]]//gi; # Bit brutal probably + if ($keep_identifier_case) + { + $code =~ s/[\[\]]/"/gi; # Bit brutal probably + } + else + { + $code =~ s/[\[\]]//gi; # Bit brutal probably + } $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; @@ -560,7 +567,7 @@ sub next_col_pos unless ($norelabel_dbo) { $relabel_schemas{'dbo'}='public'; - } + } # dbo can be overwritten in relabel_schema (the user will probably forget to deactivate the relabel). # so we do the real relabeling after the norelabel_dbo, to overwrite if (defined $relabel_schemas) @@ -575,7 +582,7 @@ sub next_col_pos $relabel_schemas{$pair[0]}=$pair[1]; } } - + } @@ -737,7 +744,7 @@ sub generate_kettle { $newincrementaltemplate=$incremental_template; } - + # Build the column list of the table to put into the SQL Server query my @colsdef; @@ -817,7 +824,7 @@ sub generate_kettle $keys.="$pk\n"; } $newincrementaltemplate =~ s/__KEYS_MERGE__/$keys/g; - + my $sortkeys=''; my $synckeys=''; foreach my $pk(@pk) @@ -834,7 +841,7 @@ sub generate_kettle $newincrementaltemplate =~ s/__SORT_KEYS_SQLSERVER__/$sortkeys/g; $newincrementaltemplate =~ s/__SORT_KEYS_PG__/$sortkeys/g; $newincrementaltemplate =~ s/__KEYS_SYNC__/$synckeys/g; - + # We also need to tell the merge step to compare all columns my $valuesmerge=''; my $valuessync=''; @@ -856,13 +863,13 @@ sub generate_kettle $newincrementaltemplate =~ s/__VALUES_MERGE__/$valuesmerge/g; $newincrementaltemplate =~ s/__VALUES_SYNC__/$valuessync/g; - + # Produce the incremental transformation open FILE, ">$dir/incremental-$schema-$table.ktr" or die "Cannot write to $dir/incremental-$schema-$table.ktr"; binmode(FILE,":utf8"); print FILE $newincrementaltemplate; - close FILE; + close FILE; } else { @@ -1014,7 +1021,7 @@ sub generate_kettle $afterscript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ENABLE TRIGGER ALL;\n"; } } - + # This is for the SQL Scripts. We also need to specify the PG connection $job_header =~ s/__SQL_SCRIPT_INIT__/$beforescript/g; @@ -1105,7 +1112,7 @@ sub add_column_to_table if ($coltype eq 'xml') { $colqual = undef - ; # ignoring sql server xml schema since its not supported in pg + ; # ignoring sql server xml schema since its not supported in pg } elsif ($colqual eq '(max)') { @@ -1228,7 +1235,7 @@ sub parse_dump TABLE: while (my $line = read_and_clean($file)) { # Here is a col definition. - # We ignore ROWGUIDCOL as it has no meaning in PostgreSQL and cannot be emulated + # We ignore ROWGUIDCOL as it has no meaning in PostgreSQL and cannot be emulated # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ @@ -1427,8 +1434,8 @@ sub parse_dump next MAIN; } } - - + + } elsif ($line =~ /^CREATE SCHEMA \[(.*)\]/) { @@ -1676,7 +1683,7 @@ sub parse_dump $def.=$idx; } print STDERR "This spatial index won't be migrated:\n$def\n"; - + } # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. @@ -1716,7 +1723,7 @@ sub parse_dump $constraint->{NAME} = $3; } - + CONS: while (my $consline= read_and_clean($file)) { next if ($consline =~ /^\($/); @@ -2011,7 +2018,7 @@ sub parse_dump { next; } - + # Ignore xml schema collections since they are not supported in pg elsif ($line =~ /^CREATE XML SCHEMA COLLECTION/) { @@ -2021,7 +2028,7 @@ sub parse_dump elsif ($line =~ /^ALTER XML SCHEMA COLLECTION/) { next; - } + } # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views elsif ($line =~ /^IF NOT EXISTS/) @@ -2087,7 +2094,7 @@ sub parse_dump { $line =read_and_clean($file); } - + next; } @@ -2142,7 +2149,7 @@ sub generate_schema { print BEFORE "CREATE EXTENSION IF NOT EXISTS citext;\n"; } - + # Do we require PostGIS ? if ($requires_postgis) { @@ -2169,7 +2176,7 @@ sub generate_schema # The tables, columns, etc... will be created in the before script, so there is no dependancy # problem with constraints, that will be in the after script, except foreign keys which depend on unique indexes # We have to do all domains and types before all tables - # Don't care for dependancy + # Don't care for dependancy while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # The user-defined types (domains, etc) @@ -2339,7 +2346,7 @@ sub generate_schema } } } - } + } # Other constraints while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { @@ -2454,7 +2461,7 @@ sub generate_schema my $seqref = $refschema->{SEQUENCES}->{$sequence}; # This may not be an identity. Skip it then next unless defined ($seqref->{OWNERCOL}); - + print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select max(". format_identifier($seqref->{OWNERCOL}) .") from " . format_identifier($seqref->{OWNERSCHEMA}) . '.'. format_identifier($seqref->{OWNERTABLE}) . ")::bigint);\n"; } } @@ -3493,28 +3500,28 @@ BEGIN $incremental_template= < - migration__sqlserver_table_name__ - - - - Normal - 0 - / - - - - - - - - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD - - -
- - + migration__sqlserver_table_name__ + + + + Normal + 0 + / + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD + + +
+ + ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS @@ -3856,7 +3863,7 @@ BEGIN __postgres_db__ - 100 + 100 N N @@ -3920,28 +3927,28 @@ BEGIN $incremental_template_sortable_pk= < - migration__sqlserver_table_name__ - - - - Normal - 0 - / - - - - - -
- - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD - - -
- - + migration__sqlserver_table_name__ + + + + Normal + 0 + / + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD + + +
+ + ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS @@ -4185,7 +4192,7 @@ BEGIN __postgres_db__ - 100 + 100 N N From 6ac4809726c74583f6198ee1875bcc63aace7db4 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 27 Feb 2017 14:49:59 +0100 Subject: [PATCH 127/216] Support kettle 7 (change of sql server driver). Avoid running truncate directly on table output, as it is run in all threads --- sqlserver2pgsql.pl | 196 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 184 insertions(+), 12 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index daa24f3..aa8c4e7 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2837,7 +2837,7 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQL + MSSQLNATIVE Native __sqlserver_database__ __sqlserver_port__ @@ -2847,14 +2847,16 @@ BEGIN - EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN + MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ + PRESERVE_RESERVED_WORD_CASEY QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN + SUPPORTS_TIMESTAMP_DATA_TYPEN USE_POOLINGN @@ -2884,7 +2886,89 @@ BEGIN - Table inputUser Defined Java ClassY User Defined Java ClassTable outputY + + User Defined Java Class + Table output + Y + + + Table input + Block this step until steps finish + Y + + + Block this step until steps finish + User Defined Java Class + Y + + + + Execute SQL script + ExecSQL + + Y + + 1 + + none + + + __postgres_db__ + N + N + N + N + truncate table __postgres_schema_name__.__postgres_table_name__ + N + + + + + + + + + + + + + + + 64 + 160 + Y + + + + Block this step until steps finish + BlockUntilStepsFinish + + Y + + 1 + + none + + + + + Execute SQL script + 0 + + + + + + + + + + + 240 + 240 + Y + + User Defined Java Class UserDefinedJavaClass @@ -2991,7 +3075,7 @@ BEGIN N - 122 + 182 160 Y @@ -3115,7 +3199,7 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQL + MSSQLNATIVE Native __sqlserver_database__ __sqlserver_port__ @@ -3125,14 +3209,16 @@ BEGIN - EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN + MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ + PRESERVE_RESERVED_WORD_CASEY QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN + SUPPORTS_TIMESTAMP_DATA_TYPEN USE_POOLINGN @@ -3161,7 +3247,89 @@ BEGIN - Table inputUser Defined Java ClassY User Defined Java ClassTable outputY + + User Defined Java Class + Table output + Y + + + Table input + Block this step until steps finish + Y + + + Block this step until steps finish + User Defined Java Class + Y + + + + Execute SQL script + ExecSQL + + Y + + 1 + + none + + + __postgres_db__ + N + N + N + N + truncate table __postgres_schema_name__.__postgres_table_name__ + N + + + + + + + + + + + + + + + 64 + 160 + Y + + + + Block this step until steps finish + BlockUntilStepsFinish + + Y + + 1 + + none + + + + + Execute SQL script + 0 + + + + + + + + + + + 240 + 240 + Y + + User Defined Java Class UserDefinedJavaClass @@ -3267,7 +3435,7 @@ BEGIN N - 122 + 182 160 Y @@ -3593,7 +3761,7 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQL + MSSQLNATIVE Native __sqlserver_database__ __sqlserver_port__ @@ -3603,14 +3771,16 @@ BEGIN - EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN + MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ + PRESERVE_RESERVED_WORD_CASEY QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN + SUPPORTS_TIMESTAMP_DATA_TYPEN USE_POOLINGN @@ -4020,7 +4190,7 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQL + MSSQLNATIVE Native __sqlserver_database__ __sqlserver_port__ @@ -4030,14 +4200,16 @@ BEGIN - EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN + MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ + PRESERVE_RESERVED_WORD_CASEY QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN + SUPPORTS_TIMESTAMP_DATA_TYPEN USE_POOLINGN From 2a55e54bc85368e6b75ee0802ad805d4dabb9cf6 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 27 Feb 2017 18:19:14 +0100 Subject: [PATCH 128/216] Remove truncate from the table output step. Silly me --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index aa8c4e7..b9dd820 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -3095,7 +3095,7 @@ BEGIN __postgres_schema_name__
__postgres_table_name__
100 - Y + N N Y N @@ -3455,7 +3455,7 @@ BEGIN __postgres_schema_name__ __postgres_table_name__
100 - Y + N N Y N From 2b356864dffcdb70b814910cbb308110552618d1 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 1 Mar 2017 11:44:17 +0100 Subject: [PATCH 129/216] Turn the KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL message into a warning (in case the script is not run in the target environment) --- sqlserver2pgsql.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index b9dd820..29745ff 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -608,8 +608,8 @@ sub is_windows sub kettle_die { my ($file) = @_; - die - "You have to set up KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in $file.\nIf this file doesn't exist yet, start spoon from the kettle directory once."; + print STDERR + "Important WARNIN: You have to set up KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in $file.\nIf this file doesn't exist yet, start spoon from the kettle directory once."; } # This sub checks ~/.kettle/kettle.properties to be sure From 8c9caadd99a29b9dc5530bd526da3de9e0ae3c68 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 1 Mar 2017 11:45:10 +0100 Subject: [PATCH 130/216] And correct a typo in previous patch --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 29745ff..0cd48bc 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -609,7 +609,7 @@ sub kettle_die { my ($file) = @_; print STDERR - "Important WARNIN: You have to set up KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in $file.\nIf this file doesn't exist yet, start spoon from the kettle directory once."; + "Important WARNING: You have to set up KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in $file.\nIf this file doesn't exist yet, start spoon from the kettle directory once."; } # This sub checks ~/.kettle/kettle.properties to be sure From 2c2c1b33c6b7fde06543e4adbc78ce5ac2f014b0 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Wed, 1 Mar 2017 11:55:17 +0100 Subject: [PATCH 131/216] Fix the kettle check, try to correct paths for windows jobs --- sqlserver2pgsql.pl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 0cd48bc..f138e21 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -605,7 +605,7 @@ sub is_windows } # Die if kettle is not set up correctly -sub kettle_die +sub kettle_warn { my ($file) = @_; print STDERR @@ -627,7 +627,11 @@ sub check_kettle_properties { $file = $ENV{'USERPROFILE'} . '/.kettle/kettle.properties'; } - open FILE, $file or kettle_die($file); + unless (open FILE, $file ) + { + kettle_warn($file); + return 1; + } while () { next unless (/^KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y$/); @@ -636,7 +640,8 @@ sub check_kettle_properties close FILE; if (not $ok) { - kettle_die($file); + kettle_warn($file); + return 1; } return 0; } @@ -930,7 +935,7 @@ sub generate_kettle # The only difference between normal and incremental job is the filename of the transformation my $JOBFILEname; my $INCJOBFILEname; - if ($dir =~ /^(\\|\/)/) # Absolute path + if ($dir =~ /^([A-Za-z]:\\|\/)/) # Absolute path { $JOBFILEname = $dir . '/' . $schema . '-' . $table . '.ktr'; $INCJOBFILEname = $dir . '/' . 'incremental-' . $schema . '-' . $table . '.ktr'; From 2df280ef336786a395a7b1592522156e72d27ee5 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 16 Mar 2017 21:07:51 +0100 Subject: [PATCH 132/216] Add support for -camel_to_snake --- README.md | 2 ++ sqlserver2pgsql.pl | 56 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 13 deletions(-) mode change 100755 => 100644 sqlserver2pgsql.pl diff --git a/README.md b/README.md index dbbe2bb..4593b59 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,8 @@ so the scale is often not important. `-keep_identifier_case`: don't convert the dump to all lower case. This is not recommended, as you'll have to put every identifier (column, table…) in double quotes… +`-camel_to_snake`: convert the object name (table, column, index...) from CamelCase to snake_case. Only do this if you are willing to change all your queries (or you use an ORM for instance) + `-validate_constraints=yes/after/no`: for foreign keys, if yes: foreign keys are created as valid in the after script (default) if no: they are created as not valid (enforced only for new rows) if after: they are created as not valid, but the statements to validate them are put in the unsure file diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl old mode 100755 new mode 100644 index f138e21..1fe8b13 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -40,7 +40,9 @@ our $before_file; our $after_file; our $unsure_file; +our $case_treatment=1; # 1=convert to lowercase, 2=convert to snake_case, 0 do nothing our $keep_identifier_case; +our $camel_to_snake; our $validate_constraints; our $parallelism; our $sort_size; @@ -91,6 +93,7 @@ sub parse_conf_file 'convert numeric to int' => 'convert_numeric_to_int', 'relabel schemas' => 'relabel_schemas', 'keep identifier case' => 'keep_identifier_case', + 'camelcasetosnake' => 'camel_to_snake', 'validate constraints' => 'validate_constraints', 'sort size' => 'sort_size', 'use pk if possible' => 'use_pk_if_possible', @@ -130,7 +133,8 @@ sub set_default_conf_values $case_insensitive=0 unless (defined ($case_insensitive)); $norelabel_dbo=0 unless (defined ($norelabel_dbo)); $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); - $keep_identifier_case=0 unless (defined ($keep_identifier_case)); + $case_treatment=0 if (defined ($keep_identifier_case)); + $case_treatment=2 if (defined ($camel_to_snake)); $parallelism=8 unless (defined ($parallelism)); $sort_size=10000 unless (defined ($sort_size)); $use_pk_if_possible=0 unless (defined ($use_pk_if_possible)); @@ -388,6 +392,16 @@ sub postgres_convert_column } } +# This is used to convert camelCase to snake_case. The latter is more usual with PostgreSQL + +sub camel_to_snake +{ + my ($string)=@_; + $string =~ s/([[:lower:]])([[:upper:]]+)/$1_\l$2/g; + $string=lc($string); + return $string; +} + # This function is used to determine if a PK will be sorted the same in SQL Server and PG # It means that it doesn't depend on collation orders or other internals. # For now, only numeric and date data types are considered OK @@ -417,6 +431,20 @@ sub is_pk_sort_order_safe return $isok; } +# This function renames the identifiers +sub rename_identifier +{ + my ($identifier)=@_; + if ($case_treatment==1) + { + $identifier=lc($identifier); + } + elsif ($case_treatment==2) + { + $identifier=camel_to_snake($identifier); + } + return $identifier; +} # This function formats the identifiers (object name), putting double quotes around it # It also converts case if asked @@ -424,10 +452,8 @@ sub format_identifier { my ($identifier)=@_; croak "identifier not defined in format_identifier" unless (defined $identifier); - unless ($keep_identifier_case) - { - $identifier=lc($identifier); - } + $identifier=rename_identifier($identifier); + # Now, we protect the identifier (similar to quote_ident in PG) $identifier=~ s/"/""/g; $identifier='"'.$identifier.'"'; @@ -451,13 +477,13 @@ sub format_identifier_cols_index sub convert_transactsql_code { my ($code)=@_; - if ($keep_identifier_case) + if ($case_treatment==0) { $code =~ s/[\[\]]/"/gi; # Bit brutal probably } else { - $code =~ s/[\[\]]//gi; # Bit brutal probably + $code =~ s/[\(.*)[\]]/rename_identifier($1)/gie; # Bit brutal probably } $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; @@ -664,6 +690,8 @@ sub usage print " -nr simply cancels the default dbo=>public remapping. Don't forget to put the remapping between quotes\n"; print "-keep_identifier_case tells $0 to keep the case of sql server database objects (not advised). Default is to lowercase everything.\n"; + print + "-camel_to_snake tells $0 to convert the object names from camelCase to camel_case, which is more often used in PostgreSQL.\nDon't use this unless you are ready to do changes in the client.\n"; print "before_file contains the structure\n"; print "after_file contains index, constraints\n"; print "validate_constraints validates the constraints that have been created\n"; @@ -836,10 +864,7 @@ sub generate_kettle { $sortkeys.="\n$pk\nY\nY\n\n"; my $outcol=$pk; - unless ($keep_identifier_case) - { - $outcol=lc($outcol); - } + $outcol=rename_identifier($outcol); $synckeys.="\n$pk\n$outcol\n=\n\n\n"; } @@ -858,9 +883,13 @@ sub generate_kettle $valuesmerge.="$colname\n"; # we need to use the correct case for postgresql output my $outcol=$colname; - unless ($keep_identifier_case) + if ($case_treatment==1) + { + $outcol=lc($outcol); + } + elsif ($case_treatment==2) { - $outcol=lc($outcol); + $outcol=camel_to_snake($outcol); } $valuessync.="\n$outcol\n$colname\nY\n\n"; # } @@ -2683,6 +2712,7 @@ sub resolve_name_conflicts "num" => \$convert_numeric_to_int, "relabel_schemas=s" => \$relabel_schemas, "keep_identifier_case" =>\$keep_identifier_case, + "camel_to_snake" => \$camel_to_snake, "validate_constraints=s" =>\$validate_constraints, "sort_size=i" =>\$sort_size, "use_pk_if_possible=s" =>\$use_pk_if_possible,); From f0d7d2954de20f7440e5706d842462d557e33aa3 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Thu, 16 Mar 2017 22:02:45 +0100 Subject: [PATCH 133/216] Fix changes introduced by 774472497d7801f5bf071ebfabcc0dea12d9f897 and made visible by 2df280ef336786a395a7b1592522156e72d27ee5 --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 sqlserver2pgsql.pl diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl old mode 100644 new mode 100755 index 1fe8b13..cace905 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -483,7 +483,7 @@ sub convert_transactsql_code } else { - $code =~ s/[\(.*)[\]]/rename_identifier($1)/gie; # Bit brutal probably + $code =~ s/\[(.*)\]/rename_identifier($1)/gie; # Bit brutal probably } $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; From 7f8425bbcd65f335c9e9130ba970f20413eec66c Mon Sep 17 00:00:00 2001 From: marco44 Date: Wed, 22 Mar 2017 19:12:32 +0100 Subject: [PATCH 134/216] Create contributors --- contributors | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 contributors diff --git a/contributors b/contributors new file mode 100644 index 0000000..1c40269 --- /dev/null +++ b/contributors @@ -0,0 +1,17 @@ +Marc Cousin, main developper + +Bill Ruddock (binarii) +Julien Rouhaud (rjuju) +Yann Verry (yanntech) +Konstantin Mosolov (kmosolov) +Javier Callico (JCallico) +Joshua F. Rountree (joshuairl) + +And the following unindentified github users (please tell me if you want your name added/replaced) + +stuey1978 +sebpcspkr +keyjote + + +Thanks to everyone who helped improving this script ! From 75f88996d0120bede9205edd49c60a46ec530a97 Mon Sep 17 00:00:00 2001 From: marco44 Date: Wed, 22 Mar 2017 19:28:59 +0100 Subject: [PATCH 135/216] Update contributors --- contributors | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors b/contributors index 1c40269..9775146 100644 --- a/contributors +++ b/contributors @@ -13,5 +13,6 @@ stuey1978 sebpcspkr keyjote +If I forgot your name in this list, please tell me. Thanks to everyone who helped improving this script ! From 3ccb0ea90080860f9fd2570f495162d252654226 Mon Sep 17 00:00:00 2001 From: marco44 Date: Sat, 6 May 2017 09:46:50 +0200 Subject: [PATCH 136/216] support some sql server 2000-isms (spaces are not at the same place...) --- sqlserver2pgsql.pl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index cace905..e7774d9 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1172,7 +1172,7 @@ sub add_column_to_table # We have an identity field. We remember the default value and # initialize the sequence correctly in the after script - $isidentity =~ /IDENTITY\((\d+),\s*(\d+)\)/ + $isidentity =~ /IDENTITY\s*\((\d+),\s*(\d+)\)/ or die "Cannot understand <$isidentity>"; my $startseq = $1; my $stepseq = $2; @@ -1258,7 +1258,7 @@ sub parse_dump { # Create table, obviously. There will be other lines below for the rest of the table definition - if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\(/) + if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\s*\(/) { my $schemaname = relabel_schemas($1); my $orig_schema = $1; @@ -1273,7 +1273,7 @@ sub parse_dump # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ - /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE )?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ + /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\]\s*(\(.+?\))?(?: COLLATE (\S+))?( IDENTITY\s*\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE )?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ ) { #Deported into a function because we can also meet alter table add columns on their own @@ -1281,9 +1281,10 @@ sub parse_dump my $coltypeschema = $2; my $coltype = $3; my $colqual =$4; - my $isidentity =$5; - my $colisnull =$6; - my $default =$7; + my $colcollate =$5; # Ignore for now + my $isidentity =$6; + my $colisnull =$7; + my $default =$8; add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); if (defined $default) { @@ -1644,7 +1645,7 @@ sub parse_dump } elsif ($line =~ - /^CREATE (UNIQUE )?(NONCLUSTERED|CLUSTERED) INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/ + /^\s*CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/ ) { # Index creation. Index are namespaced per table in SQL Server, not in PostgreSQL From e2c4f54d17147b56ca6f9ece45242a81894c5c89 Mon Sep 17 00:00:00 2001 From: marco44 Date: Sat, 6 May 2017 09:58:40 +0200 Subject: [PATCH 137/216] Ignore "if" tests --- sqlserver2pgsql.pl | 94 ++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index e7774d9..32a720b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1710,16 +1710,16 @@ sub parse_dump } } } - elsif ($line =~ /^CREATE SPATIAL INDEX/) - { - my $def=$line; - while (my $idx = read_and_clean($file)) + elsif ($line =~ /^CREATE SPATIAL INDEX/) { - $def.=$idx; - } - print STDERR "This spatial index won't be migrated:\n$def\n"; + my $def=$line; + while (my $idx = read_and_clean($file)) + { + $def.=$idx; + } + print STDERR "This spatial index won't be migrated:\n$def\n"; - } + } # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table @@ -2025,6 +2025,7 @@ sub parse_dump } } + # Ignore USE, GO, and things that have no meaning for postgresql elsif ($line =~ /^USE\s|^GO\s*$|\/\*\*\*\*|^SET ANSI_NULLS (ON|OFF)|^SET QUOTED_IDENTIFIER|^SET ANSI_PADDING|CHECK CONSTRAINT|^BEGIN|^END/ @@ -2066,48 +2067,53 @@ sub parse_dump } # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views - elsif ($line =~ /^IF NOT EXISTS/) + # Also ignore version tests + elsif ($line =~ /^IF EXISTS|^IF \(\@\@microsoftversion/i) { - next; + # just read until next go + while ($line !~ /^GO$/) + { + $line =read_and_clean($file); + } } # Ignore CREATE DATABASE: we hope that we are given a single database as an option. It is multiline. # Ignore everything until next GO # Ignore ALTER DATABASE for the same reason. The given parameters have no meaning in PG anyway - # Except for SET ARITHABORT OFF, for which we print a warning because it probably means the database contents are weird (10/0 = null) - elsif ($line =~ - /^ALTER DATABASE.* SET ARITHABORT OFF/) - { - print STDERR "WARNING: the source database is set as ARITHABORT OFF.\n"; - print STDERR " It means that for SQL Server, 10/0 = NULL.\n"; - print STDERR " You'll probably have problems porting that to PostgreSQL.\n"; - while ($line !~ /^GO$/) - { - $line =read_and_clean($file); - } - # We read everything in the CREATE DATABASE. Back to work ! - next; - } - # Sometimes, when there is a ALTER DATABASE SET ARITHABORT OFF, there are SET ARITHABORT ON. Just ignore them - elsif ($line =~ /^SET ARITHABORT ON/) - { - next; - } - # Sometimes we meet this: SET CONCAT_NULL_YIELDS_NULL ON. That's the normal behaviour for a SQL database. Just ignore - elsif ($line =~ /^SET CONCAT_NULL_YIELDS_NULL ON/) - { - next; - } - # Same more or less - elsif ($line =~ /^SET ANSI_WARNINGS ON/) - { - next; - } - # What the hell does this do in a dump ??? - elsif ($line =~ /^SET NUMERIC_ROUNDABORT OFF/) - { - next; - } + # Except for SET ARITHABORT OFF, for which we print a warning because it probably means the database contents are weird (10/0 = null) + elsif ($line =~ + /^ALTER DATABASE.* SET ARITHABORT OFF/) + { + print STDERR "WARNING: the source database is set as ARITHABORT OFF.\n"; + print STDERR " It means that for SQL Server, 10/0 = NULL.\n"; + print STDERR " You'll probably have problems porting that to PostgreSQL.\n"; + while ($line !~ /^GO$/) + { + $line =read_and_clean($file); + } + # We read everything in the CREATE DATABASE. Back to work ! + next; + } + # Sometimes, when there is a ALTER DATABASE SET ARITHABORT OFF, there are SET ARITHABORT ON. Just ignore them + elsif ($line =~ /^SET ARITHABORT ON/) + { + next; + } + # Sometimes we meet this: SET CONCAT_NULL_YIELDS_NULL ON. That's the normal behaviour for a SQL database. Just ignore + elsif ($line =~ /^SET CONCAT_NULL_YIELDS_NULL ON/) + { + next; + } + # Same more or less + elsif ($line =~ /^SET ANSI_WARNINGS ON/) + { + next; + } + # What the hell does this do in a dump ??? + elsif ($line =~ /^SET NUMERIC_ROUNDABORT OFF/) + { + next; + } # Same for tests about full text search. elsif ($line =~ From cb728bd496db414786722cc4161ae71622c4a45c Mon Sep 17 00:00:00 2001 From: marco44 Date: Sat, 6 May 2017 10:05:32 +0200 Subject: [PATCH 138/216] Manage IF NOT EXISTS too --- sqlserver2pgsql.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 32a720b..5e6eb4e 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2068,10 +2068,10 @@ sub parse_dump # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views # Also ignore version tests - elsif ($line =~ /^IF EXISTS|^IF \(\@\@microsoftversion/i) + elsif ($line =~ /^IF (NOT )?EXISTS|^IF \(\@\@microsoftversion/i) { - # just read until next go - while ($line !~ /^GO$/) + # just read until next go (or EOF) + while (defined $line and $line !~ /^GO$/) { $line =read_and_clean($file); } From 47a2e9c3ec793eb06001d451ea16d61c53590c9b Mon Sep 17 00:00:00 2001 From: marco44 Date: Sat, 6 May 2017 11:05:17 +0200 Subject: [PATCH 139/216] Accept indexes where column definition is on first line --- sqlserver2pgsql.pl | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 5e6eb4e..43205ce 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1645,7 +1645,7 @@ sub parse_dump } elsif ($line =~ - /^\s*CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/ + /^\s*CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*?)\] ON \[(.*?)\]\.\[(.*?)\](\(\[.*?\]\))?/ ) { # Index creation. Index are namespaced per table in SQL Server, not in PostgreSQL @@ -1658,6 +1658,7 @@ sub parse_dump my $idxname = $3; my $schemaname = relabel_schemas($4); my $tablename = $5; + my $maybecols = $6; # Some versions of sql server put column definitions on the first line if ($isunique) { $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} @@ -1668,6 +1669,24 @@ sub parse_dump $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} ->{$idxname}->{UNIQUE} = 0; } + if (defined $maybecols) + { + my @maybecols = split (',',$maybecols); + foreach my $coldef(@maybecols) + { + $coldef=~/\[(.*)\](?: (ASC|DESC))?/ or die "Cannot understand coldef $coldef in index"; + if (defined $2) + { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{COLS}}, ("$1 $2"); + } + else + { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{COLS}}, ("$1"); + } + } + } while (my $idx = read_and_clean($file)) { @@ -2068,7 +2087,7 @@ sub parse_dump # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views # Also ignore version tests - elsif ($line =~ /^IF (NOT )?EXISTS|^IF \(\@\@microsoftversion/i) + elsif ($line =~ /^IF EXISTS|^IF \(\@\@microsoftversion/i) { # just read until next go (or EOF) while (defined $line and $line !~ /^GO$/) @@ -2076,6 +2095,13 @@ sub parse_dump $line =read_and_clean($file); } } + elsif ($line =~ /^IF NOT EXISTS/i) + { + # Just ignore the line + next; + + } + # Ignore CREATE DATABASE: we hope that we are given a single database as an option. It is multiline. # Ignore everything until next GO From b91a755fb6f3c3cc741b639df60cf6da62338b9f Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 9 May 2017 10:39:59 +0200 Subject: [PATCH 140/216] Correct a bug introduced by #4019450e6c9a921a24f4ecdd832647155ce9fe06 --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 43205ce..84f51b0 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -104,7 +104,7 @@ sub parse_conf_file while (my $line = ) { $line =~ s/#.*//; # Remove comments - $line =~ s/\s+=\s+//; # Remove whitespaces around the = + $line =~ s/\s+=\s+/=/; # Remove whitespaces around the = $line =~ s/\s+$//; # Remove trailing whitespaces next if ($line =~ /^$/); # Empty line after comments have been removed From 446e5224196ebeff852db9bfdae4effa29fee52c Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 12 May 2017 12:11:41 +0200 Subject: [PATCH 141/216] correct handling of an empty instance name --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 84f51b0..8592328 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -802,7 +802,7 @@ sub generate_kettle my $pgschema=format_identifier($targetschema); my $sqlinstancename = ''; - if (length $si) { + if (defined $si) { $sqlinstancename = $si; } From 0c26295c46301103f600419e8679ccd79c5154b8 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 23 May 2017 16:15:49 +0200 Subject: [PATCH 142/216] Reduce default parallelism to 4 --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 8592328..e8a156c 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -135,7 +135,7 @@ sub set_default_conf_values $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); $case_treatment=0 if (defined ($keep_identifier_case)); $case_treatment=2 if (defined ($camel_to_snake)); - $parallelism=8 unless (defined ($parallelism)); + $parallelism=4 unless (defined ($parallelism));# the MS jdbc driver errors when there are more than 5 sessions $sort_size=10000 unless (defined ($sort_size)); $use_pk_if_possible=0 unless (defined ($use_pk_if_possible)); $validate_constraints='yes' unless (defined ($validate_constraints)); From 0235c1c1f5f0c8066c964ba4169a46940e5080fd Mon Sep 17 00:00:00 2001 From: marco44 Date: Thu, 25 May 2017 17:54:50 +0200 Subject: [PATCH 143/216] correction of the timestamp type conversion. Per #64 --- sqlserver2pgsql.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index e8a156c..f700833 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -197,7 +197,8 @@ sub add_cast 'datetime2' => 'timestamp', 'smalldatetime' => 'timestamp', 'time' => 'time', - 'timestamp' => 'timestamp', + 'timestamp' => 'bytea', + 'rowversion' => 'bytea', 'datetimeoffset' => 'timestamp with time zone', 'image' => 'bytea', 'binary' => 'bytea', From 02b3b6f043709d25e602d0743b378abe0ca2c577 Mon Sep 17 00:00:00 2001 From: marco44 Date: Fri, 26 May 2017 12:14:29 +0200 Subject: [PATCH 144/216] Update INSTALL.md --- INSTALL.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 89b17ef..62efd3f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -8,8 +8,15 @@ will tell you what to do). If you are trying to run the script under Windows, you probably won't have a Perl interpreter. I recommand you use the latest Strawberry Perl (use either the installer or the portable version). +Kettle: +========================== If you want to migrate the data, you'll need "Kettle", an Open Source ETL. Get the latest version from here: http://kettle.pentaho.com/ . + +On newest versions of Kettle, the SQL Server java driver isn't included. You'll need this one: http://jtds.sourceforge.net/. Just download the zip file (jtds-1.3.1-dist.zip at the time of this writing), extract the jar file (jtds-1.3.1.jar) and put this file in the lib directory of Kettle. + +If you don't want to bother with installing a JVM system-wide for Kettle, just download Sun's JVM and put in in Kettle's directory, in a "java" subdirectory. Latest versions of Kettle require Java 8. + You'll also need a SQL Server account with the permission to SELECT from the tables you want to migrate. -As Kettle is a Java program, you'll also need a recent JVM (Java 6 or 7 should do the trick). + From 988e26e4994a8605d48851941eace27acedcca62 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 12 Jun 2017 15:39:52 +0200 Subject: [PATCH 145/216] Add support for -ignore_errors option --- README.md | 2 +- example_conf_file | 3 +++ sqlserver2pgsql.pl | 14 +++++++++++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4593b59..43806eb 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ cleartext, so don't make this directory public): `-pu` : postgresql username `-pw` : postgresql password `-f` : the SQL Server structure dump file - +-ignore_errors : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) `-p` : The parallelism used in kettle jobs: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 `-sort_size=100000`: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). diff --git a/example_conf_file b/example_conf_file index 0d06b69..76d42c6 100644 --- a/example_conf_file +++ b/example_conf_file @@ -32,3 +32,6 @@ validate constraints = yes # yes, after or no, should the constraints be validat # Incremental job sort size=10000 # drives the amount of memory and temporary files that will be created by an incremental job use pk if possible=0 # 1/list of tables, for tables where you want to try getting already sorted records + +# Ignore errors ? (will be slower, and you'll have to read the migration job's log throroughly). Ignored for incremental jobs +ignore errors=0 diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index f700833..6668ec4 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -41,6 +41,7 @@ our $after_file; our $unsure_file; our $case_treatment=1; # 1=convert to lowercase, 2=convert to snake_case, 0 do nothing +our $ignore_errors; our $keep_identifier_case; our $camel_to_snake; our $validate_constraints; @@ -97,6 +98,7 @@ sub parse_conf_file 'validate constraints' => 'validate_constraints', 'sort size' => 'sort_size', 'use pk if possible' => 'use_pk_if_possible', + 'ignore errors' => 'ignore_errors', ); # Open the conf file or die @@ -139,6 +141,7 @@ sub set_default_conf_values $sort_size=10000 unless (defined ($sort_size)); $use_pk_if_possible=0 unless (defined ($use_pk_if_possible)); $validate_constraints='yes' unless (defined ($validate_constraints)); + $ignore_errors=0 unless (defined ($ignore_errors)); # Default ports for PostgreSQL and SQL Server $pp=5432 unless (defined ($pp)); $sp=1433 unless (defined ($sp)); @@ -717,6 +720,8 @@ sub usage print "-pu: postgresql username\n"; print "-pw: postgresql password\n"; print "-p: parallelism level for the kettle job\n"; + print "You may also choose to ignore insert errors (inserting will be much slower)\n"; + print "-ignore_errors\n"; } # This function generates kettle transformations, and a kettle job running all these @@ -826,6 +831,12 @@ sub generate_kettle $newtemplate =~ s/__postgres_schema_name__/$pgschema/g; $newtemplate =~ s/__PARALLELISM__/$parallelism/g; + if ($ignore_errors) + { + $newtemplate =~ s/N<\/ignore_errors>/Y<\/ignore_errors>/g; + $newtemplate =~ s/Y<\/use_batch>/N<\/use_batch>/g; # Cannot use batch mode with ignore errors + } + $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; @@ -2749,7 +2760,8 @@ sub resolve_name_conflicts "camel_to_snake" => \$camel_to_snake, "validate_constraints=s" =>\$validate_constraints, "sort_size=i" =>\$sort_size, - "use_pk_if_possible=s" =>\$use_pk_if_possible,); + "use_pk_if_possible=s" =>\$use_pk_if_possible, + "ignore_errors" => \$ignore_errors); # We don't understand command line or have been asked for usage if (not $options or $help) From 8ddfe763bc95add6a8e96ac6dade3fe10bd66e9b Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 19 Jun 2017 15:03:13 +0200 Subject: [PATCH 146/216] Split parallelism in IN and OUT --- README.md | 3 ++- example_conf_file | 3 ++- sqlserver2pgsql.pl | 43 ++++++++++++++++++++++++++----------------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 43806eb..f1be992 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,8 @@ cleartext, so don't make this directory public): `-f` : the SQL Server structure dump file -ignore_errors : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) -`-p` : The parallelism used in kettle jobs: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 +`-pi` : The parallelism used in kettle jobs to read from SQL Server (1 by default, the jdbc driver frequently errors out when larger) +`-po` : The parallelism used in kettle jobs to write to PostgresSQL: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 `-sort_size=100000`: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). We don't sort in databases for two reasons: the sort order (collation for strings for example) can be different between SQL Server diff --git a/example_conf_file b/example_conf_file index 76d42c6..a2b734f 100644 --- a/example_conf_file +++ b/example_conf_file @@ -19,7 +19,8 @@ postgresql host=bar_host postgresql port=5432 postgresql username=bar_user postgresql password=bar_password -parallelism=8 # Default value is 8. Number of parallel connections used by kettle to insert data into the PostgreSQL database +parallelism_in=8 # Parallelism reading from SQL Server (where available) Default value is 1 +parallelism_out=8 # Default value is 8. Number of parallel connections used by kettle to insert data into the PostgreSQL database # Optional behaviour case insensitive=0 # set it to 1 to generate a dump with citext and check constraints all over the place diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 6668ec4..794fe9d 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -45,7 +45,8 @@ our $keep_identifier_case; our $camel_to_snake; our $validate_constraints; -our $parallelism; +our $parallelism_in; +our $parallelism_out; our $sort_size; our $use_pk_if_possible; @@ -84,7 +85,8 @@ sub parse_conf_file 'postgresql username' => 'pu', 'postgresql password' => 'pw', 'kettle directory' => 'kettle', - 'parallelism' => 'parallelism', + 'parallelism_in' => 'parallelism_in', + 'parallelism_out' => 'parallelism_out', 'before file' => 'before_file', 'after file' => 'after_file', 'unsure file' => 'unsure_file', @@ -137,7 +139,8 @@ sub set_default_conf_values $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); $case_treatment=0 if (defined ($keep_identifier_case)); $case_treatment=2 if (defined ($camel_to_snake)); - $parallelism=4 unless (defined ($parallelism));# the MS jdbc driver errors when there are more than 5 sessions + $parallelism_in=1 unless (defined ($parallelism_in));# the jdbc driver often errors when there are several sessions to sql server + $parallelism_out=8 unless (defined ($parallelism_out)); $sort_size=10000 unless (defined ($sort_size)); $use_pk_if_possible=0 unless (defined ($use_pk_if_possible)); $validate_constraints='yes' unless (defined ($validate_constraints)); @@ -719,7 +722,8 @@ sub usage print "-pp: postgresql port\n"; print "-pu: postgresql username\n"; print "-pw: postgresql password\n"; - print "-p: parallelism level for the kettle job\n"; + print "-pi: parallelism level for the kettle job (input, sql server)\n"; + print "-po: parallelism level for the kettle job (output, postgresql)\n"; print "You may also choose to ignore insert errors (inserting will be much slower)\n"; print "-ignore_errors\n"; } @@ -829,7 +833,8 @@ sub generate_kettle $newtemplate =~ s/__sqlserver_table_cols__/$colsdef/g; $newtemplate =~ s/__postgres_table_name__/$pgtable/g; $newtemplate =~ s/__postgres_schema_name__/$pgschema/g; - $newtemplate =~ s/__PARALLELISM__/$parallelism/g; + $newtemplate =~ s/__PARALLELISM_IN__/$parallelism_in/g; + $newtemplate =~ s/__PARALLELISM_OUT__/$parallelism_out/g; if ($ignore_errors) { @@ -855,7 +860,8 @@ sub generate_kettle $newincrementaltemplate =~ s/__postgres_table_name__/$pgtable/g; $newincrementaltemplate =~ s/__postgres_schema_name__/$pgschema/g; $newincrementaltemplate =~ s/__postgres_table_cols__/$pgcolsdef/g; - $newincrementaltemplate =~ s/__PARALLELISM__/$parallelism/g; + $newincrementaltemplate =~ s/__PARALLELISM_IN__/$parallelism_in/g; + $newincrementaltemplate =~ s/__PARALLELISM_OUT__/$parallelism_out/g; $newincrementaltemplate =~ s/__sort_size__/$sort_size/g; # We have a bit of work to do on primary keys for the incremental template: we need them @@ -2734,7 +2740,8 @@ sub resolve_name_conflicts my $help = 0; my $options = GetOptions("k=s" => \$kettle, - "p=i" => \$parallelism, + "pi=i" => \$parallelism_in, + "po=i" => \$parallelism_out, "b=s" => \$before_file, "a=s" => \$after_file, "u=s" => \$unsure_file, @@ -2928,6 +2935,7 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3055,7 +3063,7 @@ BEGIN UserDefinedJavaClass Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -3167,7 +3175,7 @@ BEGIN TableOutput Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -3290,6 +3298,7 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3416,7 +3425,7 @@ BEGIN UserDefinedJavaClass Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -3502,7 +3511,7 @@ BEGIN TableInput Y - __PARALLELISM__ + __PARALLELISM_IN__ none @@ -3527,7 +3536,7 @@ BEGIN TableOutput Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -3899,7 +3908,7 @@ BEGIN UserDefinedJavaClass Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -4010,7 +4019,7 @@ BEGIN SortRows Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -4038,7 +4047,7 @@ BEGIN SortRows Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -4108,7 +4117,7 @@ BEGIN SynchronizeAfterMerge Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -4439,7 +4448,7 @@ BEGIN SynchronizeAfterMerge Y - __PARALLELISM__ + __PARALLELISM_OUT__ none From 129f69f35826fd0d16b1f881b397b321d4198fa8 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 19 Jun 2017 15:21:04 +0200 Subject: [PATCH 147/216] Revert to jtds. The doc was already ok, but not the code. --- sqlserver2pgsql.pl | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 794fe9d..44612e8 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2925,7 +2925,7 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQLNATIVE + MSSQL Native __sqlserver_database__ __sqlserver_port__ @@ -2939,13 +2939,10 @@ BEGIN FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN - MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ - PRESERVE_RESERVED_WORD_CASEY QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN - SUPPORTS_TIMESTAMP_DATA_TYPEN USE_POOLINGN @@ -3288,7 +3285,7 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQLNATIVE + MSSQL Native __sqlserver_database__ __sqlserver_port__ @@ -3302,13 +3299,10 @@ BEGIN FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN - MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ - PRESERVE_RESERVED_WORD_CASEY QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN - SUPPORTS_TIMESTAMP_DATA_TYPEN USE_POOLINGN @@ -3851,7 +3845,7 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQLNATIVE + MSSQL Native __sqlserver_database__ __sqlserver_port__ @@ -3864,13 +3858,10 @@ BEGIN FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN - MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ - PRESERVE_RESERVED_WORD_CASEY QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN - SUPPORTS_TIMESTAMP_DATA_TYPEN USE_POOLINGN @@ -4280,7 +4271,7 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQLNATIVE + MSSQL Native __sqlserver_database__ __sqlserver_port__ @@ -4293,13 +4284,10 @@ BEGIN FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN - MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN PORT_NUMBER__sqlserver_port__ - PRESERVE_RESERVED_WORD_CASEY QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN - SUPPORTS_TIMESTAMP_DATA_TYPEN USE_POOLINGN From 9f3d916366415150e80ad0e25accb9d4c4097484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zekai=20O=C4=9Fuz=20=C3=96zalp?= Date: Thu, 29 Jun 2017 21:09:46 +0300 Subject: [PATCH 148/216] Update sqlserver2pgsql.pl Bug fixed on SET IDENTITY_INSERT commands. --- sqlserver2pgsql.pl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 44612e8..a9e875b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1525,7 +1525,17 @@ sub parse_dump next MAIN if ($contline =~ /^GO$/); } } + elsif ($line =~ /SET\s+IDENTITY_INSERT\s+\[(.*)\]/i) + { + print STDERR "Warning: SET IDENTITY_INSERT ignored\n"; + # We have to find next GO to know we are out of the procedure + while (my $contline = read_and_clean($file)) + { + next MAIN if ($contline eq ''); + } + } + # Now we parse the create view. It is multi-line, so the code looks like like create table: we parse everything until a line # containing only a single quote (end of the dbo.sp_executesql) # The problem is that SQL Server seems to be spitting the original query used to create the view, not a normalized version From 3e07c7b42724dffcc7f447cf10e13cbfeb9411fe Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Fri, 30 Jun 2017 14:31:54 +0200 Subject: [PATCH 149/216] Fix #67 --- sqlserver2pgsql.pl | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index a9e875b..a01ce68 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1535,7 +1535,7 @@ sub parse_dump next MAIN if ($contline eq ''); } } - + # Now we parse the create view. It is multi-line, so the code looks like like create table: we parse everything until a line # containing only a single quote (end of the dbo.sp_executesql) # The problem is that SQL Server seems to be spitting the original query used to create the view, not a normalized version @@ -2040,6 +2040,18 @@ sub parse_dump $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{COLS} ->{$subobjname}->{COMMENT} = $comment; } + elsif ($obj eq 'TABLE' and $subobj eq 'CONSTRAINT') + { + # It's a bit tedious: constraints are stored in an array for each table + # Find the one + foreach my $constraint (@{$objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{CONSTRAINTS}}) + { + if (defined ($constraint->{NAME}) and $constraint->{NAME} eq $subobjname) + { + $constraint->{COMMENT}=$comment; + } + } + } else { croak "Cannot understand this comment: $sqlproperty"; @@ -2521,6 +2533,23 @@ sub generate_schema } } } + # Another pass at constraints. This time we want to produce the comments + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) + { + + # We have all we need for FKs now. We can put all other constraints (except PK of course) + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + foreach my $constraint ( + @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) + { + next unless defined ($constraint->{COMMENT}); + print UNSURE "COMMENT ON CONSTRAINT " . format_identifier($constraint->{NAME}) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " IS '" + . $constraint->{COMMENT} . "';\n"; + + } + } + } # Default values while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) From b42f3959fba1eeb58a64e12acdb366a8823bc2cb Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 31 Jul 2017 14:41:46 +0200 Subject: [PATCH 150/216] Support an option to force SSL connection to the PostgreSQL DB --- sqlserver2pgsql.pl | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index a01ce68..6a764f4 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -49,6 +49,7 @@ our $parallelism_out; our $sort_size; our $use_pk_if_possible; +our $pforce_ssl; # Will be set if we detect GIS objects our $requires_postgis=0; @@ -101,6 +102,7 @@ sub parse_conf_file 'sort size' => 'sort_size', 'use pk if possible' => 'use_pk_if_possible', 'ignore errors' => 'ignore_errors', + 'postgresql force ssl' => 'pforce_ssl', ); # Open the conf file or die @@ -148,6 +150,7 @@ sub set_default_conf_values # Default ports for PostgreSQL and SQL Server $pp=5432 unless (defined ($pp)); $sp=1433 unless (defined ($sp)); + $pforce_ssl=0 unless (defined ($pforce_ssl)); } # Converts numeric(4,0) and similar to int, bigint, smallint @@ -724,6 +727,7 @@ sub usage print "-pw: postgresql password\n"; print "-pi: parallelism level for the kettle job (input, sql server)\n"; print "-po: parallelism level for the kettle job (output, postgresql)\n"; + print "-pforce_ssl: force a SSL session to PostgreSQL\n"; print "You may also choose to ignore insert errors (inserting will be much slower)\n"; print "-ignore_errors\n"; } @@ -842,6 +846,15 @@ sub generate_kettle $newtemplate =~ s/Y<\/use_batch>/N<\/use_batch>/g; # Cannot use batch mode with ignore errors } + if ($pforce_ssl) + { + $newtemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; + } + else + { + $newtemplate =~ s/__pforce_ssl__//g; + } + $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; @@ -864,7 +877,15 @@ sub generate_kettle $newincrementaltemplate =~ s/__PARALLELISM_OUT__/$parallelism_out/g; $newincrementaltemplate =~ s/__sort_size__/$sort_size/g; - # We have a bit of work to do on primary keys for the incremental template: we need them + if ($pforce_ssl) + { + $newincrementaltemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; + } + else + { + $newincrementaltemplate =~ s/__pforce_ssl__//g; + } +# We have a bit of work to do on primary keys for the incremental template: we need them # to compare the tables… if (defined($refschema->{TABLES}->{$table}->{PK}->{COLS})) { @@ -2807,7 +2828,8 @@ sub resolve_name_conflicts "validate_constraints=s" =>\$validate_constraints, "sort_size=i" =>\$sort_size, "use_pk_if_possible=s" =>\$use_pk_if_possible, - "ignore_errors" => \$ignore_errors); + "ignore_errors" => \$ignore_errors, + "pforce_ssl" => \$pforce_ssl); # We don't understand command line or have been asked for usage if (not $options or $help) @@ -2975,6 +2997,7 @@ BEGIN EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -2999,6 +3022,7 @@ BEGIN EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3335,6 +3359,7 @@ BEGIN EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN From 839fee12a10d80356417ce4e14d21861d801d9e8 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 31 Jul 2017 14:44:00 +0200 Subject: [PATCH 151/216] update README for -pforce_ssl --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f1be992..b594d09 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ cleartext, so don't make this directory public): `-pp` : postgresql port `-pu` : postgresql username `-pw` : postgresql password +`-pforce_ssl` : force a SSL connection to your PostgreSQL database. ssl=on should be set on the PostgreSQL server `-f` : the SQL Server structure dump file -ignore_errors : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) From d2d8ab0ec74a0c74399c0a7151f904148d8e93b4 Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Mon, 21 Aug 2017 17:21:27 +0200 Subject: [PATCH 152/216] Correct issue #70 --- sqlserver2pgsql.pl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 6a764f4..6d3194f 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2996,8 +2996,6 @@ BEGIN - EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue - __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3358,8 +3356,6 @@ BEGIN - EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue - __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3383,6 +3379,8 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3662,6 +3660,8 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3896,6 +3896,8 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -4322,6 +4324,8 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN From c0cb1c55d092fdc64a3f05b05f3fb99ddc983aba Mon Sep 17 00:00:00 2001 From: Marc Cousin Date: Tue, 22 Aug 2017 09:37:39 +0200 Subject: [PATCH 153/216] Should finally solve #70 --- sqlserver2pgsql.pl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 6d3194f..dcac9d9 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1091,7 +1091,8 @@ sub generate_kettle keys %{$refschema->{TABLES}}) { $beforescript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " DISABLE TRIGGER ALL;\n"; - $afterscript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ENABLE TRIGGER ALL;\n"; + + $afterscript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ENABLE TRIGGER ALL;\n"; } } @@ -1104,6 +1105,14 @@ sub generate_kettle $job_header =~ s/__postgres_port__/$pp/g; $job_header =~ s/__postgres_username__/$pu/g; $job_header =~ s/__postgres_password__/$pw/g; + if ($pforce_ssl) + { + $job_header =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; + } + else + { + $job_header =~ s/__pforce_ssl__//g; + } print JOBFILE $job_header; print JOBFILE $entries; From 17ea8330cf7dd4c9abedb9a4bf4bac110a48b246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=CC=8Cygimantas=20Kazlauskas?= Date: Sun, 3 Dec 2017 19:40:39 +0200 Subject: [PATCH 154/216] sql server host instance fix --- sqlserver2pgsql.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index dcac9d9..571169d 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -3005,6 +3005,7 @@ BEGIN + EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3365,6 +3366,7 @@ BEGIN + EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3930,6 +3932,7 @@ BEGIN + EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -4358,6 +4361,7 @@ BEGIN + EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN From 117c662a9f6cb211bf41777d91c51057f8a16ae3 Mon Sep 17 00:00:00 2001 From: Bradley Sacks Date: Fri, 15 Dec 2017 09:00:13 -0800 Subject: [PATCH 155/216] Found that pattern for 'NOT FOR REPLICATION' wasn't matching. Adjusted regex. --- sqlserver2pgsql.pl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index dcac9d9..791895d 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1973,10 +1973,10 @@ sub parse_dump { $constraint->{ON_UPD_CASC} = 1; } - elsif ($fk =~ /^NOT FOR REPLICATION$/) - { - next; # We don't care for this, it has no meaning for PostgreSQL - } + elsif ($fk =~ /^NOT FOR REPLICATION\s*$/) + { + next; # We don't care for this, it has no meaning for PostgreSQL + } else { croak "Cannot parse $fk $., in a FK. This is a bug"; From 5b7869e263f736908471e153f574684d346177fc Mon Sep 17 00:00:00 2001 From: madtibo Date: Thu, 22 Mar 2018 18:17:43 +0100 Subject: [PATCH 156/216] parse where clause and AND/OR structures --- sqlserver2pgsql.pl | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index dcac9d9..c621045 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -445,6 +445,7 @@ sub is_pk_sort_order_safe sub rename_identifier { my ($identifier)=@_; + if ($case_treatment==1) { $identifier=lc($identifier); @@ -483,21 +484,36 @@ sub format_identifier_cols_index } # This one will try to convert what can obviously be converted from transact to (or embedded WHERE in indexes for instance) PG +# We check if we have several blocks separated by logical operators # Things such as getdate() which can become CURRENT_TIMESTAMP sub convert_transactsql_code { my ($code)=@_; - if ($case_treatment==0) - { - $code =~ s/[\[\]]/"/gi; # Bit brutal probably + # print STDERR "convert: $code\n"; + + if ($code =~ /^\((.*)\)\s+(AND|OR)\s+\((.*)\)$/) { + my ($lhs,$op,$rhs)=($1,$2,$3); + $code = "(".convert_transactsql_code("$lhs").") $op (".convert_transactsql_code("$rhs").")"; } - else - { - $code =~ s/\[(.*)\]/rename_identifier($1)/gie; # Bit brutal probably - } - $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; - $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; - $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; + elsif ($code =~ /^(.*)\s+(AND|OR)\s+(.*)$/) { + my ($lhs,$op,$rhs)=($1,$2,$3); + $code = convert_transactsql_code("$lhs")." $op ".convert_transactsql_code("$rhs"); + } + else { + if ($case_treatment==0) + { + $code =~ s/[\[\]]/"/gi; # Bit brutal probably + } + else + { + $code =~ s/\[(.*)\]/rename_identifier($1)/gie; # Bit brutal probably + } + $code =~ s/ISNULL\s*\(/COALESCE(/gi; + $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; + $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; + $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; + } + # print STDERR "to: $code\n\n"; return $code; } @@ -1924,7 +1940,7 @@ sub parse_dump # FK constraint. It's multi line, we have to look for references, and what to do on update, delete, etc (I have only seen delete cascade for now) # Constraint name is optionnal elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? FOREIGN KEY\((.*?)\)/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK(?: NOT FOR REPLICATION)?\s+ADD(?:\s+CONSTRAINT \[(.*)\])? FOREIGN KEY\((.*?)\)/ ) { # This is a FK definition. We have the foreign table definition in next line. @@ -1973,7 +1989,7 @@ sub parse_dump { $constraint->{ON_UPD_CASC} = 1; } - elsif ($fk =~ /^NOT FOR REPLICATION$/) + elsif ($fk =~ /^NOT FOR REPLICATION\s*$/) { next; # We don't care for this, it has no meaning for PostgreSQL } @@ -1986,7 +2002,7 @@ sub parse_dump # Check constraint. As it can be arbitrary code, we just get this code, and hope it will work on PG (it will be stored in a special script file) elsif ($line =~ - /ALTER TABLE \[(.*)\]\.\[(.*)\] WITH (?:NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? CHECK \(\((.*)\)\)/ + /ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? CHECK(?: NOT FOR REPLICATION)?\s+\(\((.*)\)\)/ ) { # Check constraint. We'll do what we can, syntax may be different. @@ -2031,7 +2047,7 @@ sub parse_dump or croak "Cannot find a name for this extended property: $sqlproperty"; my $propertyname = $1; - if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1|Display Name|Description|Example Values|Source System|Table Description|Table Type|ETL Rules|Display Folder|SCD Type|Source Datatype)$/) + if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1|MS_DiagramPane2|Display Name|Description|Example Values|Source System|Table Description|Table Type|ETL Rules|Display Folder|SCD Type|Source Datatype)$/) { # We don't dump these. They are graphical descriptions of the GUI next; From 7988448c2d89018af1626ef1a195acb960b676a8 Mon Sep 17 00:00:00 2001 From: madtibo Date: Fri, 23 Mar 2018 17:24:44 +0100 Subject: [PATCH 157/216] there can be 'WITH (NO)CHECK' option in the primary key definition on alter table --- sqlserver2pgsql.pl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index dcac9d9..8a5eee2 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1823,7 +1823,7 @@ sub parse_dump # Table constraints # Primary key. Multiline elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED|NONCLUSTERED)?/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+(?:WITH (?:NO)?CHECK )?ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED|NONCLUSTERED)?/ ) { my $schemaname=relabel_schemas($1); @@ -1833,13 +1833,12 @@ sub parse_dump if (defined $3) { $constraint->{NAME} = $3; - } - + } CONS: while (my $consline= read_and_clean($file)) { next if ($consline =~ /^\($/); - if ($consline =~ /^\t\[(.*)\] ASC,?$/) + if ($consline =~ /^\t\[(.*)\](?:ASC)?,?$/) { push @{$constraint->{COLS}}, ($1); } From 0444f0a62592e912ba242047ff36adb6b5f637b6 Mon Sep 17 00:00:00 2001 From: madtibo Date: Wed, 4 Apr 2018 15:37:23 +0200 Subject: [PATCH 158/216] add a regression test file --- regression/reg_tests.sql | Bin 0 -> 4086 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 regression/reg_tests.sql diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql new file mode 100644 index 0000000000000000000000000000000000000000..910c537cb8f5fb9164230ab831d67e67761b7966 GIT binary patch literal 4086 zcmeHK+iuf95S>@@6@GLiglG$bN)XAUcCeB#$AW#xiD1x5{&e^fMoB9%$ zQX~*s#cOA0W@qPm_WKX%%8`T;$OKOh&p@8Zz`W1oL|)5U=5kMFSUHtX@)`3n{yoFj zTzqNC9jqlXl|xA-k|B6wJm-)#lclV{!S@9JP0Tz6#X#<6oR%X6-$7%(186uly2mnt zWbT<{+zWXN9nbOQ%R{*@?UH>%P^aJ@Lh~a?Qhi;{odTa{=o3&+z!PB4#GGjc4f>Eq z3v!KbxfLmQ1nL;}2{6CHcOP_|X_Evp2hf3hA=X3Kz=Lh-EgOKo3#zuffaHlh2ak_^ z0&vrg3Fc5<;$Kgm;x~k6BG}wB`_R|1d5a-s3=V3-y=jj_$e9{#irlgEE@@NjF$HaU zd7jOk0^{c)G)y5|=K{PTMjD zH?2Q6R-;9#R(n`$VKmK)V`RxYxxjNN3&h<5^BQr_Y_S=##CnRgwR`~WDx*Cy*|j}W zsw3~CyocO+9c~tZ^_u^W#M_TV=7v^>>(*z_sSjOL31)a7S;0ENI>US~t7ql)+?C*? z-uM-jzk{0NXIU6PK6fhb5l;-|zSz+=RadjQUj8n2u!A+e$zFR74?Ba77V3>={hzmW zSqJ<#EzPRSDqZi_2Uk4Q%&+Y8cHEX<*(Y7!Y|A6px96zRD|usZ6En|dV~&kTP5ZDD zkC5~1&f2TBTWQ9*G8Hj(V9{OK1J-$*+2E1MBAX9I9?iU*Ysc^@m+CNcuw0JJ+R;Pz$&BPW4I%ZxH+B=#4R4v?|X;??w1=Cb}agWc_oo^tOIY z>2-bAwcRF8sm7EpDpcuTar)vJEH-jAE<^torItyt$e@KT#Av5QnEk#%pX@P2aOIXH zR#oKQ!5gu7^_eZ&zG~-O>k)1POCT@a3ckVXr+8Xy*5uwntZgk*#FW3*Kg#uTBl@}( zC%=j4U9C#~rv7zdG-adaRy}0TkTD%bJL2fbX8SY SV=eEMMVaN*w@hYf#r+AKdp0rv literal 0 HcmV?d00001 From a7ccf6cb36875423b1d9f429c02dc6891721e655 Mon Sep 17 00:00:00 2001 From: madtibo Date: Wed, 4 Apr 2018 15:37:39 +0200 Subject: [PATCH 159/216] update contributors --- contributors | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/contributors b/contributors index 9775146..78dee8c 100644 --- a/contributors +++ b/contributors @@ -1,17 +1,20 @@ Marc Cousin, main developper Bill Ruddock (binarii) -Julien Rouhaud (rjuju) -Yann Verry (yanntech) -Konstantin Mosolov (kmosolov) Javier Callico (JCallico) Joshua F. Rountree (joshuairl) +Julien Rouhaud (rjuju) +Konstantin Mosolov (kmosolov) +Thibaut Madelaine (madtibo) +Yann Verry (yanntech) +Žygimantas Kazlauskas (zygimantaskazlauskas) And the following unindentified github users (please tell me if you want your name added/replaced) -stuey1978 -sebpcspkr +bsacks99 keyjote +sebpcspkr +stuey1978 If I forgot your name in this list, please tell me. From 03a104e496ff8fa602845a7a336ae4d0583136a9 Mon Sep 17 00:00:00 2001 From: madtibo Date: Mon, 28 May 2018 09:59:59 +0200 Subject: [PATCH 160/216] add default values test --- regression/reg_tests.sql | Bin 4086 -> 5646 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index 910c537cb8f5fb9164230ab831d67e67761b7966..f8c96080ff095ba78cfd6ca34aff0fe2366b7de9 100644 GIT binary patch literal 5646 zcmeHLYfls`P$Diwpf5s*@6uXm5F7jF+nzJSFthH?vO;ar zrrEF$_P*x6=G^`9Qz~*IRSBena~|hF_N1ZidvYOXGM2u)kRIM#%B@UsJ;lE_cQ^|gU(1@vvlvQXLOQga;#ux0=WKKdF~z&xsfwgPKuzh3M1EaLPn zb1>@tzVaG9;(L35cR3tc>*5qu@=5M+9?CiL?i|-K@}AXVD`bfG1H2o{7toGov=^$n z7FUWt^S;Vw$W6ClwG2$R{C_4seI~Ltba!}cf6hDop@J^K3O_#LBbPXs{8f z=^yUIEz~@BXT4YJ-AY%SD^nw;U3j!4Wni7HnGZHq71?@-^XSSmTx*I?b7>oT4wf^> z%+=#^WY?o>mU?*no~m2a8>?HG28KXhoC>}p>X$fMY?fv2Al9~?iDSy;_K#${ z+>Cx~$1#(Lp4EKvqxPGeW~Ibu5i^NsFKIv6sUscz{0$z{b@mOpaRX$|sdkm_oV6os zI$syoxn}mtxXk44n>^=If6 z$zLrv5p%NU)XDC}FB5SU{ zJto`A+Ny`%z=PTJ-p=V))tpV=s#+RqhoCRB>0_o`E8nh?FUheyhAsE8++04ACE&vi zwU3${@N8$SRJMgxTa From b074c5b08c02abc023b1bbfa7ff02e44d09d8311 Mon Sep 17 00:00:00 2001 From: madtibo Date: Wed, 30 May 2018 14:04:44 +0200 Subject: [PATCH 161/216] issue #59 add MS_SSMA_SOURCE as a comment propertyname --- regression/issue_59.sql | 27 +++++++++++++++++++++++++++ sqlserver2pgsql.pl | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 regression/issue_59.sql diff --git a/regression/issue_59.sql b/regression/issue_59.sql new file mode 100644 index 0000000..57d7432 --- /dev/null +++ b/regression/issue_59.sql @@ -0,0 +1,27 @@ +CREATE TABLE [dbo].[ACCOUNT]( + [ID] [char](36) NOT NULL, + [VERSION] [numeric](10, 0) NOT NULL, + [APPLICATION_ID] [char](36) NOT NULL, + [ACCOUNT_NUMBER] [nvarchar](35) NOT NULL, + [DELETED] [numeric](1, 0) NOT NULL, + [DELETED_BY] [char](36) NULL, + [DELETED_ON] [datetime2](0) NULL, + [CREATED_ON] [datetime2](0) NULL, + [CREATED_BY] [char](36) NULL, + [DAILY_LIMIT] [numeric](21, 7) NULL, + [BIC] [varchar](35) NULL, + [IBAN] [varchar](35) NULL, + [BACK_OFFICE_ACCOUNT_NUMBER] [varchar](35) NULL, + [BANK_ACCOUNT_NUMBER] [varchar](35) NULL, + [OWNER_ID] [char](36) NULL, + [ALLOW_PENDING] [numeric](1, 0) NOT NULL DEFAULT ((0)), + CONSTRAINT [SYS_C0010802] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] + + + +EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT.ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT', @level2type=N'COLUMN',@level2name=N'ID' +GO diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index fb8de27..3542155 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2052,7 +2052,7 @@ sub parse_dump next; } - elsif ($propertyname eq 'MS_Description') + elsif ($propertyname =~ /^(MS_Description|MS_SSMA_SOURCE)$/) { # This is a comment. We parse it. From c709fe8cc3f102a697b5d721ac91bfd7610ea59f Mon Sep 17 00:00:00 2001 From: madtibo Date: Fri, 1 Jun 2018 12:25:49 +0200 Subject: [PATCH 162/216] issue #59 add index comments --- regression/issue_59.sql | 31 ++++++++++++++++++++- sqlserver2pgsql.pl | 62 ++++++++++++++++++++++++++++++----------- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/regression/issue_59.sql b/regression/issue_59.sql index 57d7432..ef11c9c 100644 --- a/regression/issue_59.sql +++ b/regression/issue_59.sql @@ -22,6 +22,35 @@ CREATE TABLE [dbo].[ACCOUNT]( ) ON [PRIMARY] - EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT.ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT', @level2type=N'COLUMN',@level2name=N'ID' GO + + +CREATE NONCLUSTERED INDEX [IDX_ACCOUNT_ID] ON [dbo].[ACCOUNT] +( + [ID] ASC +) +GO +EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT.ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT', @level2type=N'INDEX',@level2name=N'IDX_ACCOUNT_ID' +GO + +CREATE NONCLUSTERED INDEX [IDX_ACCOUNT_VERSION] ON [dbo].[ACCOUNT] +( + [VERSION] ASC +) +WHERE ((ISNULL([VERSION], 0) > 1)) +GO +EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT.VERSION' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT', @level2type=N'INDEX',@level2name=N'IDX_ACCOUNT_VERSION' +GO + +CREATE TABLE [dbo].[ACCOUNT_CATEGORY]( + [ID] [char](36) NOT NULL, + [VERSION] [numeric](10, 0) NOT NULL, + CONSTRAINT [SYS_C0010844] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT_CATEGORY.UQ_INDEX' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT_CATEGORY', @level2type=N'INDEX',@level2name=N'UQ_INDEX' +GO diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3542155..5312991 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2096,6 +2096,11 @@ sub parse_dump $constraint->{COMMENT}=$comment; } } + } + elsif ($obj eq 'TABLE' and $subobj eq 'INDEX') + { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{INDEXES} + ->{$subobjname}->{COMMENT} = $comment; } else { @@ -2473,9 +2478,11 @@ sub generate_schema # They don't have a schema qualifier. But their table has, and they are in the same schema as their table foreach my $table (sort keys %{$refschema->{TABLES}}) { - foreach my $index ( - sort keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) + foreach + my $index ( + sort keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) { + my $index_created = 0; my $idxref = $refschema->{TABLES}->{$table}->{INDEXES}->{$index}; my $idxdef = "CREATE"; @@ -2483,22 +2490,44 @@ sub generate_schema { $idxdef .= " UNIQUE"; } - $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" - . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ")"; - if (not defined $idxref->{WHERE}) + if (defined $idxref->{COLS}) { - $idxdef .= ";\n"; - print AFTER $idxdef; - } - else - { - print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; - $idxdef .= "\nWHERE (" . convert_transactsql_code($idxref->{WHERE}) . ");\n"; - print UNSURE $idxdef; + $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" + . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ")"; + if (not defined $idxref->{WHERE}) + { + $idxdef .= ";\n"; + print AFTER $idxdef; + # the possible comment would go to after file + $index_created = 1; + } + else + { + print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; + $idxdef .= "\nWHERE (" . convert_transactsql_code($idxref->{WHERE}) . ");\n"; + print UNSURE $idxdef; + # the possible comment would go to unsure file + $index_created = 2; + } + + # Produce the comments for indexes + if (defined $idxref->{COMMENT}) + { + my $idxcomment = "COMMENT ON INDEX ". format_identifier($schema) . '.' . format_identifier($index) . " IS '" . $idxref->{COMMENT} . "';\n"; + if ($index_created == 1) + { + print AFTER $idxcomment; + } + elsif ($index_created == 2) + { + print UNSURE $idxcomment; + } + } + } - } - } - } + } + } + } # Other constraints while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { @@ -2635,7 +2664,6 @@ sub generate_schema } } - # Comments on tables and columns while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { From 6c8a3fd621aca11a28ab10b6b5ec43e7b54a5c94 Mon Sep 17 00:00:00 2001 From: madtibo Date: Tue, 5 Jun 2018 13:17:59 +0200 Subject: [PATCH 163/216] space before ASC on constraint --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 5312991..c77a1d6 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1854,7 +1854,7 @@ sub parse_dump CONS: while (my $consline= read_and_clean($file)) { next if ($consline =~ /^\($/); - if ($consline =~ /^\t\[(.*)\](?:ASC)?,?$/) + if ($consline =~ /^\t\[(.*)\] (?:ASC)?,?$/) { push @{$constraint->{COLS}}, ($1); } From 9631eead5819cb4941498c1cada1e2d963ea1438 Mon Sep 17 00:00:00 2001 From: "magzhan.karasayev" Date: Thu, 24 May 2018 13:41:04 +0600 Subject: [PATCH 164/216] Upd: typo fixed --- INSTALL.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 62efd3f..1490edf 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -6,7 +6,7 @@ has no dependancy on fancy Perl modules, it only uses modules from the base perl Just run the script with your perl interpreter, providing it with the requested options (--help will tell you what to do). -If you are trying to run the script under Windows, you probably won't have a Perl interpreter. I recommand you use the latest Strawberry Perl (use either the installer or the portable version). +If you are trying to run the script under Windows, you probably won't have a Perl interpreter. I recommend you to use the latest Strawberry Perl (use either the installer or the portable version). Kettle: ========================== @@ -18,5 +18,3 @@ On newest versions of Kettle, the SQL Server java driver isn't included. You'll If you don't want to bother with installing a JVM system-wide for Kettle, just download Sun's JVM and put in in Kettle's directory, in a "java" subdirectory. Latest versions of Kettle require Java 8. You'll also need a SQL Server account with the permission to SELECT from the tables you want to migrate. - - From dd107c2342586252dcf26c93bca6c3581824cfca Mon Sep 17 00:00:00 2001 From: "magzhan.karasayev" Date: Tue, 5 Jun 2018 18:18:42 +0600 Subject: [PATCH 165/216] Fix: column definition fixed because microsoft generated a column definition with an extra space --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index fb8de27..2513ec9 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1337,7 +1337,7 @@ sub parse_dump # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ - /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\]\s*(\(.+?\))?(?: COLLATE (\S+))?( IDENTITY\s*\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE )?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ + /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\]\s*(\(.+?\))?(?: COLLATE (\S+))?( IDENTITY\s*\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE +)?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ ) { #Deported into a function because we can also meet alter table add columns on their own From 70e28ea23295a7de61539050b10d6ac257cf3de8 Mon Sep 17 00:00:00 2001 From: Philippe Beaudoin Date: Wed, 20 Jun 2018 09:51:02 +0200 Subject: [PATCH 166/216] Fix a bug in the generation of .ktr kettle job files for incremental data migration. The problem appears when column names are in uppercase in the source database but lowercase in the postgres database. When preparing the sql statement reading the postgres table, one should call the format_identifier function (which itself calls the format_identifier function, which takes into account the value of the -keep_identifier_case parameter), instead of just putting double quotes around the column names. --- sqlserver2pgsql.pl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 9a2a508..0906964 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -392,13 +392,13 @@ sub postgres_convert_column 'timestamp with time zone' => 'to_char({colname} AT TIME ZONE \'UTC\', \'YYYY-MM-DD HH:MI:SS.US+00\')'); if (defined ($functions{$coltype})) { - my $tmpcol = $functions{$coltype}; - $tmpcol =~ s/\{colname\}/"$colname"/; + my $tmpcol = $functions{$coltype}; + $tmpcol =~ s/\{colname\}/$colname/; return $tmpcol; } else { - return "\"$colname\""; + return $colname; } } @@ -821,7 +821,7 @@ sub generate_kettle { my $coldef = sql_convert_column($col,$refschema->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}) . " AS " . format_identifier($col); - my $pgcoldef = postgres_convert_column($col,$refschema->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}) . " AS " . format_identifier($col); + my $pgcoldef = postgres_convert_column(format_identifier($col),$refschema->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}) . " AS " . format_identifier($col); push @colsdef,($coldef); push @pgcolsdef,($pgcoldef); } From bebf86dd712427b027e9e5383559d2e09a55db6d Mon Sep 17 00:00:00 2001 From: madtibo Date: Wed, 20 Jun 2018 10:51:38 +0200 Subject: [PATCH 167/216] update contributors --- contributors | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contributors b/contributors index 78dee8c..8e419fd 100644 --- a/contributors +++ b/contributors @@ -5,6 +5,7 @@ Javier Callico (JCallico) Joshua F. Rountree (joshuairl) Julien Rouhaud (rjuju) Konstantin Mosolov (kmosolov) +Philippe Baudoin (beaud76) Thibaut Madelaine (madtibo) Yann Verry (yanntech) Žygimantas Kazlauskas (zygimantaskazlauskas) @@ -13,6 +14,7 @@ And the following unindentified github users (please tell me if you want your na bsacks99 keyjote +mark-jay sebpcspkr stuey1978 From 264091941e56b9f45c835a8393fe3f9d7a016941 Mon Sep 17 00:00:00 2001 From: madtibo Date: Wed, 20 Jun 2018 13:35:14 +0200 Subject: [PATCH 168/216] parse primary key constraint with no order on column --- FAQ.md | 2 +- regression/reg_tests.sql | Bin 5646 -> 8814 bytes sqlserver2pgsql.pl | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/FAQ.md b/FAQ.md index 6b909aa..1d5686b 100644 --- a/FAQ.md +++ b/FAQ.md @@ -39,4 +39,4 @@ doesn't exist either in PG. So more constraints will fail. Can this tool migrate functions and stored procedures? ---------------------------------- -No, Transact-SQL is very different from PostgreSQL's many PL languages. You could give a look at https://bitbucket.org/openscg/pgtsql but I can't help you with this… and I don't think it is a long term viable option. +No, Transact-SQL is very different from PostgreSQL's many PL languages. These would need a manual migration. \ No newline at end of file diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index f8c96080ff095ba78cfd6ca34aff0fe2366b7de9..16280717a6310a586a5858584c440884b62a5119 100644 GIT binary patch literal 8814 zcmeHMZBJrH6h2?;uizslP16Opu1PgP1ed5_1p2bX5LewTtt&*(+CSg+dFC+8jCba8 z5qFz5%?;cOch1Y4*Joz_{7dR`CJm`c58owx*JNLg&GSI6f;Cps7J- z+stVI4UWK#7UVlVSu2w7DMq(opBnBzZ_ha1a7~l0YYa6}%(VWL2v>Ad!YJ3wjU>r3u zwhnEnzs_}M?s3|dF&J9^+*pkkNxeP9yA51L^Wp+o@~hnAdnA{LyGz{1h%a9S? z5Akj+pD^}zKK9CF*YZqBN8Ut!1K)fd){DS=&Hswy1FOS9^-O6U7^>xqYo`IUX%U7yRZ?32E4mgSM}+dEY0 zTlr{k6En|cqs2xXP5ZDDpCaekowZkMx6+LBc`9PsgGDP+1=e{px52i_B9{+Y8qGXQ z^~%sGmFh6?u$)C^z8s&TxE#${%Hj1gS+}glrK?;MwQzawRIfDP4PqaQ-WbDKt8z~C zUK1bQiSCFAS^s=2y{uo8dtTr5Z5QH{G^X@Xp>lu3?6Y%l*vQv7hW;H&jd5|vU;|x< z(N2pn`^yG>vd0j?#Vtv!s>pp0PsHNsIXkrd)XrAxQ>+F?KwexG`~k0Ds05W6C~$?C-&4c0S7l6Frcl+rP4!S!Wm4YV?P(D4>ktcC9x z=0RL>mz>dTK03W0Q*35kQ$ufH!D4c!i}CA9E+%hXDaWQm(3ZvIF;YGn+pZHYabKQ5 zm&Z(OKHrlS;KSFZkJ=bYc{46~9bZ;~Pq*beeAvgW{Lsd6sjGXmg8bitq%EM~DrEB6 zmzwf%T>f)Cyn`7NT~{HUHJF;bf&Hn?_8h5)dNq+%WB7cQ%;FD=5v)^o)nIW{eg+rC zFU9);_l&tue1_^vJi$Fi?#VSgWX0~mzRKEE$Ly}~q&4=zWSYO&D_11AxG28?18v8sL>g%<+)nMf- z*qaD&id;z9+u118CZw;dNnc%=z79Lg3_d54h{Xx37~?@@vIn^i`ek}SxwN5 zjrSO*Y0oiNvobp}F}`EgYBWuORAwf&XYeHdw-JBk{d%f9*C}_1XzK2G6{lC3GoZ9- z&K$BQ8a%scLL&F9Z}6erd-!lxXPro{n=M*}ywoMjMu5-SPWM}HU?VYK$tJ~Sqy_6m&(1Ry6 zw9GZ0_H!zYn@iY{){ESf2C=ARKC?0mO{TABLES}->{$table}->{PK}->{COLS})) { my @pk=@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}; - my $keys; + my $keys=""; foreach my $pk(@pk) { $keys.="$pk\n"; @@ -1854,7 +1854,7 @@ sub parse_dump CONS: while (my $consline= read_and_clean($file)) { next if ($consline =~ /^\($/); - if ($consline =~ /^\t\[(.*)\] (?:ASC)?,?$/) + if ($consline =~ /^\t\[(.*)\]\s+(?:ASC)?,?$/) { push @{$constraint->{COLS}}, ($1); } From 7e7d94265de8846b9b2275af5cb551b2deae71b3 Mon Sep 17 00:00:00 2001 From: madtibo Date: Mon, 25 Jun 2018 17:40:46 +0200 Subject: [PATCH 169/216] #86 treat disabled indexes. Write them down in unsure file --- regression/reg_tests.sql | Bin 8814 -> 9556 bytes sqlserver2pgsql.pl | 67 ++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index 16280717a6310a586a5858584c440884b62a5119..19337028a3438cc1d4a70acaf4bee267c5bda9a6 100644 GIT binary patch delta 461 zcmaFoa>Z*yp3-Cix_+)8j zzR4H4HNuf)d>D*?>J=FLfhrY%x(Jv9RLjMn0mPg@wVn(vV7-nE!3@p}T$Ar`NJe`C z&2eFHWrzTqMwv|tK$ARyW;-%C0e#^*S%6s*%yQ3 fbh8~08!{M7zQrM(i{CHg+KT2YU|6~XUC#voDZ@>P delta 7 OcmccO_0DBOo)Q2MQv;Cz diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3118e2f..43f8fbd 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1802,18 +1802,29 @@ sub parse_dump ->{INDEXES}->{$idxname}->{WHERE}=$filter; } } + } + + # we do not take migrate spatial indexes + elsif ($line =~ /^CREATE SPATIAL INDEX/) + { + my $def=$line; + while (my $idx = read_and_clean($file)) + { + $def.=$idx; + } + print STDERR "This spatial index won't be migrated:\n$def\n"; } - elsif ($line =~ /^CREATE SPATIAL INDEX/) - { - my $def=$line; - while (my $idx = read_and_clean($file)) - { - $def.=$idx; - } - print STDERR "This spatial index won't be migrated:\n$def\n"; - } + elsif ($line =~ /^ALTER INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\] DISABLE$/) + { + my $idxname = $1; + my $schemaname = relabel_schemas($2); + my $tablename = $3; + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} + ->{$idxname}->{DISABLE} = 1; + } + # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table elsif ($line =~ @@ -2485,7 +2496,12 @@ sub generate_schema my $index_created = 0; my $idxref = $refschema->{TABLES}->{$table}->{INDEXES}->{$index}; - my $idxdef = "CREATE"; + my $idxdef .= ""; + if ($idxref->{DISABLE}) + { + $idxdef .= "-- "; + } + $idxdef .= "CREATE"; if ($idxref->{UNIQUE}) { $idxdef .= " UNIQUE"; @@ -2494,20 +2510,35 @@ sub generate_schema { $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ")"; - if (not defined $idxref->{WHERE}) + if (not defined $idxref->{WHERE} and not defined $idxref->{DISABLE}) { $idxdef .= ";\n"; print AFTER $idxdef; - # the possible comment would go to after file - $index_created = 1; + # the possible comment would go to after file + $index_created = 1; } else - { - print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; - $idxdef .= "\nWHERE (" . convert_transactsql_code($idxref->{WHERE}) . ");\n"; + { + # this is either a disabled index or an index with a where declaration + if (defined $idxref->{WHERE}) + { + print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; + if ($idxref->{DISABLE}) + { + # if disabled, will be on the same line + $idxdef .= " "; + } + else + { + # otherwise, write condition on a new line + $idxdef .= "\n"; + } + $idxdef .= "WHERE (" . convert_transactsql_code($idxref->{WHERE}) . ")"; + } + $idxdef .= ";\n"; print UNSURE $idxdef; - # the possible comment would go to unsure file - $index_created = 2; + # the possible comment would go to unsure file + $index_created = 2; } # Produce the comments for indexes From 7a088ab6be1b05743003524265a296b3e3e08c8e Mon Sep 17 00:00:00 2001 From: madtibo Date: Fri, 27 Jul 2018 11:45:58 +0200 Subject: [PATCH 170/216] add circleci tests - lint script --- .circleci/config.yml | 27 +++++++++++++++++++++++++++ t/lint.t | 5 +++++ 2 files changed, 32 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 t/lint.t diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..dc55a09 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,27 @@ +version: 2 + +jobs: + convert: + docker: [{image: "perl:5.24-threaded"}] + steps: + working_directory: /tmp/sqlserver2pgsql + steps: + - run: + name: Install debian packages + command: > + apt update -y && + apt install -y --no-install-recommends bats git openssh-client + - checkout + - run: + name: Executing tests + environment: + # Inject APT packaged dependencies. + PERL5LIB: /usr/lib/x86_64-linux-gnu/perl5/5.24:/usr/share/perl5 + command: prove --verbose + +workflows: + version: 2 + pipeline: + jobs: + - convert + #- test diff --git a/t/lint.t b/t/lint.t new file mode 100644 index 0000000..56dac54 --- /dev/null +++ b/t/lint.t @@ -0,0 +1,5 @@ +#!/usr/bin/env bats + +@test "PERL syntax check" { + perl -c sqlserver2pgsql.pl +} From ae80a7259879d1febc68d649f86837f5c2863000 Mon Sep 17 00:00:00 2001 From: madtibo Date: Fri, 27 Jul 2018 11:56:34 +0200 Subject: [PATCH 171/216] lint all scripts --- t/{lint.t => lint-sqlserver2pgsql.t} | 0 t/lint_kettle.t | 5 +++++ 2 files changed, 5 insertions(+) rename t/{lint.t => lint-sqlserver2pgsql.t} (100%) create mode 100644 t/lint_kettle.t diff --git a/t/lint.t b/t/lint-sqlserver2pgsql.t similarity index 100% rename from t/lint.t rename to t/lint-sqlserver2pgsql.t diff --git a/t/lint_kettle.t b/t/lint_kettle.t new file mode 100644 index 0000000..8ce0f0f --- /dev/null +++ b/t/lint_kettle.t @@ -0,0 +1,5 @@ +#!/usr/bin/env bats + +@test "PERL syntax check" { + perl -c kettle_report.pl +} From 83406c9eedf92261478637a8e159f054052e281f Mon Sep 17 00:00:00 2001 From: madtibo Date: Fri, 27 Jul 2018 12:08:46 +0200 Subject: [PATCH 172/216] first basic non reg test using regression/reg_tests.sql --- t/convert.t | 5 +++++ t/{lint_kettle.t => lint-kettle.t} | 0 2 files changed, 5 insertions(+) create mode 100644 t/convert.t rename t/{lint_kettle.t => lint-kettle.t} (100%) diff --git a/t/convert.t b/t/convert.t new file mode 100644 index 0000000..11054ac --- /dev/null +++ b/t/convert.t @@ -0,0 +1,5 @@ +#!/usr/bin/env bats + +@test "schema conversion test" { + ./sqlserver2pgsql.pl -f regression/reg_tests.sql -b /tmp/before -a /tmp/after -u /tmp/unsure -k /tmp/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 +} diff --git a/t/lint_kettle.t b/t/lint-kettle.t similarity index 100% rename from t/lint_kettle.t rename to t/lint-kettle.t From 4b0a11b0c0b49fe25ff0d01f9f1b6e9d5e816ac1 Mon Sep 17 00:00:00 2001 From: madtibo Date: Wed, 29 Aug 2018 00:22:12 +0200 Subject: [PATCH 173/216] format usage text and reorder options --- README.md | 8 +-- sqlserver2pgsql.pl | 156 ++++++++++++++++++++++++++++++--------------- 2 files changed, 108 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index b594d09..bebf4f4 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,10 @@ run sqlserver2pgsql.pl from. If you just want to convert this schema, run: ``` -./sqlserver2pgsql.pl -f sqlserver_sql_dump \ - -b output_before_script\ - -a output_after_script\ - -u output_unsure_script +./sqlserver2pgsql.pl -f sqlserver_sql_dump \ + -b output_before_script \ + -a output_after_script \ + -u output_unsure_script ``` The sqlserver2pgsql Perl script processes your SQL raw dump "sqlserver_sql_dump" and produces these three scripts: diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 43f8fbd..d65a64d 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -86,8 +86,8 @@ sub parse_conf_file 'postgresql username' => 'pu', 'postgresql password' => 'pw', 'kettle directory' => 'kettle', - 'parallelism_in' => 'parallelism_in', - 'parallelism_out' => 'parallelism_out', + 'parallelism_in' => 'parallelism_in', + 'parallelism_out' => 'parallelism_out', 'before file' => 'before_file', 'after file' => 'after_file', 'unsure file' => 'unsure_file', @@ -102,7 +102,7 @@ sub parse_conf_file 'sort size' => 'sort_size', 'use pk if possible' => 'use_pk_if_possible', 'ignore errors' => 'ignore_errors', - 'postgresql force ssl' => 'pforce_ssl', + 'postgresql force ssl' => 'pforce_ssl', ); # Open the conf file or die @@ -412,11 +412,13 @@ sub camel_to_snake return $string; } -# This function is used to determine if a PK will be sorted the same in SQL Server and PG +# This function is used to determine if a PK will be sorted the same in SQL +# Server and PostgreSQL. # It means that it doesn't depend on collation orders or other internals. -# For now, only numeric and date data types are considered OK -# Used for incremental jobs, to know if we can ask the databases to send us pre-sorted data -# We also filter on $use_pk_if_possible +# For now, only numeric and date data types are considered OK. +# Used for incremental jobs, to know if we can ask the databases to send us +# pre-sorted data. +# We also filter on $use_pk_if_possible. sub is_pk_sort_order_safe { my ($schema,$table)=@_; @@ -701,51 +703,101 @@ sub check_kettle_properties # Usage, obviously. Has to be kept in sync with new command line options sub usage { - print - "$0 [-k kettle_output_directory] -b before_file -a after_file -u unsure_file -f sql_server_schema_file[-h] [-i]\n"; - print - "\nExpects a SQL Server SQL structure dump as -f (preferably unicode)\n"; - print "-conf uses a conf file. All options below can also be set there. Command line options will overwrite conf options\n"; - print "-i tells $0 to create a case-insensitive PostgreSQL schema\n"; - print - "-nr tells $0 not to convert the dbo schema to public. dbo will stay dbo\n"; - print - "-num tells $0 to convert numeric xxx,0 to int, bigint, etc. Will not keep numeric scale and precision for the converted\n"; - print - "-relabel_schemas gives a list of schemas to rename. For instance -relabel_schemas 'source1=>dest1;source2=>dest2'\n"; - print " -nr simply cancels the default dbo=>public remapping. Don't forget to put the remapping between quotes\n"; - print - "-keep_identifier_case tells $0 to keep the case of sql server database objects (not advised). Default is to lowercase everything.\n"; - print - "-camel_to_snake tells $0 to convert the object names from camelCase to camel_case, which is more often used in PostgreSQL.\nDon't use this unless you are ready to do changes in the client.\n"; - print "before_file contains the structure\n"; - print "after_file contains index, constraints\n"; - print "validate_constraints validates the constraints that have been created\n"; - print "sort_size will change size of sort batch for the incremental job. Too small and it will be slow, too big and you will get Java Out of Heap Memory errors.\n"; - print "sort_size is 10000, which is very low, to try to avoid problems. First, raise java heap memory (in the kitchen script), then try higher values if you need more speed\n"; - print "use_pk_if_possible is false (0) by default. You can put it to 1 (true), or give a comma separated list of tables (with schema). Compared case insensitively\n"; - print - "unsure_file contains things we cannot guarantee will work, such as views\n"; - print "\n"; - print - "If you are generating for kettle, you'll need to provide connection information\n"; - print "for connecting to both databases:\n"; - print "-sd: sqlserver database\n"; - print "-sh: sqlserver host\n"; - print "-si: sqlserver host instance\n"; - print "-sp: sqlserver port\n"; - print "-su: sqlserver username\n"; - print "-sw: sqlserver password\n"; - print "-pd: postgresql database\n"; - print "-ph: postgresql host\n"; - print "-pp: postgresql port\n"; - print "-pu: postgresql username\n"; - print "-pw: postgresql password\n"; - print "-pi: parallelism level for the kettle job (input, sql server)\n"; - print "-po: parallelism level for the kettle job (output, postgresql)\n"; - print "-pforce_ssl: force a SSL session to PostgreSQL\n"; - print "You may also choose to ignore insert errors (inserting will be much slower)\n"; - print "-ignore_errors\n"; + print qq{ +Usage: + sqlserver2pgsql -b BEFORE_FILE -a AFTER_FILE -u UNSURE_FILE -f SQLSERVER_SCHEMA_FILE + +Description: + + This script is a migration tool to convert a Microsoft SQL Server Database + into a PostgreSQL database, as automatically as possible. It takes a Sql + Server SQL schema dump, and creates a postgresql dump in 3 parts: + before_file, after_file and unsure_file. + Optionnaly, using the '-k' option, it will generate a kettle job to + transfer all data. + +Mandatory options: + + SQL Server schema input file: + -f SQLSERVER_SCHEMA_FILE + a readable SQL Server SQL structure dump. + + PostgreSQL output schema files: + -b BEFORE_SCRIPT + contains what is needed to import data: types, tables and columns. + -a AFTER_SCRIPT + contains the rest: indexes, constraints. + -u UNSURE_SCRIPT + contains objects we attempt to migrate, but cannot guarantee, such + as views or complex indexes. + +Options: + + -conf CONFIGURATION_FILE + uses a configuration file. All options can be set there. Command + line options will overwrite conf options. + -i the resulting PostgreSQL names will be case-insensitive. + -nr the SQL Server 'dbo' schema will not be translated to PostgreSQL + 'public' schema. 'dbo' will stay 'dbo'. + -camel_to_snake + all object names are converted from 'camelCase' to 'camel_case', + which is more often used in PostgreSQL. Do not use this unless you + are ready to do SQL query changes in the client. + -relabel_schemas 'SOURCE1=>DEST1;SOURCE2=>DEST2' + gives a list of schemas to rename. Quote this option to prevent the + shell to alter it. The '-nr' option cancels the default 'dbo' to + 'public' remapping. + -keep_identifier_case + keep the case of SQL server database objects. This option is not + advised. Default is to lowercase everything. + -num convert numeric 'xxx,0' to int, bigint, etc. Will not keep numeric + scale and precision for the converted. + -validate_constraints {yes|no|after} + should the constraints be validated. Set to "yes" by default. If + set to "no", the constraints will be set as 'NOT VALID'. If set to + "after", the constraints will be set as + 'NOT VALID', but the validation queries will be enforced in the + unsure file. + -use_pk_if_possible {0|1|LIST_OF_TABLES} (Default 0) + determine if a primary key will be sorted in the same way in SQL + Server and PostgreSQL. Default '0', do not sort tables. If set to + '1' sort all tables. LIST_OF_TABLES gives a comma separated list of + tables to sort in the form 'schema1.table1,schema2.table2'. Cases + are compared insensitively. + + Kettle options: + if you are generating for kettle, you must provide connection information. + + -sd SQLSERVER_DATABASE + -sh SQLSERVER_HOST + -si SQLSERVER_HOST + -sp SQLSERVER_PORT + -su SQLSERVER_USERNAME + -sw SQLSERVER_PASSWORD + -pd POSTGRESQL_DATABASE + -ph POSTGRESQL_HOST + -pp POSTGRESQL_PORxT + -pu POSTGRESQL_USERNAME + -pw POSTGRESQL_PASSWORD + -pi PARALLELISM_IN + parallelism level for the kettle job (input, SQL Server). Default 1. + -po PARALLELISM_OUT + parallelism level for the kettle job (output, PostgreSQL). Default 8. + -pforce_ssl + force a SSL session to PostgreSQL + -k KETTLE_OUTPUT_DIRECTORY + -sort_size INTEGER + set the size of the sort batch for the incremental job. If set to a + small value, the process will be slow. If set to a too large value, + you will get Java 'Out of Heap Memory' errors. + The default value is 10000, which is very low, in order to avoid + problems. First, raise Java 'Heap Memory' in the kitchen script, + then try higher values if you need more speed. + -ignore_errors + choose to ignore insert errors. If this option is used, inserting + will be much slower. +}; + exit 0; } # This function generates kettle transformations, and a kettle job running all these From a09a35615ebb08bc5199a594f5ded33c883150d6 Mon Sep 17 00:00:00 2001 From: madtibo Date: Fri, 27 Jul 2018 12:20:40 +0200 Subject: [PATCH 174/216] Setup CI to create PG scripts and load them in a DB --- .circleci/config.yml | 37 +++++++++++++----------- sqlserver2pgsql.pl | 68 ++++++++++++++++++++++---------------------- t/convert.t | 5 ---- t/test_convert.t | 47 ++++++++++++++++++++++++++++++ t/test_db_creation.t | 15 ++++++++++ 5 files changed, 116 insertions(+), 56 deletions(-) delete mode 100644 t/convert.t create mode 100644 t/test_convert.t create mode 100644 t/test_db_creation.t diff --git a/.circleci/config.yml b/.circleci/config.yml index dc55a09..e677c50 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,27 +1,30 @@ version: 2 jobs: - convert: - docker: [{image: "perl:5.24-threaded"}] - steps: + test: + docker: + - image: perl:5.24-threaded + - image: postgres:10-alpine + environment: + # Inject APT packaged dependencies. + PERL5LIB: /usr/lib/x86_64-linux-gnu/perl5/5.24:/usr/share/perl5 + PGHOST: localhost + PGDATABASE: postgres + PGUSER: postgres working_directory: /tmp/sqlserver2pgsql steps: - - run: - name: Install debian packages - command: > - apt update -y && - apt install -y --no-install-recommends bats git openssh-client - - checkout - - run: - name: Executing tests - environment: - # Inject APT packaged dependencies. - PERL5LIB: /usr/lib/x86_64-linux-gnu/perl5/5.24:/usr/share/perl5 - command: prove --verbose + - run: + name: Install debian packages + command: | + apt update -y + apt install -y --no-install-recommends bats git openssh-client postgresql-client + - checkout + - run: + name: Executing tests + command: prove --verbose workflows: version: 2 pipeline: jobs: - - convert - #- test + - test diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 43f8fbd..3c12b07 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -489,15 +489,15 @@ sub format_identifier_cols_index sub convert_transactsql_code { my ($code)=@_; - # print STDERR "convert: $code\n"; + print STDERR "convert: $code\n"; - if ($code =~ /^\((.*)\)\s+(AND|OR)\s+\((.*)\)$/) { + if ($code =~ /^\((.+?)\)\s+(AND|OR)\s+\((.+?)\)$/) { my ($lhs,$op,$rhs)=($1,$2,$3); $code = "(".convert_transactsql_code("$lhs").") $op (".convert_transactsql_code("$rhs").")"; } - elsif ($code =~ /^(.*)\s+(AND|OR)\s+(.*)$/) { + elsif ($code =~ /^(.+?)\s+(AND|OR)\s+(.+?)$/) { my ($lhs,$op,$rhs)=($1,$2,$3); - $code = convert_transactsql_code("$lhs")." $op ".convert_transactsql_code("$rhs"); + $code = "(".convert_transactsql_code("$lhs")." $op ".convert_transactsql_code("$rhs").")"; } else { if ($case_treatment==0) @@ -513,7 +513,7 @@ sub convert_transactsql_code $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; } - # print STDERR "to: $code\n\n"; + print STDERR "to: $code\n\n"; return $code; } @@ -1799,7 +1799,7 @@ sub parse_dump # This is a where clause. PostgreSQL has them too. But we cannot be sure this will be exactly the same. So if an index as a WHERE clause, it has to go to unsure my $filter=$1; $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} - ->{INDEXES}->{$idxname}->{WHERE}=$filter; + ->{INDEXES}->{$idxname}->{WHERE}="(".$filter.")"; } } } @@ -2467,7 +2467,7 @@ sub generate_schema foreach my $table (sort keys %{$refschema->{TABLES}}) { foreach my $constraint ( - @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) + @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) { next unless ($constraint->{TYPE} eq 'UNIQUE'); my $consdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; @@ -2518,28 +2518,28 @@ sub generate_schema $index_created = 1; } else - { - # this is either a disabled index or an index with a where declaration - if (defined $idxref->{WHERE}) - { - print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; - if ($idxref->{DISABLE}) - { - # if disabled, will be on the same line - $idxdef .= " "; - } - else - { - # otherwise, write condition on a new line - $idxdef .= "\n"; - } - $idxdef .= "WHERE (" . convert_transactsql_code($idxref->{WHERE}) . ")"; - } - $idxdef .= ";\n"; - print UNSURE $idxdef; - # the possible comment would go to unsure file - $index_created = 2; - } + { + # this is either a disabled index or an index with a where declaration + if (defined $idxref->{WHERE}) + { + print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; + if ($idxref->{DISABLE}) + { + # if disabled, will be on the same line + $idxdef .= " "; + } + else + { + # otherwise, write condition on a new line + $idxdef .= "\n"; + } + $idxdef .= "WHERE (" . convert_transactsql_code($idxref->{WHERE}) . ")"; + } + $idxdef .= ";\n"; + print UNSURE $idxdef; + # the possible comment would go to unsure file + $index_created = 2; + } # Produce the comments for indexes if (defined $idxref->{COMMENT}) @@ -2617,11 +2617,11 @@ sub generate_schema } } elsif ($constraint->{TYPE} eq 'CHECK') - { - $consdef .= " CHECK (" . convert_transactsql_code($constraint->{TEXT}) . ");\n"; - print UNSURE $consdef - ; # Check constraints are SQL, so cannot be sure - } + { + $consdef .= " CHECK (" . convert_transactsql_code($constraint->{TEXT}) . ");\n"; + print UNSURE $consdef + ; # Check constraints are SQL, so cannot be sure + } elsif ($constraint->{TYPE} eq 'CHECK_CITEXT') { diff --git a/t/convert.t b/t/convert.t deleted file mode 100644 index 11054ac..0000000 --- a/t/convert.t +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bats - -@test "schema conversion test" { - ./sqlserver2pgsql.pl -f regression/reg_tests.sql -b /tmp/before -a /tmp/after -u /tmp/unsure -k /tmp/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 -} diff --git a/t/test_convert.t b/t/test_convert.t new file mode 100644 index 0000000..3e5791c --- /dev/null +++ b/t/test_convert.t @@ -0,0 +1,47 @@ +#!/usr/bin/env bats + +@test "schema conversion test" { + WORK_DIR="/tmp/tests" + + # we must add the KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in kettle conf file + if [ ! -d ~/.kettle ]; then + mkdir ~/.kettle + fi + if [ ! -f ~/.kettle/kettle.properties ]; then + touch ~/.kettle/kettle.properties + fi + if [ -z $(grep "KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y" ~/.kettle/kettle.properties) ]; then + echo -e "\n# This line was added for sqlserver2pgsql tests" >> ~/.kettle/kettle.properties + echo -e "KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y" >> ~/.kettle/kettle.properties + fi + + # create the array of command to try + options_to_try=( "-i" "-nr" "-num" "-keep_identifier_case" "-validate_constraints=after" ) + declare -a all_combinations=("blank ") + + for option_to_try in ${options_to_try[@]} ; do + for exiting_combination in ${all_combinations[@]}; do + all_combinations+="${option_to_try}_sep_${exiting_combination} " + done + done + + # clear the possible leftovers + if [ -d $WORK_DIR ]; then + rm -rf $WORK_DIR + fi + mkdir -p $WORK_DIR + + # run the sqlserver2pgsql script for all files and confs + declare -i test_nb=1 + for reg_file in regression/*.sql ; do + for option in ${all_combinations[@]} ; do + CURRENT_DIR=$WORK_DIR/$test_nb + mkdir -p $CURRENT_DIR + real_option=$(echo $option | sed 's/blank//' | sed 's/_sep_/ /g') + command_line="./sqlserver2pgsql.pl -f $reg_file -b $CURRENT_DIR/before.sql -a $CURRENT_DIR/after.sql -u $CURRENT_DIR/unsure.sql -k $CURRENT_DIR/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 $real_option" + echo $command_line > $CURRENT_DIR/command_line + eval $command_line > $CURRENT_DIR/command_output 2>&1 + test_nb+=1 + done + done +} diff --git a/t/test_db_creation.t b/t/test_db_creation.t new file mode 100644 index 0000000..c9423b7 --- /dev/null +++ b/t/test_db_creation.t @@ -0,0 +1,15 @@ +#!/usr/bin/env bats + +@test "PostgreSQL database creation test" { + WORK_DIR="/tmp/tests" + cd $WORK_DIR + for test_dir in * ; do + cd $test_dir + createdb reg + psql reg < before.sql + psql reg < after.sql + psql reg < unsure.sql + dropdb reg + cd .. + done +} From 3e6ad2e4e54d83f91b6f25a19713c32cbbef3402 Mon Sep 17 00:00:00 2001 From: madtibo Date: Fri, 28 Sep 2018 12:39:11 +0200 Subject: [PATCH 175/216] add a docker-compose.yml file for local tests and remove debug traces --- docker-compose.yml | 20 ++++++++++++++++++++ sqlserver2pgsql.pl | 4 ++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d0843dd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3" + +services: + sqlserver2pgsql: + image: perl:5.24-threaded + environment: + # Inject APT packaged dependencies. + PERL5LIB: /usr/lib/x86_64-linux-gnu/perl5/5.24:/usr/share/perl5 + volumes: + - .:/tmp/workspace + command: perl ./sqlserver2pgsql.pl -f ./regression/reg_tests.sql -b /tmp/workspace/test/before.sql -a /tmp/workspace/test/after.sql -u /tmp/workspace/test/unsure.sql -k /tmp/workspace/test/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 + + postgres: + image: postgres:10-alpine + environment: + PGHOST: localhost + PGDATABASE: postgres + PGUSER: postgres + volumes: + - .:/tmp/workspace diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3c12b07..74bf102 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -489,7 +489,7 @@ sub format_identifier_cols_index sub convert_transactsql_code { my ($code)=@_; - print STDERR "convert: $code\n"; + #print STDERR "convert: $code\n"; if ($code =~ /^\((.+?)\)\s+(AND|OR)\s+\((.+?)\)$/) { my ($lhs,$op,$rhs)=($1,$2,$3); @@ -513,7 +513,7 @@ sub convert_transactsql_code $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; } - print STDERR "to: $code\n\n"; + #print STDERR "to: $code\n\n"; return $code; } From 1658eb5d7827868b78a291171826f21ead66e28a Mon Sep 17 00:00:00 2001 From: madtibo Date: Mon, 8 Oct 2018 12:00:56 +0200 Subject: [PATCH 176/216] add a length chek for binary column --- sqlserver2pgsql.pl | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 81f37f6..adb82f8 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -232,8 +232,19 @@ sub convert_type if ((defined $sqlqual and defined($unqual{$types{$sqlstype}})) or not defined $sqlqual) { - # This is one of the few types that have to be unqualified (binary type) - $rettype = $types{$sqlstype}; + # This is one of the few types that have to be unqualified (binary type) + $rettype = $types{$sqlstype}; + + # but we might add a check constraint for binary data + if ($sqlstype =~ 'binary' and defined $sqlqual) { + print STDERR "convert_type: $sqlstype, $sqlqual, $colname\n"; + my $constraint; + $constraint->{TYPE} = 'CHECK_BINARY_LENGTH'; + $constraint->{TABLE} = $tablename; + $constraint->{TEXT} = "octet_length(" . format_identifier($colname) . ") <= $sqlqual"; + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{CONSTRAINTS}}, ($constraint); + } } elsif (defined $sqlqual) { @@ -2676,12 +2687,18 @@ sub generate_schema } elsif ($constraint->{TYPE} eq 'CHECK_CITEXT') { - # These have been generated here, for citext mostly. So we know their syntax is ok $consdef .= " CHECK (" . $constraint->{TEXT} . ");\n"; print BEFORE $consdef ; # These are for citext. So they should be checked asap } + elsif ($constraint->{TYPE} eq 'CHECK_BINARY_LENGTH') + { + # These have been generated here. Their syntax are ok. + $consdef .= " CHECK (" . $constraint->{TEXT} . ");\n"; + print BEFORE $consdef + ; # These are for bytea length, checked them asap + } else { # Shouldn't get there. it would mean I have forgotten a type of constraint From d2fc7ad6346dff4e8875c8fd40a475078b36fcbb Mon Sep 17 00:00:00 2001 From: madtibo Date: Mon, 8 Oct 2018 12:01:32 +0200 Subject: [PATCH 177/216] add a test file dor binary max length --- regression/issue_91.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 regression/issue_91.sql diff --git a/regression/issue_91.sql b/regression/issue_91.sql new file mode 100644 index 0000000..45721da --- /dev/null +++ b/regression/issue_91.sql @@ -0,0 +1,14 @@ + +CREATE TABLE [candidate].[PersonalIdentityHashes]( + [OrganizationId] [int] NOT NULL, + [PersonalIdentity] [char](64) NOT NULL, + [PersonalIdentityHash] [binary](64) NOT NULL, + [PersonalIdentityHash2] [varbinary](64) NOT NULL, + [CandidateId] [bigint] NOT NULL, + CONSTRAINT [PK_candidatePersonalIdentityHashes] PRIMARY KEY CLUSTERED +( + [OrganizationId] ASC, + [PersonalIdentityHash] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO From 384c51b5d2739017f15f8845b70757426660c110 Mon Sep 17 00:00:00 2001 From: madtibo Date: Thu, 4 Oct 2018 19:58:17 +0200 Subject: [PATCH 178/216] issue #72 - add the '-stringtype_unspecified' switch to insert the option in the ktr files --- sqlserver2pgsql.pl | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 81f37f6..0b24bf3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -50,6 +50,7 @@ our $sort_size; our $use_pk_if_possible; our $pforce_ssl; +our $stringtype_unspecified; # Will be set if we detect GIS objects our $requires_postgis=0; @@ -103,6 +104,7 @@ sub parse_conf_file 'use pk if possible' => 'use_pk_if_possible', 'ignore errors' => 'ignore_errors', 'postgresql force ssl' => 'pforce_ssl', + 'stringtype unspecified' => 'stringtype_unspecified', ); # Open the conf file or die @@ -151,6 +153,7 @@ sub set_default_conf_values $pp=5432 unless (defined ($pp)); $sp=1433 unless (defined ($sp)); $pforce_ssl=0 unless (defined ($pforce_ssl)); + $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); } # Converts numeric(4,0) and similar to int, bigint, smallint @@ -785,6 +788,10 @@ sub usage parallelism level for the kettle job (output, PostgreSQL). Default 8. -pforce_ssl force a SSL session to PostgreSQL + -stringtype_unspecified + By default, the data are sent as varchar. This can block date field + migration. By using this option, make PostgreSQL infer the data type. + -k KETTLE_OUTPUT_DIRECTORY -sort_size INTEGER set the size of the sort batch for the incremental job. If set to a @@ -923,6 +930,12 @@ sub generate_kettle $newtemplate =~ s/__pforce_ssl__//g; } + if ($stringtype_unspecified) { + $newtemplate =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; + } + else { + $newtemplate =~ s/__stringtype_unspecified__//g; + } $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; @@ -953,7 +966,13 @@ sub generate_kettle { $newincrementaltemplate =~ s/__pforce_ssl__//g; } -# We have a bit of work to do on primary keys for the incremental template: we need them + if ($stringtype_unspecified) { + $newincrementaltemplate =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; + } + else { + $newincrementaltemplate =~ s/__stringtype_unspecified__//g; + } + # We have a bit of work to do on primary keys for the incremental template: we need them # to compare the tables… if (defined($refschema->{TABLES}->{$table}->{PK}->{COLS})) { @@ -1181,6 +1200,12 @@ sub generate_kettle { $job_header =~ s/__pforce_ssl__//g; } + if ($stringtype_unspecified) { + $job_header =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; + } + else { + $job_header =~ s/__stringtype_unspecified__//g; + } print JOBFILE $job_header; print JOBFILE $entries; @@ -2957,14 +2982,16 @@ sub resolve_name_conflicts "i" => \$case_insensitive, "nr" => \$norelabel_dbo, "num" => \$convert_numeric_to_int, - "relabel_schemas=s" => \$relabel_schemas, - "keep_identifier_case" =>\$keep_identifier_case, - "camel_to_snake" => \$camel_to_snake, + "relabel_schemas=s" => \$relabel_schemas, + "keep_identifier_case" =>\$keep_identifier_case, + "camel_to_snake" => \$camel_to_snake, "validate_constraints=s" =>\$validate_constraints, "sort_size=i" =>\$sort_size, "use_pk_if_possible=s" =>\$use_pk_if_possible, "ignore_errors" => \$ignore_errors, - "pforce_ssl" => \$pforce_ssl); + "pforce_ssl" => \$pforce_ssl, + "stringtype_unspecified" => \$stringtype_unspecified + ); # We don't understand command line or have been asked for usage if (not $options or $help) @@ -3166,6 +3193,7 @@ BEGIN USE_POOLINGN SQL_CONNECTset synchronous_commit to off; PRESERVE_RESERVED_WORD_CASEY + __stringtype_unspecified__ @@ -3527,6 +3555,7 @@ BEGIN USE_POOLINGN SQL_CONNECTset synchronous_commit to off; PRESERVE_RESERVED_WORD_CASEY + __stringtype_unspecified__ @@ -3808,6 +3837,7 @@ BEGIN USE_POOLINGN SQL_CONNECTset synchronous_commit to off; PRESERVE_RESERVED_WORD_CASEY + __stringtype_unspecified__ @@ -4043,6 +4073,7 @@ BEGIN SQL_CONNECTset synchronous_commit to off; SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN + __stringtype_unspecified__ @@ -4472,6 +4503,7 @@ BEGIN SQL_CONNECTset synchronous_commit to off; SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN + __stringtype_unspecified__ From 0f333e7aff047a50170971275eb38252a56cdf65 Mon Sep 17 00:00:00 2001 From: madtibo Date: Wed, 20 Mar 2019 12:04:00 +0100 Subject: [PATCH 179/216] Relax parsing on leading white spaces --- regression/reg_tests.sql | Bin 9556 -> 11704 bytes sqlserver2pgsql.pl | 36 +++++++++++++++++++----------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index 19337028a3438cc1d4a70acaf4bee267c5bda9a6..2dee81da88c0c613d7a22988f0ef3f8021dacd2a 100644 GIT binary patch delta 1451 zcmb7EO-sW-5FJ6m7Ay#wq9CnQ!BVT#>dm_#ejLi4ymMJ+2qjx zfA8o4e+H!E+yG*Sid4n=#WBL5F?NVHNf+y@_3~~CE;_rCXPn2vO0bcGqu-NBJ#3E3 ze=g{T&afm`2WtqWE>MnmQ_77rR8pIUN?7k-jY*U)J~!GmnJ0P1I(0-CA%-O!Wvw+U z=MY#G8aZ#wYR+PfXhII^i?#YvkArA7{NeksfY6{Qt7k4aDqOQsM=s_HlCs@4(t1H5 z2*IYSX{C5T8ejzSGmA6mDX>Yh2M`u z)&mx5(gEsX8_{SE#bc$J7#Myi#yAfpOpR7%AEsk)Z}W|Chw~E_K1JaWj^VXSn)px( gOH+!)cePO~1`lZA-{@~ delta 7 OcmdlHeZ^}-h$;XNS_4}E diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 47b59e3..48538a1 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -112,10 +112,10 @@ sub parse_conf_file while (my $line = ) { $line =~ s/#.*//; # Remove comments - $line =~ s/\s+=\s+/=/; # Remove whitespaces around the = + $line =~ s/\s+=\s+/=/; # Remove whitespaces around the = $line =~ s/\s+$//; # Remove trailing whitespaces next - if ($line =~ /^$/); # Empty line after comments have been removed + if ($line =~ /^$/); # Empty line after comments have been removed $line =~ /^(.*?)=(.*)$/ or die "Cannot parse $line from $conf_file"; my ($param, $value) = ($1, $2); no strict 'refs'; # Using references by name, temporarily @@ -1277,6 +1277,7 @@ sub generate_kettle $line =~ s/\/\*.*//; # Remove everything after the comment } } + return $line; } } @@ -1421,14 +1422,15 @@ sub parse_dump TABLE: while (my $line = read_and_clean($file)) { # Here is a col definition. - # We ignore ROWGUIDCOL as it has no meaning in PostgreSQL and cannot be emulated - # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) - # To make matters even worse, they seem to systematically add a space after it :) + # We ignore ROWGUIDCOL as it has no meaning in PostgreSQL and cannot be emulated + # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the + # column name, typical microsoft stuff :( ) + # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ - /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\]\s*(\(.+?\))?(?: COLLATE (\S+))?( IDENTITY\s*\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE +)?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ + /^\s+\[(.*)\]\s*(?:\[(.*)\]\.)?\[(.*)\]\s*(\(.+?\))?(?: COLLATE (\S+))?( IDENTITY\s*\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE +)?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ ) { - #Deported into a function because we can also meet alter table add columns on their own + # Deported into a function because we can also meet alter table add columns on their own my $colname = $1; my $coltypeschema = $2; my $coltype = $3; @@ -1438,16 +1440,16 @@ sub parse_dump my $colisnull =$7; my $default =$8; add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); - if (defined $default) - { - store_default_value($schemaname,$tablename,$colname,$default,$line); - } - } + if (defined $default) + { + store_default_value($schemaname,$tablename,$colname,$default,$line); + } + } # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) # So just put it as a varchar, and issue a warning is STDOUT - elsif ($line =~ /^\t\[(.*)\]\s+AS\s+\((.*)\)/) + elsif ($line =~ /^\s*\[(.*)\]\s+AS\s+\((.*)\)/) { # We just get the column name @@ -1498,7 +1500,7 @@ sub parse_dump } elsif ($line =~ - /^(?: CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED/) + /^\s*(?:CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED/) { my $constraint ; # We put everything inside this hashref, we'll push it into the constraint list later @@ -1557,7 +1559,7 @@ sub parse_dump } } - elsif ($line =~ /^\) ON \[PRIMARY\]/) + elsif ($line =~ /^\s*\) ON \[PRIMARY\]/) { # End of the table next MAIN; @@ -1674,7 +1676,7 @@ sub parse_dump # containing only a single quote (end of the dbo.sp_executesql) # The problem is that SQL Server seems to be spitting the original query used to create the view, not a normalized version # of it, as PostgreSQL does. So we capture the query, and hope it works for now. - elsif ($line =~/^\s*(create\s*view)/i) + elsif ($line =~/^(create\s*view)/i) { my $viewname; my $schemaname; @@ -1807,7 +1809,7 @@ sub parse_dump } elsif ($line =~ - /^\s*CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*?)\] ON \[(.*?)\]\.\[(.*?)\](\(\[.*?\]\))?/ + /^CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*?)\] ON \[(.*?)\]\.\[(.*?)\](\(\[.*?\]\))?/ ) { # Index creation. Index are namespaced per table in SQL Server, not in PostgreSQL From e5d4f64d8a59bd3c8109f6da91c844eac8f0f8d7 Mon Sep 17 00:00:00 2001 From: madtibo Date: Wed, 20 Mar 2019 14:58:42 +0100 Subject: [PATCH 180/216] add an option to drop rowversion/timestamp columns --- regression/reg_tests.sql | Bin 11704 -> 12596 bytes sqlserver2pgsql.pl | 156 +++++++++++++++++++++------------------ 2 files changed, 86 insertions(+), 70 deletions(-) diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index 2dee81da88c0c613d7a22988f0ef3f8021dacd2a..3e6b517666c89a927d837cb32f7c13678edee699 100644 GIT binary patch delta 318 zcmdlHy(MYG4&BLoV)Blu3@Hpr46zJ)4ABfh4EYS@3}FnZ3`GpZ44FV)9zzI2B2YdD zC>G10z@Wjv#lQ(v830s~$>0qXt(-VPQx&4c8K@x#tSlNR4Kk_>*{I14@}kT-3|x~7 z1dK(wuo< 'sd', - 'sql server host' => 'sh', - 'sql server host instance' => 'si', - 'sql server port' => 'sp', - 'sql server username' => 'su', - 'sql server password' => 'sw', - 'postgresql database' => 'pd', - 'postgresql host' => 'ph', - 'postgresql port' => 'pp', - 'postgresql username' => 'pu', - 'postgresql password' => 'pw', - 'kettle directory' => 'kettle', - 'parallelism_in' => 'parallelism_in', - 'parallelism_out' => 'parallelism_out', - 'before file' => 'before_file', - 'after file' => 'after_file', - 'unsure file' => 'unsure_file', - 'sql server dump filename' => 'filename', - 'case insensitive' => 'case_insensitive', - 'no relabel dbo' => 'norelabel_dbo', - 'convert numeric to int' => 'convert_numeric_to_int', - 'relabel schemas' => 'relabel_schemas', - 'keep identifier case' => 'keep_identifier_case', - 'camelcasetosnake' => 'camel_to_snake', - 'validate constraints' => 'validate_constraints', - 'sort size' => 'sort_size', - 'use pk if possible' => 'use_pk_if_possible', - 'ignore errors' => 'ignore_errors', - 'postgresql force ssl' => 'pforce_ssl', - 'stringtype unspecified' => 'stringtype_unspecified', - ); + # Correspondance between conf_file parameter and program variable + # This is also used as the list of accepted parameters in the configuration file + my %parameters = ( + 'sql server database' => 'sd', + 'sql server host' => 'sh', + 'sql server host instance' => 'si', + 'sql server port' => 'sp', + 'sql server username' => 'su', + 'sql server password' => 'sw', + 'postgresql database' => 'pd', + 'postgresql host' => 'ph', + 'postgresql port' => 'pp', + 'postgresql username' => 'pu', + 'postgresql password' => 'pw', + 'kettle directory' => 'kettle', + 'parallelism_in' => 'parallelism_in', + 'parallelism_out' => 'parallelism_out', + 'before file' => 'before_file', + 'after file' => 'after_file', + 'unsure file' => 'unsure_file', + 'sql server dump filename' => 'filename', + 'case insensitive' => 'case_insensitive', + 'no relabel dbo' => 'norelabel_dbo', + 'convert numeric to int' => 'convert_numeric_to_int', + 'drop rowversion' => 'drop_rowversion', + 'relabel schemas' => 'relabel_schemas', + 'keep identifier case' => 'keep_identifier_case', + 'camelcasetosnake' => 'camel_to_snake', + 'validate constraints' => 'validate_constraints', + 'sort size' => 'sort_size', + 'use pk if possible' => 'use_pk_if_possible', + 'ignore errors' => 'ignore_errors', + 'postgresql force ssl' => 'pforce_ssl', + 'stringtype unspecified' => 'stringtype_unspecified', + ); # Open the conf file or die open CONF, $conf_file or die "Cannot open $conf_file"; @@ -141,6 +144,7 @@ sub set_default_conf_values $case_insensitive=0 unless (defined ($case_insensitive)); $norelabel_dbo=0 unless (defined ($norelabel_dbo)); $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); + $drop_rowversion=0 unless (defined ($drop_rowversion)); $case_treatment=0 if (defined ($keep_identifier_case)); $case_treatment=2 if (defined ($camel_to_snake)); $parallelism_in=1 unless (defined ($parallelism_in));# the jdbc driver often errors when there are several sessions to sql server @@ -210,7 +214,7 @@ sub add_cast 'smalldatetime' => 'timestamp', 'time' => 'time', 'timestamp' => 'bytea', - 'rowversion' => 'bytea', + 'rowversion' => 'bytea', 'datetimeoffset' => 'timestamp with time zone', 'image' => 'bytea', 'binary' => 'bytea', @@ -766,6 +770,8 @@ sub usage advised. Default is to lowercase everything. -num convert numeric 'xxx,0' to int, bigint, etc. Will not keep numeric scale and precision for the converted. + -drop_rowversion (Default 0) + should we drop the rowversion/timestamp columns when converting -validate_constraints {yes|no|after} should the constraints be validated. Set to "yes" by default. If set to "no", the constraints will be set as 'NOT VALID'. If set to @@ -1313,7 +1319,16 @@ sub add_column_to_table or die "Cannot parse colqual <$colqual>"; $colqual = "$1"; } - } + } + + # in case of a rowversion or timestamp columns, check if we want to keep it + if ($drop_rowversion + and ($coltype eq 'rowversion' or $coltype eq 'timestamp')) + { + # do nothing + return; + } + my $newtype = convert_type($coltype, $colqual, $colname, $tablename, undef, $schemaname); @@ -1358,8 +1373,7 @@ sub add_column_to_table # use a special kettle transformation for it if generating kettle # (see generate_kettle() ) if ( $newtype eq 'bytea' - or $coltype eq - 'ntext') # Ntext is very slow, stored out of page + or $coltype eq 'ntext') # Ntext is very slow, stored out of page { $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename} ->{haslobs} = 1; @@ -2978,39 +2992,41 @@ sub resolve_name_conflicts # Parse command line my $help = 0; -my $options = GetOptions("k=s" => \$kettle, - "pi=i" => \$parallelism_in, - "po=i" => \$parallelism_out, - "b=s" => \$before_file, - "a=s" => \$after_file, - "u=s" => \$unsure_file, - "h" => \$help, - "conf=s" => \$conf_file, - "sd=s" => \$sd, - "sh=s" => \$sh, - "si=s" => \$si, - "sp=s" => \$sp, - "su=s" => \$su, - "sw=s" => \$sw, - "pd=s" => \$pd, - "ph=s" => \$ph, - "pp=s" => \$pp, - "pu=s" => \$pu, - "pw=s" => \$pw, - "f=s" => \$filename, - "i" => \$case_insensitive, - "nr" => \$norelabel_dbo, - "num" => \$convert_numeric_to_int, - "relabel_schemas=s" => \$relabel_schemas, - "keep_identifier_case" =>\$keep_identifier_case, - "camel_to_snake" => \$camel_to_snake, - "validate_constraints=s" =>\$validate_constraints, - "sort_size=i" =>\$sort_size, - "use_pk_if_possible=s" =>\$use_pk_if_possible, - "ignore_errors" => \$ignore_errors, - "pforce_ssl" => \$pforce_ssl, - "stringtype_unspecified" => \$stringtype_unspecified - ); +my $options = GetOptions( + "k=s" => \$kettle, + "pi=i" => \$parallelism_in, + "po=i" => \$parallelism_out, + "b=s" => \$before_file, + "a=s" => \$after_file, + "u=s" => \$unsure_file, + "h" => \$help, + "conf=s" => \$conf_file, + "sd=s" => \$sd, + "sh=s" => \$sh, + "si=s" => \$si, + "sp=s" => \$sp, + "su=s" => \$su, + "sw=s" => \$sw, + "pd=s" => \$pd, + "ph=s" => \$ph, + "pp=s" => \$pp, + "pu=s" => \$pu, + "pw=s" => \$pw, + "f=s" => \$filename, + "i" => \$case_insensitive, + "nr" => \$norelabel_dbo, + "num" => \$convert_numeric_to_int, + "drop_rowversion" => \$drop_rowversion, + "relabel_schemas=s" => \$relabel_schemas, + "keep_identifier_case" => \$keep_identifier_case, + "camel_to_snake" => \$camel_to_snake, + "validate_constraints=s" => \$validate_constraints, + "sort_size=i" => \$sort_size, + "use_pk_if_possible=s" => \$use_pk_if_possible, + "ignore_errors" => \$ignore_errors, + "pforce_ssl" => \$pforce_ssl, + "stringtype_unspecified" => \$stringtype_unspecified +); # We don't understand command line or have been asked for usage if (not $options or $help) From d539a2ae65b342be121a7a2b5639a9527579d56d Mon Sep 17 00:00:00 2001 From: madtibo Date: Wed, 20 Mar 2019 16:47:26 +0100 Subject: [PATCH 181/216] save variable and use them in default values --- sqlserver2pgsql.pl | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 48538a1..dfa0b93 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1809,7 +1809,7 @@ sub parse_dump } elsif ($line =~ - /^CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*?)\] ON \[(.*?)\]\.\[(.*?)\](\(\[.*?\]\))?/ + /^\s*CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*?)\] ON \[(.*?)\]\.\[(.*?)\](\(\[.*?\]\))?/ ) { # Index creation. Index are namespaced per table in SQL Server, not in PostgreSQL @@ -2235,6 +2235,14 @@ sub parse_dump } } + # Save variable for future use + elsif ($line =~ /^:setvar\s+(\S+)\s+"(.*)"/) + { + my $varname = $1; + my $varvalue = $2; + $objects->{VARIABLES}->{$varname} = $varvalue; + next; + } # Ignore USE, GO, and things that have no meaning for postgresql elsif ($line =~ @@ -2763,9 +2771,15 @@ sub generate_schema { my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; next unless (defined $colref->{DEFAULT}); + my $default_value = $colref->{DEFAULT}->{VALUE}; + if ($default_value =~ /\(\$\((\S+)\)\)/) + { + $default_value = $objects->{VARIABLES}->{$1} + } my $definition = - "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ALTER COLUMN " . format_identifier($col) . " SET DEFAULT " - . $colref->{DEFAULT}->{VALUE} . ";\n"; + "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) + . " ALTER COLUMN " . format_identifier($col) + . " SET DEFAULT " . $default_value . ";\n"; if ($colref->{DEFAULT}->{UNSURE}) { print UNSURE $definition; From 5ac16878d2fa04959a228789a5d6f3c66ba80bef Mon Sep 17 00:00:00 2001 From: madtibo Date: Fri, 22 Mar 2019 16:36:08 +0100 Subject: [PATCH 182/216] do not replace default value --- sqlserver2pgsql.pl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index dfa0b93..af46768 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1500,7 +1500,7 @@ sub parse_dump } elsif ($line =~ - /^\s*(?:CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED/) + /^\s*(?:CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED(?: HASH)?/) { my $constraint ; # We put everything inside this hashref, we'll push it into the constraint list later @@ -2438,6 +2438,12 @@ sub generate_schema } } + # Set psql variables in UNSURE + foreach my $varname (sort keys %{$objects->{VARIABLES}}) + { + print UNSURE "\\set $varname '$objects->{VARIABLES}->{$varname}'\n"; + } + # For the rest, we iterate over schemas, except for array types (no point in complicating this) # The tables, columns, etc... will be created in the before script, so there is no dependancy # problem with constraints, that will be in the after script, except foreign keys which depend on unique indexes @@ -2774,7 +2780,7 @@ sub generate_schema my $default_value = $colref->{DEFAULT}->{VALUE}; if ($default_value =~ /\(\$\((\S+)\)\)/) { - $default_value = $objects->{VARIABLES}->{$1} + $default_value = ":$1"; } my $definition = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) From 4f78b3aa63b22a391563d8f4e2104cf6b5f09d54 Mon Sep 17 00:00:00 2001 From: madtibo Date: Tue, 9 Jul 2019 11:36:31 +0200 Subject: [PATCH 183/216] correct usage --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 46b1004..17c6840 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -723,7 +723,7 @@ sub usage { print qq{ Usage: - sqlserver2pgsql -b BEFORE_FILE -a AFTER_FILE -u UNSURE_FILE -f SQLSERVER_SCHEMA_FILE + sqlserver2pgsql.pl -b BEFORE_FILE -a AFTER_FILE -u UNSURE_FILE -f SQLSERVER_SCHEMA_FILE Description: From 5de0d54e2997aa047f7b6d76901650a2f42e1096 Mon Sep 17 00:00:00 2001 From: madtibo Date: Tue, 9 Jul 2019 14:49:40 +0200 Subject: [PATCH 184/216] add an example install script for linux --- example_install_script.sh | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 example_install_script.sh diff --git a/example_install_script.sh b/example_install_script.sh new file mode 100644 index 0000000..e2435bc --- /dev/null +++ b/example_install_script.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# note this script does not install spoon dependencies +# ignore libwebkitgtk-1.0 errors when running kitchen. +# libwebkitgtk-1.0 is not available for RedHat/CentOS 7 or debian buster and is +# not needed for kitchen to run correctly + +# on RedHat 7 / CentOS 7 +# yum -y install wget unzip perl perl-MLDBM java-1.8.0-openjdk + +# on debian stretch +# apt install -y wget unzip perl libmldbm-perl openjdk-8-jdk +# (on debian buster you need to install openjdk-8-jdk from sid) + +MIGRATIONDIR=/opt/data_migration +mkdir -p $MIGRATIONDIR/kettlejobs + +# install sqlserver2pgsql +if [ ! -f "$MIGRATIONDIR/sqlserver2pgsql.pl" ]; then + wget https://raw.githubusercontent.com/dalibo/sqlserver2pgsql/master/sqlserver2pgsql.pl -P $MIGRATIONDIR + # make executable + chmod u+x,g+x,a+x $MIGRATIONDIR/sqlserver2pgsql.pl +fi + +# install kettle +if [ ! -f "$MIGRATIONDIR/data-integration/kitchen.sh" ]; then + wget https://sourceforge.net/projects/pentaho/files/latest/download?source=files -O /tmp/kettle.zip + unzip /tmp/kettle.zip -d /tmp/kettle + cp -R /tmp/kettle/data-integration $MIGRATIONDIR + rm -Rf /tmp/kettle;rm -f /tmp/kettle.zip + # make all shell scripts executable + chmod -R u+x,g+x,a+x $MIGRATIONDIR/data-integration/*.sh +fi + +# install JDBC driver (JTDS version works fine with MSSQL) +set -- $MIGRATIONDIR/data-integration/lib/jtds* +if [ ! -f "$1" ]; then + wget https://sourceforge.net/projects/jtds/files/latest/download?source=files -O /tmp/jtds.zip + unzip /tmp/jtds.zip -d /tmp/jtds + # copy to kettle lib directory + cp /tmp/jtds/jtds-*.jar $MIGRATIONDIR/data-integration/lib/ + rm -Rf /tmp/jtds;rm -f /tmp/jtds.zip +fi + + From b2374c4bdcfc8ec486b6eeb7647e62dc7a17f5ae Mon Sep 17 00:00:00 2001 From: madtibo Date: Tue, 9 Jul 2019 14:53:42 +0200 Subject: [PATCH 185/216] update contributors --- contributors | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors b/contributors index 8e419fd..88b21c5 100644 --- a/contributors +++ b/contributors @@ -15,6 +15,7 @@ And the following unindentified github users (please tell me if you want your na bsacks99 keyjote mark-jay +mikes-gh sebpcspkr stuey1978 From e3f34e12a65d432b0b39db07611b1e26b87b7d2f Mon Sep 17 00:00:00 2001 From: madtibo Date: Mon, 15 Jul 2019 10:42:02 +0200 Subject: [PATCH 186/216] indent and whiteline --- sqlserver2pgsql.pl | 766 ++++++++++++++++++++++----------------------- 1 file changed, 380 insertions(+), 386 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 17c6840..75f9a9b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -74,46 +74,46 @@ sub parse_conf_file { - # Correspondance between conf_file parameter and program variable - # This is also used as the list of accepted parameters in the configuration file - my %parameters = ( - 'sql server database' => 'sd', - 'sql server host' => 'sh', - 'sql server host instance' => 'si', - 'sql server port' => 'sp', - 'sql server username' => 'su', - 'sql server password' => 'sw', - 'postgresql database' => 'pd', - 'postgresql host' => 'ph', - 'postgresql port' => 'pp', - 'postgresql username' => 'pu', - 'postgresql password' => 'pw', - 'kettle directory' => 'kettle', - 'parallelism_in' => 'parallelism_in', - 'parallelism_out' => 'parallelism_out', - 'before file' => 'before_file', - 'after file' => 'after_file', - 'unsure file' => 'unsure_file', - 'sql server dump filename' => 'filename', - 'case insensitive' => 'case_insensitive', - 'no relabel dbo' => 'norelabel_dbo', - 'convert numeric to int' => 'convert_numeric_to_int', - 'drop rowversion' => 'drop_rowversion', - 'relabel schemas' => 'relabel_schemas', - 'keep identifier case' => 'keep_identifier_case', - 'camelcasetosnake' => 'camel_to_snake', - 'validate constraints' => 'validate_constraints', - 'sort size' => 'sort_size', - 'use pk if possible' => 'use_pk_if_possible', - 'ignore errors' => 'ignore_errors', - 'postgresql force ssl' => 'pforce_ssl', - 'stringtype unspecified' => 'stringtype_unspecified', - ); - - # Open the conf file or die - open CONF, $conf_file or die "Cannot open $conf_file"; - while (my $line = ) - { + # Correspondance between conf_file parameter and program variable + # This is also used as the list of accepted parameters in the configuration file + my %parameters = ( + 'sql server database' => 'sd', + 'sql server host' => 'sh', + 'sql server host instance' => 'si', + 'sql server port' => 'sp', + 'sql server username' => 'su', + 'sql server password' => 'sw', + 'postgresql database' => 'pd', + 'postgresql host' => 'ph', + 'postgresql port' => 'pp', + 'postgresql username' => 'pu', + 'postgresql password' => 'pw', + 'kettle directory' => 'kettle', + 'parallelism_in' => 'parallelism_in', + 'parallelism_out' => 'parallelism_out', + 'before file' => 'before_file', + 'after file' => 'after_file', + 'unsure file' => 'unsure_file', + 'sql server dump filename' => 'filename', + 'case insensitive' => 'case_insensitive', + 'no relabel dbo' => 'norelabel_dbo', + 'convert numeric to int' => 'convert_numeric_to_int', + 'drop rowversion' => 'drop_rowversion', + 'relabel schemas' => 'relabel_schemas', + 'keep identifier case' => 'keep_identifier_case', + 'camelcasetosnake' => 'camel_to_snake', + 'validate constraints' => 'validate_constraints', + 'sort size' => 'sort_size', + 'use pk if possible' => 'use_pk_if_possible', + 'ignore errors' => 'ignore_errors', + 'postgresql force ssl' => 'pforce_ssl', + 'stringtype unspecified' => 'stringtype_unspecified', + ); + + # Open the conf file or die + open CONF, $conf_file or die "Cannot open $conf_file"; + while (my $line = ) + { $line =~ s/#.*//; # Remove comments $line =~ s/\s+=\s+/=/; # Remove whitespaces around the = $line =~ s/\s+$//; # Remove trailing whitespaces @@ -140,7 +140,7 @@ sub parse_conf_file sub set_default_conf_values { - # Hard coded default values, set only if not passed or found in configuration + # Hard coded default values, set only if not passed or found in configuration $case_insensitive=0 unless (defined ($case_insensitive)); $norelabel_dbo=0 unless (defined ($norelabel_dbo)); $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); @@ -410,9 +410,9 @@ sub postgres_convert_column 'timestamp with time zone' => 'to_char({colname} AT TIME ZONE \'UTC\', \'YYYY-MM-DD HH:MI:SS.US+00\')'); if (defined ($functions{$coltype})) { - my $tmpcol = $functions{$coltype}; - $tmpcol =~ s/\{colname\}/$colname/; - return $tmpcol; + my $tmpcol = $functions{$coltype}; + $tmpcol =~ s/\{colname\}/$colname/; + return $tmpcol; } else { @@ -508,33 +508,31 @@ sub format_identifier_cols_index # Things such as getdate() which can become CURRENT_TIMESTAMP sub convert_transactsql_code { - my ($code)=@_; - #print STDERR "convert: $code\n"; - - if ($code =~ /^\((.+?)\)\s+(AND|OR)\s+\((.+?)\)$/) { - my ($lhs,$op,$rhs)=($1,$2,$3); - $code = "(".convert_transactsql_code("$lhs").") $op (".convert_transactsql_code("$rhs").")"; - } - elsif ($code =~ /^(.+?)\s+(AND|OR)\s+(.+?)$/) { - my ($lhs,$op,$rhs)=($1,$2,$3); - $code = "(".convert_transactsql_code("$lhs")." $op ".convert_transactsql_code("$rhs").")"; - } - else { - if ($case_treatment==0) - { - $code =~ s/[\[\]]/"/gi; # Bit brutal probably - } - else - { - $code =~ s/\[(.*)\]/rename_identifier($1)/gie; # Bit brutal probably - } - $code =~ s/ISNULL\s*\(/COALESCE(/gi; - $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; - $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; - $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; - } - #print STDERR "to: $code\n\n"; - return $code; + my ($code)=@_; + #print STDERR "convert: $code\n"; + + if ($code =~ /^\((.+?)\)\s+(AND|OR)\s+\((.+?)\)$/) { + my ($lhs,$op,$rhs)=($1,$2,$3); + $code = "(".convert_transactsql_code("$lhs").") $op (".convert_transactsql_code("$rhs").")"; + } + elsif ($code =~ /^(.+?)\s+(AND|OR)\s+(.+?)$/) { + my ($lhs,$op,$rhs)=($1,$2,$3); + $code = "(".convert_transactsql_code("$lhs")." $op ".convert_transactsql_code("$rhs").")"; + } + else { + if ($case_treatment==0) { + $code =~ s/[\[\]]/"/gi; # Bit brutal probably + } + else { + $code =~ s/\[(.*)\]/rename_identifier($1)/gie; # Bit brutal probably + } + $code =~ s/ISNULL\s*\(/COALESCE(/gi; + $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; + $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; + $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; + } + #print STDERR "to: $code\n\n"; + return $code; } # This function does its best to convert MS's weird default values syntax into something logical @@ -543,60 +541,51 @@ sub store_default_value my ($schema,$table,$col,$value,$line)=@_; if ($value =~ /^\(?(\d+(\.\d+)?)\)?$/) # Value is numeric { - $value = $1; # Get rid of parenthesis - if ($objects->{SCHEMAS}->{relabel_schemas($schema)}->{TABLES}->{$table}->{COLS}->{$col}->{TYPE} eq 'boolean') - { - # Ok, it IS a boolean, and we have received a number - if ($value eq '0') - { - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'false'; - } - elsif ($value eq '1') - { - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'true'; - } - else - { - # We should not get here: we have a numeric which isn't 0 or 1, and is supposed to be a boolean - die "Got an unexpected boolean : $value, for line $line\n"; - } - } - else - { - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} - = $value; - } - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} - = 0; - + $value = $1; # Get rid of parenthesis + if ($objects->{SCHEMAS}->{relabel_schemas($schema)}->{TABLES}->{$table}->{COLS}->{$col}->{TYPE} eq 'boolean') { + # Ok, it IS a boolean, and we have received a number + if ($value eq '0') { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'false'; + } + elsif ($value eq '1') { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'true'; + } + else { + # We should not get here: we have a numeric which isn't 0 or 1, and is supposed to be a boolean + die "Got an unexpected boolean : $value, for line $line\n"; + } + } + else { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = $value; + } + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 0; } elsif ($value =~ /^NULL$/) # A NULL value { - # NULL WITHOUT quotes around it ! - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} - = 'NULL'; - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} - = 0; + # NULL WITHOUT quotes around it ! + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = 'NULL'; + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 0; } elsif ($value =~ /^N?'(.*)'$/) # There is sometimes an N before a string. { - $value = $1; # Get rid of junk - # Default text value, text, between commas - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} - = "'$1'"; - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} - = 0; + $value = $1; # Get rid of junk + # Default text value, text, between commas + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = "'$1'"; + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 0; } - else - { - #This must be a function call... - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} - = convert_transactsql_code($value); - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} - = 1; - + else { + #This must be a function call... + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = convert_transactsql_code($value); + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 1; } - } @@ -938,21 +927,21 @@ sub generate_kettle $newtemplate =~ s/Y<\/use_batch>/N<\/use_batch>/g; # Cannot use batch mode with ignore errors } - if ($pforce_ssl) - { - $newtemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; - } - else - { - $newtemplate =~ s/__pforce_ssl__//g; - } + if ($pforce_ssl) + { + $newtemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; + } + else + { + $newtemplate =~ s/__pforce_ssl__//g; + } - if ($stringtype_unspecified) { - $newtemplate =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; - } - else { - $newtemplate =~ s/__stringtype_unspecified__//g; - } + if ($stringtype_unspecified) { + $newtemplate =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; + } + else { + $newtemplate =~ s/__stringtype_unspecified__//g; + } $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; @@ -976,30 +965,30 @@ sub generate_kettle $newincrementaltemplate =~ s/__sort_size__/$sort_size/g; if ($pforce_ssl) - { - $newincrementaltemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; - } - else - { - $newincrementaltemplate =~ s/__pforce_ssl__//g; - } - if ($stringtype_unspecified) { - $newincrementaltemplate =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; - } - else { - $newincrementaltemplate =~ s/__stringtype_unspecified__//g; - } - # We have a bit of work to do on primary keys for the incremental template: we need them + { + $newincrementaltemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; + } + else + { + $newincrementaltemplate =~ s/__pforce_ssl__//g; + } + if ($stringtype_unspecified) { + $newincrementaltemplate =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; + } + else { + $newincrementaltemplate =~ s/__stringtype_unspecified__//g; + } + # We have a bit of work to do on primary keys for the incremental template: we need them # to compare the tables… if (defined($refschema->{TABLES}->{$table}->{PK}->{COLS})) { - my @pk=@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}; - my $keys=""; - foreach my $pk(@pk) - { - $keys.="$pk\n"; - } - $newincrementaltemplate =~ s/__KEYS_MERGE__/$keys/g; + my @pk=@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}; + my $keys=""; + foreach my $pk(@pk) + { + $keys.="$pk\n"; + } + $newincrementaltemplate =~ s/__KEYS_MERGE__/$keys/g; my $sortkeys=''; my $synckeys=''; @@ -1300,7 +1289,7 @@ sub add_column_to_table # It has already been declared before. We just need to find it $coltype = relabel_schemas($coltypeschema) . '.' . $coltype; } - if ($colqual) + if ($colqual) { if ($coltype eq 'xml') { @@ -1319,15 +1308,15 @@ sub add_column_to_table or die "Cannot parse colqual <$colqual>"; $colqual = "$1"; } - } + } - # in case of a rowversion or timestamp columns, check if we want to keep it - if ($drop_rowversion - and ($coltype eq 'rowversion' or $coltype eq 'timestamp')) - { - # do nothing - return; - } + # in case of a rowversion or timestamp columns, check if we want to keep it + if ($drop_rowversion + and ($coltype eq 'rowversion' or $coltype eq 'timestamp')) + { + # do nothing + return; + } my $newtype = convert_type($coltype, $colqual, $colname, @@ -1454,12 +1443,11 @@ sub parse_dump my $colisnull =$7; my $default =$8; add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); - if (defined $default) - { - store_default_value($schemaname,$tablename,$colname,$default,$line); - } - } - + if (defined $default) + { + store_default_value($schemaname,$tablename,$colname,$default,$line); + } + } # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) # So just put it as a varchar, and issue a warning is STDOUT @@ -1589,57 +1577,55 @@ sub parse_dump ################################################################ elsif ($line =~ /^CREATE SEQUENCE \[(.*)\]\.\[(.*)\]/) { - my $schemaname = relabel_schemas($1); - my $orig_schema = $1; - my $seqname = $2; - while (my $contline = read_and_clean($file)) - { - if ($contline =~ /^\s*AS \[.*\]\s*$/) - { - next; # We don't care, sequences are always bigint in PostgreSQL - } - elsif ($contline =~ /^\s*START WITH (\d+)\s*$/) - { - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{START}=$1; - } - elsif ($contline =~ /^\s*INCREMENT BY (\d+)\s*$/) - { - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP}=$1; - } - elsif ($contline =~ /^\s*MINVALUE (-?\d+)\s*$/) - { - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MIN}=$1; - } - elsif ($contline =~ /^\s*MAXVALUE (-?\d+)\s*$/) - { - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MAX}=$1; - } - elsif ($contline =~ /^\s*(NO)?CACHE( \d+)?\s*$/) - { - if (defined $1) - { - # It's a no cache. Equivalent to CACHE = 1 in PostgreSQL - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=1; - } - elsif (defined $2) - { - # We have a specified value - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=$2; - } - else - { - # Cache, but not specified. SQL Server isn't very clear on the size of the cache. Let's say 100 - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=100; - } - } - elsif ($contline =~ /^GO$/) - { - next MAIN; - } - } - - - } + my $schemaname = relabel_schemas($1); + my $orig_schema = $1; + my $seqname = $2; + while (my $contline = read_and_clean($file)) + { + if ($contline =~ /^\s*AS \[.*\]\s*$/) + { + next; # We don't care, sequences are always bigint in PostgreSQL + } + elsif ($contline =~ /^\s*START WITH (\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{START}=$1; + } + elsif ($contline =~ /^\s*INCREMENT BY (\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP}=$1; + } + elsif ($contline =~ /^\s*MINVALUE (-?\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MIN}=$1; + } + elsif ($contline =~ /^\s*MAXVALUE (-?\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MAX}=$1; + } + elsif ($contline =~ /^\s*(NO)?CACHE( \d+)?\s*$/) + { + if (defined $1) + { + # It's a no cache. Equivalent to CACHE = 1 in PostgreSQL + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=1; + } + elsif (defined $2) + { + # We have a specified value + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=$2; + } + else + { + # Cache, but not specified. SQL Server isn't very clear on the size of the cache. Let's say 100 + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=100; + } + } + elsif ($contline =~ /^GO$/) + { + next MAIN; + } + } + } elsif ($line =~ /^CREATE SCHEMA \[(.*)\]/) { $objects->{SCHEMAS}->{relabel_schemas($1)} = undef @@ -1849,21 +1835,21 @@ sub parse_dump } if (defined $maybecols) { - my @maybecols = split (',',$maybecols); - foreach my $coldef(@maybecols) - { - $coldef=~/\[(.*)\](?: (ASC|DESC))?/ or die "Cannot understand coldef $coldef in index"; - if (defined $2) - { - push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} - ->{INDEXES}->{$idxname}->{COLS}}, ("$1 $2"); - } - else - { - push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} - ->{INDEXES}->{$idxname}->{COLS}}, ("$1"); - } - } + my @maybecols = split (',',$maybecols); + foreach my $coldef(@maybecols) + { + $coldef=~/\[(.*)\](?: (ASC|DESC))?/ or die "Cannot understand coldef $coldef in index"; + if (defined $2) + { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{COLS}}, ("$1 $2"); + } + else + { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{COLS}}, ("$1"); + } + } } while (my $idx = read_and_clean($file)) { @@ -1906,9 +1892,9 @@ sub parse_dump ->{INDEXES}->{$idxname}->{WHERE}="(".$filter.")"; } } - } + } - # we do not take migrate spatial indexes + # we do not take migrate spatial indexes elsif ($line =~ /^CREATE SPATIAL INDEX/) { my $def=$line; @@ -1927,8 +1913,8 @@ sub parse_dump $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} ->{$idxname}->{DISABLE} = 1; - } - + } + # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table elsif ($line =~ @@ -1947,7 +1933,7 @@ sub parse_dump add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); if (defined $default) { - store_default_value($schemaname,$tablename,$colname,$default,$line); + store_default_value($schemaname,$tablename,$colname,$default,$line); } } @@ -2026,13 +2012,13 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((\(?(?:-)?\d+(?:\.\d+)?\))?\) FOR \[(.*)\]/ ) { - store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(('.*')\) FOR \[(.*)\]/ ) { - store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } # Yes, we also get default NULL (what for ? :) ), and sometimes with a different case @@ -2040,7 +2026,7 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(((?i)NULL)\) FOR \[(.*)\]/ ) { - store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } # And there are also constraints with functions and other strange code in them. Put them as unsure @@ -2048,7 +2034,7 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/ ) { - store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } # FK constraint. It's multi line, we have to look for references, and what to do on update, delete, etc (I have only seen delete cascade for now) @@ -2249,13 +2235,13 @@ sub parse_dump } } - # Save variable for future use + # Save variable for future use elsif ($line =~ /^:setvar\s+(\S+)\s+"(.*)"/) - { - my $varname = $1; - my $varvalue = $2; - $objects->{VARIABLES}->{$varname} = $varvalue; - next; + { + my $varname = $1; + my $varvalue = $2; + $objects->{VARIABLES}->{$varname} = $varvalue; + next; } # Ignore USE, GO, and things that have no meaning for postgresql @@ -2302,57 +2288,58 @@ sub parse_dump # Also ignore version tests elsif ($line =~ /^IF EXISTS|^IF \(\@\@microsoftversion/i) { - # just read until next go (or EOF) - while (defined $line and $line !~ /^GO$/) - { - $line =read_and_clean($file); - } + # just read until next go (or EOF) + while (defined $line and $line !~ /^GO$/) + { + $line =read_and_clean($file); + } } elsif ($line =~ /^IF NOT EXISTS/i) { - # Just ignore the line - next; - + # Just ignore the line + next; } - - # Ignore CREATE DATABASE: we hope that we are given a single database as an option. It is multiline. + # Ignore CREATE DATABASE: we hope that we are given a single database as + # an option. It is multiline. # Ignore everything until next GO - # Ignore ALTER DATABASE for the same reason. The given parameters have no meaning in PG anyway - # Except for SET ARITHABORT OFF, for which we print a warning because it probably means the database contents are weird (10/0 = null) - elsif ($line =~ - /^ALTER DATABASE.* SET ARITHABORT OFF/) - { - print STDERR "WARNING: the source database is set as ARITHABORT OFF.\n"; - print STDERR " It means that for SQL Server, 10/0 = NULL.\n"; - print STDERR " You'll probably have problems porting that to PostgreSQL.\n"; - while ($line !~ /^GO$/) - { - $line =read_and_clean($file); - } - # We read everything in the CREATE DATABASE. Back to work ! - next; - } - # Sometimes, when there is a ALTER DATABASE SET ARITHABORT OFF, there are SET ARITHABORT ON. Just ignore them - elsif ($line =~ /^SET ARITHABORT ON/) - { - next; - } - # Sometimes we meet this: SET CONCAT_NULL_YIELDS_NULL ON. That's the normal behaviour for a SQL database. Just ignore - elsif ($line =~ /^SET CONCAT_NULL_YIELDS_NULL ON/) - { - next; - } - # Same more or less - elsif ($line =~ /^SET ANSI_WARNINGS ON/) - { - next; - } - # What the hell does this do in a dump ??? - elsif ($line =~ /^SET NUMERIC_ROUNDABORT OFF/) - { - next; - } + # Ignore ALTER DATABASE for the same reason. The given parameters have no + # meaning in PG anyway + # Except for SET ARITHABORT OFF, for which we print a warning because it + # probably means the database contents are weird (10/0 = null) + elsif ($line =~ + /^ALTER DATABASE.* SET ARITHABORT OFF/) + { + print STDERR "WARNING: the source database is set as ARITHABORT OFF.\n"; + print STDERR " It means that for SQL Server, 10/0 = NULL.\n"; + print STDERR " You'll probably have problems porting that to PostgreSQL.\n"; + while ($line !~ /^GO$/) + { + $line =read_and_clean($file); + } + # We read everything in the CREATE DATABASE. Back to work ! + next; + } + # Sometimes, when there is a ALTER DATABASE SET ARITHABORT OFF, there are SET ARITHABORT ON. Just ignore them + elsif ($line =~ /^SET ARITHABORT ON/) + { + next; + } + # Sometimes we meet this: SET CONCAT_NULL_YIELDS_NULL ON. That's the normal behaviour for a SQL database. Just ignore + elsif ($line =~ /^SET CONCAT_NULL_YIELDS_NULL ON/) + { + next; + } + # Same more or less + elsif ($line =~ /^SET ANSI_WARNINGS ON/) + { + next; + } + # What the hell does this do in a dump ??? + elsif ($line =~ /^SET NUMERIC_ROUNDABORT OFF/) + { + next; + } # Same for tests about full text search. elsif ($line =~ @@ -2452,12 +2439,12 @@ sub generate_schema } } - # Set psql variables in UNSURE - foreach my $varname (sort keys %{$objects->{VARIABLES}}) - { - print UNSURE "\\set $varname '$objects->{VARIABLES}->{$varname}'\n"; - } - + # Set psql variables in UNSURE + foreach my $varname (sort keys %{$objects->{VARIABLES}}) + { + print UNSURE "\\set $varname '$objects->{VARIABLES}->{$varname}'\n"; + } + # For the rest, we iterate over schemas, except for array types (no point in complicating this) # The tables, columns, etc... will be created in the before script, so there is no dependancy # problem with constraints, that will be in the after script, except foreign keys which depend on unique indexes @@ -2524,34 +2511,34 @@ sub generate_schema { my $seqref = $refschema->{SEQUENCES}->{$sequence}; print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence); - if (defined $seqref->{STEP}) - { - print AFTER " INCREMENT BY ",$seqref->{STEP}; - } - if (defined $seqref->{MIN}) - { - print AFTER " MINVALUE ",$seqref->{MIN}; - } - if (defined $seqref->{MAX}) - { - print AFTER " MAXVALUE ",$seqref->{MAX}; - } - if (defined $seqref->{START}) - { - print AFTER " START WITH ",$seqref->{START}; - } - if (defined $seqref->{CACHE}) - { - print AFTER " CACHE ",$seqref->{CACHE}; - } - if (defined $seqref->{OWNERTABLE}) - { - print AFTER " OWNED BY ",format_identifier($seqref->{OWNERSCHEMA}), - '.',format_identifier($seqref->{OWNERTABLE}), - '.',format_identifier($seqref->{OWNERCOL}); - } - print AFTER ";\n"; - } + if (defined $seqref->{STEP}) + { + print AFTER " INCREMENT BY ",$seqref->{STEP}; + } + if (defined $seqref->{MIN}) + { + print AFTER " MINVALUE ",$seqref->{MIN}; + } + if (defined $seqref->{MAX}) + { + print AFTER " MAXVALUE ",$seqref->{MAX}; + } + if (defined $seqref->{START}) + { + print AFTER " START WITH ",$seqref->{START}; + } + if (defined $seqref->{CACHE}) + { + print AFTER " CACHE ",$seqref->{CACHE}; + } + if (defined $seqref->{OWNERTABLE}) + { + print AFTER " OWNED BY ",format_identifier($seqref->{OWNERSCHEMA}), + '.',format_identifier($seqref->{OWNERTABLE}), + '.',format_identifier($seqref->{OWNERCOL}); + } + print AFTER ";\n"; + } # Now PK. We have to go through all tables foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -2636,42 +2623,42 @@ sub generate_schema $index_created = 1; } else + { + # this is either a disabled index or an index with a where declaration + if (defined $idxref->{WHERE}) { - # this is either a disabled index or an index with a where declaration - if (defined $idxref->{WHERE}) - { - print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; - if ($idxref->{DISABLE}) - { - # if disabled, will be on the same line - $idxdef .= " "; - } - else - { - # otherwise, write condition on a new line - $idxdef .= "\n"; - } - $idxdef .= "WHERE (" . convert_transactsql_code($idxref->{WHERE}) . ")"; - } - $idxdef .= ";\n"; - print UNSURE $idxdef; - # the possible comment would go to unsure file - $index_created = 2; + print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; + if ($idxref->{DISABLE}) + { + # if disabled, will be on the same line + $idxdef .= " "; + } + else + { + # otherwise, write condition on a new line + $idxdef .= "\n"; + } + $idxdef .= "WHERE (" . convert_transactsql_code($idxref->{WHERE}) . ")"; } + $idxdef .= ";\n"; + print UNSURE $idxdef; + # the possible comment would go to unsure file + $index_created = 2; + } - # Produce the comments for indexes - if (defined $idxref->{COMMENT}) - { - my $idxcomment = "COMMENT ON INDEX ". format_identifier($schema) . '.' . format_identifier($index) . " IS '" . $idxref->{COMMENT} . "';\n"; - if ($index_created == 1) - { - print AFTER $idxcomment; - } - elsif ($index_created == 2) - { - print UNSURE $idxcomment; - } - } + # Produce the comments for indexes + if (defined $idxref->{COMMENT}) + { + my $idxcomment = "COMMENT ON INDEX ". format_identifier($schema) . '.' . format_identifier($index) . " IS '" . $idxref->{COMMENT} . "';\n"; + if ($index_created == 1) + { + print AFTER $idxcomment; + } + elsif ($index_created == 2) + { + print UNSURE $idxcomment; + } + } } } @@ -2735,11 +2722,11 @@ sub generate_schema } } elsif ($constraint->{TYPE} eq 'CHECK') - { - $consdef .= " CHECK (" . convert_transactsql_code($constraint->{TEXT}) . ");\n"; - print UNSURE $consdef - ; # Check constraints are SQL, so cannot be sure - } + { + $consdef .= " CHECK (" . convert_transactsql_code($constraint->{TEXT}) . ");\n"; + print UNSURE $consdef + ; # Check constraints are SQL, so cannot be sure + } elsif ($constraint->{TYPE} eq 'CHECK_CITEXT') { # These have been generated here, for citext mostly. So we know their syntax is ok @@ -2772,10 +2759,11 @@ sub generate_schema foreach my $constraint ( @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) { - next unless defined ($constraint->{COMMENT}); - print UNSURE "COMMENT ON CONSTRAINT " . format_identifier($constraint->{NAME}) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " IS '" - . $constraint->{COMMENT} . "';\n"; - + next unless defined ($constraint->{COMMENT}); + print UNSURE "COMMENT ON CONSTRAINT " + . format_identifier($constraint->{NAME}) . " ON " + . format_identifier($schema) . '.' . format_identifier($table) + . " IS '" . $constraint->{COMMENT} . "';\n"; } } } @@ -2791,17 +2779,18 @@ sub generate_schema { my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; next unless (defined $colref->{DEFAULT}); - my $default_value = $colref->{DEFAULT}->{VALUE}; - if ($default_value =~ /\(\$\((\S+)\)\)/) - { - $default_value = ":$1"; - } + my $default_value = $colref->{DEFAULT}->{VALUE}; + if ($default_value =~ /\(\$\((\S+)\)\)/) + { + $default_value = ":$1"; + } my $definition = - "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) - . " ALTER COLUMN " . format_identifier($col) - . " SET DEFAULT " . $default_value . ";\n"; + "ALTER TABLE " . format_identifier($schema) . '.' + . format_identifier($table) + . " ALTER COLUMN " . format_identifier($col) + . " SET DEFAULT " . $default_value . ";\n"; if ($colref->{DEFAULT}->{UNSURE}) - { + { print UNSURE $definition; } else @@ -2817,11 +2806,14 @@ sub generate_schema { foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) { - my $seqref = $refschema->{SEQUENCES}->{$sequence}; - # This may not be an identity. Skip it then - next unless defined ($seqref->{OWNERCOL}); - - print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select max(". format_identifier($seqref->{OWNERCOL}) .") from " . format_identifier($seqref->{OWNERSCHEMA}) . '.'. format_identifier($seqref->{OWNERTABLE}) . ")::bigint);\n"; + my $seqref = $refschema->{SEQUENCES}->{$sequence}; + # This may not be an identity. Skip it then + next unless defined ($seqref->{OWNERCOL}); + print AFTER "select setval('" . format_identifier($schema) . '.' + . format_identifier($sequence) . "',(select max(" + . format_identifier($seqref->{OWNERCOL}) .") from " + . format_identifier($seqref->{OWNERSCHEMA}) . '.' + . format_identifier($seqref->{OWNERTABLE}) . ")::bigint);\n"; } } @@ -2842,8 +2834,10 @@ sub generate_schema my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; if (defined($colref->{COMMENT})) { - print AFTER "COMMENT ON COLUMN " . format_identifier($schema) . '.' . format_identifier($table) . '.' . format_identifier($col) . " IS '" - . $colref->{COMMENT} . "';\n"; + print AFTER "COMMENT ON COLUMN " . format_identifier($schema) + . '.' . format_identifier($table) . '.' + . format_identifier($col) . " IS '" + . $colref->{COMMENT} . "';\n"; } } } @@ -3044,7 +3038,7 @@ sub resolve_name_conflicts "sort_size=i" => \$sort_size, "use_pk_if_possible=s" => \$use_pk_if_possible, "ignore_errors" => \$ignore_errors, - "pforce_ssl" => \$pforce_ssl, + "pforce_ssl" => \$pforce_ssl, "stringtype_unspecified" => \$stringtype_unspecified ); From afddc60019c9703ebd7a7255baa82a07b2ad5e6d Mon Sep 17 00:00:00 2001 From: Thibaut Date: Mon, 7 Oct 2019 17:22:08 +0200 Subject: [PATCH 187/216] 113 constraint not understood (#122) * first commit for parse view query (#107) * better parse constraint on alter table --- regression/basic_test/views.sql | Bin 0 -> 6546 bytes regression/issue_59.sql | 4 ++ regression/issue_91.sql | 14 ---- regression/issues.sql | 51 +++++++++++++++ sqlserver2pgsql.pl | 112 ++++++++++++++++++++++++++------ t/test_convert.t | 25 +++++-- t/test_db_creation.t | 1 + 7 files changed, 167 insertions(+), 40 deletions(-) create mode 100644 regression/basic_test/views.sql delete mode 100644 regression/issue_91.sql create mode 100644 regression/issues.sql diff --git a/regression/basic_test/views.sql b/regression/basic_test/views.sql new file mode 100644 index 0000000000000000000000000000000000000000..093198a9d6485d163a802a6a2eeb81948a2c7139 GIT binary patch literal 6546 zcmeI1Yfsx&7{{Nto3!uX7n?#0q9lQCRGPF949kFIB?-{cD&mrMObn*NZqh#cw%`9e zKG;r^&`XNesj?jV*w6j$lYjp1Th~%c?9hI;BfGL|8`?{IW}n%owrQPkePH(sN5a!% z`w6%cpdW#o*gCk^oX3I2FwpA(A$se!20cOP=z{gq&g>W7M7&|3@})g6oi$sss;%<2 z%Kal-u{ys?Toe1=cK9ty^B_prv|Y4lSTE?Z9_||OZQN*AmPXno)~iQ$m1D86mREOeFZ@0&NwbKjPoY|YnxyF> z%@4r2?8{hsm1Ehyg5pEl!io+jtq(?^``BeWSfmAy6kiwh+eCtn?epf_jCvL7HE284 zdzp2&f!YMZt^B$+J;ydL(4xq(jg0bR?uWau=)hc-MXr-AyMpe^9CaeOc8gt=1MZ1UXXTGljOQSuy{fkr{{J9TPEre?|QnkOR)?I^&e*X{=+a zTi=kos&r>yUu2q9z_{S~F)|I2M=`4>rzLy5trr?(m9lcc)h#%{iz$|Agta`?VI^ye zbF}~L_;ueK2gTMphbpe;{QJ;jA1=Faa4xggMqNNvO+BU);&;49@`${$KZ)W+PDP$# zaX=M6qL!Z$IVVJy`tLbU)iW=7eneysSw9aey;a9-6K%!V_WGx%s-7N+F&^^XUc`9Q z-rh$d^_ttn!FH{=J+W9llc%u0&!qCLuUpC-Ar zlI_RoJ{Kr_f_Iv9D#h+0-tk;Ku^+LY=3jNO6}pU1f~vT&7B3X=Yg6!R;4fzsyowhw z>lvnc*LAqp#Q7?p(TQ^lOK9?JqNg&!yS!``r%~CbsGVluW{~+FvzXT{b!o4}vUEGx z`Tz>*>)u&>GL%K-0r64S?&2BAscBo&vu1^UkYNapm0B;O2PRK~bO3UXXv)EkZd?H<7Eb6Le{F_>A=(I6euMu~c<(EKN3E>F3D7 zTn6t$xfF^{`9_^LhZDBCTZF4&iSoC3d1jexZJ#Y>XYIu1kt^XF;V(|zX&vRPhSm6Y z(aW-m?}ajY#rY(<4cG##k@s zWo^o2JvmC3Lmj-0c8maJq@qq=svd<}HSwz^%()5C;rU+y$8%0KWi<9QZOh;PXFsmI zmaU_0=Wnu0R`*KmRX6v>JU0EDx7q>IzC*&Qf`?*WDuN?sxk5)zR_K#ge8c!on)dK= k40X2;EvNE5lga;wyzM?)LY36N6#Bm1=ic|s%X@hL0zM^5w*UYD literal 0 HcmV?d00001 diff --git a/regression/issue_59.sql b/regression/issue_59.sql index ef11c9c..3eda87a 100644 --- a/regression/issue_59.sql +++ b/regression/issue_59.sql @@ -21,6 +21,10 @@ CREATE TABLE [dbo].[ACCOUNT]( )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY] +ALTER TABLE [dbo].[ACCOUNT] WITH CHECK ADD CHECK (APPLICATION_ID IN('1.0', '2.1', '2.2', '4.12', '10.3', 'None')); +GO +ALTER TABLE [dbo].[ACCOUNT] ADD CONSTRAINT deletor_list CHECK (DELETED_BY IN('Jacques', 'Philippe', 'Pierre', 'None')); +GO EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT.ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT', @level2type=N'COLUMN',@level2name=N'ID' GO diff --git a/regression/issue_91.sql b/regression/issue_91.sql deleted file mode 100644 index 45721da..0000000 --- a/regression/issue_91.sql +++ /dev/null @@ -1,14 +0,0 @@ - -CREATE TABLE [candidate].[PersonalIdentityHashes]( - [OrganizationId] [int] NOT NULL, - [PersonalIdentity] [char](64) NOT NULL, - [PersonalIdentityHash] [binary](64) NOT NULL, - [PersonalIdentityHash2] [varbinary](64) NOT NULL, - [CandidateId] [bigint] NOT NULL, - CONSTRAINT [PK_candidatePersonalIdentityHashes] PRIMARY KEY CLUSTERED -( - [OrganizationId] ASC, - [PersonalIdentityHash] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] -) ON [PRIMARY] -GO diff --git a/regression/issues.sql b/regression/issues.sql new file mode 100644 index 0000000..32089f8 --- /dev/null +++ b/regression/issues.sql @@ -0,0 +1,51 @@ +USE [master] +GO +/****** Issue 91 ******/ +CREATE TABLE [candidate].[PersonalIdentityHashes]( + [OrganizationId] [int] NOT NULL, + [PersonalIdentity] [char](64) NOT NULL, + [PersonalIdentityHash] [binary](64) NOT NULL, + [PersonalIdentityHash2] [varbinary](64) NOT NULL, + [CandidateId] [bigint] NOT NULL, + CONSTRAINT [PK_candidatePersonalIdentityHashes] PRIMARY KEY CLUSTERED +( + [OrganizationId] ASC, + [PersonalIdentityHash] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO + +/****** Issue 98 ******/ +CREATE TABLE [etl].[account_handle]( + [acct_event_id] [bigint] NOT NULL , + [acct_id] [bigint] NOT NULL , + [comment_id] [bigint] NULL, + [descrip] [varchar] NULL, + [event_cat_id] [bigint] NOT NULL , + [event_date] [datetime] NOT NULL , + [event_end_time] [varchar] NULL, + [event_start_time] [varchar] NULL, + [event_type_id] [bigint] NOT NULL , + [problem_cd] [bigint] NULL, + [reason_cd] [bigint] NULL, + [resolution] [varchar] NULL, + [user_id] [bigint] NULL, + [ext_user_name] [varchar] NULL, + [level1_id] [bigint] NULL, + [level2_id] [bigint] NULL, + [level3_id] [bigint] NULL, + [level4_id] [bigint] NULL, + [group_id] [bigint] NULL, + [escalation_id] [bigint] NULL, + [level5_id] [bigint] NULL, + [reference_id] [bigint] NULL, + [entity] [varchar] NULL, + [modified_ts] [datetime] NOT NULL , + [device_type_id] [bigint] NULL, + [device_error_codes] [varchar] NULL, + PRIMARY KEY CLUSTERED +( + [acct_event_id] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 75f9a9b..0fa7a99 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -421,7 +421,6 @@ sub postgres_convert_column } # This is used to convert camelCase to snake_case. The latter is more usual with PostgreSQL - sub camel_to_snake { my ($string)=@_; @@ -503,9 +502,24 @@ sub format_identifier_cols_index return $formatted . ' ' . $order; } -# This one will try to convert what can obviously be converted from transact to (or embedded WHERE in indexes for instance) PG -# We check if we have several blocks separated by logical operators +# This one will try to convert what can obviously be converted from transact to PG # Things such as getdate() which can become CURRENT_TIMESTAMP +sub convert_transact_function +{ + my ($code)=@_; + $code =~ s/ISNULL\s*\(/COALESCE(/gi; + $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; + $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; + $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; + $code =~ s/CONVERT\s*\(\s*NVARCHAR\s*(.*?)\s*\(\s*(.*?)\s*\s*\)\,\s*(.*?)\s*\)/CAST($3 AS varchar($2))/gi; + $code =~ s/CONVERT\s*\(\s*(.*?)\s*\(\s*(.*?)\s*\s*\)\,\s*(.*?)\s*\)/CAST($3 AS $1($2))/gi; + $code =~ s/CONVERT\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/CAST($2 AS $1)/gi; + return $code; +} + +# This one will try to convert from transact to PG +# We check if we have several blocks separated by logical operators +# Code going through this function will go to UNSURE file sub convert_transactsql_code { my ($code)=@_; @@ -526,10 +540,7 @@ sub convert_transactsql_code else { $code =~ s/\[(.*)\]/rename_identifier($1)/gie; # Bit brutal probably } - $code =~ s/ISNULL\s*\(/COALESCE(/gi; - $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; - $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; - $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; + $code = convert_transact_function($code); } #print STDERR "to: $code\n\n"; return $code; @@ -1450,10 +1461,10 @@ sub parse_dump } # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) - # So just put it as a varchar, and issue a warning is STDOUT + # So just put it as a varchar, and issue a warning in STDOUT + # FIXME this should exist in PG12 elsif ($line =~ /^\s*\[(.*)\]\s+AS\s+\((.*)\)/) { - # We just get the column name my $colnumber=next_col_pos($schemaname,$tablename); my $colname = $1; @@ -1546,7 +1557,6 @@ sub parse_dump # Unique key definition. We read following lines until the end of the constraint while (my $uk = read_and_clean($file)) { - # Exit when read a line beginning with ). The constraint is complete if ($uk =~ /^\)/) { @@ -1713,8 +1723,9 @@ sub parse_dump } $schemaname = relabel_schemas($schemaname); - my $sql = 'CREATE VIEW ' . $schemaname . '.' . $viewname . ' ' . $supplement . "\n"; - while (my $line_cont = read_and_clean($file)) + my $sql = $supplement; + + while (my $line_cont = read_and_clean($file)) { if ($line_cont =~ /^\s*'\s*$|^GO$/ ) # We may have a quote if the view is 'quoted', or a real sql query @@ -1723,11 +1734,58 @@ sub parse_dump # We get rid of dbo. schemas $sql =~ s/(dbo)\./relabel_schemas($1) . '.'/eg ; # We put this in the replacement schema + # print STDERR "code view: ".$sql."\n"; + # parse the query view + if ( $sql =~ /^\s*\(([^\)]+)\)\s*AS\s+SELECT\s+(.*)\s+FROM\s+(.*)$/i) { + my $view_columns = $1; + my $query_columns = $2; + my $query_end = $3; + my @rebuilt_view_columns = (); + my @rebuilt_query_columns = (); + my @string_column = (); + + # format columns names + foreach my $view_col (split (',',$view_columns)) { + $view_col =~ s/^\s+|\s+$//g; + push @rebuilt_view_columns, format_identifier($view_col); + } + $objects->{SCHEMAS}->{$schemaname}->{VIEWS}->{$viewname}->{COLS} = + join(',', @rebuilt_view_columns); + + # format view query columns + foreach my $view_query_col (split (',',$query_columns)) { + if ($view_query_col + =~ /^.*\+\s*N?'.*'\s*|\s*N?'.*'\s*\+.*|.*\+\s*N?'.*'\s*\+.*$/i) { + # PG use '||' to concatenate strings, change '+' to '||' + @string_column = (); + my $lhs; + while ($view_query_col =~ /^\s*(N?'.*?'|[^']+?)\s*\+\s*(.*)$/i) { + $lhs = $1; + $view_query_col = $2; + $lhs = $1 if ($lhs =~ /N('.*?')/); + push @string_column, $lhs; + } + $view_query_col = $1 if ($view_query_col =~ /N('.*?')/); + push @string_column, $view_query_col; + push @rebuilt_query_columns, join('||', @string_column); + } + else { + push @rebuilt_query_columns, $view_query_col; + } + } + $objects->{SCHEMAS}->{$schemaname}->{VIEWS}->{$viewname}->{QUERYCOLS} = + convert_transact_function(join(',', @rebuilt_query_columns)); + + $objects->{SCHEMAS}->{$schemaname}->{VIEWS}->{$viewname}->{QUERY} = + convert_transact_function($query_end); + } + else { + $objects->{SCHEMAS}->{$schemaname}->{VIEWS}->{$viewname}->{SQL} = + $sql; + } # Views will be stored without the full schema in them. We will # have to generate the schema in the output file - $objects->{SCHEMAS}->{$schemaname}->{'VIEWS'}->{$viewname}->{SQL} = - $sql; my @view_array=($schemaname,$viewname); push @view_list,(\@view_array); # adds another schema/view to the list next MAIN; @@ -2102,7 +2160,7 @@ sub parse_dump # Check constraint. As it can be arbitrary code, we just get this code, and hope it will work on PG (it will be stored in a special script file) elsif ($line =~ - /ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? CHECK(?: NOT FOR REPLICATION)?\s+\(\((.*)\)\)/ + /ALTER TABLE \[(.*)\]\.\[(.*)\]\s+(?:WITH? (?:NO)?CHECK? )?ADD(?:\s+CONSTRAINT (.*))?\s+CHECK(?: NOT FOR REPLICATION)?\s+\((\((.*)\)|(.*))\)/ ) { # Check constraint. We'll do what we can, syntax may be different. @@ -2111,9 +2169,10 @@ sub parse_dump my $constxt = $4; my $schema = relabel_schemas($1); $constraint->{TABLE} = $table; - if (defined $3) - { - $constraint->{NAME} = $3; + if (defined $3) { + my $constraint_name = $3; + $constraint_name = $1 if ($constraint_name =~ /\[(.*)\]/); + $constraint->{NAME} = $constraint_name; } $constraint->{TYPE} = 'CHECK'; $constraint->{TEXT} = $constxt; @@ -2848,11 +2907,22 @@ sub generate_schema { my ($schema,$view)=@$viewref; my $refschema=$objects->{SCHEMAS}->{$schema}; - print UNSURE $refschema->{VIEWS}->{$view}->{SQL}, ";\n"; + print UNSURE "CREATE VIEW " + . format_identifier($schema) . '.' . format_identifier($view) . " "; + if (not defined $refschema->{VIEWS}->{$view}->{SQL}) { + my $view_columns = $refschema->{VIEWS}->{$view}->{COLS}; + my $query_columns = $refschema->{VIEWS}->{$view}->{QUERYCOLS}; + my $query = $refschema->{VIEWS}->{$view}->{QUERY}; + print UNSURE "($view_columns) AS\n\tSELECT\n\t\t" + . $query_columns . "\n\t" . "FROM $query;\n\n"; + } + else { + print UNSURE $refschema->{VIEWS}->{$view}->{SQL} . ";\n\n"; + } if (defined $refschema->{VIEWS}->{$view}->{COMMENT}) { - print UNSURE "COMMENT ON VIEW $schema.$view IS '" - . $refschema->{VIEWS}->{$view}->{COMMENT} . "';\n"; + print UNSURE "COMMENT ON VIEW $schema.$view IS '" + . $refschema->{VIEWS}->{$view}->{COMMENT} . "';\n"; } } # Trigger functions diff --git a/t/test_convert.t b/t/test_convert.t index 3e5791c..61013c0 100644 --- a/t/test_convert.t +++ b/t/test_convert.t @@ -16,15 +16,19 @@ fi # create the array of command to try - options_to_try=( "-i" "-nr" "-num" "-keep_identifier_case" "-validate_constraints=after" ) - declare -a all_combinations=("blank ") + options_to_try=( "-i" "-nr" "-num" "-validate_constraints=after" ) + declare -a many_combinations=("blank ") for option_to_try in ${options_to_try[@]} ; do - for exiting_combination in ${all_combinations[@]}; do - all_combinations+="${option_to_try}_sep_${exiting_combination} " + for existing_combination in ${many_combinations[@]}; do + all_combinations+="${option_to_try}_sep_${existing_combination} " done done - + declare -a all_combinations=many_combinations + for existing_combination in ${all_combinations[@]}; do + all_combinations+="-keep_identifier_case_sep_${existing_combination} " + done + # clear the possible leftovers if [ -d $WORK_DIR ]; then rm -rf $WORK_DIR @@ -44,4 +48,15 @@ test_nb+=1 done done + for reg_file in regression/basic_test/*.sql ; do + for option in ${many_combinations[@]} ; do + CURRENT_DIR=$WORK_DIR/$test_nb + mkdir -p $CURRENT_DIR + real_option=$(echo $option | sed 's/blank//' | sed 's/_sep_/ /g') + command_line="./sqlserver2pgsql.pl -f $reg_file -b $CURRENT_DIR/before.sql -a $CURRENT_DIR/after.sql -u $CURRENT_DIR/unsure.sql -k $CURRENT_DIR/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 $real_option" + echo $command_line > $CURRENT_DIR/command_line + eval $command_line > $CURRENT_DIR/command_output 2>&1 + test_nb+=1 + done + done } diff --git a/t/test_db_creation.t b/t/test_db_creation.t index c9423b7..2b24be5 100644 --- a/t/test_db_creation.t +++ b/t/test_db_creation.t @@ -5,6 +5,7 @@ cd $WORK_DIR for test_dir in * ; do cd $test_dir + cat command_line createdb reg psql reg < before.sql psql reg < after.sql From 06391890d8d76692a2806306a22ef09116794a8c Mon Sep 17 00:00:00 2001 From: madtibo Date: Tue, 10 Dec 2019 11:42:17 +0100 Subject: [PATCH 188/216] help correction: si = SQLSERVER_INSTANCE --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 0fa7a99..cfab8da 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -790,7 +790,7 @@ sub usage -sd SQLSERVER_DATABASE -sh SQLSERVER_HOST - -si SQLSERVER_HOST + -si SQLSERVER_INSTANCE -sp SQLSERVER_PORT -su SQLSERVER_USERNAME -sw SQLSERVER_PASSWORD From 78054012a2bc7a5417798e44589e6eb5418a3bf7 Mon Sep 17 00:00:00 2001 From: Thibaut Date: Fri, 3 Apr 2020 17:23:55 +0200 Subject: [PATCH 189/216] permit other partition_scheme or filegroup than PRIMARY for object creation (#127) * permit other partition_scheme or filegroup than PRIMARY for object creation * specific env for PG to solve CI container pb with PG access --- .circleci/config.yml | 4 ++++ regression/reg_tests.sql | Bin 12596 -> 12608 bytes sqlserver2pgsql.pl | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e677c50..001c829 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,10 @@ jobs: docker: - image: perl:5.24-threaded - image: postgres:10-alpine + environment: + POSTGRES_USER: postgres + POSTGRES_POSTGRES: postgres + POSTGRES_HOST_AUTH_METHOD: trust environment: # Inject APT packaged dependencies. PERL5LIB: /usr/lib/x86_64-linux-gnu/perl5/5.24:/usr/share/perl5 diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index 3e6b517666c89a927d837cb32f7c13678edee699..06af866668160b3d1df00f6e817c60b87df906f5 100644 GIT binary patch delta 46 zcmdmzbRcQNL|tYD2E)l8nB+n9=5@NI%%Z^zt_;o${tSK$E)0$gK@5=$u?$=QMOh0S delta 28 kcmX?*v?Xc7MBT~0N=lReDv3>&7vtM}U)P6uaxSYN0IuH(jQ{`u diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index cfab8da..f804ee9 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1571,7 +1571,7 @@ sub parse_dump } } - elsif ($line =~ /^\s*\) ON \[PRIMARY\]/) + elsif ($line =~ /^\s*\) ON \[.*\]/) { # End of the table next MAIN; From 5e3e73838812835b397cc483d96a6724b9ab60ee Mon Sep 17 00:00:00 2001 From: Thibaut Date: Fri, 3 Apr 2020 17:34:42 +0200 Subject: [PATCH 190/216] update CI container to PG 12 (#128) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 001c829..2cab6cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ jobs: test: docker: - image: perl:5.24-threaded - - image: postgres:10-alpine + - image: postgres:12-alpine environment: POSTGRES_USER: postgres POSTGRES_POSTGRES: postgres From 7e65e475eddb4ff58cdd7ede80074bc0d9981b54 Mon Sep 17 00:00:00 2001 From: madtibo Date: Tue, 23 Jun 2020 10:16:33 +0200 Subject: [PATCH 191/216] exclude some more sp_addextendedproperty --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index f804ee9..7c6ce76 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2206,7 +2206,7 @@ sub parse_dump or croak "Cannot find a name for this extended property: $sqlproperty"; my $propertyname = $1; - if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1|MS_DiagramPane2|Display Name|Description|Example Values|Source System|Table Description|Table Type|ETL Rules|Display Folder|SCD Type|Source Datatype)$/) + if ($propertyname =~ /^(AggregateType|AllowZeroLength|AppendOnly|Attributes|CollatingOrder|ColumnHidden|ColumnOrder|ColumnWidth|DataUpdatable|DateCreated|DefaultValue|Description|Display Folder|Display Name|DisplayViewsOnSharePointSite|ETL Rules|Example Values|FilterOnLoad|GUID|HideNewField|LastUpdated|MS_DecimalPlaces|MS_DefaultView|MS_DiagramPane1|MS_DiagramPane2|MS_DiagramPaneCount|MS_DisplayControl|MS_Format|MS_Hyperlink|MS_IMEMode|MS_IMESentMode|MS_InputMask|MS_OrderByOn|MS_Orientation|Name|OrderByOnLoad|OrdinalPosition|RecordCount|Required|SCD Type|ShowDatePicker|Size|Source Datatype|Source System|SourceField|SourceTable|Table Description|Table Type|TextAlign|TextFormat|TotalsRow|Type|UnicodeCompression|Updatable)$/) { # We don't dump these. They are graphical descriptions of the GUI next; From 2ef2175d56a5a5c954b1ce32a9ab97216f1f4a35 Mon Sep 17 00:00:00 2001 From: Thibaut Date: Fri, 16 Oct 2020 16:30:11 +0200 Subject: [PATCH 192/216] 121 generated columns (#133) * Add generated columns support Many thanks to @alchemistmatt! Co-authored-by: alchemistmatt --- regression/reg_tests.sql | Bin 12608 -> 13250 bytes sqlserver2pgsql.pl | 71 +++++++++++++++++---------------------- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index 06af866668160b3d1df00f6e817c60b87df906f5..0ac88624209231fcf714edc7cdcd73024ded527b 100644 GIT binary patch delta 413 zcmX?*bSQlTtHI>IN@A1c#rP)Q*X7x)Zy?AxnM2Ib(48TbArFX)7!nyufOHCjGebT@ z4nrwJE>OIfA(labL4$#dfs-Mcp$w=dXYvDaX-31zaO-{=q)HR0E`Y(6AqeO;PaqBfvR%+zVM3H?U^giNy%PosFb154Pd=q&0RS?_ BO-29! delta 16 XcmX?}n}4FnkhK6(ZQ diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 7c6ce76..456fab8 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1460,57 +1460,48 @@ sub parse_dump } } - # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) - # So just put it as a varchar, and issue a warning in STDOUT - # FIXME this should exist in PG12 - elsif ($line =~ /^\s*\[(.*)\]\s+AS\s+\((.*)\)/) + # This is a computed column. PostgreSQL supports this as a generated column, starting with PG12 + # Will assume the data type is varchar, but this will need to be changed if the source columns are int, numeric, float, etc. + elsif ($line =~ /^\s*\[(.*)\]\s+AS\s+\((.*)\)(.*)/) { - # We just get the column name + # Get the column name my $colnumber=next_col_pos($schemaname,$tablename); my $colname = $1; my $code = $2; my $coltype = 'varchar'; + my $other_param = $3; + + # Replace square brackets in $code with double quotes + my $codequoted = $code =~ s/[\[\]]/"/gr; + my $generatedcode = " /* GENERATED ALWAYS AS ($codequoted)"; + if ($other_param =~ /PERSISTED/) { + $generatedcode .= " STORED"; + } + $generatedcode .= " */"; + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{POS} = $colnumber; $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{TYPE} = $coltype; - $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{NOT_NULL} = 0; + ->{$colname}->{TYPE} = $coltype . $generatedcode; + + if ($other_param =~ /NOT NULL/) { + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 1; + } + else { + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 0; + } - # Big fat warning + # Show a warning print STDERR - "Warning: There is a calculated column: $schemaname.$tablename.$colname. This isn't done the same way in PG at all\n"; + "\nWarning: There is a computed column: $schemaname.$tablename.$colname\n"; print STDERR - "\tFor now it has been declared as a varchar in PG, so that the values can be copied\n"; + "\tPostgreSQL 12 supports this via GENERATED ALWAYS AS (...)\n"; print STDERR - "\tYou should change its type manually in the dump (sorry for that),\n"; - print STDERR "\tA trigger has been written in the unsure file. It probably won't work as is.\n"; - print STDERR "\tPlease review it.\n"; - - # Try to correct what can be corrected from the AS : replace [COL] with NEW.COL - # It is obviously not going to work for anything a bit complicated - $code =~ s/\[(.*?)\]/NEW.$1/g; - my $triggerfunc = <{SCHEMAS}->{$schemaname}->{'TRIG_FUNCTIONS'} - ->{'trig_func_ins_or_upd' || $tablename}->{DEF} = - $triggerfunc; - $objects->{SCHEMAS}->{$schemaname}->{'TRIG_FUNCTIONS'} - ->{'trig_func_ins_or_upd' || $tablename}->{LANG} = - 'plpgsql'; - my %trigger; - $trigger{EVENTS} = 'before insert or update'; - $trigger{WHEN} = 'for each row'; - $trigger{FUNCTION} = - 'trig_func_ins_or_upd' || $tablename; # In the same schema - $trigger{NAME} = 'trig_ins_or_upd' || $tablename; - push @{$objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename} - ->{TRIGGERS}}, (\%trigger); - + "\tFor now it has been declared as a varchar and the calculation formula has been commented.\n"; + print STDERR + "\tThe formula will likely need to be manually fixed to properly refer to other columns.\n"; } elsif ($line =~ /^\s*(?:CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED(?: HASH)?/) @@ -2704,7 +2695,7 @@ sub generate_schema # the possible comment would go to unsure file $index_created = 2; } - + # Produce the comments for indexes if (defined $idxref->{COMMENT}) { From 8a3faa4b82f69611f2263aed07c0d7abf3543ce0 Mon Sep 17 00:00:00 2001 From: Thibaut Date: Fri, 16 Oct 2020 16:43:07 +0200 Subject: [PATCH 193/216] 116 ignore grants (#134) * Ignore GRANT statements * add GRANT orders to regression tests Co-authored-by: alchemistmatt --- regression/basic_test/views.sql | Bin 6546 -> 6666 bytes regression/reg_tests.sql | Bin 13250 -> 13502 bytes sqlserver2pgsql.pl | 10 ++++++++++ 3 files changed, 10 insertions(+) diff --git a/regression/basic_test/views.sql b/regression/basic_test/views.sql index 093198a9d6485d163a802a6a2eeb81948a2c7139..f92587755af4174b18c7332453caa9f2f13f150f 100644 GIT binary patch delta 89 zcmbPa+-0(1lB6*&0~dokLlA=_gC9c(g91YsgC~P4LpYG-0;Ju5JU<`~0b+k3O94bq fo-HmZ8Ukb~Fhn!>GNdxZ0$Gj>!3+wM4LQXD&_NF# delta 7 OcmeA&nPj|Sk|Y2N_yXSm diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index 0ac88624209231fcf714edc7cdcd73024ded527b..f9c40f36fe419ddab81338b2a03829355b0ece1a 100644 GIT binary patch delta 192 zcmX? Date: Fri, 16 Oct 2020 16:52:35 +0200 Subject: [PATCH 194/216] 120 warn long name (#135) * Warn the user if a column or table name is more than 63 characters long Co-authored-by: alchemistmatt --- regression/reg_tests.sql | Bin 13502 -> 13694 bytes sqlserver2pgsql.pl | 5 +++++ 2 files changed, 5 insertions(+) diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index f9c40f36fe419ddab81338b2a03829355b0ece1a..bf771fc02b53c1b30aa3da8dadc1973b86d528be 100644 GIT binary patch delta 173 zcmdm&`7djOE@M3>Lo`DegDXQ2LnK2ygAap0gCB!CkmU?y`7nes_yYOyK 63) + { + print STDERR "WARNING: $identifier is more than 63 characters long; PostgreSQL will truncate the name internally\n"; + } + # Now, we protect the identifier (similar to quote_ident in PG) $identifier=~ s/"/""/g; $identifier='"'.$identifier.'"'; From bf08ca9ae5cedaaadcf4ccdc4a4c726e59b72d95 Mon Sep 17 00:00:00 2001 From: Thibaut Date: Fri, 16 Oct 2020 18:06:36 +0200 Subject: [PATCH 195/216] Add option to skip checking the maximum length of citext columns (#136) * Add option to skip checking the maximum length of citext columns and camelcasetosnake option to the example conf file These options are commented out, to retain the current behaviour of the example conf file Co-authored-by: alchemistmatt --- example_conf_file | 12 +++++++----- sqlserver2pgsql.pl | 5 ++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/example_conf_file b/example_conf_file index a2b734f..da617bb 100644 --- a/example_conf_file +++ b/example_conf_file @@ -23,12 +23,14 @@ parallelism_in=8 # Parallelism reading from SQL Server (where available) Default parallelism_out=8 # Default value is 8. Number of parallel connections used by kettle to insert data into the PostgreSQL database # Optional behaviour -case insensitive=0 # set it to 1 to generate a dump with citext and check constraints all over the place -no relabel dbo=1 # set it to 0 to convert the dbo schema to public -convert numeric to int=1 # set it to 0 to keep numeric(xx,0) as numeric(xx,0). Will be converted to smallint, int or bigint by default +case insensitive=0 # set it to 1 to generate a dump with citext and check constraints all over the place +no relabel dbo=1 # set it to 0 to convert the dbo schema to public +convert numeric to int=1 # set it to 0 to keep numeric(xx,0) as numeric(xx,0). Will be converted to smallint, int or bigint by default relabel schemas=dbo=>foo;schema1=>bar -keep identifier case=1 # keep case of database objects -validate constraints = yes # yes, after or no, should the constraints be validated by the dump ? (yes=validate during load, after after the load, no keep invalidated) +keep identifier case=1 # keep case of database objects; comment out to convert names to lowercase +#camelcasetosnake=1 # Uncomment to convert to snake case; comment out to leave names unchanged (or lowercase) +validate constraints = yes # yes, after or no, should the constraints be validated by the dump ? (yes=validate during load, after after the load, no keep invalidated) +#skip citext length check=1 # When defined, do not add a CHECK (char_length()) check for citext fields # Incremental job sort size=10000 # drives the amount of memory and temporary files that will be created by an incremental job diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index a5fd9f7..6ffe989 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -52,6 +52,7 @@ our $use_pk_if_possible; our $pforce_ssl; our $stringtype_unspecified; +our $skip_citext_length_check; # Will be set if we detect GIS objects our $requires_postgis=0; @@ -108,6 +109,7 @@ sub parse_conf_file 'ignore errors' => 'ignore_errors', 'postgresql force ssl' => 'pforce_ssl', 'stringtype unspecified' => 'stringtype_unspecified', + 'skip citext length check' => 'skip_citext_length_check', ); # Open the conf file or die @@ -158,6 +160,7 @@ sub set_default_conf_values $sp=1433 unless (defined ($sp)); $pforce_ssl=0 unless (defined ($pforce_ssl)); $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); + $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); } # Converts numeric(4,0) and similar to int, bigint, smallint @@ -326,7 +329,7 @@ sub convert_type $rettype = "citext"; # Do we have a SQL qualifier ? (we'll have to do check constraints then) - if ($sqlqual) + if ($sqlqual and not defined($skip_citext_length_check) or $sqlqual and $skip_citext_length_check == 0) { # Check we have a table name and a colname, or a typname From 898044b5128cffafac100f17c68bf071541df072 Mon Sep 17 00:00:00 2001 From: Thibaut Date: Wed, 21 Oct 2020 11:53:23 +0200 Subject: [PATCH 196/216] parse index included columns (available for PG 11 onward) (#132) * parse index included columns (available for PG 11 onward) The created index will be written to AFTER file. --- regression/issue_59.sql | 48 +++++++++++++ sqlserver2pgsql.pl | 147 ++++++++++++++++++++-------------------- 2 files changed, 122 insertions(+), 73 deletions(-) diff --git a/regression/issue_59.sql b/regression/issue_59.sql index 3eda87a..db1073d 100644 --- a/regression/issue_59.sql +++ b/regression/issue_59.sql @@ -47,6 +47,15 @@ GO EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT.VERSION' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT', @level2type=N'INDEX',@level2name=N'IDX_ACCOUNT_VERSION' GO +CREATE NONCLUSTERED INDEX [IDX_ACCOUNT_BIC_IBAN] ON [dbo].[ACCOUNT] +( + [BIC] ASC, + [IBAN] ASC +) +INCLUDE ( [BACK_OFFICE_ACCOUNT_NUMBER], +[BANK_ACCOUNT_NUMBER]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO + CREATE TABLE [dbo].[ACCOUNT_CATEGORY]( [ID] [char](36) NOT NULL, [VERSION] [numeric](10, 0) NOT NULL, @@ -58,3 +67,42 @@ CREATE TABLE [dbo].[ACCOUNT_CATEGORY]( GO EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT_CATEGORY.UQ_INDEX' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT_CATEGORY', @level2type=N'INDEX',@level2name=N'UQ_INDEX' GO + +CREATE NONCLUSTERED INDEX [IDX_ACCOUNT_CATEGORY_ID] ON [dbo].[ACCOUNT_CATEGORY] +( + [ID] ASC +) +INCLUDE ( [VERSION]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO +/****** Object: Table [dbo].[IDX_TESTS] Script Date: 15/10/2020 14:34:08 ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[IDX_TESTS]( + [I] [int] NULL, + [J] [int] NULL, + [K] [int] NULL, + [L] [int] NULL +) ON [PRIMARY] +GO +/****** Object: Index [idx_IDX_TESTS_i_part] Script Date: 15/10/2020 14:34:09 ******/ +CREATE NONCLUSTERED INDEX [IDX_IDX_TESTS_I_INCL_K_PART] ON [dbo].[IDX_TESTS] +( + [I] ASC +) +INCLUDE ( [K]) +WHERE ([L]>(10)) +WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO +/****** Object: Index [IDX_IDX_TESTS_I_J_PART] Script Date: 15/10/2020 14:34:09 ******/ +CREATE NONCLUSTERED INDEX [IDX_IDX_TESTS_I_J_INCL_K_L_PART] ON [dbo].[IDX_TESTS] +( + [I] ASC, + [J] ASC +) +INCLUDE ( [K], + [L]) +WHERE ([J]>(1)) +WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 6ffe989..ca229cd 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1917,41 +1917,45 @@ sub parse_dump next MAIN; } next - if ($idx =~ /^\(|^\)/) - ; # Begin/end of the columns declaration - if ($idx =~ /\t\[(.*)\] (ASC|DESC)(,)?/) - { - if (defined $2) - { - push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + if ($idx =~ /^\(|^\)/) + ; # Begin/end of the columns declaration + if ($idx =~ /\t\[(.*)\] (ASC|DESC)(,)?/) { + if (defined $2) { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} ->{INDEXES}->{$idxname}->{COLS}}, ("$1 $2"); - } - else - { - push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + } + else { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} ->{INDEXES}->{$idxname}->{COLS}}, ("$1"); - } + } } - if ($idx =~ /^INCLUDE \(/) - { - print STDERR - "Warning: This index ($schemaname.$tablename.$idxname) has some include columns. This isn't supported in PostgreSQL.\n"; - print STDERR - "\tThe columns in the INCLUDE clause have been ignored.\n"; - next - ; # Nothing equivalent in PG. Maybe if the index isn't unique, these columns should be added? + if ($idx =~ /^INCLUDE\s*\(\s*\[(.*?)\](.*)/) { + # INCLUDE coluns in indexes are available on PG11 onward + # if multiple included columns, there are declared one per line + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{INCLUDE}}, ($1); + if (index($2, ')') == -1) { + while (my $incl_line = read_and_clean($file)) { + if ($incl_line =~ /^\s*\[(.*?)\](.*)/) { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{INCLUDE}}, ($1); + last if (index($2, ')') != -1); + } + } + } } - if ($idx =~ /^WHERE\s*\((.*)\)$/) - { - # This is a where clause. PostgreSQL has them too. But we cannot be sure this will be exactly the same. So if an index as a WHERE clause, it has to go to unsure - my $filter=$1; - $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} - ->{INDEXES}->{$idxname}->{WHERE}="(".$filter.")"; + if ($idx =~ /^WHERE\s*\((.*)\)$/) { + # This is a where clause. PostgreSQL has them too. But we + # cannot be sure this will be exactly the same. So if an + # index as a WHERE clause, it has to go to unsure + my $filter=$1; + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{WHERE}="(".$filter.")"; } - } - } + } + } - # we do not take migrate spatial indexes + # we do not take migrate spatial indexes elsif ($line =~ /^CREATE SPATIAL INDEX/) { my $def=$line; @@ -2640,7 +2644,7 @@ sub generate_schema foreach my $table (sort keys %{$refschema->{TABLES}}) { foreach my $constraint ( - @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) + @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) { next unless ($constraint->{TYPE} eq 'UNIQUE'); my $consdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; @@ -2683,51 +2687,48 @@ sub generate_schema { $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ")"; - if (not defined $idxref->{WHERE} and not defined $idxref->{DISABLE}) - { - $idxdef .= ";\n"; - print AFTER $idxdef; - # the possible comment would go to after file - $index_created = 1; + + if (defined $idxref->{INCLUDE}) { + $idxdef .= " INCLUDE (" . + join(",", map{format_identifier_cols_index($_)} @{$idxref->{INCLUDE}}) + . ")"; } - else - { - # this is either a disabled index or an index with a where declaration - if (defined $idxref->{WHERE}) - { - print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; - if ($idxref->{DISABLE}) - { - # if disabled, will be on the same line - $idxdef .= " "; - } - else - { - # otherwise, write condition on a new line - $idxdef .= "\n"; - } - $idxdef .= "WHERE (" . convert_transactsql_code($idxref->{WHERE}) . ")"; - } - $idxdef .= ";\n"; - print UNSURE $idxdef; - # the possible comment would go to unsure file - $index_created = 2; - } - - # Produce the comments for indexes - if (defined $idxref->{COMMENT}) - { - my $idxcomment = "COMMENT ON INDEX ". format_identifier($schema) . '.' . format_identifier($index) . " IS '" . $idxref->{COMMENT} . "';\n"; - if ($index_created == 1) - { - print AFTER $idxcomment; - } - elsif ($index_created == 2) - { - print UNSURE $idxcomment; - } - } + if (not defined $idxref->{WHERE} and not defined $idxref->{DISABLE}) { + $idxdef .= ";\n"; + print AFTER $idxdef; + # the possible comment would go to after file + $index_created = 1; + } + else { + + # this is either a disabled index or an index with a where declaration + if (defined $idxref->{WHERE}) { + print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; + if ($idxref->{DISABLE}) { + # if disabled, will be on the same line + $idxdef .= " "; + } else { + # otherwise, write condition on a new line + $idxdef .= "\n"; + } + $idxdef .= "WHERE (" . convert_transactsql_code($idxref->{WHERE}) . ")"; + } + $idxdef .= ";\n"; + print UNSURE $idxdef; + # the possible comment would go to unsure file + $index_created = 2; + } + + # Produce the comments for indexes + if (defined $idxref->{COMMENT}) { + my $idxcomment = "COMMENT ON INDEX ". format_identifier($schema) . '.' . format_identifier($index) . " IS '" . $idxref->{COMMENT} . "';\n"; + if ($index_created == 1) { + print AFTER $idxcomment; + } elsif ($index_created == 2) { + print UNSURE $idxcomment; + } + } } } } From bfb908f9adbd4b1f6fb0449d4cc362d60337fa29 Mon Sep 17 00:00:00 2001 From: Thibaut Date: Thu, 14 Jan 2021 14:54:06 +0100 Subject: [PATCH 197/216] exclude extendedproperty microsoft_database_tools_support (#138) --- sqlserver2pgsql.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index ca229cd..e5a5ea8 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2209,7 +2209,7 @@ sub parse_dump or croak "Cannot find a name for this extended property: $sqlproperty"; my $propertyname = $1; - if ($propertyname =~ /^(AggregateType|AllowZeroLength|AppendOnly|Attributes|CollatingOrder|ColumnHidden|ColumnOrder|ColumnWidth|DataUpdatable|DateCreated|DefaultValue|Description|Display Folder|Display Name|DisplayViewsOnSharePointSite|ETL Rules|Example Values|FilterOnLoad|GUID|HideNewField|LastUpdated|MS_DecimalPlaces|MS_DefaultView|MS_DiagramPane1|MS_DiagramPane2|MS_DiagramPaneCount|MS_DisplayControl|MS_Format|MS_Hyperlink|MS_IMEMode|MS_IMESentMode|MS_InputMask|MS_OrderByOn|MS_Orientation|Name|OrderByOnLoad|OrdinalPosition|RecordCount|Required|SCD Type|ShowDatePicker|Size|Source Datatype|Source System|SourceField|SourceTable|Table Description|Table Type|TextAlign|TextFormat|TotalsRow|Type|UnicodeCompression|Updatable)$/) + if ($propertyname =~ /^(AggregateType|AllowZeroLength|AppendOnly|Attributes|CollatingOrder|ColumnHidden|ColumnOrder|ColumnWidth|DataUpdatable|DateCreated|DefaultValue|Description|Display Folder|Display Name|DisplayViewsOnSharePointSite|ETL Rules|Example Values|FilterOnLoad|GUID|HideNewField|LastUpdated|microsoft_database_tools_support|MS_DecimalPlaces|MS_DefaultView|MS_DiagramPane1|MS_DiagramPane2|MS_DiagramPaneCount|MS_DisplayControl|MS_Format|MS_Hyperlink|MS_IMEMode|MS_IMESentMode|MS_InputMask|MS_OrderByOn|MS_Orientation|Name|OrderByOnLoad|OrdinalPosition|RecordCount|Required|SCD Type|ShowDatePicker|Size|Source Datatype|Source System|SourceField|SourceTable|Table Description|Table Type|TextAlign|TextFormat|TotalsRow|Type|UnicodeCompression|Updatable)$/) { # We don't dump these. They are graphical descriptions of the GUI next; From 97688656adeef78c2f5858c2c35070d7f90993ff Mon Sep 17 00:00:00 2001 From: Thibaut Date: Tue, 9 Feb 2021 11:50:59 +0100 Subject: [PATCH 198/216] add a section for operation order in the FAQ (#141) --- FAQ.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/FAQ.md b/FAQ.md index 1d5686b..d712aba 100644 --- a/FAQ.md +++ b/FAQ.md @@ -16,6 +16,31 @@ variable to a higher value (4096) for 4GB for instance. There is another big advantage of using Kettle: you can tailor the scripts produced by sqlserver2pgsql to your needs, such as adding some conversions, schema changes. As Kettle is an ETL, it is a good tool for doing such conversions on the fly. +In which order should I run the operations? +---------------------------------- + +sqlserver2pgsql outputs several files: before, after and unsure SQL files, plus +the kettle jobs files. + +If you load all the SQL files before the data migration, you can experience +problems. For example, you can have errors when the kettle job truncates the +table at the start of the process. If a foreign key constraint is enforced, +PostgreSQL cannot truncate a table referenced in a foreign key constraint and +the job would error out. + +You should first check the unsure file, verify that the SQL is fine or correct +it if needed. Some SQL orders from the unsure files are to be run before the +data migration, for example, default column values, procedures or functions, +triggers. So move them to the before file. + +You can then load the before file (`before.sql`). Then use the kettle jobs to +migrate the data (`migration.kjb`). When this is done, load the rest of the +unsure and the after file (`unsure.sql` and `after.sql`). + +In case you are still using the original database, a specific kettle job is +created so that you can feed the change periodically to your PostgreSQL +database (`incremental.kjb`). + What is this IGNORE NULLS I have to change in kettle.properties ? ---------------------------------- @@ -39,4 +64,5 @@ doesn't exist either in PG. So more constraints will fail. Can this tool migrate functions and stored procedures? ---------------------------------- -No, Transact-SQL is very different from PostgreSQL's many PL languages. These would need a manual migration. \ No newline at end of file +No, Transact-SQL is very different from PostgreSQL's many PL languages. These would need a manual migration. + From e5c3c276063441a86280bec921e5407fb854eeff Mon Sep 17 00:00:00 2001 From: Thibaut Date: Wed, 24 Mar 2021 10:12:51 +0100 Subject: [PATCH 199/216] permit max in typequal in CREATE TYPE (#145) --- sqlserver2pgsql.pl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index e5a5ea8..5311002 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1824,14 +1824,18 @@ sub parse_dump my $newtype; TYPE: while (my $typeline= read_and_clean($file)) { - if ($typeline =~ /^\t\[(.*)\] \[(.*)\](?:\s*?\((\d+(?:,\d+)?)\))?(?:\s+?(?:NOT\s+?)?NULL),?$/) + if ($typeline =~ /^\t\[(.*)\] \[(.+?)\](?:\s*?\((\d+|max(?:,\d+)?)\))?(?:\s+?(?:NOT\s+?)?NULL),?$/) { # This is another column for this type $colname=$1; $type=$2; $typequal=$3; + if (defined $typequal and $typequal eq 'max') { + # max in SqlServer is the same as no typequal in pg + $typequal = undef; + } $newtype = - convert_type($type, $typequal, undef, undef, undef, undef); + convert_type($type, $typequal, undef, undef, undef, undef); push @cols_newbasetype,(format_identifier($colname) . ' ' . $newtype); } elsif ( $typeline =~ /PRIMARY KEY/) From 11f82dbd8a2f236727f31b3645981fd1916b3436 Mon Sep 17 00:00:00 2001 From: Thibaut Date: Wed, 24 Mar 2021 15:30:11 +0100 Subject: [PATCH 200/216] update docker versions for CI (PG and perl) (#146) --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2cab6cf..dd16792 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,8 +3,8 @@ version: 2 jobs: test: docker: - - image: perl:5.24-threaded - - image: postgres:12-alpine + - image: perl:5-threaded + - image: postgres:13-alpine environment: POSTGRES_USER: postgres POSTGRES_POSTGRES: postgres From d93f0966e48442e21f4c38f07aa7e604c8841812 Mon Sep 17 00:00:00 2001 From: Thibaut Date: Thu, 29 Apr 2021 11:14:54 +0200 Subject: [PATCH 201/216] #112 negative argument as IDENTITY (#149) * support IDENTITY with negative arguments; detect end of table more generically * Add tests * rebase patch by potrusil-osi Co-authored-by: Tomas Potrusil --- contributors | 1 + regression/issue_112.sql | 46 ++++++++++++++++++++++++++++++++++++++++ sqlserver2pgsql.pl | 10 ++++----- 3 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 regression/issue_112.sql diff --git a/contributors b/contributors index 88b21c5..64b0657 100644 --- a/contributors +++ b/contributors @@ -16,6 +16,7 @@ bsacks99 keyjote mark-jay mikes-gh +postrusil-osi sebpcspkr stuey1978 diff --git a/regression/issue_112.sql b/regression/issue_112.sql new file mode 100644 index 0000000..181156c --- /dev/null +++ b/regression/issue_112.sql @@ -0,0 +1,46 @@ +CREATE TABLE [dbo].[AFElementAttributeCategory]( + [rid] [bigint] IDENTITY(-1,-1) NOT NULL, + [id] [uniqueidentifier] NOT NULL, + [rowversion] [timestamp] NOT NULL, + [fkelementversionid] [bigint] NOT NULL, + [fkparentattributeid] [uniqueidentifier] NULL, + [fkcategoryid] [uniqueidentifier] NOT NULL, + [changedby] [int] NOT NULL, + CONSTRAINT [PK_AFElementAttributeCategory] PRIMARY KEY CLUSTERED +( + [rid] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [ASSETS] +) ON [ASSETS] +GO + +CREATE TABLE [dbo].[AFCaseAdjustment]( + [rid] [bigint] IDENTITY(-1,-1) NOT NULL, + [id] [uniqueidentifier] NOT NULL, + [rowversion] [timestamp] NOT NULL, + [fkcaseid] [bigint] NOT NULL, + [attributeid] [uniqueidentifier] NOT NULL, + [adjustedvalue] [varbinary](max) NULL, + [comment] [nvarchar](1000) NULL, + [previousvalue] [varbinary](max) NULL, + [creator] [nvarchar](50) NULL, + [creationdate] [datetime2](7) NULL, + [changedby] [int] NOT NULL, + CONSTRAINT [PK_AFCaseAdjustment] PRIMARY KEY NONCLUSTERED +( + [rid] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [ANALYSIS] +) ON [ANALYSIS] TEXTIMAGE_ON [ANALYSIS] +GO + +CREATE TABLE [dbo].[sd]( + [rid] [int] IDENTITY(1000,1) NOT NULL, + [rowversion] [timestamp] NOT NULL, + [sd] [nvarchar](max) NOT NULL, + [ownerRights] [int] NOT NULL, + [lupd] [datetime2](7) NULL, + CONSTRAINT [pk_sd] PRIMARY KEY CLUSTERED +( + [rid] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [ASSETS] +) ON [ASSETS] TEXTIMAGE_ON [ASSETS] +GO \ No newline at end of file diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 5311002..2e70cc9 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1348,7 +1348,7 @@ sub add_column_to_table # We have an identity field. We remember the default value and # initialize the sequence correctly in the after script - $isidentity =~ /IDENTITY\s*\((\d+),\s*(\d+)\)/ + $isidentity =~ /IDENTITY\s*\((-?\d+),\s*(-?\d+)\)/ or die "Cannot understand <$isidentity>"; my $startseq = $1; my $stepseq = $2; @@ -1365,8 +1365,6 @@ sub add_column_to_table $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{START} = $startseq; - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MIN} - = $startseq; $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} = $stepseq; $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname} @@ -1449,7 +1447,7 @@ sub parse_dump # column name, typical microsoft stuff :( ) # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ - /^\s+\[(.*)\]\s*(?:\[(.*)\]\.)?\[(.*)\]\s*(\(.+?\))?(?: COLLATE (\S+))?( IDENTITY\s*\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE +)?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ + /^\s+\[(.*)\]\s*(?:\[(.*)\]\.)?\[(.*)\]\s*(\(.+?\))?(?: COLLATE (\S+))?( IDENTITY\s*\(-?\d+,\s*-?\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE +)?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ ) { # Deported into a function because we can also meet alter table add columns on their own @@ -1983,7 +1981,7 @@ sub parse_dump # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD \[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))? (NOT NULL|NULL)(?: CONSTRAINT \[.*\] )?(?: DEFAULT \(.*\))?$/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD \[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(-?\d+,\s*-?\d+\))? (NOT NULL|NULL)(?: CONSTRAINT \[.*\] )?(?: DEFAULT \(.*\))?$/ ) { my $schemaname=relabel_schemas($1); @@ -2883,7 +2881,7 @@ sub generate_schema # This may not be an identity. Skip it then next unless defined ($seqref->{OWNERCOL}); print AFTER "select setval('" . format_identifier($schema) . '.' - . format_identifier($sequence) . "',(select max(" + . format_identifier($sequence) . "',(select " . ($seqref->{STEP} > 0 ? "max" : "min") . "(" . format_identifier($seqref->{OWNERCOL}) .") from " . format_identifier($seqref->{OWNERSCHEMA}) . '.' . format_identifier($seqref->{OWNERTABLE}) . ")::bigint);\n"; From e4aa635046065574321e3da2bcd463547517918d Mon Sep 17 00:00:00 2001 From: Thibaut Date: Tue, 4 May 2021 11:57:11 +0200 Subject: [PATCH 202/216] Add option to convert Identity columns to "GENERATED ALWAYS AS IDENTITY" (#150) * Add option (`use_identity_column`) to convert Identity columns to "GENERATED ALWAYS AS IDENTITY" columns unset the option to use `CREATE SEQUENCE` statements instead. Co-authored-by: alchemistmatt --- example_conf_file | 1 + sqlserver2pgsql.pl | 131 ++++++++++++++++++++++++++++++--------------- 2 files changed, 90 insertions(+), 42 deletions(-) diff --git a/example_conf_file b/example_conf_file index da617bb..a93dea3 100644 --- a/example_conf_file +++ b/example_conf_file @@ -31,6 +31,7 @@ keep identifier case=1 # keep case of database objects; comment out to conv #camelcasetosnake=1 # Uncomment to convert to snake case; comment out to leave names unchanged (or lowercase) validate constraints = yes # yes, after or no, should the constraints be validated by the dump ? (yes=validate during load, after after the load, no keep invalidated) #skip citext length check=1 # When defined, do not add a CHECK (char_length()) check for citext fields +use identity column=1 # if set, use identity columns statements ('CREATE GENERATED ALWAYS') instead of creating a dedicated sequence ('CREATE SEQUENCE') # Incremental job sort size=10000 # drives the amount of memory and temporary files that will be created by an incremental job diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2e70cc9..96b78fb 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -53,6 +53,7 @@ our $pforce_ssl; our $stringtype_unspecified; our $skip_citext_length_check; +our $use_identity_column; # Will be set if we detect GIS objects our $requires_postgis=0; @@ -109,7 +110,8 @@ sub parse_conf_file 'ignore errors' => 'ignore_errors', 'postgresql force ssl' => 'pforce_ssl', 'stringtype unspecified' => 'stringtype_unspecified', - 'skip citext length check' => 'skip_citext_length_check', + 'skip citext length check' => 'skip_citext_length_check', + 'use identity column' => 'use_identity_column', ); # Open the conf file or die @@ -161,6 +163,7 @@ sub set_default_conf_values $pforce_ssl=0 unless (defined ($pforce_ssl)); $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); + $use_identity_column=0 unless (defined ($use_identity_column)); } # Converts numeric(4,0) and similar to int, bigint, smallint @@ -792,6 +795,11 @@ sub usage '1' sort all tables. LIST_OF_TABLES gives a comma separated list of tables to sort in the form 'schema1.table1,schema2.table2'. Cases are compared insensitively. + -skip_citext_length_check (Default 0) + if set, do not add a CHECK (char_length()) check for citext fields + -use_identity_column (Default 1) + if set, use identity columns statements (GENERATED ALWAYS AS + IDENTITY) instead of creating a dedicated sequence (CREATE SEQUENCE) Kettle options: if you are generating for kettle, you must provide connection information. @@ -2584,35 +2592,63 @@ sub generate_schema foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) { my $seqref = $refschema->{SEQUENCES}->{$sequence}; - print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence); - if (defined $seqref->{STEP}) - { - print AFTER " INCREMENT BY ",$seqref->{STEP}; - } - if (defined $seqref->{MIN}) - { - print AFTER " MINVALUE ",$seqref->{MIN}; - } - if (defined $seqref->{MAX}) - { - print AFTER " MAXVALUE ",$seqref->{MAX}; - } - if (defined $seqref->{START}) - { - print AFTER " START WITH ",$seqref->{START}; - } - if (defined $seqref->{CACHE}) - { - print AFTER " CACHE ",$seqref->{CACHE}; - } - if (defined $seqref->{OWNERTABLE}) - { - print AFTER " OWNED BY ",format_identifier($seqref->{OWNERSCHEMA}), - '.',format_identifier($seqref->{OWNERTABLE}), - '.',format_identifier($seqref->{OWNERCOL}); - } - print AFTER ";\n"; - } + + if ($use_identity_column and defined $seqref->{OWNERTABLE}) + { + # Add a statement of the form + # ALTER TABLE "schema"."table_name" ALTER COLUMN "column_name" ADD GENERATED ALWAYS AS IDENTITY (start 1000); + + print AFTER "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($seqref->{OWNERTABLE}) . " "; + print AFTER "ALTER COLUMN " . format_identifier($seqref->{OWNERCOL}) . " ADD GENERATED ALWAYS AS IDENTITY"; + + if (defined $seqref->{START} or defined $seqref->{STEP}) + { + print AFTER " ("; + if (defined $seqref->{START}) + { + print AFTER " START WITH ",$seqref->{START}; + } + + if (defined $seqref->{STEP}) + { + print AFTER " INCREMENT BY ",$seqref->{STEP}; + } + print AFTER ")"; + } + } + else + { + print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence); + if (defined $seqref->{STEP}) + { + print AFTER " INCREMENT BY ",$seqref->{STEP}; + } + if (defined $seqref->{MIN}) + { + print AFTER " MINVALUE ",$seqref->{MIN}; + } + if (defined $seqref->{MAX}) + { + print AFTER " MAXVALUE ",$seqref->{MAX}; + } + if (defined $seqref->{START}) + { + print AFTER " START WITH ",$seqref->{START}; + } + if (defined $seqref->{CACHE}) + { + print AFTER " CACHE ",$seqref->{CACHE}; + } + if (defined $seqref->{OWNERTABLE}) + { + print AFTER " OWNED BY ",format_identifier($seqref->{OWNERSCHEMA}), + '.',format_identifier($seqref->{OWNERTABLE}), + '.',format_identifier($seqref->{OWNERCOL}); + } + } + + print AFTER ";\n"; + } # Now PK. We have to go through all tables foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -2861,12 +2897,20 @@ sub generate_schema . " ALTER COLUMN " . format_identifier($col) . " SET DEFAULT " . $default_value . ";\n"; if ($colref->{DEFAULT}->{UNSURE}) - { + { print UNSURE $definition; } else { - print AFTER $definition; + if ($use_identity_column and ($definition =~ /nextval.+_seq/i)) + { + # Skip this set default item + } + else + { + print AFTER $definition; + } + } } } @@ -2880,6 +2924,7 @@ sub generate_schema my $seqref = $refschema->{SEQUENCES}->{$sequence}; # This may not be an identity. Skip it then next unless defined ($seqref->{OWNERCOL}); + next if defined ($use_identity_column); print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select " . ($seqref->{STEP} > 0 ? "max" : "min") . "(" . format_identifier($seqref->{OWNERCOL}) .") from " @@ -3112,16 +3157,18 @@ sub resolve_name_conflicts "i" => \$case_insensitive, "nr" => \$norelabel_dbo, "num" => \$convert_numeric_to_int, - "drop_rowversion" => \$drop_rowversion, - "relabel_schemas=s" => \$relabel_schemas, - "keep_identifier_case" => \$keep_identifier_case, - "camel_to_snake" => \$camel_to_snake, - "validate_constraints=s" => \$validate_constraints, - "sort_size=i" => \$sort_size, - "use_pk_if_possible=s" => \$use_pk_if_possible, - "ignore_errors" => \$ignore_errors, - "pforce_ssl" => \$pforce_ssl, - "stringtype_unspecified" => \$stringtype_unspecified + "drop_rowversion" => \$drop_rowversion, + "relabel_schemas=s" => \$relabel_schemas, + "keep_identifier_case" => \$keep_identifier_case, + "camel_to_snake" => \$camel_to_snake, + "validate_constraints=s" => \$validate_constraints, + "sort_size=i" => \$sort_size, + "use_pk_if_possible=s" => \$use_pk_if_possible, + "ignore_errors" => \$ignore_errors, + "pforce_ssl" => \$pforce_ssl, + "stringtype_unspecified" => \$stringtype_unspecified, + "skip_citext_length_check" => \$skip_citext_length_check, + "use_identity_column" => \$use_identity_column ); # We don't understand command line or have been asked for usage From e01340a7bc74188168df939ea23c59037087778d Mon Sep 17 00:00:00 2001 From: Thibaut Date: Tue, 4 May 2021 12:20:06 +0200 Subject: [PATCH 203/216] test using sequences for identity columns (#151) --- regression/reg.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression/reg.pl b/regression/reg.pl index f76e24d..c29b716 100755 --- a/regression/reg.pl +++ b/regression/reg.pl @@ -7,7 +7,7 @@ foreach my $file (<*.sql>) { - my @options_to_try=('-i','-nr','-num', '-keep_identifier_case', '-validate_constraints=after'); + my @options_to_try=('-i','-nr','-num', '-keep_identifier_case', '-validate_constraints=after', '-use_identity_column=0'); my @all_combinations=(''); foreach my $option (@options_to_try) { From 9027003f21fd4296cd2a9d80d652990c35a6706a Mon Sep 17 00:00:00 2001 From: madtibo Date: Tue, 4 May 2021 12:32:05 +0200 Subject: [PATCH 204/216] update contributors with alchemistmatt who made many great contributions to sqlserver2pgsql! Thank you very much for all of them! --- contributors | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors b/contributors index 64b0657..7c4c9fb 100644 --- a/contributors +++ b/contributors @@ -5,6 +5,7 @@ Javier Callico (JCallico) Joshua F. Rountree (joshuairl) Julien Rouhaud (rjuju) Konstantin Mosolov (kmosolov) +Matthew Monroe (alchemistmatt) Philippe Baudoin (beaud76) Thibaut Madelaine (madtibo) Yann Verry (yanntech) From 340b38137b5b405b9b48dded5182fb39e2871843 Mon Sep 17 00:00:00 2001 From: Thibaut Date: Wed, 5 May 2021 12:53:44 +0200 Subject: [PATCH 205/216] View parsing upgrade (#152) * Add 3 functions in transact parsing (space, charindex and dateadd). * Update the regex to parse view to make it multiline --- regression/basic_test/views.sql | Bin 6666 -> 7868 bytes sqlserver2pgsql.pl | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/regression/basic_test/views.sql b/regression/basic_test/views.sql index f92587755af4174b18c7332453caa9f2f13f150f..972c756cd9b9d6d09f7c63bdbe9967341c139df3 100644 GIT binary patch delta 310 zcmeA&*<-swN^o)suL83vgVAJbLD$K5c(;JrAGyRiQy3B%N*GcZVkawds|&(-B@CGi zxlmF0%`*HK7$=WMhmVW=xKkkmdvi4A5wxy2%&W#V5;1z5@VZ=`xrA delta 20 ccmdmE+hwvrN^tWkkss`n@3C`DJ}3JQ09vI8@c;k- diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 96b78fb..5f47f5b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -521,7 +521,10 @@ sub convert_transact_function $code =~ s/ISNULL\s*\(/COALESCE(/gi; $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; + $code =~ s/SPACE\s*\(/REPEAT(' ', /gi; + $code =~ s/charindex\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/dPOSITION('$1' in $2)/gi; $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; + $code =~ s/DATEADD\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\,\s*(.*?)\s*\)/$3 + INTERVAL '$2 $1'/gi; $code =~ s/CONVERT\s*\(\s*NVARCHAR\s*(.*?)\s*\(\s*(.*?)\s*\s*\)\,\s*(.*?)\s*\)/CAST($3 AS varchar($2))/gi; $code =~ s/CONVERT\s*\(\s*(.*?)\s*\(\s*(.*?)\s*\s*\)\,\s*(.*?)\s*\)/CAST($3 AS $1($2))/gi; $code =~ s/CONVERT\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/CAST($2 AS $1)/gi; @@ -1739,9 +1742,8 @@ sub parse_dump # We get rid of dbo. schemas $sql =~ s/(dbo)\./relabel_schemas($1) . '.'/eg ; # We put this in the replacement schema - # print STDERR "code view: ".$sql."\n"; # parse the query view - if ( $sql =~ /^\s*\(([^\)]+)\)\s*AS\s+SELECT\s+(.*)\s+FROM\s+(.*)$/i) { + if ( $sql =~ m/^\s*\(([^\)]+)\)\s*AS\s+SELECT\s+(.*)\s+FROM\s+(.*)$/is) { my $view_columns = $1; my $query_columns = $2; my $query_end = $3; From 397634493829d90f1f11cf29ea2aeceaaa3e4bcc Mon Sep 17 00:00:00 2001 From: Thibaut Date: Wed, 12 May 2021 09:23:37 +0200 Subject: [PATCH 206/216] Convert views (#153) * parse more transact functions * parse multiline views * update views test From 4e14df7ed8349fcd4b1df28bc7a2d6d620d07be1 Mon Sep 17 00:00:00 2001 From: "Amine B. Hassouna" <9788130+aminosbh@users.noreply.github.com> Date: Wed, 26 May 2021 14:19:53 +0100 Subject: [PATCH 207/216] Add support for 'ON UPDATE SET NULL' clause in FK (#156) --- sqlserver2pgsql.pl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 5f47f5b..7dfd8c4 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2162,6 +2162,10 @@ sub parse_dump { $constraint->{ON_UPD_CASC} = 1; } + elsif ($fk =~ /^ON UPDATE SET NULL\s*$/) + { + $constraint->{ON_UPD_SET_NULL} = 1; + } elsif ($fk =~ /^NOT FOR REPLICATION\s*$/) { next; # We don't care for this, it has no meaning for PostgreSQL @@ -2817,6 +2821,11 @@ sub generate_schema { $consdef .= " ON UPDATE CASCADE"; } + if (defined $constraint->{ON_UPD_SET_NULL} + and $constraint->{ON_UPD_SET_NULL}) + { + $consdef .= " ON UPDATE SET NULL"; + } # We need a name on the constraint to be able to validate it later. Maybe it would be better to generate one # FIXME: we'll see later if a generator is needed (probably) if ($constraint->{TYPE} eq 'FK' and ($validate_constraints =~ /^after|no$/) and defined($constraint->{NAME})) From 2aaecc82b507aad379191976e6cf2a03af40f321 Mon Sep 17 00:00:00 2001 From: Ben Mares <15216687+maresb@users.noreply.github.com> Date: Wed, 26 May 2021 17:44:54 +0200 Subject: [PATCH 208/216] Remove UTF-8 BOM when necessary (#155) --- sqlserver2pgsql.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 7dfd8c4..3a75ea3 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -1270,7 +1270,8 @@ sub generate_kettle my ($fd) = @_; my $line = <$fd>; return undef if (not defined $line); - $line =~ s/\r//g; # Remove \r from windows output + $line =~ s/^\x{FEFF}//; # Remove UTF-8 BOM + $line =~ s/\r//g; # Remove \r from windows output $line =~ s/EXEC(ute)?\s*(dbo|sys)\.sp_executesql( \@statement =)? N'//i ; # Remove executesql… it's a bit weird in the SQL Server's dump From 5b07d1ef0dddbb2ac309fbc3ed3d5678c9241da3 Mon Sep 17 00:00:00 2001 From: Philippe Beaudoin Date: Thu, 1 Jul 2021 10:30:35 +0200 Subject: [PATCH 209/216] Refactor the example_conf_file. Improve the global readibility by aligning the parameter values and the comments; move some parameters to a more convenient place; all optional parameters are commented with their default value; and add 3 missing parameters. Also fix a bug when either "keep identifier case" or "camelcasetosnake" are set to 0 --- example_conf_file | 104 +++++++++++++++++++++++++++------------------ sqlserver2pgsql.pl | 16 +++++-- 2 files changed, 75 insertions(+), 45 deletions(-) diff --git a/example_conf_file b/example_conf_file index a93dea3..1088657 100644 --- a/example_conf_file +++ b/example_conf_file @@ -1,41 +1,63 @@ -# Source SQL Server Dump. Obviously compulsory -sql server dump filename=/tmp/dump - -# These are used to generate SQL scripts (this is the only thing that is always done) -before file=/tmp/before -after file=/tmp/after -unsure file=/tmp/unsure - -kettle directory=/tmp/kettle # Comment this line if you don't want a kettle script to be generated -# These are ignored as long as kettle is not set -sql server database=foo -sql server host=foo_host -sql server host instance=my_instance # You can omit this if you use the default instance -sql server port=1433 -sql server username=foo_user -sql server password=foo_password -postgresql database=bar -postgresql host=bar_host -postgresql port=5432 -postgresql username=bar_user -postgresql password=bar_password -parallelism_in=8 # Parallelism reading from SQL Server (where available) Default value is 1 -parallelism_out=8 # Default value is 8. Number of parallel connections used by kettle to insert data into the PostgreSQL database - -# Optional behaviour -case insensitive=0 # set it to 1 to generate a dump with citext and check constraints all over the place -no relabel dbo=1 # set it to 0 to convert the dbo schema to public -convert numeric to int=1 # set it to 0 to keep numeric(xx,0) as numeric(xx,0). Will be converted to smallint, int or bigint by default -relabel schemas=dbo=>foo;schema1=>bar -keep identifier case=1 # keep case of database objects; comment out to convert names to lowercase -#camelcasetosnake=1 # Uncomment to convert to snake case; comment out to leave names unchanged (or lowercase) -validate constraints = yes # yes, after or no, should the constraints be validated by the dump ? (yes=validate during load, after after the load, no keep invalidated) -#skip citext length check=1 # When defined, do not add a CHECK (char_length()) check for citext fields -use identity column=1 # if set, use identity columns statements ('CREATE GENERATED ALWAYS') instead of creating a dedicated sequence ('CREATE SEQUENCE') - -# Incremental job -sort size=10000 # drives the amount of memory and temporary files that will be created by an incremental job -use pk if possible=0 # 1/list of tables, for tables where you want to try getting already sorted records - -# Ignore errors ? (will be slower, and you'll have to read the migration job's log throroughly). Ignored for incremental jobs -ignore errors=0 +# +# This file is an example of a sqlserver2pgsql configuration file. +# + +# +# Files location. +# + +# Input file. +sql server dump filename = /tmp/dump # the source SQL-Server Dump (obviously compulsory) + +# Output files. +before file = /tmp/before # SQL script to execute before loading the data +after file = /tmp/after # SQL script to execute after having loaded the data +unsure file = /tmp/unsure # SQL script containing statements to check or to adjust manually +kettle directory = /tmp/kettle # comment this line if you don't want kettle components to be generated + +# +# Optional parameters to setup specific behaviour. +# + +#case insensitive = 0 # set it to 1 to generate a dump with citext and check constraints all over the place +#skip citext length check = 0 # set it to 1 to not add a CHECK (char_length()) constraint for citext fields (when case insensitive is set to 1) +#no relabel dbo = 1 # set it to 0 to convert the dbo schema into public +#relabel schemas = dbo=>foo;schema1=>bar +#convert numeric to int = 1 # set it to 0 to keep numeric(xx,0) types as numeric(xx,0); they will be converted to smallint, int or bigint by default +#keep identifier case = 0 # set it to 1 to keep the case of database objects; by default identifier names are converted to lowercase +#camelcasetosnake = 0 # set it to 1 to convert identifiers from camelCase to snake_case; keep identifier case and camelcasetosnake cannot be both set to 1 +#validate constraints = yes # should the constraints be validated by the DDL scripts ? 'yes' = validated at constraint creation, + # 'no' = kept NOT VALID, 'after' = validated after the data load; 'yes' by default +#use identity column = 1 # set it to 0 to create an explicite SEQUENCE for identity columns (the old technic); by default, + # GENERATED ALWAYS clauses are generated +#drop rowversion = 0 # set it to 1 to ignore columns of SQL-Server type 'rowversion' or 'timestamp'; 0 by default + +# +# Parameters used for the data migration with Kettle. +# They are ignored as long as the 'kettle directory' parameter is not set. +# + +sql server database = foo +sql server host = foo_host +sql server host instance = my_instance # optional when the default instance is used +sql server port = 1433 +sql server username = foo_user +sql server password = foo_password + +postgresql database = bar +postgresql host = bar_host +postgresql port = 5432 +postgresql username = bar_user +postgresql password = bar_password + +#postgresql force ssl = 0 # set it to 1 to force a SSL session to PostgreSQL; 0 by default + +#parallelism_in = 1 # parallelism degree when reading from SQL-Server (where available); 1 by default +#parallelism_out = 8 # number of parallel connections used by kettle to insert data into the PostgreSQL database; 8 by default + +#stringtype unspecified = 0 # set it to 1 to let kettle process textual data as "not necessarily a strict PostgreSQL VARCHAR data"; 0 by default +#ignore errors = 0 # set it to 1 to not abort the data migration job when an error occurs; the parameter is ignored for incremental jobs + # warning: the migration will be slower and the job's log will need to be throroughly examined +# Incremental job parameters. +#sort size = 10000 # drives the amount of memory and temporary files that will be created by an incremental job; 10000 by default +#use pk if possible = 0 # set to either 1 or a space separated list of schema qualified table names, for tables candidated for getting already sorted rows diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 3a75ea3..c73ebfa 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -36,12 +36,12 @@ our $norelabel_dbo; # Passed as arg: should we convert DBO to public ? our $relabel_schemas; our $convert_numeric_to_int; # Should we convert numerics to int when possible ? (numeric (4,0) could be converted an int, for instance) -our $drop_rowversion; # Should we remove MSSQL timestamp/rowversion columns when converting +our $drop_rowversion; # Should we remove MSSQL timestamp/rowversion columns when converting our $kettle; our $before_file; our $after_file; our $unsure_file; -our $case_treatment=1; # 1=convert to lowercase, 2=convert to snake_case, 0 do nothing +our $case_treatment; # 0 do nothing, 1 = convert to lowercase (the default), 2 = convert to snake_case our $ignore_errors; our $keep_identifier_case; our $camel_to_snake; @@ -149,8 +149,8 @@ sub set_default_conf_values $norelabel_dbo=0 unless (defined ($norelabel_dbo)); $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); $drop_rowversion=0 unless (defined ($drop_rowversion)); - $case_treatment=0 if (defined ($keep_identifier_case)); - $case_treatment=2 if (defined ($camel_to_snake)); + $keep_identifier_case=0 unless (defined ($keep_identifier_case)); + $camel_to_snake=0 unless (defined ($camel_to_snake)); $parallelism_in=1 unless (defined ($parallelism_in));# the jdbc driver often errors when there are several sessions to sql server $parallelism_out=8 unless (defined ($parallelism_out)); $sort_size=10000 unless (defined ($sort_size)); @@ -164,6 +164,14 @@ sub set_default_conf_values $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); $use_identity_column=0 unless (defined ($use_identity_column)); + + # Compute the case_treatment flag + $case_treatment = 1; + $case_treatment = 0 if ($keep_identifier_case); + $case_treatment = 2 if ($camel_to_snake); + if ($keep_identifier_case && $camel_to_snake) { + die "keep_identifier_case and camel_to_snake parameters cannot be both set to 1.\n"; + } } # Converts numeric(4,0) and similar to int, bigint, smallint From 83de15eb771b4a1618e500401c0938bc32468415 Mon Sep 17 00:00:00 2001 From: Philippe Beaudoin Date: Thu, 1 Jul 2021 10:55:18 +0200 Subject: [PATCH 210/216] group all the checks on parameters into a new dedicated process_check_parameters() function. --- sqlserver2pgsql.pl | 74 +++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index c73ebfa..c640bb2 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -142,6 +142,7 @@ sub parse_conf_file close CONF; } +# Set the default value for all parameters not set either in the configuration file or in the command line. sub set_default_conf_values { # Hard coded default values, set only if not passed or found in configuration @@ -164,13 +165,46 @@ sub set_default_conf_values $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); $use_identity_column=0 unless (defined ($use_identity_column)); +} + +# Process and check the parameters. +sub process_check_parameters +{ + # We have no before, after, or unsure file + if (not $before_file or not $after_file or not $unsure_file or not $filename) { + usage(); + exit 1; + } + + if ($validate_constraints !~ '^(yes|after|no)$') { + die "'validate_constraints' should be either yes, after or no (default yes)\n"; + } + + # We have been asked for kettle, but the compulsory parameters are not there + if ($kettle + and ( not $sd + or not $sh + or not $sp + or not $su + or not defined($sw) # password can be empty, it just has to be defined + or not $pd + or not $ph + or not $pp + or not $pu + or not defined($pw) # password can be empty, it just has to be defined + )) { + usage(); + print + "You have to provide all connection information, if using -k or kettle directory set in configuration file\n"; + exit 1; + } # Compute the case_treatment flag $case_treatment = 1; $case_treatment = 0 if ($keep_identifier_case); $case_treatment = 2 if ($camel_to_snake); if ($keep_identifier_case && $camel_to_snake) { - die "keep_identifier_case and camel_to_snake parameters cannot be both set to 1.\n"; + die "'keep_identifier_case' and 'camel_to_snake parameters' cannot be both set to 1.\n"; } } @@ -3208,46 +3242,12 @@ sub resolve_name_conflicts # Set default values for anything not set yet set_default_conf_values(); -# We have no before, after, or unsure -if ( not $before_file - or not $after_file - or not $unsure_file - or not $filename) -{ - usage(); - exit 1; -} - -if ($validate_constraints !~ '^(yes|after|no)$') -{ - croak "validate_constraints should be yes, after or no (default yes)\n"; -} - -# We have been asked for kettle, but the compulsory parameters are not there -if ($kettle - and ( not $sd - or not $sh - or not $sp - or not $su - or not defined($sw) # password can be empty, it just has to be defined - or not $pd - or not $ph - or not $pp - or not $pu - or not defined($pw) # password can be empty, it just has to be defined - ) -) -{ - usage(); - print - "You have to provide all connection information, if using -k or kettle directory set in configuration file\n"; - exit 1; -} +# Perform checks on parameters +process_check_parameters(); # We need to build %relabel_schemas from $relabel_schemas build_relabel_schemas(); - # Read SQL Server's dump file parse_dump(); From 27014faba3a498508d1daeb6a56b6947a18db788 Mon Sep 17 00:00:00 2001 From: Philippe Beaudoin Date: Sat, 3 Jul 2021 20:59:04 +0200 Subject: [PATCH 211/216] Add options to let sqlserver2pgsql produce a file containing the list of all columns, with the SQL-Server and PostgreSQL names of the schemas, tables and columns names. "col_map_file" defines the output file name. No file is produced if the parameter is not set. When "col_map_file_header" is set, a header line is added to the file. "col_map_file_delimiter" defines the field delimiter. The parameter is a string of 1 or several characters, and may be \t, \n or \r. The default is \t. Per a proposal from alchemistmatt. Improved by me. --- README.md | 8 ++++- example_conf_file | 3 ++ sqlserver2pgsql.pl | 84 +++++++++++++++++++++++++++++++++++----------- 3 files changed, 74 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index bebf4f4..c426cfe 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,13 @@ so the scale is often not important. `-keep_identifier_case`: don't convert the dump to all lower case. This is not recommended, as you'll have to put every identifier (column, table…) in double quotes… -`-camel_to_snake`: convert the object name (table, column, index...) from CamelCase to snake_case. Only do this if you are willing to change all your queries (or you use an ORM for instance) +`-camel_to_snake`: convert the object name (table, column, index...) from CamelCase to snake_case. Only do this if you are willing to change all your queries (or you use an ORM for instance). + +`-col_map_file`: specifies an output text file containing SQL-Server and PostgreSQL schemas, tables and columns names (1 line per column). + +`-col_map_file_header`: add a header line to the col_map_file (no header by default). + +`-col_map_file_delimiter`: specify a field delimiter for the col_map_file (TAB by default). `-validate_constraints=yes/after/no`: for foreign keys, if yes: foreign keys are created as valid in the after script (default) if no: they are created as not valid (enforced only for new rows) diff --git a/example_conf_file b/example_conf_file index 1088657..93fefa2 100644 --- a/example_conf_file +++ b/example_conf_file @@ -26,6 +26,9 @@ kettle directory = /tmp/kettle # comment this line if you don't w #convert numeric to int = 1 # set it to 0 to keep numeric(xx,0) types as numeric(xx,0); they will be converted to smallint, int or bigint by default #keep identifier case = 0 # set it to 1 to keep the case of database objects; by default identifier names are converted to lowercase #camelcasetosnake = 0 # set it to 1 to convert identifiers from camelCase to snake_case; keep identifier case and camelcasetosnake cannot be both set to 1 +#col map file = /tmp/map # text file with SQL-Server and PostgreSQL schemas, tables and columns names +#col map file header = 0 # set if to 1 to add a header line to the col map file +#col map file delimiter = \t # the fields delimiter in the col map file (TAB by default) #validate constraints = yes # should the constraints be validated by the DDL scripts ? 'yes' = validated at constraint creation, # 'no' = kept NOT VALID, 'after' = validated after the data load; 'yes' by default #use identity column = 1 # set it to 0 to create an explicite SEQUENCE for identity columns (the old technic); by default, diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index c640bb2..c1bff1b 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -41,6 +41,9 @@ our $before_file; our $after_file; our $unsure_file; +our $col_map_file; +our $col_map_file_header; +our $col_map_file_delimiter; our $case_treatment; # 0 do nothing, 1 = convert to lowercase (the default), 2 = convert to snake_case our $ignore_errors; our $keep_identifier_case; @@ -99,11 +102,14 @@ sub parse_conf_file 'sql server dump filename' => 'filename', 'case insensitive' => 'case_insensitive', 'no relabel dbo' => 'norelabel_dbo', - 'convert numeric to int' => 'convert_numeric_to_int', - 'drop rowversion' => 'drop_rowversion', 'relabel schemas' => 'relabel_schemas', 'keep identifier case' => 'keep_identifier_case', 'camelcasetosnake' => 'camel_to_snake', + 'col map file' => 'col_map_file', + 'col map file header' => 'col_map_file_header', + 'col map file delimiter' => 'col_map_file_delimiter', + 'convert numeric to int' => 'convert_numeric_to_int', + 'drop rowversion' => 'drop_rowversion', 'validate constraints' => 'validate_constraints', 'sort size' => 'sort_size', 'use pk if possible' => 'use_pk_if_possible', @@ -145,22 +151,23 @@ sub parse_conf_file # Set the default value for all parameters not set either in the configuration file or in the command line. sub set_default_conf_values { - # Hard coded default values, set only if not passed or found in configuration $case_insensitive=0 unless (defined ($case_insensitive)); $norelabel_dbo=0 unless (defined ($norelabel_dbo)); - $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); - $drop_rowversion=0 unless (defined ($drop_rowversion)); $keep_identifier_case=0 unless (defined ($keep_identifier_case)); $camel_to_snake=0 unless (defined ($camel_to_snake)); - $parallelism_in=1 unless (defined ($parallelism_in));# the jdbc driver often errors when there are several sessions to sql server + $col_map_file = "" unless (defined($col_map_file)); + $col_map_file_header = 0 unless (defined($col_map_file_header)); + $col_map_file_delimiter = '\t' unless (defined($col_map_file_delimiter)); + $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); + $drop_rowversion=0 unless (defined ($drop_rowversion)); + $parallelism_in=1 unless (defined ($parallelism_in)); # the jdbc driver often errors when there are several sessions to sql server $parallelism_out=8 unless (defined ($parallelism_out)); $sort_size=10000 unless (defined ($sort_size)); $use_pk_if_possible=0 unless (defined ($use_pk_if_possible)); $validate_constraints='yes' unless (defined ($validate_constraints)); $ignore_errors=0 unless (defined ($ignore_errors)); - # Default ports for PostgreSQL and SQL Server - $pp=5432 unless (defined ($pp)); - $sp=1433 unless (defined ($sp)); + $pp=5432 unless (defined ($pp)); # Default port for PostgreSQL + $sp=1433 unless (defined ($sp)); # Default port for SQL-Server $pforce_ssl=0 unless (defined ($pforce_ssl)); $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); @@ -206,6 +213,11 @@ sub process_check_parameters if ($keep_identifier_case && $camel_to_snake) { die "'keep_identifier_case' and 'camel_to_snake parameters' cannot be both set to 1.\n"; } + + # In $col_map_file_delimiter, replace \t, \n and \r by the real equivalent characters + $col_map_file_delimiter =~ s/\\t/\t/g; + $col_map_file_delimiter =~ s/\\n/\n/g; + $col_map_file_delimiter =~ s/\\r/\r/g; } # Converts numeric(4,0) and similar to int, bigint, smallint @@ -779,7 +791,7 @@ sub usage { print qq{ Usage: - sqlserver2pgsql.pl -b BEFORE_FILE -a AFTER_FILE -u UNSURE_FILE -f SQLSERVER_SCHEMA_FILE + sqlserver2pgsql.pl -f SQLSERVER_SCHEMA_FILE -b BEFORE_FILE -a AFTER_FILE -u UNSURE_FILE ... OPTIONS Description: @@ -790,11 +802,11 @@ sub usage Optionnaly, using the '-k' option, it will generate a kettle job to transfer all data. -Mandatory options: +Mandatory parameters: SQL Server schema input file: -f SQLSERVER_SCHEMA_FILE - a readable SQL Server SQL structure dump. + a readable SQL-Server SQL structure dump. PostgreSQL output schema files: -b BEFORE_SCRIPT @@ -805,18 +817,13 @@ sub usage contains objects we attempt to migrate, but cannot guarantee, such as views or complex indexes. -Options: +Other options: -conf CONFIGURATION_FILE uses a configuration file. All options can be set there. Command line options will overwrite conf options. - -i the resulting PostgreSQL names will be case-insensitive. -nr the SQL Server 'dbo' schema will not be translated to PostgreSQL 'public' schema. 'dbo' will stay 'dbo'. - -camel_to_snake - all object names are converted from 'camelCase' to 'camel_case', - which is more often used in PostgreSQL. Do not use this unless you - are ready to do SQL query changes in the client. -relabel_schemas 'SOURCE1=>DEST1;SOURCE2=>DEST2' gives a list of schemas to rename. Quote this option to prevent the shell to alter it. The '-nr' option cancels the default 'dbo' to @@ -824,6 +831,17 @@ sub usage -keep_identifier_case keep the case of SQL server database objects. This option is not advised. Default is to lowercase everything. + -camel_to_snake + all object names are converted from 'camelCase' to 'camel_case', + which is more often used in PostgreSQL. Do not use this unless you + are ready to do SQL query changes in the client. + -col_map_file + optional text file with old and new schema, table and column names + -col_map_file_header + add a header line to the col_map_file + -col_map_file_delimiter + the field delimiter used in the col_map_file (TAB by default) + -i the resulting PostgreSQL names will be case-insensitive. -num convert numeric 'xxx,0' to int, bigint, etc. Will not keep numeric scale and precision for the converted. -drop_rowversion (Default 0) @@ -2530,6 +2548,11 @@ sub generate_schema open BEFORE, ">:utf8", $before_file or die "Cannot open $before_file, $!"; open AFTER, ">:utf8", $after_file or die "Cannot open $after_file, $!"; open UNSURE, ">:utf8", $unsure_file or die "Cannot open $unsure_file, $!"; + if ($col_map_file) + { + open NAMEMAP, ">:utf8", $col_map_file or die "Cannot open $col_map_file, $!"; + } + print BEFORE "\\set ON_ERROR_STOP\n"; print BEFORE "\\set ECHO all\n"; print BEFORE "BEGIN;\n"; @@ -2539,6 +2562,15 @@ sub generate_schema print UNSURE "\\set ON_ERROR_STOP\n"; print AFTER "\\set ECHO all\n"; print UNSURE "BEGIN;\n"; + if ($col_map_file && $col_map_file_header) + { + print NAMEMAP "Source_schema" . $col_map_file_delimiter . + "Source_table" . $col_map_file_delimiter . + "Source_column" . $col_map_file_delimiter . + "Schema" . $col_map_file_delimiter . + "Table" . $col_map_file_delimiter . + "Column\n"; + } # Are we case insensitive ? We have to install citext then # Won't work on pre-9.1 database. But as this is a migration tool @@ -2610,6 +2642,8 @@ sub generate_schema # The tables foreach my $table (sort keys %{$refschema->{TABLES}}) { + my $origschema = $refschema->{TABLES}->{$table}->{origschema}; + my $newtablename = format_identifier($table); my @colsdef; foreach my $col ( sort { @@ -2620,14 +2654,20 @@ sub generate_schema { my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; - my $coldef = format_identifier($col) . " " . $colref->{TYPE}; + my $newcolname = format_identifier($col); + my $coldef = $newcolname . " " . $colref->{TYPE}; if ($colref->{NOT_NULL}) { $coldef .= ' NOT NULL'; } push @colsdef, ($coldef); + if ($col_map_file) + { + print NAMEMAP $origschema . $col_map_file_delimiter . $table . $col_map_file_delimiter . $col . $col_map_file_delimiter + . $schema . $col_map_file_delimiter . $newtablename . $col_map_file_delimiter . $newcolname . "\n"; + } } - print BEFORE "CREATE TABLE " . format_identifier($schema) . '.' . format_identifier($table) . "( \n\t" + print BEFORE "CREATE TABLE " . format_identifier($schema) . '.' . $newtablename . "( \n\t" . join(",\n\t", @colsdef) . ");\n\n"; } @@ -3084,6 +3124,7 @@ sub generate_schema close BEFORE; close AFTER; close UNSURE; + close NAMEMAP if ($col_map_file); } @@ -3211,6 +3252,9 @@ sub resolve_name_conflicts "i" => \$case_insensitive, "nr" => \$norelabel_dbo, "num" => \$convert_numeric_to_int, + "col_map_file=s" => \$col_map_file, + "col_map_file_header" => \$col_map_file_header, + "col_map_file_delimiter=s" => \$col_map_file_delimiter, "drop_rowversion" => \$drop_rowversion, "relabel_schemas=s" => \$relabel_schemas, "keep_identifier_case" => \$keep_identifier_case, From 0f9e080738b595ba69632a6f01a99a181aa7cecd Mon Sep 17 00:00:00 2001 From: Philippe Beaudoin Date: Fri, 6 Aug 2021 09:18:23 +0200 Subject: [PATCH 212/216] When a constraint name is greater than 63 characters, do not use it in the generated scripts and let PostgreSQL rebuild the name at constraint creation time. --- regression/reg_tests.sql | Bin 13694 -> 13792 bytes sqlserver2pgsql.pl | 35 ++++++++++++++++++++++------------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql index bf771fc02b53c1b30aa3da8dadc1973b86d528be..123dd9098157e8ba5c62d66e4ba0c54048de81ed 100644 GIT binary patch delta 108 zcmeyD^&p$+|G&u>#WW_L=M>p!U&lS!kWpfCsiMH-Kt?gwa0XX~cm^LJ83LpO82lN6 xfh>21AfQMvLm*Hl9?W)P2xagD@?3y?Cx%EM=?PR93{>UA;I{c5_X15u1^}288R7r{ delta 37 tcmaEm{V$8@-~Y*m;+m8AI41vN=h 63) + { + print STDERR "Warning: because of its length, the constraint name $name is ignored and will be set by Postgres at execution time.\n"; + return 0; + } + return 1; +} + # This one will try to convert what can obviously be converted from transact to PG # Things such as getdate() which can become CURRENT_TIMESTAMP sub convert_transact_function @@ -2753,7 +2765,7 @@ sub generate_schema next; } my $pkdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; - if (defined $refpk->{NAME}) + if (defined $refpk->{NAME} and is_constraint_name_valid($refpk->{NAME})) { $pkdef .= " CONSTRAINT " . format_identifier($refpk->{NAME}); } @@ -2775,7 +2787,7 @@ sub generate_schema { next unless ($constraint->{TYPE} eq 'UNIQUE'); my $consdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; - if (defined $constraint->{NAME}) + if (defined $constraint->{NAME} and is_constraint_name_valid($constraint->{NAME})) { $consdef .= " CONSTRAINT " . format_identifier($constraint->{NAME}); } @@ -2872,12 +2884,11 @@ sub generate_schema { next if ($constraint->{TYPE} =~ /^UNIQUE|PK$/); my $consdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; - if (defined $constraint->{NAME}) + if (defined $constraint->{NAME} and is_constraint_name_valid($constraint->{NAME})) { $consdef .= " CONSTRAINT " . format_identifier($constraint->{NAME}); } - if ($constraint->{TYPE} eq - 'FK') # COLS are already a comma separated list + if ($constraint->{TYPE} eq 'FK') # COLS are already a comma separated list { # We need to convert the column list to protected names my @localcollist=map{format_identifier($_)} @{$constraint->{LOCAL_COLS}}; @@ -2911,23 +2922,22 @@ sub generate_schema } # We need a name on the constraint to be able to validate it later. Maybe it would be better to generate one # FIXME: we'll see later if a generator is needed (probably) - if ($constraint->{TYPE} eq 'FK' and ($validate_constraints =~ /^after|no$/) and defined($constraint->{NAME})) + if (($validate_constraints =~ /^after|no$/) and defined($constraint->{NAME}) and is_constraint_name_valid($constraint->{NAME})) { $consdef .= " NOT VALID"; } $consdef .= ";\n"; print AFTER $consdef; - if ($constraint->{TYPE} eq 'FK' and $validate_constraints eq 'after' and defined $constraint->{NAME}) + if ($validate_constraints eq 'after' and defined $constraint->{NAME} and is_constraint_name_valid($constraint->{NAME})) { print UNSURE "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " VALIDATE CONSTRAINT " . format_identifier($constraint->{NAME}) . ";\n"; } } elsif ($constraint->{TYPE} eq 'CHECK') - { - $consdef .= " CHECK (" . convert_transactsql_code($constraint->{TEXT}) . ");\n"; - print UNSURE $consdef - ; # Check constraints are SQL, so cannot be sure - } + { + $consdef .= " CHECK (" . convert_transactsql_code($constraint->{TEXT}) . ");\n"; + print UNSURE $consdef; # Check constraints are SQL, so cannot be sure + } elsif ($constraint->{TYPE} eq 'CHECK_CITEXT') { # These have been generated here, for citext mostly. So we know their syntax is ok @@ -3182,7 +3192,6 @@ sub resolve_name_conflicts } $known_names{format_identifier($domain."2pgd")}=1; } - } # Then we scan all indexes From 6f5af05b52a8361df4cd5c713c4c1fbcbf1b6dcc Mon Sep 17 00:00:00 2001 From: spanevin Date: Wed, 13 Jul 2022 13:36:11 +0300 Subject: [PATCH 213/216] Add support of schema-level comments --- sqlserver2pgsql.pl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 08d7935..2f0a672 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2313,7 +2313,7 @@ sub parse_dump # I hope it will be sufficient (won't be if someone decides to end a comment with a quote) unless ($sqlproperty =~ - /^EXEC sys.sp_addextendedproperty \@name=N'(.*?)'\s*,\s*\@value=N'(.*)'\s*,\s*\@level0type=N'(.*?)'\s*,\s*\@level0name=N'(.*?)'\s*(?:,\s*\@level1type=N'(.*?)'\s*,\s*\@level1name=N'(.*?)')\s*?(?:,\s*\@level2type=N'(.*?)'\s*,\s*\@level2name=N'(.*?)')?/s) + /^EXEC sys.sp_addextendedproperty \@name=N'(.*?)'\s*(?:,\s*\@value=N'(.*?)'\s*)?(?:,\s*\@level0type=N'(.*?)'\s*)?(?:,\s*\@level0name=N'(.*?)'\s*)?(?:,\s*\@level1type=N'(.*?)'\s*,\s*\@level1name=N'(.*?)')?\s*?(?:,\s*\@level2type=N'(.*?)'\s*,\s*\@level2name=N'(.*?)')?/s) { # Not parsing a comment should not stop print STDERR "Could not parse <$sqlproperty>. Ignored.\n"; @@ -2322,7 +2322,11 @@ sub parse_dump my ($comment, $schema, $obj, $objname, $subobj, $subobjname) = ($2, $4, $5, $6, $7, $8); $schema=relabel_schemas($schema); - if ($obj eq 'TABLE' and not defined $subobj) + if (not defined $obj) + { + $objects->{SCHEMAS}->{$schema}->{COMMENT} = $comment; + } + elsif ($obj eq 'TABLE' and not defined $subobj) { $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{COMMENT} = $comment; @@ -3040,6 +3044,13 @@ sub generate_schema # Comments on tables and columns while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { + # Comments on schemas + if (defined($refschema->{COMMENT})) + { + print AFTER "COMMENT ON SCHEMA " . format_identifier($schema) . " IS '" + . $refschema->{COMMENT} . "';\n"; + } + # Comments on tables foreach my $table (sort keys %{$refschema->{TABLES}}) { From eea47867d755249ed73fa24d4f08df22a4ead60f Mon Sep 17 00:00:00 2001 From: Vladislav Moiseev Date: Thu, 17 Aug 2023 07:13:26 +0400 Subject: [PATCH 214/216] Added sforce_ssl flag for Kettle --- sqlserver2pgsql.pl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2f0a672..a25144d 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -53,6 +53,7 @@ our $parallelism_out; our $sort_size; our $use_pk_if_possible; +our $sforce_ssl; our $pforce_ssl; our $stringtype_unspecified; our $skip_citext_length_check; @@ -114,6 +115,7 @@ sub parse_conf_file 'sort size' => 'sort_size', 'use pk if possible' => 'use_pk_if_possible', 'ignore errors' => 'ignore_errors', + 'sql server force ssl' => 'sforce_ssl', 'postgresql force ssl' => 'pforce_ssl', 'stringtype unspecified' => 'stringtype_unspecified', 'skip citext length check' => 'skip_citext_length_check', @@ -168,6 +170,7 @@ sub set_default_conf_values $ignore_errors=0 unless (defined ($ignore_errors)); $pp=5432 unless (defined ($pp)); # Default port for PostgreSQL $sp=1433 unless (defined ($sp)); # Default port for SQL-Server + $sforce_ssl=0 unless (defined ($sforce_ssl)); $pforce_ssl=0 unless (defined ($pforce_ssl)); $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); @@ -894,6 +897,8 @@ sub usage parallelism level for the kettle job (input, SQL Server). Default 1. -po PARALLELISM_OUT parallelism level for the kettle job (output, PostgreSQL). Default 8. + -sforce_ssl + force a SSL session to SQL Server -pforce_ssl force a SSL session to PostgreSQL -stringtype_unspecified @@ -1029,6 +1034,14 @@ sub generate_kettle $newtemplate =~ s/Y<\/use_batch>/N<\/use_batch>/g; # Cannot use batch mode with ignore errors } + if ($sforce_ssl) + { + $newtemplate =~ s/__sforce_ssl__/EXTRA_OPTION_MSSQL.ssl<\/code>require<\/attribute><\/attribute>/g; + } + else + { + $newtemplate =~ s/__sforce_ssl__//g; + } if ($pforce_ssl) { $newtemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; @@ -1066,6 +1079,14 @@ sub generate_kettle $newincrementaltemplate =~ s/__PARALLELISM_OUT__/$parallelism_out/g; $newincrementaltemplate =~ s/__sort_size__/$sort_size/g; + if ($sforce_ssl) + { + $newincrementaltemplate =~ s/__sforce_ssl__/EXTRA_OPTION_MSSQL.ssl<\/code>require<\/attribute><\/attribute>/g; + } + else + { + $newincrementaltemplate =~ s/__sforce_ssl__//g; + } if ($pforce_ssl) { $newincrementaltemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; @@ -3283,6 +3304,7 @@ sub resolve_name_conflicts "sort_size=i" => \$sort_size, "use_pk_if_possible=s" => \$use_pk_if_possible, "ignore_errors" => \$ignore_errors, + "sforce_ssl" => \$sforce_ssl, "pforce_ssl" => \$pforce_ssl, "stringtype_unspecified" => \$stringtype_unspecified, "skip_citext_length_check" => \$skip_citext_length_check, @@ -3420,6 +3442,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -3782,6 +3805,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -4351,6 +4375,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -4781,6 +4806,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN From 266465b178db3b5c755a1384fb05911b0ccc0c31 Mon Sep 17 00:00:00 2001 From: Vladislav Moiseev Date: Mon, 9 Oct 2023 20:21:55 +0400 Subject: [PATCH 215/216] Add info about -sforce_ssl flag into README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c426cfe..d8f04bc 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ cleartext, so don't make this directory public): `-pp` : postgresql port `-pu` : postgresql username `-pw` : postgresql password +`-sforce_ssl` : force a SSL connection to your SQL Server database. Required if ForceEncryption option is set to 'Yes' `-pforce_ssl` : force a SSL connection to your PostgreSQL database. ssl=on should be set on the PostgreSQL server `-f` : the SQL Server structure dump file -ignore_errors : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) From d09f6cdf9fafda25c117e35fa5fe826e55d3c878 Mon Sep 17 00:00:00 2001 From: Florent Jardin Date: Tue, 10 Oct 2023 15:17:38 +0200 Subject: [PATCH 216/216] Update README.md --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d8f04bc..2b56f93 100644 --- a/README.md +++ b/README.md @@ -124,24 +124,41 @@ one for each table to copy, plus the one for the job) You'll also need to specify the connection parameters. They will be stored inside the kettle files (in cleartext, so don't make this directory public): + `-sd` : sql server database + `-sh` : sql server host + `-si` : sql server host instance + `-sp` : sql server port (usually 1433) + `-su` : sql server username + `-sw` : sql server password + `-pd` : postgresql database + `-ph` : postgresql host + `-pp` : postgresql port + `-pu` : postgresql username + `-pw` : postgresql password + `-sforce_ssl` : force a SSL connection to your SQL Server database. Required if ForceEncryption option is set to 'Yes' + `-pforce_ssl` : force a SSL connection to your PostgreSQL database. ssl=on should be set on the PostgreSQL server + `-f` : the SQL Server structure dump file --ignore_errors : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) + +`-ignore_errors` : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) `-pi` : The parallelism used in kettle jobs to read from SQL Server (1 by default, the jdbc driver frequently errors out when larger) + `-po` : The parallelism used in kettle jobs to write to PostgresSQL: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 + `-sort_size=100000`: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). We don't sort in databases for two reasons: the sort order (collation for strings for example) can be different between SQL Server