diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2f0a672..4bd2600 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -26,11 +26,14 @@ # If you have to hack and want to understand its structure, just uncomment the call to Dumper() in the code # There is just too many things in it, and it evolves all the time my $objects; +my @itf_array; # These are global variables, from configuration file or command line arguments our ($sd, $sh, $si, $sp, $su, $sw, $pd, $ph, $pp, $pu, $pw); # Connection args our $conf_file; our $filename; # Filename passed as arg +our $includedatafilename; #IDT Filename passed as arg +our $includetablesfilename; #parse only tables from this file our $case_insensitive; # Passed as arg: was SQL Server installation case insensitive ? PostgreSQL can't ignore accents anyway # If yes, we will generate citext with CHECK constraints, that's the best we can do our $norelabel_dbo; # Passed as arg: should we convert DBO to public ? @@ -73,6 +76,8 @@ ; # These are used to create the dynamic parts of the job (XML file) my @view_list; # array to keep view ordering from sql server's dump (a view may depend on another view) +#my @idf_array; # array of tablenames which data have to be migrated + # Opens the configuration file # Sets $sd $sh $si $sp $su $sw $pd $ph $pp $pu $pw when they are not set in the command line already # Also gets kettle parameters... @@ -594,6 +599,7 @@ sub convert_transact_function $code =~ s/CONVERT\s*\(\s*NVARCHAR\s*(.*?)\s*\(\s*(.*?)\s*\s*\)\,\s*(.*?)\s*\)/CAST($3 AS varchar($2))/gi; $code =~ s/CONVERT\s*\(\s*(.*?)\s*\(\s*(.*?)\s*\s*\)\,\s*(.*?)\s*\)/CAST($3 AS $1($2))/gi; $code =~ s/CONVERT\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/CAST($2 AS $1)/gi; + $code =~ s/newid\s*\(/uuid_generate_v4(/gi; return $code; } @@ -605,11 +611,11 @@ sub convert_transactsql_code my ($code)=@_; #print STDERR "convert: $code\n"; - if ($code =~ /^\((.+?)\)\s+(AND|OR)\s+\((.+?)\)$/) { + if ($code =~ /^\((.+?)\)\s+(AND|OR)\s+\((.+?)\)$/i) { my ($lhs,$op,$rhs)=($1,$2,$3); $code = "(".convert_transactsql_code("$lhs").") $op (".convert_transactsql_code("$rhs").")"; } - elsif ($code =~ /^(.+?)\s+(AND|OR)\s+(.+?)$/) { + elsif ($code =~ /^(.+?)\s+(AND|OR)\s+(.+?)$/i) { my ($lhs,$op,$rhs)=($1,$2,$3); $code = "(".convert_transactsql_code("$lhs")." $op ".convert_transactsql_code("$rhs").")"; } @@ -926,6 +932,13 @@ sub generate_kettle { mkdir($dir) or die "Cannot create $dir"; } + +# if ($includedatafilename) +# { + open my $handle, '<', $includedatafilename; + chomp(my @idf_array = <$handle>); + close $handle; +# } # For each table in each schema in $objects, we generate a kettle file in the directory # We also create an incremental transformation @@ -937,6 +950,13 @@ sub generate_kettle foreach my $table (sort keys %{$refschema->{TABLES}}) { + # check table and skip if needed :-) + + unless (grep(/^$table\s*$/i, @idf_array) and $includedatafilename) { + print STDOUT "Skip creating kettle-jobs for $table\n"; + next; + } + my $origschema=$refschema->{TABLES}->{$table}->{origschema}; # First, does this table have LOBs ? The template depends on this and is this # table having an int PK ? @@ -950,9 +970,9 @@ sub generate_kettle my $wherefilter; $newtemplate = $template_lob; $wherefilter = - 'WHERE ' + 'WHERE [' . $refschema->{TABLES}->{$table}->{PK}->{COLS}->[0] - . '% ${Internal.Step.Unique.Count} = ${Internal.Step.Unique.Number}'; + . '] % ${Internal.Step.Unique.Count} = ${Internal.Step.Unique.Number}'; $newtemplate =~ s/__sqlserver_where_filter__/$wherefilter/; } @@ -1188,6 +1208,11 @@ sub generate_kettle foreach my $table (sort { lc($a) cmp lc($b) } keys %{$refschema->{TABLES}}) { + unless (grep(/^$table\s*$/i, @idf_array) and $includedatafilename) { + print STDOUT "Skip creating kettle-jobs for $table\n"; + next; + } + my $tmp_entry = $job_entry; # We build the entries with regexp substitutions. The tablename contains the schema @@ -1485,6 +1510,7 @@ sub add_column_to_table } } + # Reads the dump passed as -f # Generates the $object structure # That's THE MAIN FUNCTION @@ -1509,17 +1535,31 @@ sub parse_dump # or at least, perl thinks it has :) open $file, "<:encoding(" . $decoder->name . ")", $filename or die "Cannot open $filename"; + + #prepare array with tablenames + open my $handle, '<', $includetablesfilename; + chomp(@itf_array = <$handle>); + close $handle; # Tagged because sql statements are often multi-line, so there are inner loops in some conditions MAIN: while (my $line = read_and_clean($file)) { - # Create table, obviously. There will be other lines below for the rest of the table definition if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\s*\(/) { my $schemaname = relabel_schemas($1); my $orig_schema = $1; my $tablename = $2; + unless (grep(/^$tablename\s*$/i, @itf_array) and $includetablesfilename) + { + print STDERR "INFO: Table $tablename ignored\n"; + + # We have to find next GO to know we are out of the procedure + while (my $mline = read_and_clean($file)) + { + next MAIN if ($mline =~ /^GO$/); + } + } $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{haslobs} = 0; $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{origschema} = $orig_schema; # We are in a create table. Read everything until its end... @@ -1716,7 +1756,7 @@ sub parse_dump next MAIN; } } - } + } elsif ($line =~ /^CREATE SCHEMA \[(.*)\]/) { $objects->{SCHEMAS}->{relabel_schemas($1)} = undef @@ -1762,7 +1802,6 @@ sub parse_dump next MAIN if ($contline eq ''); } } - # Now we parse the create view. It is multi-line, so the code looks like like create table: we parse everything until a line # containing only a single quote (end of the dbo.sp_executesql) # The problem is that SQL Server seems to be spitting the original query used to create the view, not a normalized version @@ -1874,9 +1913,7 @@ sub parse_dump } } # These are domains with PostgreSQL - elsif ($line =~ - /^CREATE TYPE \[(.*?)\]\.\[(.*?)\] FROM \[(.*?)](?:\((\d+(?:,\s*\d+)?)?\))?/ - ) + elsif ($line =~/^CREATE TYPE \[(.*?)\]\.\[(.*?)\] FROM \[(.*?)](?:\((\d+(?:,\s*\d+)?)?\))?/) { # Dependency between types is not done for now. If the problem arises, it may be added my ($schema, $type, $origtype, $quals) = ($1, $2, $3, $4); @@ -1949,10 +1986,7 @@ sub parse_dump } } } - - elsif ($line =~ - /^\s*CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*?)\] ON \[(.*?)\]\.\[(.*?)\](\(\[.*?\]\))?/ - ) + elsif ($line =~/^\s*CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*?)\] ON \[(.*?)\]\.\[(.*?)\](\(\[.*?\]\))?/) { # Index creation. Index are namespaced per table in SQL Server, not in PostgreSQL # In PostgreSQL they are in the same namespace as the tables, and in the same @@ -2038,9 +2072,8 @@ sub parse_dump ->{INDEXES}->{$idxname}->{WHERE}="(".$filter.")"; } } - } - - # we do not take migrate spatial indexes + } + # we do not take migrate spatial indexes elsif ($line =~ /^CREATE SPATIAL INDEX/) { my $def=$line; @@ -2050,7 +2083,6 @@ sub parse_dump } print STDERR "This spatial index won't be migrated:\n$def\n"; } - elsif ($line =~ /^ALTER INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\] DISABLE$/) { my $idxname = $1; @@ -2059,13 +2091,10 @@ sub parse_dump $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} ->{$idxname}->{DISABLE} = 1; - } - + } # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table - elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD \[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(-?\d+,\s*-?\d+\))? (NOT NULL|NULL)(?: CONSTRAINT \[.*\] )?(?: DEFAULT \(.*\))?$/ - ) + elsif ($line =~/^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD \[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(-?\d+,\s*-?\d+\))? (NOT NULL|NULL)(?: CONSTRAINT \[.*\] )?(?: DEFAULT \(.*\))?$/) { my $schemaname=relabel_schemas($1); my $tablename=$2; @@ -2082,12 +2111,9 @@ sub parse_dump store_default_value($schemaname,$tablename,$colname,$default,$line); } } - # Table constraints # Primary key. Multiline - elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+(?:WITH (?:NO)?CHECK )?ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED|NONCLUSTERED)?/ - ) + elsif ($line =~/^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+(?:WITH (?:NO)?CHECK )?ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED|NONCLUSTERED)?/) { my $schemaname=relabel_schemas($1); my $tablename=$2; @@ -2121,9 +2147,7 @@ sub parse_dump } } } - elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[(.*)\])? UNIQUE (?:CLUSTERED|NONCLUSTERED)?/ - ) + elsif ($line =~/^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[(.*)\])? UNIQUE (?:CLUSTERED|NONCLUSTERED)?/) { my $schemaname=relabel_schemas($1); my $tablename=$2; @@ -2150,44 +2174,30 @@ sub parse_dump } } - # Default values. numeric, then text. These are 100% sure, they will parse in PG - # Sometimes there is a second pair of parenthesis. I don't even want to know why... - # Bit just need a little bit of work to be converted to 'true'/'false' - elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((\(?(?:-)?\d+(?:\.\d+)?\))?\) FOR \[(.*)\]/ - ) + # Sometimes there is a second pair of parenthesis. I don't even want to know why... + # Bit just need a little bit of work to be converted to 'true'/'false' + elsif ($line =~/^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((\(?(?:-)?\d+(?:\.\d+)?\))?\) FOR \[(.*)\]/) { store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } - elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(('.*')\) FOR \[(.*)\]/ - ) + elsif ($line =~/^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(('.*')\) FOR \[(.*)\]/) { store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } - # Yes, we also get default NULL (what for ? :) ), and sometimes with a different case - elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(((?i)NULL)\) FOR \[(.*)\]/ - ) + elsif ($line =~/^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(((?i)NULL)\) FOR \[(.*)\]/) { store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } - # And there are also constraints with functions and other strange code in them. Put them as unsure - elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/ - ) + elsif ($line =~/^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/) { store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } - # FK constraint. It's multi line, we have to look for references, and what to do on update, delete, etc (I have only seen delete cascade for now) # Constraint name is optionnal - elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK(?: NOT FOR REPLICATION)?\s+ADD(?:\s+CONSTRAINT \[(.*)\])? FOREIGN KEY\((.*?)\)/ - ) + elsif ($line =~/^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK(?: NOT FOR REPLICATION)?\s+ADD(?:\s+CONSTRAINT \[(.*)\])? FOREIGN KEY\((.*?)\)/) { # This is a FK definition. We have the foreign table definition in next line. my $constraint; @@ -2249,11 +2259,8 @@ sub parse_dump } } } - # Check constraint. As it can be arbitrary code, we just get this code, and hope it will work on PG (it will be stored in a special script file) - elsif ($line =~ - /ALTER TABLE \[(.*)\]\.\[(.*)\]\s+(?:WITH? (?:NO)?CHECK? )?ADD(?:\s+CONSTRAINT (.*))?\s+CHECK(?: NOT FOR REPLICATION)?\s+\((\((.*)\)|(.*))\)/ - ) + elsif ($line =~/ALTER TABLE \[(.*)\]\.\[(.*)\]\s+(?:WITH? (?:NO)?CHECK? )?ADD(?:\s+CONSTRAINT (.*))?\s+CHECK(?: NOT FOR REPLICATION)?\s+\((\((.*)\)|(.*))\)/) { # Check constraint. We'll do what we can, syntax may be different. my $constraint; @@ -2271,7 +2278,6 @@ sub parse_dump push @{$objects->{SCHEMAS}->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}}, ($constraint); } - # These are comments or extended attributes on objets. They can be multiline, so aggregate everything # Until a line that ends with a quote (but not two of them). We remove pair of quotes to make it simpler # If fact in can be a lot of things. So we have to ignore things like MS_DiagramPaneCount @@ -2389,20 +2395,16 @@ sub parse_dump "Don't know what to do with this extendedproperty: $sqlproperty"; } } - - # Save variable for future use + # Save variable for future use elsif ($line =~ /^:setvar\s+(\S+)\s+"(.*)"/) - { + { my $varname = $1; my $varvalue = $2; $objects->{VARIABLES}->{$varname} = $varvalue; next; } - # Ignore USE, GO, and things that have no meaning for postgresql - elsif ($line =~ - /^USE\s|^GO\s*$|\/\*\*\*\*|^SET ANSI_NULLS (ON|OFF)|^SET QUOTED_IDENTIFIER|^SET ANSI_PADDING|CHECK CONSTRAINT|^BEGIN|^END/ - ) + elsif ($line =~/^USE\s|^GO\s*$|\/\*\*\*\*|^SET ANSI_NULLS (ON|OFF)|^SET QUOTED_IDENTIFIER|^SET ANSI_PADDING|CHECK CONSTRAINT|^BEGIN|^END/) { next; } @@ -2410,19 +2412,16 @@ sub parse_dump { next; } - # Don't know what it is. If you know, and it is worth converting, tell me :) elsif ($line =~ /^EXEC .*bindrule/) { next; } - # Ignore users and roles. Security models will probably be very different between the two databases elsif ($line =~ /^CREATE (ROLE|USER)/) { next; } - # Ignore grant statements elsif ($line =~ /^GRANT ([^ ]+) ON (.+) TO \[[^ ]+\]/) { @@ -2432,23 +2431,19 @@ sub parse_dump { next; } - elsif ($line =~ /^ALTER (ROLE|USER)/) { next; } - # Ignore xml schema collections since they are not supported in pg elsif ($line =~ /^CREATE XML SCHEMA COLLECTION/) { next; } - elsif ($line =~ /^ALTER XML SCHEMA COLLECTION/) { next; } - # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views # Also ignore version tests elsif ($line =~ /^IF EXISTS|^IF \(\@\@microsoftversion/i) @@ -2464,51 +2459,47 @@ sub parse_dump # Just ignore the line next; } - # Ignore CREATE DATABASE: we hope that we are given a single database as - # an option. It is multiline. + # an option. It is multiline. # Ignore everything until next GO # Ignore ALTER DATABASE for the same reason. The given parameters have no - # meaning in PG anyway - # Except for SET ARITHABORT OFF, for which we print a warning because it - # probably means the database contents are weird (10/0 = null) - elsif ($line =~ - /^ALTER DATABASE.* SET ARITHABORT OFF/) - { - print STDERR "WARNING: the source database is set as ARITHABORT OFF.\n"; - print STDERR " It means that for SQL Server, 10/0 = NULL.\n"; - print STDERR " You'll probably have problems porting that to PostgreSQL.\n"; - while ($line !~ /^GO$/) - { - $line =read_and_clean($file); - } - # We read everything in the CREATE DATABASE. Back to work ! - next; - } - # Sometimes, when there is a ALTER DATABASE SET ARITHABORT OFF, there are SET ARITHABORT ON. Just ignore them - elsif ($line =~ /^SET ARITHABORT ON/) - { - next; - } - # Sometimes we meet this: SET CONCAT_NULL_YIELDS_NULL ON. That's the normal behaviour for a SQL database. Just ignore - elsif ($line =~ /^SET CONCAT_NULL_YIELDS_NULL ON/) - { - next; - } - # Same more or less - elsif ($line =~ /^SET ANSI_WARNINGS ON/) - { - next; - } - # What the hell does this do in a dump ??? - elsif ($line =~ /^SET NUMERIC_ROUNDABORT OFF/) - { - next; - } - + # meaning in PG anyway + # Except for SET ARITHABORT OFF, for which we print a warning because it + # probably means the database contents are weird (10/0 = null) + elsif ($line =~/^ALTER DATABASE.* SET ARITHABORT OFF/) + { + print STDERR "WARNING: the source database is set as ARITHABORT OFF.\n"; + print STDERR " It means that for SQL Server, 10/0 = NULL.\n"; + print STDERR " You'll probably have problems porting that to PostgreSQL.\n"; + while ($line !~ /^GO$/) + { + $line =read_and_clean($file); + } + # We read everything in the CREATE DATABASE. Back to work ! + next; + } + # Sometimes, when there is a ALTER DATABASE SET ARITHABORT OFF, there are SET ARITHABORT ON. Just ignore them + elsif ($line =~ /^SET ARITHABORT ON/) + { + next; + } + # Sometimes we meet this: SET CONCAT_NULL_YIELDS_NULL ON. That's the normal behaviour for a SQL database. Just ignore + elsif ($line =~ /^SET CONCAT_NULL_YIELDS_NULL ON/) + { + next; + } + # Same more or less + elsif ($line =~ /^SET ANSI_WARNINGS ON/) + { + next; + } + # What the hell does this do in a dump ??? + elsif ($line =~ /^SET NUMERIC_ROUNDABORT OFF/) + { + next; + } # Same for tests about full text search. - elsif ($line =~ - /^(CREATE|ALTER) DATABASE|^IF \(1 = FULLTEXTSERVICEPROPERTY/) + elsif ($line =~/^(CREATE|ALTER) DATABASE|^IF \(1 = FULLTEXTSERVICEPROPERTY/) { while ($line !~ /^GO$/) { @@ -2518,7 +2509,6 @@ sub parse_dump # We read everything in the CREATE DATABASE. Back to work ! next; } - # Ignore CREATE and ALTER statements for full text search objects, such as CATALOG, INDEX or STOPLIST. elsif ($line =~ /^(CREATE|ALTER) FULLTEXT/) { @@ -2529,7 +2519,6 @@ sub parse_dump next; } - # Ignore EXEC dbo.sp_executesql, for now only seen for a create view. Views sql command aren't executed directly, don't know why elsif ($line =~ /^EXEC dbo.sp_executesql/) { @@ -2540,12 +2529,16 @@ sub parse_dump { next; } - # Still on views: there are empty lines, and C-style comments elsif ($line =~ /^\s*$/) { next; } + # skip LOCK_ESCALATION + elsif ($line =~ /^ALTER TABLE.*SET.*LOCK_ESCALATION/) + { + next; + } else { die "Line <$line> ($.) not understood. This is a bug"; @@ -2603,10 +2596,24 @@ sub generate_schema print BEFORE "CREATE EXTENSION IF NOT EXISTS postgis_topology;\n"; } + print UNSURE 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +'; + # Ok, we have parsed everything, and definitions are in $objects # We will put in the BEFORE file only table and columns definitions. # The rest will go in the AFTER script (check constraints, put default values, etc...) + + foreach my $sch (sort keys %{$objects->{SCHEMAS}}) + { + foreach my $tbl (sort keys %{$objects->{SCHEMAS}->{$sch}->{TABLES}}) + { + if (not grep(/^$tbl\s*$/i, @itf_array) and $includetablesfilename){ + delete($objects->{SCHEMAS}->{$sch}->{TABLES}->{$tbl}); + } + } + } + # The schemas. don't create empty schema, sql server creates a schema per user, even if it ends empty foreach my $schema (sort keys %{$objects->{SCHEMAS}}) { @@ -2614,7 +2621,7 @@ sub generate_schema or not defined $objects->{SCHEMAS}->{$schema}) { # Not compatible before 9.3. This is the logical target for this tool anyway - print BEFORE "CREATE SCHEMA IF NOT EXISTS ",format_identifier($schema),";\n"; + print BEFORE "--CREATE SCHEMA IF NOT EXISTS ",format_identifier($schema),";\n"; } } @@ -3269,6 +3276,8 @@ sub resolve_name_conflicts "pu=s" => \$pu, "pw=s" => \$pw, "f=s" => \$filename, + "idt=s" => \$includedatafilename, + "itf=s" => \$includetablesfilename, "i" => \$case_insensitive, "nr" => \$norelabel_dbo, "num" => \$convert_numeric_to_int,