diff --git a/hola.txt b/hola.txt new file mode 100644 index 0000000..9b46f4d --- /dev/null +++ b/hola.txt @@ -0,0 +1 @@ +ey diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl old mode 100755 new mode 100644 index 3659e02..db5e9ca --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -48,7 +48,7 @@ our $requires_postgis=0; # These three variables are loaded in the BEGIN block at the end of this file (they are very big -my $template; +my $template; my $template_lob; my $incremental_template; my $incremental_template_sortable_pk; @@ -141,7 +141,7 @@ sub convert_numeric_to_int # This is a list of the types that require a cast to be imported in kettle # C = using CREATE CAST -# S = updating system catalog +# S = updating system catalog my %types_to_cast = ('uuid' => 'C','date' => 'C','timestamp with time zone' => 'C','xml' => 'S'); # This sub adds a cast (if not defined already) if @@ -178,7 +178,7 @@ sub add_cast 'datetime2' => 'timestamp', 'smalldatetime' => 'timestamp', 'time' => 'time', - 'timestamp' => 'timestamp', + 'timestamp' => 'bytea', 'datetimeoffset' => 'timestamp with time zone', 'image' => 'bytea', 'binary' => 'bytea', @@ -248,7 +248,7 @@ sub convert_type # Special case. This is an internal type, and should seldom be used in production. Converted to varchar(128) $rettype='varchar(128)'; } - + # We special case also the geometry and geography data types elsif ( $sqlstype =~ /^geography$|^geometry$/i ) { @@ -334,8 +334,6 @@ sub sql_convert_column my ($colname,$coltype)=@_; my %functions = ( 'uuid' => 'lower({colname})', - 'date' => 'convert(varchar(50), {colname}, 120)', - 'timestamp with time zone(7)' => 'convert(varchar(50), {colname}, 121)', 'xml' => 'case when datalength({colname}) > 5 then {colname} else null end'); if (defined ($functions{$coltype})) { @@ -428,7 +426,7 @@ sub format_identifier_cols_index } # This one will try to convert what can obviously be converted from transact to PG -# Things such as getdate() which can become CURRENT_TIMESTAMP +# Things such as getdate() which can become CURRENT_TIMESTAMP sub convert_transactsql_code { my ($code)=@_; @@ -541,7 +539,7 @@ sub next_col_pos unless ($norelabel_dbo) { $relabel_schemas{'dbo'}='public'; - } + } # dbo can be overwritten in relabel_schema (the user will probably forget to deactivate the relabel). # so we do the real relabeling after the norelabel_dbo, to overwrite if (defined $relabel_schemas) @@ -556,7 +554,7 @@ sub next_col_pos $relabel_schemas{$pair[0]}=$pair[1]; } } - + } @@ -717,13 +715,16 @@ sub generate_kettle { $newincrementaltemplate=$incremental_template; } - + # Build the column list of the table to put into the SQL Server query my @colsdef; my @pgcolsdef; foreach my $col ( sort { + scalar(grep { /^$b$/ } @{$refschema->{TABLES}->{$table}->{PK}->{COLS}}) + <=> scalar(grep { /^$a$/ } @{$refschema->{TABLES}->{$table}->{PK}->{COLS}}) + || $refschema->{TABLES}->{$table}->{COLS}->{$a}->{POS} <=> $refschema->{TABLES}->{$table}->{COLS}->{$b} ->{POS} @@ -797,7 +798,7 @@ sub generate_kettle $keys.="$pk\n"; } $newincrementaltemplate =~ s/__KEYS_MERGE__/$keys/g; - + my $sortkeys=''; my $synckeys=''; foreach my $pk(@pk) @@ -814,7 +815,7 @@ sub generate_kettle $newincrementaltemplate =~ s/__SORT_KEYS_SQLSERVER__/$sortkeys/g; $newincrementaltemplate =~ s/__SORT_KEYS_PG__/$sortkeys/g; $newincrementaltemplate =~ s/__KEYS_SYNC__/$synckeys/g; - + # We also need to tell the merge step to compare all columns my $valuesmerge=''; my $valuessync=''; @@ -836,13 +837,13 @@ sub generate_kettle $newincrementaltemplate =~ s/__VALUES_MERGE__/$valuesmerge/g; $newincrementaltemplate =~ s/__VALUES_SYNC__/$valuessync/g; - + # Produce the incremental transformation open FILE, ">$dir/incremental-$schema-$table.ktr" or die "Cannot write to $dir/incremental-$schema-$table.ktr"; binmode(FILE,":utf8"); print FILE $newincrementaltemplate; - close FILE; + close FILE; } else { @@ -865,8 +866,10 @@ sub generate_kettle # tables where we cannot do incremental (no PK...) open JOBFILE, ">$dir/migration.kjb" or die "Cannot write to $dir/migration.kjb"; + binmode(JOBFILE,":utf8"); open INCFILE, ">$dir/incremental.kjb" or die "Cannot write to $dir/incremental.kjb"; + binmode(INCFILE,":utf8"); my $real_dir = getcwd; my $entries = ''; my $incentries = ''; @@ -970,13 +973,7 @@ sub generate_kettle { foreach my $cast (keys %{$objects->{CASTS}}) { - if ($objects->{CASTS}->{$cast} eq "C") - { - $beforescript.= "DROP CAST IF EXISTS (varchar as $cast);\n"; - $beforescript.= "CREATE CAST (varchar as $cast) with inout as implicit;\n"; - $afterscript.= "DROP CAST (varchar as $cast);\n"; - } - elsif ($objects->{CASTS}->{$cast} eq "S") + if ($objects->{CASTS}->{$cast} eq "S") { $beforescript.= "UPDATE pg_cast SET castcontext='i' WHERE castsource='character varying'::regtype AND casttarget='$cast'::regtype;\n"; $afterscript.= "UPDATE pg_cast SET castcontext='e' WHERE castsource='character varying'::regtype AND casttarget='$cast'::regtype;\n"; @@ -994,7 +991,7 @@ sub generate_kettle $afterscript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ENABLE TRIGGER ALL;\n"; } } - + # This is for the SQL Scripts. We also need to specify the PG connection $job_header =~ s/__SQL_SCRIPT_INIT__/$beforescript/g; @@ -1085,7 +1082,7 @@ sub add_column_to_table if ($coltype eq 'xml') { $colqual = undef - ; # ignoring sql server xml schema since its not supported in pg + ; # ignoring sql server xml schema since its not supported in pg } elsif ($colqual eq '(max)') { @@ -1182,7 +1179,7 @@ sub parse_dump close $file; # We now ask guess... - my $decoder = guess_encoding($data, qw/iso8859-15/); + my $decoder = guess_encoding($data, qw/utf8/); #qw/iso8859-15/); die $decoder unless ref($decoder); # If we got to here, it means we have found the right decoder @@ -1206,7 +1203,7 @@ sub parse_dump TABLE: while (my $line = read_and_clean($file)) { # Here is a col definition. - # We ignore ROWGUIDCOL as it has no meaning in PostgreSQL and cannot be emulated + # We ignore ROWGUIDCOL as it has no meaning in PostgreSQL and cannot be emulated # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ @@ -1572,7 +1569,7 @@ sub parse_dump $def.=$idx; } print STDERR "This spatial index won't be migrated:\n$def\n"; - + } # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. @@ -1612,7 +1609,7 @@ sub parse_dump $constraint->{NAME} = $3; } - + CONS: while (my $consline= read_and_clean($file)) { next if ($consline =~ /^\($/); @@ -1701,16 +1698,17 @@ sub parse_dump # FK constraint. It's multi line, we have to look for references, and what to do on update, delete, etc (I have only seen delete cascade for now) # Constraint name is optionnal elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? FOREIGN KEY\((.*?)\)/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? FOREIGN KEY\((.*?)\)/ ) { # This is a FK definition. We have the foreign table definition in next line. my $constraint; + my $nocheck = (defined $3 && $3 eq "NO"); my $table = $2; my $schema = relabel_schemas($1); - my $consname= $3; + my $consname= $4; $constraint->{TYPE} = 'FK'; - my @local_cols = split (/\s*,\s*/,$4); # Split around the comma. There may be whitespaces + my @local_cols = split (/\s*,\s*/,$5); # Split around the comma. There may be whitespaces @local_cols=map{s/^\[//;s/]$//;$_;} @local_cols; # Remove the brackets around the columns $constraint->{LOCAL_COLS}=\@local_cols; $constraint->{LOCAL_TABLE} = $2; @@ -1719,12 +1717,26 @@ sub parse_dump $constraint->{NAME}=$consname; } + # Ignore constraint if NOCHECK + if ($nocheck) + { + print STDERR "Warning: NOCHECK constraint <$consname> on table <$table> ignored.\n"; + while (my $fk = read_and_clean($file)) + { + if ($fk =~ /^GO/) + { + next MAIN; + } + } + } + while (my $fk = read_and_clean($file)) { if ($fk =~ /^GO/) { push @{$objects->{SCHEMAS}->{$schema}->{'TABLES'}->{$table} ->{CONSTRAINTS}}, ($constraint); + next MAIN; } elsif ($fk =~ /^REFERENCES \[(.*)\]\.\[(.*)\] \((.*?)\)/) @@ -1750,10 +1762,10 @@ sub parse_dump { $constraint->{ON_UPD_CASC} = 1; } - elsif ($fk =~ /^NOT FOR REPLICATION$/) - { - next; # We don't care for this, it has no meaning for PostgreSQL - } + elsif ($fk =~ /^NOT FOR REPLICATION$/) + { + next; # We don't care for this, it has no meaning for PostgreSQL + } else { die "Cannot parse $fk $., in a FK. This is a bug"; @@ -1810,13 +1822,13 @@ sub parse_dump or die "Cannot find a name for this extended property: $sqlproperty"; my $propertyname = $1; - if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1|Display Name|Description|Example Values|Source System|Table Description|Table Type|ETL Rules|Display Folder|SCD Type|Source Datatype)$/) + if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1|Display Name|Description|Example Values|Source System|Table Description|Table Type|ETL Rules|Display Folder|SCD Type|Source Datatype|AllowZeroLength)$/) { # We don't dump these. They are graphical descriptions of the GUI next; } - elsif ($propertyname eq 'MS_Description') + elsif ($propertyname eq 'MS_Description' || $propertyname eq 'MS_SSMA_SOURCE') { # This is a comment. We parse it. @@ -1909,7 +1921,7 @@ sub parse_dump { next; } - + # Ignore xml schema collections since they are not supported in pg elsif ($line =~ /^CREATE XML SCHEMA COLLECTION/) { @@ -1919,7 +1931,7 @@ sub parse_dump elsif ($line =~ /^ALTER XML SCHEMA COLLECTION/) { next; - } + } # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views elsif ($line =~ /^IF NOT EXISTS/) @@ -1985,7 +1997,7 @@ sub parse_dump { $line =read_and_clean($file); } - + next; } @@ -2040,7 +2052,7 @@ sub generate_schema { print BEFORE "CREATE EXTENSION IF NOT EXISTS citext;\n"; } - + # Do we require PostGIS ? if ($requires_postgis) { @@ -2067,7 +2079,7 @@ sub generate_schema # The tables, columns, etc... will be created in the before script, so there is no dependancy # problem with constraints, that will be in the after script, except foreign keys which depend on unique indexes # We have to do all domains and types before all tables - # Don't care for dependancy + # Don't care for dependancy while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # The user-defined types (domains, etc) @@ -2100,6 +2112,9 @@ sub generate_schema my @colsdef; foreach my $col ( sort { + scalar(grep { /^$b$/ } @{$refschema->{TABLES}->{$table}->{PK}->{COLS}}) + <=> scalar(grep { /^$a$/ } @{$refschema->{TABLES}->{$table}->{PK}->{COLS}}) + || $refschema->{TABLES}->{$table}->{COLS}->{$a}->{POS} <=> $refschema->{TABLES}->{$table}->{COLS}->{$b} ->{POS} @@ -2209,7 +2224,7 @@ sub generate_schema print AFTER $idxdef; } } - } + } # Other constraints while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { @@ -2693,10 +2708,10 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQL + MSSQLNATIVE Native __sqlserver_database__ - __sqlserver_port__ + __sqlserver_username__ __sqlserver_password__ @@ -2707,8 +2722,8 @@ BEGIN FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN + MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN - PORT_NUMBER__sqlserver_port__ QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN USE_POOLINGN @@ -2727,7 +2742,6 @@ BEGIN - EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -2971,10 +2985,10 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQL + MSSQLNATIVE Native __sqlserver_database__ - __sqlserver_port__ + __sqlserver_username__ __sqlserver_password__ @@ -2985,8 +2999,8 @@ BEGIN FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN + MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN - PORT_NUMBER__sqlserver_port__ QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN USE_POOLINGN @@ -3356,28 +3370,28 @@ BEGIN $incremental_template= < - migration__sqlserver_table_name__ - - - - Normal - 0 - / - - - - - - - - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD - - -
- - + migration__sqlserver_table_name__ + + + + Normal + 0 + / + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD + + +
+ + ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS @@ -3449,10 +3463,10 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQL + MSSQLNATIVE Native __sqlserver_database__ - __sqlserver_port__ + __sqlserver_username__ __sqlserver_password__ @@ -3463,8 +3477,8 @@ BEGIN FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN + MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN - PORT_NUMBER__sqlserver_port__ QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN USE_POOLINGN @@ -3719,7 +3733,7 @@ BEGIN __postgres_db__ - 100 + 100 N N @@ -3783,28 +3797,28 @@ BEGIN $incremental_template_sortable_pk= < - migration__sqlserver_table_name__ - - - - Normal - 0 - / - - - - - -
- - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD - - -
- - + migration__sqlserver_table_name__ + + + + Normal + 0 + / + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD + + +
+ + ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS @@ -3876,10 +3890,10 @@ BEGIN __sqlserver_db__ __sqlserver_host__ - MSSQL + MSSQLNATIVE Native __sqlserver_database__ - __sqlserver_port__ + __sqlserver_username__ __sqlserver_password__ @@ -3890,8 +3904,8 @@ BEGIN FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN + MSSQLUseIntegratedSecurityfalse MSSQL_DOUBLE_DECIMAL_SEPARATORN - PORT_NUMBER__sqlserver_port__ QUOTE_ALL_FIELDSY SUPPORTS_BOOLEAN_DATA_TYPEN USE_POOLINGN @@ -4048,7 +4062,7 @@ BEGIN __postgres_db__ - 100 + 100 N N