From 6f5af05b52a8361df4cd5c713c4c1fbcbf1b6dcc Mon Sep 17 00:00:00 2001 From: spanevin Date: Wed, 13 Jul 2022 13:36:11 +0300 Subject: [PATCH 1/4] Add support of schema-level comments --- sqlserver2pgsql.pl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 08d7935..2f0a672 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -2313,7 +2313,7 @@ sub parse_dump # I hope it will be sufficient (won't be if someone decides to end a comment with a quote) unless ($sqlproperty =~ - /^EXEC sys.sp_addextendedproperty \@name=N'(.*?)'\s*,\s*\@value=N'(.*)'\s*,\s*\@level0type=N'(.*?)'\s*,\s*\@level0name=N'(.*?)'\s*(?:,\s*\@level1type=N'(.*?)'\s*,\s*\@level1name=N'(.*?)')\s*?(?:,\s*\@level2type=N'(.*?)'\s*,\s*\@level2name=N'(.*?)')?/s) + /^EXEC sys.sp_addextendedproperty \@name=N'(.*?)'\s*(?:,\s*\@value=N'(.*?)'\s*)?(?:,\s*\@level0type=N'(.*?)'\s*)?(?:,\s*\@level0name=N'(.*?)'\s*)?(?:,\s*\@level1type=N'(.*?)'\s*,\s*\@level1name=N'(.*?)')?\s*?(?:,\s*\@level2type=N'(.*?)'\s*,\s*\@level2name=N'(.*?)')?/s) { # Not parsing a comment should not stop print STDERR "Could not parse <$sqlproperty>. Ignored.\n"; @@ -2322,7 +2322,11 @@ sub parse_dump my ($comment, $schema, $obj, $objname, $subobj, $subobjname) = ($2, $4, $5, $6, $7, $8); $schema=relabel_schemas($schema); - if ($obj eq 'TABLE' and not defined $subobj) + if (not defined $obj) + { + $objects->{SCHEMAS}->{$schema}->{COMMENT} = $comment; + } + elsif ($obj eq 'TABLE' and not defined $subobj) { $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{COMMENT} = $comment; @@ -3040,6 +3044,13 @@ sub generate_schema # Comments on tables and columns while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { + # Comments on schemas + if (defined($refschema->{COMMENT})) + { + print AFTER "COMMENT ON SCHEMA " . format_identifier($schema) . " IS '" + . $refschema->{COMMENT} . "';\n"; + } + # Comments on tables foreach my $table (sort keys %{$refschema->{TABLES}}) { From eea47867d755249ed73fa24d4f08df22a4ead60f Mon Sep 17 00:00:00 2001 From: Vladislav Moiseev Date: Thu, 17 Aug 2023 07:13:26 +0400 Subject: [PATCH 2/4] Added sforce_ssl flag for Kettle --- sqlserver2pgsql.pl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index 2f0a672..a25144d 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -53,6 +53,7 @@ our $parallelism_out; our $sort_size; our $use_pk_if_possible; +our $sforce_ssl; our $pforce_ssl; our $stringtype_unspecified; our $skip_citext_length_check; @@ -114,6 +115,7 @@ sub parse_conf_file 'sort size' => 'sort_size', 'use pk if possible' => 'use_pk_if_possible', 'ignore errors' => 'ignore_errors', + 'sql server force ssl' => 'sforce_ssl', 'postgresql force ssl' => 'pforce_ssl', 'stringtype unspecified' => 'stringtype_unspecified', 'skip citext length check' => 'skip_citext_length_check', @@ -168,6 +170,7 @@ sub set_default_conf_values $ignore_errors=0 unless (defined ($ignore_errors)); $pp=5432 unless (defined ($pp)); # Default port for PostgreSQL $sp=1433 unless (defined ($sp)); # Default port for SQL-Server + $sforce_ssl=0 unless (defined ($sforce_ssl)); $pforce_ssl=0 unless (defined ($pforce_ssl)); $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); @@ -894,6 +897,8 @@ sub usage parallelism level for the kettle job (input, SQL Server). Default 1. -po PARALLELISM_OUT parallelism level for the kettle job (output, PostgreSQL). Default 8. + -sforce_ssl + force a SSL session to SQL Server -pforce_ssl force a SSL session to PostgreSQL -stringtype_unspecified @@ -1029,6 +1034,14 @@ sub generate_kettle $newtemplate =~ s/Y<\/use_batch>/N<\/use_batch>/g; # Cannot use batch mode with ignore errors } + if ($sforce_ssl) + { + $newtemplate =~ s/__sforce_ssl__/EXTRA_OPTION_MSSQL.ssl<\/code>require<\/attribute><\/attribute>/g; + } + else + { + $newtemplate =~ s/__sforce_ssl__//g; + } if ($pforce_ssl) { $newtemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; @@ -1066,6 +1079,14 @@ sub generate_kettle $newincrementaltemplate =~ s/__PARALLELISM_OUT__/$parallelism_out/g; $newincrementaltemplate =~ s/__sort_size__/$sort_size/g; + if ($sforce_ssl) + { + $newincrementaltemplate =~ s/__sforce_ssl__/EXTRA_OPTION_MSSQL.ssl<\/code>require<\/attribute><\/attribute>/g; + } + else + { + $newincrementaltemplate =~ s/__sforce_ssl__//g; + } if ($pforce_ssl) { $newincrementaltemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; @@ -3283,6 +3304,7 @@ sub resolve_name_conflicts "sort_size=i" => \$sort_size, "use_pk_if_possible=s" => \$use_pk_if_possible, "ignore_errors" => \$ignore_errors, + "sforce_ssl" => \$sforce_ssl, "pforce_ssl" => \$pforce_ssl, "stringtype_unspecified" => \$stringtype_unspecified, "skip_citext_length_check" => \$skip_citext_length_check, @@ -3420,6 +3442,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -3782,6 +3805,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -4351,6 +4375,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -4781,6 +4806,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN From 266465b178db3b5c755a1384fb05911b0ccc0c31 Mon Sep 17 00:00:00 2001 From: Vladislav Moiseev Date: Mon, 9 Oct 2023 20:21:55 +0400 Subject: [PATCH 3/4] Add info about -sforce_ssl flag into README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c426cfe..d8f04bc 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ cleartext, so don't make this directory public): `-pp` : postgresql port `-pu` : postgresql username `-pw` : postgresql password +`-sforce_ssl` : force a SSL connection to your SQL Server database. Required if ForceEncryption option is set to 'Yes' `-pforce_ssl` : force a SSL connection to your PostgreSQL database. ssl=on should be set on the PostgreSQL server `-f` : the SQL Server structure dump file -ignore_errors : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) From d09f6cdf9fafda25c117e35fa5fe826e55d3c878 Mon Sep 17 00:00:00 2001 From: Florent Jardin Date: Tue, 10 Oct 2023 15:17:38 +0200 Subject: [PATCH 4/4] Update README.md --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d8f04bc..2b56f93 100644 --- a/README.md +++ b/README.md @@ -124,24 +124,41 @@ one for each table to copy, plus the one for the job) You'll also need to specify the connection parameters. They will be stored inside the kettle files (in cleartext, so don't make this directory public): + `-sd` : sql server database + `-sh` : sql server host + `-si` : sql server host instance + `-sp` : sql server port (usually 1433) + `-su` : sql server username + `-sw` : sql server password + `-pd` : postgresql database + `-ph` : postgresql host + `-pp` : postgresql port + `-pu` : postgresql username + `-pw` : postgresql password + `-sforce_ssl` : force a SSL connection to your SQL Server database. Required if ForceEncryption option is set to 'Yes' + `-pforce_ssl` : force a SSL connection to your PostgreSQL database. ssl=on should be set on the PostgreSQL server + `-f` : the SQL Server structure dump file --ignore_errors : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) + +`-ignore_errors` : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) `-pi` : The parallelism used in kettle jobs to read from SQL Server (1 by default, the jdbc driver frequently errors out when larger) + `-po` : The parallelism used in kettle jobs to write to PostgresSQL: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 + `-sort_size=100000`: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). We don't sort in databases for two reasons: the sort order (collation for strings for example) can be different between SQL Server