diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..dd16792 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,34 @@ +version: 2 + +jobs: + test: + docker: + - image: perl:5-threaded + - image: postgres:13-alpine + environment: + POSTGRES_USER: postgres + POSTGRES_POSTGRES: postgres + POSTGRES_HOST_AUTH_METHOD: trust + environment: + # Inject APT packaged dependencies. + PERL5LIB: /usr/lib/x86_64-linux-gnu/perl5/5.24:/usr/share/perl5 + PGHOST: localhost + PGDATABASE: postgres + PGUSER: postgres + working_directory: /tmp/sqlserver2pgsql + steps: + - run: + name: Install debian packages + command: | + apt update -y + apt install -y --no-install-recommends bats git openssh-client postgresql-client + - checkout + - run: + name: Executing tests + command: prove --verbose + +workflows: + version: 2 + pipeline: + jobs: + - test diff --git a/FAQ.md b/FAQ.md index 6b909aa..d712aba 100644 --- a/FAQ.md +++ b/FAQ.md @@ -16,6 +16,31 @@ variable to a higher value (4096) for 4GB for instance. There is another big advantage of using Kettle: you can tailor the scripts produced by sqlserver2pgsql to your needs, such as adding some conversions, schema changes. As Kettle is an ETL, it is a good tool for doing such conversions on the fly. +In which order should I run the operations? +---------------------------------- + +sqlserver2pgsql outputs several files: before, after and unsure SQL files, plus +the kettle jobs files. + +If you load all the SQL files before the data migration, you can experience +problems. For example, you can have errors when the kettle job truncates the +table at the start of the process. If a foreign key constraint is enforced, +PostgreSQL cannot truncate a table referenced in a foreign key constraint and +the job would error out. + +You should first check the unsure file, verify that the SQL is fine or correct +it if needed. Some SQL orders from the unsure files are to be run before the +data migration, for example, default column values, procedures or functions, +triggers. So move them to the before file. + +You can then load the before file (`before.sql`). Then use the kettle jobs to +migrate the data (`migration.kjb`). When this is done, load the rest of the +unsure and the after file (`unsure.sql` and `after.sql`). + +In case you are still using the original database, a specific kettle job is +created so that you can feed the change periodically to your PostgreSQL +database (`incremental.kjb`). + What is this IGNORE NULLS I have to change in kettle.properties ? ---------------------------------- @@ -39,4 +64,5 @@ doesn't exist either in PG. So more constraints will fail. Can this tool migrate functions and stored procedures? ---------------------------------- -No, Transact-SQL is very different from PostgreSQL's many PL languages. You could give a look at https://bitbucket.org/openscg/pgtsql but I can't help you with this… and I don't think it is a long term viable option. +No, Transact-SQL is very different from PostgreSQL's many PL languages. These would need a manual migration. + diff --git a/INSTALL.md b/INSTALL.md index 89b17ef..1490edf 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -6,10 +6,15 @@ has no dependancy on fancy Perl modules, it only uses modules from the base perl Just run the script with your perl interpreter, providing it with the requested options (--help will tell you what to do). -If you are trying to run the script under Windows, you probably won't have a Perl interpreter. I recommand you use the latest Strawberry Perl (use either the installer or the portable version). +If you are trying to run the script under Windows, you probably won't have a Perl interpreter. I recommend you to use the latest Strawberry Perl (use either the installer or the portable version). +Kettle: +========================== If you want to migrate the data, you'll need "Kettle", an Open Source ETL. Get the latest version from here: http://kettle.pentaho.com/ . -You'll also need a SQL Server account with the permission to SELECT from the tables you want to migrate. -As Kettle is a Java program, you'll also need a recent JVM (Java 6 or 7 should do the trick). +On newest versions of Kettle, the SQL Server java driver isn't included. You'll need this one: http://jtds.sourceforge.net/. Just download the zip file (jtds-1.3.1-dist.zip at the time of this writing), extract the jar file (jtds-1.3.1.jar) and put this file in the lib directory of Kettle. + +If you don't want to bother with installing a JVM system-wide for Kettle, just download Sun's JVM and put in in Kettle's directory, in a "java" subdirectory. Latest versions of Kettle require Java 8. + +You'll also need a SQL Server account with the permission to SELECT from the tables you want to migrate. diff --git a/README.md b/README.md index dbbe2bb..2b56f93 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,10 @@ run sqlserver2pgsql.pl from. If you just want to convert this schema, run: ``` -./sqlserver2pgsql.pl -f sqlserver_sql_dump \ - -b output_before_script\ - -a output_after_script\ - -u output_unsure_script +./sqlserver2pgsql.pl -f sqlserver_sql_dump \ + -b output_before_script \ + -a output_after_script \ + -u output_unsure_script ``` The sqlserver2pgsql Perl script processes your SQL raw dump "sqlserver_sql_dump" and produces these three scripts: @@ -98,6 +98,14 @@ so the scale is often not important. `-keep_identifier_case`: don't convert the dump to all lower case. This is not recommended, as you'll have to put every identifier (column, table…) in double quotes… +`-camel_to_snake`: convert the object name (table, column, index...) from CamelCase to snake_case. Only do this if you are willing to change all your queries (or you use an ORM for instance). + +`-col_map_file`: specifies an output text file containing SQL-Server and PostgreSQL schemas, tables and columns names (1 line per column). + +`-col_map_file_header`: add a header line to the col_map_file (no header by default). + +`-col_map_file_delimiter`: specify a field delimiter for the col_map_file (TAB by default). + `-validate_constraints=yes/after/no`: for foreign keys, if yes: foreign keys are created as valid in the after script (default) if no: they are created as not valid (enforced only for new rows) if after: they are created as not valid, but the statements to validate them are put in the unsure file @@ -116,21 +124,41 @@ one for each table to copy, plus the one for the job) You'll also need to specify the connection parameters. They will be stored inside the kettle files (in cleartext, so don't make this directory public): + `-sd` : sql server database + `-sh` : sql server host + `-si` : sql server host instance + `-sp` : sql server port (usually 1433) + `-su` : sql server username + `-sw` : sql server password + `-pd` : postgresql database + `-ph` : postgresql host + `-pp` : postgresql port + `-pu` : postgresql username + `-pw` : postgresql password + +`-sforce_ssl` : force a SSL connection to your SQL Server database. Required if ForceEncryption option is set to 'Yes' + +`-pforce_ssl` : force a SSL connection to your PostgreSQL database. ssl=on should be set on the PostgreSQL server + `-f` : the SQL Server structure dump file +`-ignore_errors` : ignore insert errors (not advised, you'll need to examine kettle's logs, and it will be slower) + +`-pi` : The parallelism used in kettle jobs to read from SQL Server (1 by default, the jdbc driver frequently errors out when larger) + +`-po` : The parallelism used in kettle jobs to write to PostgresSQL: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 -`-p` : The parallelism used in kettle jobs: there will be this amount of sessions used to insert into PostgreSQL. Default to 8 `-sort_size=100000`: sort size to use for incremental jobs. Default is 10000, to try to be on the safe side (see below). We don't sort in databases for two reasons: the sort order (collation for strings for example) can be different between SQL Server diff --git a/contributors b/contributors new file mode 100644 index 0000000..7c4c9fb --- /dev/null +++ b/contributors @@ -0,0 +1,26 @@ +Marc Cousin, main developper + +Bill Ruddock (binarii) +Javier Callico (JCallico) +Joshua F. Rountree (joshuairl) +Julien Rouhaud (rjuju) +Konstantin Mosolov (kmosolov) +Matthew Monroe (alchemistmatt) +Philippe Baudoin (beaud76) +Thibaut Madelaine (madtibo) +Yann Verry (yanntech) +Žygimantas Kazlauskas (zygimantaskazlauskas) + +And the following unindentified github users (please tell me if you want your name added/replaced) + +bsacks99 +keyjote +mark-jay +mikes-gh +postrusil-osi +sebpcspkr +stuey1978 + +If I forgot your name in this list, please tell me. + +Thanks to everyone who helped improving this script ! diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d0843dd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3" + +services: + sqlserver2pgsql: + image: perl:5.24-threaded + environment: + # Inject APT packaged dependencies. + PERL5LIB: /usr/lib/x86_64-linux-gnu/perl5/5.24:/usr/share/perl5 + volumes: + - .:/tmp/workspace + command: perl ./sqlserver2pgsql.pl -f ./regression/reg_tests.sql -b /tmp/workspace/test/before.sql -a /tmp/workspace/test/after.sql -u /tmp/workspace/test/unsure.sql -k /tmp/workspace/test/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 + + postgres: + image: postgres:10-alpine + environment: + PGHOST: localhost + PGDATABASE: postgres + PGUSER: postgres + volumes: + - .:/tmp/workspace diff --git a/example_conf_file b/example_conf_file index 0d06b69..93fefa2 100644 --- a/example_conf_file +++ b/example_conf_file @@ -1,34 +1,66 @@ -# Source SQL Server Dump. Obviously compulsory -sql server dump filename=/tmp/dump - -# These are used to generate SQL scripts (this is the only thing that is always done) -before file=/tmp/before -after file=/tmp/after -unsure file=/tmp/unsure - -kettle directory=/tmp/kettle # Comment this line if you don't want a kettle script to be generated -# These are ignored as long as kettle is not set -sql server database=foo -sql server host=foo_host -sql server host instance=my_instance # You can omit this if you use the default instance -sql server port=1433 -sql server username=foo_user -sql server password=foo_password -postgresql database=bar -postgresql host=bar_host -postgresql port=5432 -postgresql username=bar_user -postgresql password=bar_password -parallelism=8 # Default value is 8. Number of parallel connections used by kettle to insert data into the PostgreSQL database - -# Optional behaviour -case insensitive=0 # set it to 1 to generate a dump with citext and check constraints all over the place -no relabel dbo=1 # set it to 0 to convert the dbo schema to public -convert numeric to int=1 # set it to 0 to keep numeric(xx,0) as numeric(xx,0). Will be converted to smallint, int or bigint by default -relabel schemas=dbo=>foo;schema1=>bar -keep identifier case=1 # keep case of database objects -validate constraints = yes # yes, after or no, should the constraints be validated by the dump ? (yes=validate during load, after after the load, no keep invalidated) - -# Incremental job -sort size=10000 # drives the amount of memory and temporary files that will be created by an incremental job -use pk if possible=0 # 1/list of tables, for tables where you want to try getting already sorted records +# +# This file is an example of a sqlserver2pgsql configuration file. +# + +# +# Files location. +# + +# Input file. +sql server dump filename = /tmp/dump # the source SQL-Server Dump (obviously compulsory) + +# Output files. +before file = /tmp/before # SQL script to execute before loading the data +after file = /tmp/after # SQL script to execute after having loaded the data +unsure file = /tmp/unsure # SQL script containing statements to check or to adjust manually +kettle directory = /tmp/kettle # comment this line if you don't want kettle components to be generated + +# +# Optional parameters to setup specific behaviour. +# + +#case insensitive = 0 # set it to 1 to generate a dump with citext and check constraints all over the place +#skip citext length check = 0 # set it to 1 to not add a CHECK (char_length()) constraint for citext fields (when case insensitive is set to 1) +#no relabel dbo = 1 # set it to 0 to convert the dbo schema into public +#relabel schemas = dbo=>foo;schema1=>bar +#convert numeric to int = 1 # set it to 0 to keep numeric(xx,0) types as numeric(xx,0); they will be converted to smallint, int or bigint by default +#keep identifier case = 0 # set it to 1 to keep the case of database objects; by default identifier names are converted to lowercase +#camelcasetosnake = 0 # set it to 1 to convert identifiers from camelCase to snake_case; keep identifier case and camelcasetosnake cannot be both set to 1 +#col map file = /tmp/map # text file with SQL-Server and PostgreSQL schemas, tables and columns names +#col map file header = 0 # set if to 1 to add a header line to the col map file +#col map file delimiter = \t # the fields delimiter in the col map file (TAB by default) +#validate constraints = yes # should the constraints be validated by the DDL scripts ? 'yes' = validated at constraint creation, + # 'no' = kept NOT VALID, 'after' = validated after the data load; 'yes' by default +#use identity column = 1 # set it to 0 to create an explicite SEQUENCE for identity columns (the old technic); by default, + # GENERATED ALWAYS clauses are generated +#drop rowversion = 0 # set it to 1 to ignore columns of SQL-Server type 'rowversion' or 'timestamp'; 0 by default + +# +# Parameters used for the data migration with Kettle. +# They are ignored as long as the 'kettle directory' parameter is not set. +# + +sql server database = foo +sql server host = foo_host +sql server host instance = my_instance # optional when the default instance is used +sql server port = 1433 +sql server username = foo_user +sql server password = foo_password + +postgresql database = bar +postgresql host = bar_host +postgresql port = 5432 +postgresql username = bar_user +postgresql password = bar_password + +#postgresql force ssl = 0 # set it to 1 to force a SSL session to PostgreSQL; 0 by default + +#parallelism_in = 1 # parallelism degree when reading from SQL-Server (where available); 1 by default +#parallelism_out = 8 # number of parallel connections used by kettle to insert data into the PostgreSQL database; 8 by default + +#stringtype unspecified = 0 # set it to 1 to let kettle process textual data as "not necessarily a strict PostgreSQL VARCHAR data"; 0 by default +#ignore errors = 0 # set it to 1 to not abort the data migration job when an error occurs; the parameter is ignored for incremental jobs + # warning: the migration will be slower and the job's log will need to be throroughly examined +# Incremental job parameters. +#sort size = 10000 # drives the amount of memory and temporary files that will be created by an incremental job; 10000 by default +#use pk if possible = 0 # set to either 1 or a space separated list of schema qualified table names, for tables candidated for getting already sorted rows diff --git a/example_install_script.sh b/example_install_script.sh new file mode 100644 index 0000000..e2435bc --- /dev/null +++ b/example_install_script.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# note this script does not install spoon dependencies +# ignore libwebkitgtk-1.0 errors when running kitchen. +# libwebkitgtk-1.0 is not available for RedHat/CentOS 7 or debian buster and is +# not needed for kitchen to run correctly + +# on RedHat 7 / CentOS 7 +# yum -y install wget unzip perl perl-MLDBM java-1.8.0-openjdk + +# on debian stretch +# apt install -y wget unzip perl libmldbm-perl openjdk-8-jdk +# (on debian buster you need to install openjdk-8-jdk from sid) + +MIGRATIONDIR=/opt/data_migration +mkdir -p $MIGRATIONDIR/kettlejobs + +# install sqlserver2pgsql +if [ ! -f "$MIGRATIONDIR/sqlserver2pgsql.pl" ]; then + wget https://raw.githubusercontent.com/dalibo/sqlserver2pgsql/master/sqlserver2pgsql.pl -P $MIGRATIONDIR + # make executable + chmod u+x,g+x,a+x $MIGRATIONDIR/sqlserver2pgsql.pl +fi + +# install kettle +if [ ! -f "$MIGRATIONDIR/data-integration/kitchen.sh" ]; then + wget https://sourceforge.net/projects/pentaho/files/latest/download?source=files -O /tmp/kettle.zip + unzip /tmp/kettle.zip -d /tmp/kettle + cp -R /tmp/kettle/data-integration $MIGRATIONDIR + rm -Rf /tmp/kettle;rm -f /tmp/kettle.zip + # make all shell scripts executable + chmod -R u+x,g+x,a+x $MIGRATIONDIR/data-integration/*.sh +fi + +# install JDBC driver (JTDS version works fine with MSSQL) +set -- $MIGRATIONDIR/data-integration/lib/jtds* +if [ ! -f "$1" ]; then + wget https://sourceforge.net/projects/jtds/files/latest/download?source=files -O /tmp/jtds.zip + unzip /tmp/jtds.zip -d /tmp/jtds + # copy to kettle lib directory + cp /tmp/jtds/jtds-*.jar $MIGRATIONDIR/data-integration/lib/ + rm -Rf /tmp/jtds;rm -f /tmp/jtds.zip +fi + + diff --git a/regression/basic_test/views.sql b/regression/basic_test/views.sql new file mode 100644 index 0000000..972c756 Binary files /dev/null and b/regression/basic_test/views.sql differ diff --git a/regression/issue_112.sql b/regression/issue_112.sql new file mode 100644 index 0000000..181156c --- /dev/null +++ b/regression/issue_112.sql @@ -0,0 +1,46 @@ +CREATE TABLE [dbo].[AFElementAttributeCategory]( + [rid] [bigint] IDENTITY(-1,-1) NOT NULL, + [id] [uniqueidentifier] NOT NULL, + [rowversion] [timestamp] NOT NULL, + [fkelementversionid] [bigint] NOT NULL, + [fkparentattributeid] [uniqueidentifier] NULL, + [fkcategoryid] [uniqueidentifier] NOT NULL, + [changedby] [int] NOT NULL, + CONSTRAINT [PK_AFElementAttributeCategory] PRIMARY KEY CLUSTERED +( + [rid] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [ASSETS] +) ON [ASSETS] +GO + +CREATE TABLE [dbo].[AFCaseAdjustment]( + [rid] [bigint] IDENTITY(-1,-1) NOT NULL, + [id] [uniqueidentifier] NOT NULL, + [rowversion] [timestamp] NOT NULL, + [fkcaseid] [bigint] NOT NULL, + [attributeid] [uniqueidentifier] NOT NULL, + [adjustedvalue] [varbinary](max) NULL, + [comment] [nvarchar](1000) NULL, + [previousvalue] [varbinary](max) NULL, + [creator] [nvarchar](50) NULL, + [creationdate] [datetime2](7) NULL, + [changedby] [int] NOT NULL, + CONSTRAINT [PK_AFCaseAdjustment] PRIMARY KEY NONCLUSTERED +( + [rid] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [ANALYSIS] +) ON [ANALYSIS] TEXTIMAGE_ON [ANALYSIS] +GO + +CREATE TABLE [dbo].[sd]( + [rid] [int] IDENTITY(1000,1) NOT NULL, + [rowversion] [timestamp] NOT NULL, + [sd] [nvarchar](max) NOT NULL, + [ownerRights] [int] NOT NULL, + [lupd] [datetime2](7) NULL, + CONSTRAINT [pk_sd] PRIMARY KEY CLUSTERED +( + [rid] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [ASSETS] +) ON [ASSETS] TEXTIMAGE_ON [ASSETS] +GO \ No newline at end of file diff --git a/regression/issue_59.sql b/regression/issue_59.sql new file mode 100644 index 0000000..db1073d --- /dev/null +++ b/regression/issue_59.sql @@ -0,0 +1,108 @@ +CREATE TABLE [dbo].[ACCOUNT]( + [ID] [char](36) NOT NULL, + [VERSION] [numeric](10, 0) NOT NULL, + [APPLICATION_ID] [char](36) NOT NULL, + [ACCOUNT_NUMBER] [nvarchar](35) NOT NULL, + [DELETED] [numeric](1, 0) NOT NULL, + [DELETED_BY] [char](36) NULL, + [DELETED_ON] [datetime2](0) NULL, + [CREATED_ON] [datetime2](0) NULL, + [CREATED_BY] [char](36) NULL, + [DAILY_LIMIT] [numeric](21, 7) NULL, + [BIC] [varchar](35) NULL, + [IBAN] [varchar](35) NULL, + [BACK_OFFICE_ACCOUNT_NUMBER] [varchar](35) NULL, + [BANK_ACCOUNT_NUMBER] [varchar](35) NULL, + [OWNER_ID] [char](36) NULL, + [ALLOW_PENDING] [numeric](1, 0) NOT NULL DEFAULT ((0)), + CONSTRAINT [SYS_C0010802] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] + +ALTER TABLE [dbo].[ACCOUNT] WITH CHECK ADD CHECK (APPLICATION_ID IN('1.0', '2.1', '2.2', '4.12', '10.3', 'None')); +GO +ALTER TABLE [dbo].[ACCOUNT] ADD CONSTRAINT deletor_list CHECK (DELETED_BY IN('Jacques', 'Philippe', 'Pierre', 'None')); +GO + +EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT.ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT', @level2type=N'COLUMN',@level2name=N'ID' +GO + + +CREATE NONCLUSTERED INDEX [IDX_ACCOUNT_ID] ON [dbo].[ACCOUNT] +( + [ID] ASC +) +GO +EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT.ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT', @level2type=N'INDEX',@level2name=N'IDX_ACCOUNT_ID' +GO + +CREATE NONCLUSTERED INDEX [IDX_ACCOUNT_VERSION] ON [dbo].[ACCOUNT] +( + [VERSION] ASC +) +WHERE ((ISNULL([VERSION], 0) > 1)) +GO +EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT.VERSION' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT', @level2type=N'INDEX',@level2name=N'IDX_ACCOUNT_VERSION' +GO + +CREATE NONCLUSTERED INDEX [IDX_ACCOUNT_BIC_IBAN] ON [dbo].[ACCOUNT] +( + [BIC] ASC, + [IBAN] ASC +) +INCLUDE ( [BACK_OFFICE_ACCOUNT_NUMBER], +[BANK_ACCOUNT_NUMBER]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO + +CREATE TABLE [dbo].[ACCOUNT_CATEGORY]( + [ID] [char](36) NOT NULL, + [VERSION] [numeric](10, 0) NOT NULL, + CONSTRAINT [SYS_C0010844] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +EXEC sys.sp_addextendedproperty @name=N'MS_SSMA_SOURCE', @value=N'ONEBANK.ACCOUNT_CATEGORY.UQ_INDEX' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'ACCOUNT_CATEGORY', @level2type=N'INDEX',@level2name=N'UQ_INDEX' +GO + +CREATE NONCLUSTERED INDEX [IDX_ACCOUNT_CATEGORY_ID] ON [dbo].[ACCOUNT_CATEGORY] +( + [ID] ASC +) +INCLUDE ( [VERSION]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO +/****** Object: Table [dbo].[IDX_TESTS] Script Date: 15/10/2020 14:34:08 ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[IDX_TESTS]( + [I] [int] NULL, + [J] [int] NULL, + [K] [int] NULL, + [L] [int] NULL +) ON [PRIMARY] +GO +/****** Object: Index [idx_IDX_TESTS_i_part] Script Date: 15/10/2020 14:34:09 ******/ +CREATE NONCLUSTERED INDEX [IDX_IDX_TESTS_I_INCL_K_PART] ON [dbo].[IDX_TESTS] +( + [I] ASC +) +INCLUDE ( [K]) +WHERE ([L]>(10)) +WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO +/****** Object: Index [IDX_IDX_TESTS_I_J_PART] Script Date: 15/10/2020 14:34:09 ******/ +CREATE NONCLUSTERED INDEX [IDX_IDX_TESTS_I_J_INCL_K_L_PART] ON [dbo].[IDX_TESTS] +( + [I] ASC, + [J] ASC +) +INCLUDE ( [K], + [L]) +WHERE ([J]>(1)) +WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO diff --git a/regression/issues.sql b/regression/issues.sql new file mode 100644 index 0000000..32089f8 --- /dev/null +++ b/regression/issues.sql @@ -0,0 +1,51 @@ +USE [master] +GO +/****** Issue 91 ******/ +CREATE TABLE [candidate].[PersonalIdentityHashes]( + [OrganizationId] [int] NOT NULL, + [PersonalIdentity] [char](64) NOT NULL, + [PersonalIdentityHash] [binary](64) NOT NULL, + [PersonalIdentityHash2] [varbinary](64) NOT NULL, + [CandidateId] [bigint] NOT NULL, + CONSTRAINT [PK_candidatePersonalIdentityHashes] PRIMARY KEY CLUSTERED +( + [OrganizationId] ASC, + [PersonalIdentityHash] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO + +/****** Issue 98 ******/ +CREATE TABLE [etl].[account_handle]( + [acct_event_id] [bigint] NOT NULL , + [acct_id] [bigint] NOT NULL , + [comment_id] [bigint] NULL, + [descrip] [varchar] NULL, + [event_cat_id] [bigint] NOT NULL , + [event_date] [datetime] NOT NULL , + [event_end_time] [varchar] NULL, + [event_start_time] [varchar] NULL, + [event_type_id] [bigint] NOT NULL , + [problem_cd] [bigint] NULL, + [reason_cd] [bigint] NULL, + [resolution] [varchar] NULL, + [user_id] [bigint] NULL, + [ext_user_name] [varchar] NULL, + [level1_id] [bigint] NULL, + [level2_id] [bigint] NULL, + [level3_id] [bigint] NULL, + [level4_id] [bigint] NULL, + [group_id] [bigint] NULL, + [escalation_id] [bigint] NULL, + [level5_id] [bigint] NULL, + [reference_id] [bigint] NULL, + [entity] [varchar] NULL, + [modified_ts] [datetime] NOT NULL , + [device_type_id] [bigint] NULL, + [device_error_codes] [varchar] NULL, + PRIMARY KEY CLUSTERED +( + [acct_event_id] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO diff --git a/regression/reg.pl b/regression/reg.pl index f76e24d..c29b716 100755 --- a/regression/reg.pl +++ b/regression/reg.pl @@ -7,7 +7,7 @@ foreach my $file (<*.sql>) { - my @options_to_try=('-i','-nr','-num', '-keep_identifier_case', '-validate_constraints=after'); + my @options_to_try=('-i','-nr','-num', '-keep_identifier_case', '-validate_constraints=after', '-use_identity_column=0'); my @all_combinations=(''); foreach my $option (@options_to_try) { diff --git a/regression/reg_tests.sql b/regression/reg_tests.sql new file mode 100644 index 0000000..123dd90 Binary files /dev/null and b/regression/reg_tests.sql differ diff --git a/sqlserver2pgsql.pl b/sqlserver2pgsql.pl index b37cba0..a25144d 100755 --- a/sqlserver2pgsql.pl +++ b/sqlserver2pgsql.pl @@ -36,21 +36,34 @@ our $norelabel_dbo; # Passed as arg: should we convert DBO to public ? our $relabel_schemas; our $convert_numeric_to_int; # Should we convert numerics to int when possible ? (numeric (4,0) could be converted an int, for instance) +our $drop_rowversion; # Should we remove MSSQL timestamp/rowversion columns when converting our $kettle; our $before_file; our $after_file; our $unsure_file; +our $col_map_file; +our $col_map_file_header; +our $col_map_file_delimiter; +our $case_treatment; # 0 do nothing, 1 = convert to lowercase (the default), 2 = convert to snake_case +our $ignore_errors; our $keep_identifier_case; +our $camel_to_snake; our $validate_constraints; -our $parallelism; +our $parallelism_in; +our $parallelism_out; our $sort_size; our $use_pk_if_possible; +our $sforce_ssl; +our $pforce_ssl; +our $stringtype_unspecified; +our $skip_citext_length_check; +our $use_identity_column; # Will be set if we detect GIS objects our $requires_postgis=0; # These three variables are loaded in the BEGIN block at the end of this file (they are very big -my $template; +my $template; my $template_lob; my $incremental_template; my $incremental_template_sortable_pk; @@ -67,44 +80,57 @@ sub parse_conf_file { - # Correspondance between conf_file parameter and program variable - # This is also used as the list of accepted parameters in the configuration file - my %parameters = ('sql server database' => 'sd', - 'sql server host' => 'sh', - 'sql server host instance' => 'si', - 'sql server port' => 'sp', - 'sql server username' => 'su', - 'sql server password' => 'sw', - 'postgresql database' => 'pd', - 'postgresql host' => 'ph', - 'postgresql port' => 'pp', - 'postgresql username' => 'pu', - 'postgresql password' => 'pw', - 'kettle directory' => 'kettle', - 'parallelism' => 'parallelism', - 'before file' => 'before_file', - 'after file' => 'after_file', - 'unsure file' => 'unsure_file', - 'sql server dump filename' => 'filename', - 'case insensitive' => 'case_insensitive', - 'no relabel dbo' => 'norelabel_dbo', - 'convert numeric to int' => 'convert_numeric_to_int', - 'relabel schemas' => 'relabel_schemas', - 'keep identifier case' => 'keep_identifier_case', - 'validate constraints' => 'validate_constraints', - 'sort size' => 'sort_size', - 'use pk if possible' => 'use_pk_if_possible', - ); - - # Open the conf file or die - open CONF, $conf_file or die "Cannot open $conf_file"; - while (my $line = ) - { + # Correspondance between conf_file parameter and program variable + # This is also used as the list of accepted parameters in the configuration file + my %parameters = ( + 'sql server database' => 'sd', + 'sql server host' => 'sh', + 'sql server host instance' => 'si', + 'sql server port' => 'sp', + 'sql server username' => 'su', + 'sql server password' => 'sw', + 'postgresql database' => 'pd', + 'postgresql host' => 'ph', + 'postgresql port' => 'pp', + 'postgresql username' => 'pu', + 'postgresql password' => 'pw', + 'kettle directory' => 'kettle', + 'parallelism_in' => 'parallelism_in', + 'parallelism_out' => 'parallelism_out', + 'before file' => 'before_file', + 'after file' => 'after_file', + 'unsure file' => 'unsure_file', + 'sql server dump filename' => 'filename', + 'case insensitive' => 'case_insensitive', + 'no relabel dbo' => 'norelabel_dbo', + 'relabel schemas' => 'relabel_schemas', + 'keep identifier case' => 'keep_identifier_case', + 'camelcasetosnake' => 'camel_to_snake', + 'col map file' => 'col_map_file', + 'col map file header' => 'col_map_file_header', + 'col map file delimiter' => 'col_map_file_delimiter', + 'convert numeric to int' => 'convert_numeric_to_int', + 'drop rowversion' => 'drop_rowversion', + 'validate constraints' => 'validate_constraints', + 'sort size' => 'sort_size', + 'use pk if possible' => 'use_pk_if_possible', + 'ignore errors' => 'ignore_errors', + 'sql server force ssl' => 'sforce_ssl', + 'postgresql force ssl' => 'pforce_ssl', + 'stringtype unspecified' => 'stringtype_unspecified', + 'skip citext length check' => 'skip_citext_length_check', + 'use identity column' => 'use_identity_column', + ); + + # Open the conf file or die + open CONF, $conf_file or die "Cannot open $conf_file"; + while (my $line = ) + { $line =~ s/#.*//; # Remove comments - $line =~ s/\s+=\s+/=/; # Remove whitespaces around the = + $line =~ s/\s+=\s+/=/; # Remove whitespaces around the = $line =~ s/\s+$//; # Remove trailing whitespaces next - if ($line =~ /^$/); # Empty line after comments have been removed + if ($line =~ /^$/); # Empty line after comments have been removed $line =~ /^(.*?)=(.*)$/ or die "Cannot parse $line from $conf_file"; my ($param, $value) = ($1, $2); no strict 'refs'; # Using references by name, temporarily @@ -124,20 +150,77 @@ sub parse_conf_file close CONF; } +# Set the default value for all parameters not set either in the configuration file or in the command line. sub set_default_conf_values { - # Hard coded default values, set only if not passed or found in configuration $case_insensitive=0 unless (defined ($case_insensitive)); $norelabel_dbo=0 unless (defined ($norelabel_dbo)); - $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); $keep_identifier_case=0 unless (defined ($keep_identifier_case)); - $parallelism=8 unless (defined ($parallelism)); + $camel_to_snake=0 unless (defined ($camel_to_snake)); + $col_map_file = "" unless (defined($col_map_file)); + $col_map_file_header = 0 unless (defined($col_map_file_header)); + $col_map_file_delimiter = '\t' unless (defined($col_map_file_delimiter)); + $convert_numeric_to_int=0 unless (defined ($convert_numeric_to_int)); + $drop_rowversion=0 unless (defined ($drop_rowversion)); + $parallelism_in=1 unless (defined ($parallelism_in)); # the jdbc driver often errors when there are several sessions to sql server + $parallelism_out=8 unless (defined ($parallelism_out)); $sort_size=10000 unless (defined ($sort_size)); $use_pk_if_possible=0 unless (defined ($use_pk_if_possible)); $validate_constraints='yes' unless (defined ($validate_constraints)); - # Default ports for PostgreSQL and SQL Server - $pp=5432 unless (defined ($pp)); - $sp=1433 unless (defined ($sp)); + $ignore_errors=0 unless (defined ($ignore_errors)); + $pp=5432 unless (defined ($pp)); # Default port for PostgreSQL + $sp=1433 unless (defined ($sp)); # Default port for SQL-Server + $sforce_ssl=0 unless (defined ($sforce_ssl)); + $pforce_ssl=0 unless (defined ($pforce_ssl)); + $stringtype_unspecified=0 unless (defined ($stringtype_unspecified)); + $skip_citext_length_check=0 unless (defined ($skip_citext_length_check)); + $use_identity_column=0 unless (defined ($use_identity_column)); +} + +# Process and check the parameters. +sub process_check_parameters +{ + # We have no before, after, or unsure file + if (not $before_file or not $after_file or not $unsure_file or not $filename) { + usage(); + exit 1; + } + + if ($validate_constraints !~ '^(yes|after|no)$') { + die "'validate_constraints' should be either yes, after or no (default yes)\n"; + } + + # We have been asked for kettle, but the compulsory parameters are not there + if ($kettle + and ( not $sd + or not $sh + or not $sp + or not $su + or not defined($sw) # password can be empty, it just has to be defined + or not $pd + or not $ph + or not $pp + or not $pu + or not defined($pw) # password can be empty, it just has to be defined + )) { + usage(); + print + "You have to provide all connection information, if using -k or kettle directory set in configuration file\n"; + exit 1; + } + + # Compute the case_treatment flag + $case_treatment = 1; + $case_treatment = 0 if ($keep_identifier_case); + $case_treatment = 2 if ($camel_to_snake); + if ($keep_identifier_case && $camel_to_snake) { + die "'keep_identifier_case' and 'camel_to_snake parameters' cannot be both set to 1.\n"; + } + + # In $col_map_file_delimiter, replace \t, \n and \r by the real equivalent characters + $col_map_file_delimiter =~ s/\\t/\t/g; + $col_map_file_delimiter =~ s/\\n/\n/g; + $col_map_file_delimiter =~ s/\\r/\r/g; } # Converts numeric(4,0) and similar to int, bigint, smallint @@ -156,7 +239,7 @@ sub convert_numeric_to_int # This is a list of the types that require a cast to be imported in kettle # C = using CREATE CAST -# S = updating system catalog +# S = updating system catalog my %types_to_cast = ('uuid' => 'C','date' => 'C','timestamp with time zone' => 'C','xml' => 'S'); # This sub adds a cast (if not defined already) if @@ -193,7 +276,8 @@ sub add_cast 'datetime2' => 'timestamp', 'smalldatetime' => 'timestamp', 'time' => 'time', - 'timestamp' => 'timestamp', + 'timestamp' => 'bytea', + 'rowversion' => 'bytea', 'datetimeoffset' => 'timestamp with time zone', 'image' => 'bytea', 'binary' => 'bytea', @@ -218,8 +302,19 @@ sub convert_type if ((defined $sqlqual and defined($unqual{$types{$sqlstype}})) or not defined $sqlqual) { - # This is one of the few types that have to be unqualified (binary type) - $rettype = $types{$sqlstype}; + # This is one of the few types that have to be unqualified (binary type) + $rettype = $types{$sqlstype}; + + # but we might add a check constraint for binary data + if ($sqlstype =~ 'binary' and defined $sqlqual) { + print STDERR "convert_type: $sqlstype, $sqlqual, $colname\n"; + my $constraint; + $constraint->{TYPE} = 'CHECK_BINARY_LENGTH'; + $constraint->{TABLE} = $tablename; + $constraint->{TEXT} = "octet_length(" . format_identifier($colname) . ") <= $sqlqual"; + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{CONSTRAINTS}}, ($constraint); + } } elsif (defined $sqlqual) { @@ -263,7 +358,7 @@ sub convert_type # Special case. This is an internal type, and should seldom be used in production. Converted to varchar(128) $rettype='varchar(128)'; } - + # We special case also the geometry and geography data types elsif ( $sqlstype =~ /^geography$|^geometry$/i ) { @@ -294,7 +389,7 @@ sub convert_type $rettype = "citext"; # Do we have a SQL qualifier ? (we'll have to do check constraints then) - if ($sqlqual) + if ($sqlqual and not defined($skip_citext_length_check) or $sqlqual and $skip_citext_length_check == 0) { # Check we have a table name and a colname, or a typname @@ -378,21 +473,32 @@ sub postgres_convert_column 'timestamp with time zone' => 'to_char({colname} AT TIME ZONE \'UTC\', \'YYYY-MM-DD HH:MI:SS.US+00\')'); if (defined ($functions{$coltype})) { - my $tmpcol = $functions{$coltype}; - $tmpcol =~ s/\{colname\}/"$colname"/; - return $tmpcol; + my $tmpcol = $functions{$coltype}; + $tmpcol =~ s/\{colname\}/$colname/; + return $tmpcol; } else { - return "\"$colname\""; + return $colname; } } -# This function is used to determine if a PK will be sorted the same in SQL Server and PG +# This is used to convert camelCase to snake_case. The latter is more usual with PostgreSQL +sub camel_to_snake +{ + my ($string)=@_; + $string =~ s/([[:lower:]])([[:upper:]]+)/$1_\l$2/g; + $string=lc($string); + return $string; +} + +# This function is used to determine if a PK will be sorted the same in SQL +# Server and PostgreSQL. # It means that it doesn't depend on collation orders or other internals. -# For now, only numeric and date data types are considered OK -# Used for incremental jobs, to know if we can ask the databases to send us pre-sorted data -# We also filter on $use_pk_if_possible +# For now, only numeric and date data types are considered OK. +# Used for incremental jobs, to know if we can ask the databases to send us +# pre-sorted data. +# We also filter on $use_pk_if_possible. sub is_pk_sort_order_safe { my ($schema,$table)=@_; @@ -417,6 +523,21 @@ sub is_pk_sort_order_safe return $isok; } +# This function renames the identifiers +sub rename_identifier +{ + my ($identifier)=@_; + + if ($case_treatment==1) + { + $identifier=lc($identifier); + } + elsif ($case_treatment==2) + { + $identifier=camel_to_snake($identifier); + } + return $identifier; +} # This function formats the identifiers (object name), putting double quotes around it # It also converts case if asked @@ -424,10 +545,13 @@ sub format_identifier { my ($identifier)=@_; croak "identifier not defined in format_identifier" unless (defined $identifier); - unless ($keep_identifier_case) + $identifier=rename_identifier($identifier); + + if (length($identifier) > 63) { - $identifier=lc($identifier); + print STDERR "WARNING: $identifier is more than 63 characters long; PostgreSQL will truncate the name internally\n"; } + # Now, we protect the identifier (similar to quote_ident in PG) $identifier=~ s/"/""/g; $identifier='"'.$identifier.'"'; @@ -446,16 +570,63 @@ sub format_identifier_cols_index return $formatted . ' ' . $order; } +# This sub returns FALSE and prints a warning message when the length of a provided constraint name is greater than 63. +sub is_constraint_name_valid +{ + my ($name) = @_; + if (length($name) > 63) + { + print STDERR "Warning: because of its length, the constraint name $name is ignored and will be set by Postgres at execution time.\n"; + return 0; + } + return 1; +} + # This one will try to convert what can obviously be converted from transact to PG -# Things such as getdate() which can become CURRENT_TIMESTAMP +# Things such as getdate() which can become CURRENT_TIMESTAMP +sub convert_transact_function +{ + my ($code)=@_; + $code =~ s/ISNULL\s*\(/COALESCE(/gi; + $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; + $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; + $code =~ s/SPACE\s*\(/REPEAT(' ', /gi; + $code =~ s/charindex\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/dPOSITION('$1' in $2)/gi; + $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; + $code =~ s/DATEADD\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\,\s*(.*?)\s*\)/$3 + INTERVAL '$2 $1'/gi; + $code =~ s/CONVERT\s*\(\s*NVARCHAR\s*(.*?)\s*\(\s*(.*?)\s*\s*\)\,\s*(.*?)\s*\)/CAST($3 AS varchar($2))/gi; + $code =~ s/CONVERT\s*\(\s*(.*?)\s*\(\s*(.*?)\s*\s*\)\,\s*(.*?)\s*\)/CAST($3 AS $1($2))/gi; + $code =~ s/CONVERT\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/CAST($2 AS $1)/gi; + return $code; +} + +# This one will try to convert from transact to PG +# We check if we have several blocks separated by logical operators +# Code going through this function will go to UNSURE file sub convert_transactsql_code { - my ($code)=@_; - $code =~ s/[\[\]]/\"/gi; - $code =~ s/getdate\s*\(\)/CURRENT_TIMESTAMP/gi; - $code =~ s/user_name\s*\(\)/CURRENT_USER/gi; - $code =~ s/datepart\s*\(\s*(.*?)\s*\,\s*(.*?)\s*\)/date_part('$1', $2)/gi; - return $code; + my ($code)=@_; + #print STDERR "convert: $code\n"; + + if ($code =~ /^\((.+?)\)\s+(AND|OR)\s+\((.+?)\)$/) { + my ($lhs,$op,$rhs)=($1,$2,$3); + $code = "(".convert_transactsql_code("$lhs").") $op (".convert_transactsql_code("$rhs").")"; + } + elsif ($code =~ /^(.+?)\s+(AND|OR)\s+(.+?)$/) { + my ($lhs,$op,$rhs)=($1,$2,$3); + $code = "(".convert_transactsql_code("$lhs")." $op ".convert_transactsql_code("$rhs").")"; + } + else { + if ($case_treatment==0) { + $code =~ s/[\[\]]/"/gi; # Bit brutal probably + } + else { + $code =~ s/\[(.*)\]/rename_identifier($1)/gie; # Bit brutal probably + } + $code = convert_transact_function($code); + } + #print STDERR "to: $code\n\n"; + return $code; } # This function does its best to convert MS's weird default values syntax into something logical @@ -464,60 +635,51 @@ sub store_default_value my ($schema,$table,$col,$value,$line)=@_; if ($value =~ /^\(?(\d+(\.\d+)?)\)?$/) # Value is numeric { - $value = $1; # Get rid of parenthesis - if ($objects->{SCHEMAS}->{relabel_schemas($schema)}->{TABLES}->{$table}->{COLS}->{$col}->{TYPE} eq 'boolean') - { - # Ok, it IS a boolean, and we have received a number - if ($value eq '0') - { - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'false'; - } - elsif ($value eq '1') - { - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'true'; - } - else - { - # We should not get here: we have a numeric which isn't 0 or 1, and is supposed to be a boolean - die "Got an unexpected boolean : $value, for line $line\n"; - } - } - else - { - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} - = $value; - } - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} - = 0; - + $value = $1; # Get rid of parenthesis + if ($objects->{SCHEMAS}->{relabel_schemas($schema)}->{TABLES}->{$table}->{COLS}->{$col}->{TYPE} eq 'boolean') { + # Ok, it IS a boolean, and we have received a number + if ($value eq '0') { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'false'; + } + elsif ($value eq '1') { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} = 'true'; + } + else { + # We should not get here: we have a numeric which isn't 0 or 1, and is supposed to be a boolean + die "Got an unexpected boolean : $value, for line $line\n"; + } + } + else { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = $value; + } + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 0; } elsif ($value =~ /^NULL$/) # A NULL value { - # NULL WITHOUT quotes around it ! - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} - = 'NULL'; - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} - = 0; + # NULL WITHOUT quotes around it ! + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = 'NULL'; + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 0; } elsif ($value =~ /^N?'(.*)'$/) # There is sometimes an N before a string. { - $value = $1; # Get rid of junk - # Default text value, text, between commas - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} - = "'$1'"; - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} - = 0; + $value = $1; # Get rid of junk + # Default text value, text, between commas + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = "'$1'"; + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 0; } - else - { - #This must be a function call... - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} - = convert_transactsql_code($value); - $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} - = 1; - + else { + #This must be a function call... + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{VALUE} + = convert_transactsql_code($value); + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$table}->{COLS}->{$col}->{DEFAULT}->{UNSURE} + = 1; } - } @@ -560,7 +722,7 @@ sub next_col_pos unless ($norelabel_dbo) { $relabel_schemas{'dbo'}='public'; - } + } # dbo can be overwritten in relabel_schema (the user will probably forget to deactivate the relabel). # so we do the real relabeling after the norelabel_dbo, to overwrite if (defined $relabel_schemas) @@ -575,7 +737,7 @@ sub next_col_pos $relabel_schemas{$pair[0]}=$pair[1]; } } - + } @@ -598,11 +760,11 @@ sub is_windows } # Die if kettle is not set up correctly -sub kettle_die +sub kettle_warn { my ($file) = @_; - die - "You have to set up KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in $file.\nIf this file doesn't exist yet, start spoon from the kettle directory once."; + print STDERR + "Important WARNING: You have to set up KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in $file.\nIf this file doesn't exist yet, start spoon from the kettle directory once."; } # This sub checks ~/.kettle/kettle.properties to be sure @@ -620,7 +782,11 @@ sub check_kettle_properties { $file = $ENV{'USERPROFILE'} . '/.kettle/kettle.properties'; } - open FILE, $file or kettle_die($file); + unless (open FILE, $file ) + { + kettle_warn($file); + return 1; + } while () { next unless (/^KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y$/); @@ -629,7 +795,8 @@ sub check_kettle_properties close FILE; if (not $ok) { - kettle_die($file); + kettle_warn($file); + return 1; } return 0; } @@ -637,45 +804,120 @@ sub check_kettle_properties # Usage, obviously. Has to be kept in sync with new command line options sub usage { - print - "$0 [-k kettle_output_directory] -b before_file -a after_file -u unsure_file -f sql_server_schema_file[-h] [-i]\n"; - print - "\nExpects a SQL Server SQL structure dump as -f (preferably unicode)\n"; - print "-conf uses a conf file. All options below can also be set there. Command line options will overwrite conf options\n"; - print "-i tells $0 to create a case-insensitive PostgreSQL schema\n"; - print - "-nr tells $0 not to convert the dbo schema to public. dbo will stay dbo\n"; - print - "-num tells $0 to convert numeric xxx,0 to int, bigint, etc. Will not keep numeric scale and precision for the converted\n"; - print - "-relabel_schemas gives a list of schemas to rename. For instance -relabel_schemas 'source1=>dest1;source2=>dest2'\n"; - print " -nr simply cancels the default dbo=>public remapping. Don't forget to put the remapping between quotes\n"; - print - "-keep_identifier_case tells $0 to keep the case of sql server database objects (not advised). Default is to lowercase everything.\n"; - print "before_file contains the structure\n"; - print "after_file contains index, constraints\n"; - print "validate_constraints validates the constraints that have been created\n"; - print "sort_size will change size of sort batch for the incremental job. Too small and it will be slow, too big and you will get Java Out of Heap Memory errors.\n"; - print "sort_size is 10000, which is very low, to try to avoid problems. First, raise java heap memory (in the kitchen script), then try higher values if you need more speed\n"; - print "use_pk_if_possible is false (0) by default. You can put it to 1 (true), or give a comma separated list of tables (with schema). Compared case insensitively\n"; - print - "unsure_file contains things we cannot guarantee will work, such as views\n"; - print "\n"; - print - "If you are generating for kettle, you'll need to provide connection information\n"; - print "for connecting to both databases:\n"; - print "-sd: sqlserver database\n"; - print "-sh: sqlserver host\n"; - print "-si: sqlserver host instance\n"; - print "-sp: sqlserver port\n"; - print "-su: sqlserver username\n"; - print "-sw: sqlserver password\n"; - print "-pd: postgresql database\n"; - print "-ph: postgresql host\n"; - print "-pp: postgresql port\n"; - print "-pu: postgresql username\n"; - print "-pw: postgresql password\n"; - print "-p: parallelism level for the kettle job\n"; + print qq{ +Usage: + sqlserver2pgsql.pl -f SQLSERVER_SCHEMA_FILE -b BEFORE_FILE -a AFTER_FILE -u UNSURE_FILE ... OPTIONS + +Description: + + This script is a migration tool to convert a Microsoft SQL Server Database + into a PostgreSQL database, as automatically as possible. It takes a Sql + Server SQL schema dump, and creates a postgresql dump in 3 parts: + before_file, after_file and unsure_file. + Optionnaly, using the '-k' option, it will generate a kettle job to + transfer all data. + +Mandatory parameters: + + SQL Server schema input file: + -f SQLSERVER_SCHEMA_FILE + a readable SQL-Server SQL structure dump. + + PostgreSQL output schema files: + -b BEFORE_SCRIPT + contains what is needed to import data: types, tables and columns. + -a AFTER_SCRIPT + contains the rest: indexes, constraints. + -u UNSURE_SCRIPT + contains objects we attempt to migrate, but cannot guarantee, such + as views or complex indexes. + +Other options: + + -conf CONFIGURATION_FILE + uses a configuration file. All options can be set there. Command + line options will overwrite conf options. + -nr the SQL Server 'dbo' schema will not be translated to PostgreSQL + 'public' schema. 'dbo' will stay 'dbo'. + -relabel_schemas 'SOURCE1=>DEST1;SOURCE2=>DEST2' + gives a list of schemas to rename. Quote this option to prevent the + shell to alter it. The '-nr' option cancels the default 'dbo' to + 'public' remapping. + -keep_identifier_case + keep the case of SQL server database objects. This option is not + advised. Default is to lowercase everything. + -camel_to_snake + all object names are converted from 'camelCase' to 'camel_case', + which is more often used in PostgreSQL. Do not use this unless you + are ready to do SQL query changes in the client. + -col_map_file + optional text file with old and new schema, table and column names + -col_map_file_header + add a header line to the col_map_file + -col_map_file_delimiter + the field delimiter used in the col_map_file (TAB by default) + -i the resulting PostgreSQL names will be case-insensitive. + -num convert numeric 'xxx,0' to int, bigint, etc. Will not keep numeric + scale and precision for the converted. + -drop_rowversion (Default 0) + should we drop the rowversion/timestamp columns when converting + -validate_constraints {yes|no|after} + should the constraints be validated. Set to "yes" by default. If + set to "no", the constraints will be set as 'NOT VALID'. If set to + "after", the constraints will be set as + 'NOT VALID', but the validation queries will be enforced in the + unsure file. + -use_pk_if_possible {0|1|LIST_OF_TABLES} (Default 0) + determine if a primary key will be sorted in the same way in SQL + Server and PostgreSQL. Default '0', do not sort tables. If set to + '1' sort all tables. LIST_OF_TABLES gives a comma separated list of + tables to sort in the form 'schema1.table1,schema2.table2'. Cases + are compared insensitively. + -skip_citext_length_check (Default 0) + if set, do not add a CHECK (char_length()) check for citext fields + -use_identity_column (Default 1) + if set, use identity columns statements (GENERATED ALWAYS AS + IDENTITY) instead of creating a dedicated sequence (CREATE SEQUENCE) + + Kettle options: + if you are generating for kettle, you must provide connection information. + + -sd SQLSERVER_DATABASE + -sh SQLSERVER_HOST + -si SQLSERVER_INSTANCE + -sp SQLSERVER_PORT + -su SQLSERVER_USERNAME + -sw SQLSERVER_PASSWORD + -pd POSTGRESQL_DATABASE + -ph POSTGRESQL_HOST + -pp POSTGRESQL_PORxT + -pu POSTGRESQL_USERNAME + -pw POSTGRESQL_PASSWORD + -pi PARALLELISM_IN + parallelism level for the kettle job (input, SQL Server). Default 1. + -po PARALLELISM_OUT + parallelism level for the kettle job (output, PostgreSQL). Default 8. + -sforce_ssl + force a SSL session to SQL Server + -pforce_ssl + force a SSL session to PostgreSQL + -stringtype_unspecified + By default, the data are sent as varchar. This can block date field + migration. By using this option, make PostgreSQL infer the data type. + + -k KETTLE_OUTPUT_DIRECTORY + -sort_size INTEGER + set the size of the sort batch for the incremental job. If set to a + small value, the process will be slow. If set to a too large value, + you will get Java 'Out of Heap Memory' errors. + The default value is 10000, which is very low, in order to avoid + problems. First, raise Java 'Heap Memory' in the kitchen script, + then try higher values if you need more speed. + -ignore_errors + choose to ignore insert errors. If this option is used, inserting + will be much slower. +}; + exit 0; } # This function generates kettle transformations, and a kettle job running all these @@ -737,7 +979,7 @@ sub generate_kettle { $newincrementaltemplate=$incremental_template; } - + # Build the column list of the table to put into the SQL Server query my @colsdef; @@ -751,7 +993,7 @@ sub generate_kettle { my $coldef = sql_convert_column($col,$refschema->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}) . " AS " . format_identifier($col); - my $pgcoldef = postgres_convert_column($col,$refschema->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}) . " AS " . format_identifier($col); + my $pgcoldef = postgres_convert_column(format_identifier($col),$refschema->{TABLES}->{$table}->{COLS}->{$col}->{TYPE}) . " AS " . format_identifier($col); push @colsdef,($coldef); push @pgcolsdef,($pgcoldef); } @@ -762,7 +1004,7 @@ sub generate_kettle my $pgschema=format_identifier($targetschema); my $sqlinstancename = ''; - if (length $si) { + if (defined $si) { $sqlinstancename = $si; } @@ -783,8 +1025,38 @@ sub generate_kettle $newtemplate =~ s/__sqlserver_table_cols__/$colsdef/g; $newtemplate =~ s/__postgres_table_name__/$pgtable/g; $newtemplate =~ s/__postgres_schema_name__/$pgschema/g; - $newtemplate =~ s/__PARALLELISM__/$parallelism/g; + $newtemplate =~ s/__PARALLELISM_IN__/$parallelism_in/g; + $newtemplate =~ s/__PARALLELISM_OUT__/$parallelism_out/g; + + if ($ignore_errors) + { + $newtemplate =~ s/N<\/ignore_errors>/Y<\/ignore_errors>/g; + $newtemplate =~ s/Y<\/use_batch>/N<\/use_batch>/g; # Cannot use batch mode with ignore errors + } + + if ($sforce_ssl) + { + $newtemplate =~ s/__sforce_ssl__/EXTRA_OPTION_MSSQL.ssl<\/code>require<\/attribute><\/attribute>/g; + } + else + { + $newtemplate =~ s/__sforce_ssl__//g; + } + if ($pforce_ssl) + { + $newtemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; + } + else + { + $newtemplate =~ s/__pforce_ssl__//g; + } + if ($stringtype_unspecified) { + $newtemplate =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; + } + else { + $newtemplate =~ s/__stringtype_unspecified__//g; + } $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; $newincrementaltemplate =~ s/__sqlserver_database__/$sd/g; @@ -803,38 +1075,58 @@ sub generate_kettle $newincrementaltemplate =~ s/__postgres_table_name__/$pgtable/g; $newincrementaltemplate =~ s/__postgres_schema_name__/$pgschema/g; $newincrementaltemplate =~ s/__postgres_table_cols__/$pgcolsdef/g; - $newincrementaltemplate =~ s/__PARALLELISM__/$parallelism/g; + $newincrementaltemplate =~ s/__PARALLELISM_IN__/$parallelism_in/g; + $newincrementaltemplate =~ s/__PARALLELISM_OUT__/$parallelism_out/g; $newincrementaltemplate =~ s/__sort_size__/$sort_size/g; + if ($sforce_ssl) + { + $newincrementaltemplate =~ s/__sforce_ssl__/EXTRA_OPTION_MSSQL.ssl<\/code>require<\/attribute><\/attribute>/g; + } + else + { + $newincrementaltemplate =~ s/__sforce_ssl__//g; + } + if ($pforce_ssl) + { + $newincrementaltemplate =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; + } + else + { + $newincrementaltemplate =~ s/__pforce_ssl__//g; + } + if ($stringtype_unspecified) { + $newincrementaltemplate =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; + } + else { + $newincrementaltemplate =~ s/__stringtype_unspecified__//g; + } # We have a bit of work to do on primary keys for the incremental template: we need them # to compare the tables… if (defined($refschema->{TABLES}->{$table}->{PK}->{COLS})) { - my @pk=@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}; - my $keys; - foreach my $pk(@pk) - { - $keys.="$pk\n"; - } - $newincrementaltemplate =~ s/__KEYS_MERGE__/$keys/g; - + my @pk=@{$refschema->{TABLES}->{$table}->{PK}->{COLS}}; + my $keys=""; + foreach my $pk(@pk) + { + $keys.="$pk\n"; + } + $newincrementaltemplate =~ s/__KEYS_MERGE__/$keys/g; + my $sortkeys=''; my $synckeys=''; foreach my $pk(@pk) { $sortkeys.="\n$pk\nY\nY\n\n"; my $outcol=$pk; - unless ($keep_identifier_case) - { - $outcol=lc($outcol); - } + $outcol=rename_identifier($outcol); $synckeys.="\n$pk\n$outcol\n=\n\n\n"; } $newincrementaltemplate =~ s/__SORT_KEYS_SQLSERVER__/$sortkeys/g; $newincrementaltemplate =~ s/__SORT_KEYS_PG__/$sortkeys/g; $newincrementaltemplate =~ s/__KEYS_SYNC__/$synckeys/g; - + # We also need to tell the merge step to compare all columns my $valuesmerge=''; my $valuessync=''; @@ -846,9 +1138,13 @@ sub generate_kettle $valuesmerge.="$colname\n"; # we need to use the correct case for postgresql output my $outcol=$colname; - unless ($keep_identifier_case) + if ($case_treatment==1) + { + $outcol=lc($outcol); + } + elsif ($case_treatment==2) { - $outcol=lc($outcol); + $outcol=camel_to_snake($outcol); } $valuessync.="\n$outcol\n$colname\nY\n\n"; # } @@ -856,13 +1152,13 @@ sub generate_kettle $newincrementaltemplate =~ s/__VALUES_MERGE__/$valuesmerge/g; $newincrementaltemplate =~ s/__VALUES_SYNC__/$valuessync/g; - + # Produce the incremental transformation open FILE, ">$dir/incremental-$schema-$table.ktr" or die "Cannot write to $dir/incremental-$schema-$table.ktr"; binmode(FILE,":utf8"); print FILE $newincrementaltemplate; - close FILE; + close FILE; } else { @@ -923,7 +1219,7 @@ sub generate_kettle # The only difference between normal and incremental job is the filename of the transformation my $JOBFILEname; my $INCJOBFILEname; - if ($dir =~ /^(\\|\/)/) # Absolute path + if ($dir =~ /^([A-Za-z]:\\|\/)/) # Absolute path { $JOBFILEname = $dir . '/' . $schema . '-' . $table . '.ktr'; $INCJOBFILEname = $dir . '/' . 'incremental-' . $schema . '-' . $table . '.ktr'; @@ -1011,10 +1307,11 @@ sub generate_kettle keys %{$refschema->{TABLES}}) { $beforescript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " DISABLE TRIGGER ALL;\n"; - $afterscript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ENABLE TRIGGER ALL;\n"; + + $afterscript.= "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ENABLE TRIGGER ALL;\n"; } } - + # This is for the SQL Scripts. We also need to specify the PG connection $job_header =~ s/__SQL_SCRIPT_INIT__/$beforescript/g; @@ -1024,6 +1321,20 @@ sub generate_kettle $job_header =~ s/__postgres_port__/$pp/g; $job_header =~ s/__postgres_username__/$pu/g; $job_header =~ s/__postgres_password__/$pw/g; + if ($pforce_ssl) + { + $job_header =~ s/__pforce_ssl__/EXTRA_OPTION_POSTGRESQL.ssl<\/code>true<\/attribute><\/attribute>\nEXTRA_OPTION_POSTGRESQL.sslfactory<\/code>org.postgresql.ssl.NonValidatingFactory<\/attribute><\/attribute>/g; + } + else + { + $job_header =~ s/__pforce_ssl__//g; + } + if ($stringtype_unspecified) { + $job_header =~ s/__stringtype_unspecified__/EXTRA_OPTION_POSTGRESQL.stringtype<\/code>unspecified<\/attribute><\/attribute>\n/g; + } + else { + $job_header =~ s/__stringtype_unspecified__//g; + } print JOBFILE $job_header; print JOBFILE $entries; @@ -1052,7 +1363,8 @@ sub generate_kettle my ($fd) = @_; my $line = <$fd>; return undef if (not defined $line); - $line =~ s/\r//g; # Remove \r from windows output + $line =~ s/^\x{FEFF}//; # Remove UTF-8 BOM + $line =~ s/\r//g; # Remove \r from windows output $line =~ s/EXEC(ute)?\s*(dbo|sys)\.sp_executesql( \@statement =)? N'//i ; # Remove executesql… it's a bit weird in the SQL Server's dump @@ -1084,6 +1396,7 @@ sub generate_kettle $line =~ s/\/\*.*//; # Remove everything after the comment } } + return $line; } } @@ -1100,12 +1413,12 @@ sub add_column_to_table # It has already been declared before. We just need to find it $coltype = relabel_schemas($coltypeschema) . '.' . $coltype; } - if ($colqual) + if ($colqual) { if ($coltype eq 'xml') { $colqual = undef - ; # ignoring sql server xml schema since its not supported in pg + ; # ignoring sql server xml schema since its not supported in pg } elsif ($colqual eq '(max)') { @@ -1119,7 +1432,16 @@ sub add_column_to_table or die "Cannot parse colqual <$colqual>"; $colqual = "$1"; } + } + + # in case of a rowversion or timestamp columns, check if we want to keep it + if ($drop_rowversion + and ($coltype eq 'rowversion' or $coltype eq 'timestamp')) + { + # do nothing + return; } + my $newtype = convert_type($coltype, $colqual, $colname, $tablename, undef, $schemaname); @@ -1131,7 +1453,7 @@ sub add_column_to_table # We have an identity field. We remember the default value and # initialize the sequence correctly in the after script - $isidentity =~ /IDENTITY\((\d+),\s*(\d+)\)/ + $isidentity =~ /IDENTITY\s*\((-?\d+),\s*(-?\d+)\)/ or die "Cannot understand <$isidentity>"; my $startseq = $1; my $stepseq = $2; @@ -1148,8 +1470,6 @@ sub add_column_to_table $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{START} = $startseq; - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MIN} - = $startseq; $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP} = $stepseq; $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname} @@ -1164,8 +1484,7 @@ sub add_column_to_table # use a special kettle transformation for it if generating kettle # (see generate_kettle() ) if ( $newtype eq 'bytea' - or $coltype eq - 'ntext') # Ntext is very slow, stored out of page + or $coltype eq 'ntext') # Ntext is very slow, stored out of page { $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename} ->{haslobs} = 1; @@ -1217,7 +1536,7 @@ sub parse_dump { # Create table, obviously. There will be other lines below for the rest of the table definition - if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\(/) + if ($line =~ /^CREATE TABLE \[(.*)\]\.\[(.*)\]\s*\(/) { my $schemaname = relabel_schemas($1); my $orig_schema = $1; @@ -1228,83 +1547,75 @@ sub parse_dump TABLE: while (my $line = read_and_clean($file)) { # Here is a col definition. - # We ignore ROWGUIDCOL as it has no meaning in PostgreSQL and cannot be emulated - # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the column name, typical microsoft stuff :( ) - # To make matters even worse, they seem to systematically add a space after it :) + # We ignore ROWGUIDCOL as it has no meaning in PostgreSQL and cannot be emulated + # (it makes it possible to do a select xxx WHERE $ROWGUID, without knowing the + # column name, typical microsoft stuff :( ) + # To make matters even worse, they seem to systematically add a space after it :) if ($line =~ - /^\t\[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ + /^\s+\[(.*)\]\s*(?:\[(.*)\]\.)?\[(.*)\]\s*(\(.+?\))?(?: COLLATE (\S+))?( IDENTITY\s*\(-?\d+,\s*-?\d+\))?(?: ROWGUIDCOL ?)? (?:NOT FOR REPLICATION )?(?:SPARSE +)?(NOT NULL|NULL)(?:\s+CONSTRAINT \[.*\])?(?:\s+DEFAULT \((.*)\))?(?:,|$)?/ ) { - #Deported into a function because we can also meet alter table add columns on their own + # Deported into a function because we can also meet alter table add columns on their own my $colname = $1; my $coltypeschema = $2; my $coltype = $3; my $colqual =$4; - my $isidentity =$5; - my $colisnull =$6; - my $default =$7; + my $colcollate =$5; # Ignore for now + my $isidentity =$6; + my $colisnull =$7; + my $default =$8; add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); - if (defined $default) - { - store_default_value($schemaname,$tablename,$colname,$default,$line); - } - } - - - # This is a calculated column. It doesn't exist in PG, it is not typed (I guess its type is the type of the returning function) - # So just put it as a varchar, and issue a warning is STDOUT - elsif ($line =~ /^\t\[(.*)\]\s+AS\s+\((.*)\)/) + if (defined $default) + { + store_default_value($schemaname,$tablename,$colname,$default,$line); + } + } + + # This is a computed column. PostgreSQL supports this as a generated column, starting with PG12 + # Will assume the data type is varchar, but this will need to be changed if the source columns are int, numeric, float, etc. + elsif ($line =~ /^\s*\[(.*)\]\s+AS\s+\((.*)\)(.*)/) { - - # We just get the column name + # Get the column name my $colnumber=next_col_pos($schemaname,$tablename); my $colname = $1; my $code = $2; my $coltype = 'varchar'; + my $other_param = $3; + + # Replace square brackets in $code with double quotes + my $codequoted = $code =~ s/[\[\]]/"/gr; + my $generatedcode = " /* GENERATED ALWAYS AS ($codequoted)"; + if ($other_param =~ /PERSISTED/) { + $generatedcode .= " STORED"; + } + $generatedcode .= " */"; + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} ->{$colname}->{POS} = $colnumber; $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{TYPE} = $coltype; - $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} - ->{$colname}->{NOT_NULL} = 0; + ->{$colname}->{TYPE} = $coltype . $generatedcode; + + if ($other_param =~ /NOT NULL/) { + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 1; + } + else { + $objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename}->{COLS} + ->{$colname}->{NOT_NULL} = 0; + } - # Big fat warning + # Show a warning print STDERR - "Warning: There is a calculated column: $schemaname.$tablename.$colname. This isn't done the same way in PG at all\n"; + "\nWarning: There is a computed column: $schemaname.$tablename.$colname\n"; print STDERR - "\tFor now it has been declared as a varchar in PG, so that the values can be copied\n"; + "\tPostgreSQL 12 supports this via GENERATED ALWAYS AS (...)\n"; print STDERR - "\tYou should change its type manually in the dump (sorry for that),\n"; - print STDERR "\tA trigger has been written in the unsure file. It probably won't work as is.\n"; - print STDERR "\tPlease review it.\n"; - - # Try to correct what can be corrected from the AS : replace [COL] with NEW.COL - # It is obviously not going to work for anything a bit complicated - $code =~ s/\[(.*?)\]/NEW.$1/g; - my $triggerfunc = <{SCHEMAS}->{$schemaname}->{'TRIG_FUNCTIONS'} - ->{'trig_func_ins_or_upd' || $tablename}->{DEF} = - $triggerfunc; - $objects->{SCHEMAS}->{$schemaname}->{'TRIG_FUNCTIONS'} - ->{'trig_func_ins_or_upd' || $tablename}->{LANG} = - 'plpgsql'; - my %trigger; - $trigger{EVENTS} = 'before insert or update'; - $trigger{WHEN} = 'for each row'; - $trigger{FUNCTION} = - 'trig_func_ins_or_upd' || $tablename; # In the same schema - $trigger{NAME} = 'trig_ins_or_upd' || $tablename; - push @{$objects->{SCHEMAS}->{$schemaname}->{'TABLES'}->{$tablename} - ->{TRIGGERS}}, (\%trigger); - + "\tFor now it has been declared as a varchar and the calculation formula has been commented.\n"; + print STDERR + "\tThe formula will likely need to be manually fixed to properly refer to other columns.\n"; } elsif ($line =~ - /^(?: CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED/) + /^\s*(?:CONSTRAINT \[(.*)\] )?PRIMARY KEY (?:NON)?CLUSTERED(?: HASH)?/) { my $constraint ; # We put everything inside this hashref, we'll push it into the constraint list later @@ -1348,7 +1659,6 @@ sub parse_dump # Unique key definition. We read following lines until the end of the constraint while (my $uk = read_and_clean($file)) { - # Exit when read a line beginning with ). The constraint is complete if ($uk =~ /^\)/) { @@ -1363,14 +1673,14 @@ sub parse_dump } } - elsif ($line =~ /^\) ON \[PRIMARY\]/) + elsif ($line =~ /^\s*\) ON \[.*\]/) { # End of the table next MAIN; } else { - die "Cannot understand $line\n"; + croak "Cannot understand $line\n"; } } } @@ -1379,57 +1689,55 @@ sub parse_dump ################################################################ elsif ($line =~ /^CREATE SEQUENCE \[(.*)\]\.\[(.*)\]/) { - my $schemaname = relabel_schemas($1); - my $orig_schema = $1; - my $seqname = $2; - while (my $contline = read_and_clean($file)) - { - if ($contline =~ /^\s*AS \[.*\]\s*$/) - { - next; # We don't care, sequences are always bigint in PostgreSQL - } - elsif ($contline =~ /^\s*START WITH (\d+)\s*$/) - { - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{START}=$1; - } - elsif ($contline =~ /^\s*INCREMENT BY (\d+)\s*$/) - { - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP}=$1; - } - elsif ($contline =~ /^\s*MINVALUE (-?\d+)\s*$/) - { - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MIN}=$1; - } - elsif ($contline =~ /^\s*MAXVALUE (-?\d+)\s*$/) - { - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MAX}=$1; - } - elsif ($contline =~ /^\s*(NO)?CACHE( \d+)?\s*$/) - { - if (defined $1) - { - # It's a no cache. Equivalent to CACHE = 1 in PostgreSQL - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=1; - } - elsif (defined $2) - { - # We have a specified value - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=$2; - } - else - { - # Cache, but not specified. SQL Server isn't very clear on the size of the cache. Let's say 100 - $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=100; - } - } - elsif ($contline =~ /^GO$/) - { - next MAIN; - } - } - - - } + my $schemaname = relabel_schemas($1); + my $orig_schema = $1; + my $seqname = $2; + while (my $contline = read_and_clean($file)) + { + if ($contline =~ /^\s*AS \[.*\]\s*$/) + { + next; # We don't care, sequences are always bigint in PostgreSQL + } + elsif ($contline =~ /^\s*START WITH (\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{START}=$1; + } + elsif ($contline =~ /^\s*INCREMENT BY (\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{STEP}=$1; + } + elsif ($contline =~ /^\s*MINVALUE (-?\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MIN}=$1; + } + elsif ($contline =~ /^\s*MAXVALUE (-?\d+)\s*$/) + { + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{MAX}=$1; + } + elsif ($contline =~ /^\s*(NO)?CACHE( \d+)?\s*$/) + { + if (defined $1) + { + # It's a no cache. Equivalent to CACHE = 1 in PostgreSQL + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=1; + } + elsif (defined $2) + { + # We have a specified value + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=$2; + } + else + { + # Cache, but not specified. SQL Server isn't very clear on the size of the cache. Let's say 100 + $objects->{SCHEMAS}->{$schemaname}->{SEQUENCES}->{$seqname}->{CACHE}=100; + } + } + elsif ($contline =~ /^GO$/) + { + next MAIN; + } + } + } elsif ($line =~ /^CREATE SCHEMA \[(.*)\]/) { $objects->{SCHEMAS}->{relabel_schemas($1)} = undef @@ -1465,28 +1773,61 @@ sub parse_dump next MAIN if ($contline =~ /^GO$/); } } + elsif ($line =~ /SET\s+IDENTITY_INSERT\s+\[(.*)\]/i) + { + print STDERR "Warning: SET IDENTITY_INSERT ignored\n"; + + # We have to find next GO to know we are out of the procedure + while (my $contline = read_and_clean($file)) + { + next MAIN if ($contline eq ''); + } + } # Now we parse the create view. It is multi-line, so the code looks like like create table: we parse everything until a line # containing only a single quote (end of the dbo.sp_executesql) # The problem is that SQL Server seems to be spitting the original query used to create the view, not a normalized version # of it, as PostgreSQL does. So we capture the query, and hope it works for now. - elsif ($line =~ - /^\s*(create\s*view)\s*(?:\[(\S+)\]\.)?\[(.*?)\]\s*(.*)$/i) + elsif ($line =~/^(create\s*view)/i) { - my $viewname = $3; + my $viewname; my $schemaname; - if (defined $2) + my $supplement; + # Either we have the create view and the object name with it, or it is on next non-empty line + if ( $line =~ /^\s*(create\s*view)\s*(?:\[(\S+)\]\.)?\[(.*?)\]\s*(.*)$/i) { - $schemaname = $2; + $viewname = $3; + if (defined $2) + { + $schemaname = $2; + } + else + { + $schemaname = 'dbo'; + } + $supplement=$4; } else { - $schemaname = 'dbo'; + # Find the line containing the view name. For now, it's the next line. If not, well, correct your dump by yourself, sorry :) + my $line_name = read_and_clean($file); + $line_name=~ /^\s*(?:\[(\S+)\]\.)?\[(.*?)\]\s*(.*)$/ or croak "This line $line_name doesn't contain a view name"; + $viewname = $2; + if (defined $1) + { + $schemaname = $1; + } + else + { + $schemaname = 'dbo'; + } + $supplement=$3; } $schemaname = relabel_schemas($schemaname); - my $sql = $1 . ' ' . $schemaname . '.' . $3 . ' ' . $4 . "\n"; - while (my $line_cont = read_and_clean($file)) + my $sql = $supplement; + + while (my $line_cont = read_and_clean($file)) { if ($line_cont =~ /^\s*'\s*$|^GO$/ ) # We may have a quote if the view is 'quoted', or a real sql query @@ -1495,11 +1836,57 @@ sub parse_dump # We get rid of dbo. schemas $sql =~ s/(dbo)\./relabel_schemas($1) . '.'/eg ; # We put this in the replacement schema + # parse the query view + if ( $sql =~ m/^\s*\(([^\)]+)\)\s*AS\s+SELECT\s+(.*)\s+FROM\s+(.*)$/is) { + my $view_columns = $1; + my $query_columns = $2; + my $query_end = $3; + my @rebuilt_view_columns = (); + my @rebuilt_query_columns = (); + my @string_column = (); + + # format columns names + foreach my $view_col (split (',',$view_columns)) { + $view_col =~ s/^\s+|\s+$//g; + push @rebuilt_view_columns, format_identifier($view_col); + } + $objects->{SCHEMAS}->{$schemaname}->{VIEWS}->{$viewname}->{COLS} = + join(',', @rebuilt_view_columns); + + # format view query columns + foreach my $view_query_col (split (',',$query_columns)) { + if ($view_query_col + =~ /^.*\+\s*N?'.*'\s*|\s*N?'.*'\s*\+.*|.*\+\s*N?'.*'\s*\+.*$/i) { + # PG use '||' to concatenate strings, change '+' to '||' + @string_column = (); + my $lhs; + while ($view_query_col =~ /^\s*(N?'.*?'|[^']+?)\s*\+\s*(.*)$/i) { + $lhs = $1; + $view_query_col = $2; + $lhs = $1 if ($lhs =~ /N('.*?')/); + push @string_column, $lhs; + } + $view_query_col = $1 if ($view_query_col =~ /N('.*?')/); + push @string_column, $view_query_col; + push @rebuilt_query_columns, join('||', @string_column); + } + else { + push @rebuilt_query_columns, $view_query_col; + } + } + $objects->{SCHEMAS}->{$schemaname}->{VIEWS}->{$viewname}->{QUERYCOLS} = + convert_transact_function(join(',', @rebuilt_query_columns)); + + $objects->{SCHEMAS}->{$schemaname}->{VIEWS}->{$viewname}->{QUERY} = + convert_transact_function($query_end); + } + else { + $objects->{SCHEMAS}->{$schemaname}->{VIEWS}->{$viewname}->{SQL} = + $sql; + } # Views will be stored without the full schema in them. We will # have to generate the schema in the output file - $objects->{SCHEMAS}->{$schemaname}->{'VIEWS'}->{$viewname}->{SQL} = - $sql; my @view_array=($schemaname,$viewname); push @view_list,(\@view_array); # adds another schema/view to the list next MAIN; @@ -1539,14 +1926,18 @@ sub parse_dump my $newtype; TYPE: while (my $typeline= read_and_clean($file)) { - if ($typeline =~ /^\t\[(.*)\] \[(.*)\](?:\s*?\((\d+(?:,\d+)?)\))?(?:\s+?(?:NOT\s+?)?NULL),?$/) + if ($typeline =~ /^\t\[(.*)\] \[(.+?)\](?:\s*?\((\d+|max(?:,\d+)?)\))?(?:\s+?(?:NOT\s+?)?NULL),?$/) { # This is another column for this type $colname=$1; $type=$2; $typequal=$3; + if (defined $typequal and $typequal eq 'max') { + # max in SqlServer is the same as no typequal in pg + $typequal = undef; + } $newtype = - convert_type($type, $typequal, undef, undef, undef, undef); + convert_type($type, $typequal, undef, undef, undef, undef); push @cols_newbasetype,(format_identifier($colname) . ' ' . $newtype); } elsif ( $typeline =~ /PRIMARY KEY/) @@ -1575,13 +1966,13 @@ sub parse_dump } else { - die "Cannot understand $typeline\n"; + croak "Cannot understand $typeline\n"; } } } elsif ($line =~ - /^CREATE (UNIQUE )?(NONCLUSTERED|CLUSTERED) INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\]/ + /^\s*CREATE\s*(UNIQUE )?\s*(NONCLUSTERED|CLUSTERED)?\s*INDEX \[(.*?)\] ON \[(.*?)\]\.\[(.*?)\](\(\[.*?\]\))?/ ) { # Index creation. Index are namespaced per table in SQL Server, not in PostgreSQL @@ -1594,6 +1985,7 @@ sub parse_dump my $idxname = $3; my $schemaname = relabel_schemas($4); my $tablename = $5; + my $maybecols = $6; # Some versions of sql server put column definitions on the first line if ($isunique) { $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} @@ -1604,6 +1996,24 @@ sub parse_dump $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} ->{$idxname}->{UNIQUE} = 0; } + if (defined $maybecols) + { + my @maybecols = split (',',$maybecols); + foreach my $coldef(@maybecols) + { + $coldef=~/\[(.*)\](?: (ASC|DESC))?/ or die "Cannot understand coldef $coldef in index"; + if (defined $2) + { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{COLS}}, ("$1 $2"); + } + else + { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{COLS}}, ("$1"); + } + } + } while (my $idx = read_and_clean($file)) { @@ -1613,47 +2023,69 @@ sub parse_dump next MAIN; } next - if ($idx =~ /^\(|^\)/) - ; # Begin/end of the columns declaration - if ($idx =~ /\t\[(.*)\] (ASC|DESC)(,)?/) - { - if (defined $2) - { - push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + if ($idx =~ /^\(|^\)/) + ; # Begin/end of the columns declaration + if ($idx =~ /\t\[(.*)\] (ASC|DESC)(,)?/) { + if (defined $2) { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} ->{INDEXES}->{$idxname}->{COLS}}, ("$1 $2"); - } - else - { - push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + } + else { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} ->{INDEXES}->{$idxname}->{COLS}}, ("$1"); - } + } } - if ($idx =~ /^INCLUDE \(/) - { - print STDERR - "Warning: This index ($schemaname.$tablename.$idxname) has some include columns. This isn't supported in PostgreSQL.\n"; - print STDERR - "\tThe columns in the INCLUDE clause have been ignored.\n"; - next - ; # Nothing equivalent in PG. Maybe if the index isn't unique, these columns should be added? + if ($idx =~ /^INCLUDE\s*\(\s*\[(.*?)\](.*)/) { + # INCLUDE coluns in indexes are available on PG11 onward + # if multiple included columns, there are declared one per line + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{INCLUDE}}, ($1); + if (index($2, ')') == -1) { + while (my $incl_line = read_and_clean($file)) { + if ($incl_line =~ /^\s*\[(.*?)\](.*)/) { + push @{$objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{INCLUDE}}, ($1); + last if (index($2, ')') != -1); + } + } + } } + if ($idx =~ /^WHERE\s*\((.*)\)$/) { + # This is a where clause. PostgreSQL has them too. But we + # cannot be sure this will be exactly the same. So if an + # index as a WHERE clause, it has to go to unsure + my $filter=$1; + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename} + ->{INDEXES}->{$idxname}->{WHERE}="(".$filter.")"; + } + } + } + + # we do not take migrate spatial indexes + elsif ($line =~ /^CREATE SPATIAL INDEX/) + { + my $def=$line; + while (my $idx = read_and_clean($file)) + { + $def.=$idx; } + print STDERR "This spatial index won't be migrated:\n$def\n"; } - elsif ($line =~ /^CREATE SPATIAL INDEX/) - { - my $def=$line; - while (my $idx = read_and_clean($file)) - { - $def.=$idx; - } - print STDERR "This spatial index won't be migrated:\n$def\n"; - - } + + elsif ($line =~ /^ALTER INDEX \[(.*)\] ON \[(.*)\]\.\[(.*)\] DISABLE$/) + { + my $idxname = $1; + my $schemaname = relabel_schemas($2); + my $tablename = $3; + + $objects->{SCHEMAS}->{$schemaname}->{TABLES}->{$tablename}->{INDEXES} + ->{$idxname}->{DISABLE} = 1; + } # Added table columns… this seems to appear in SQL Server when some columns have ANSI padding, and some not. # PG follows ANSI, that is not an option. The end of the regexp is pasted from the create table elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD \[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(\d+,\s*\d+\))? (NOT NULL|NULL)(?: CONSTRAINT \[.*\] )?(?: DEFAULT \(.*\))?$/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD \[(.*)\] (?:\[(.*)\]\.)?\[(.*)\](\(.+?\))?( IDENTITY\(-?\d+,\s*-?\d+\))? (NOT NULL|NULL)(?: CONSTRAINT \[.*\] )?(?: DEFAULT \(.*\))?$/ ) { my $schemaname=relabel_schemas($1); @@ -1668,14 +2100,14 @@ sub parse_dump add_column_to_table($schemaname,$tablename,$colname,$coltypeschema,$coltype,$colqual,$isidentity,$colisnull); if (defined $default) { - store_default_value($schemaname,$tablename,$colname,$default,$line); + store_default_value($schemaname,$tablename,$colname,$default,$line); } } # Table constraints # Primary key. Multiline elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED|NONCLUSTERED)?/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+(?:WITH (?:NO)?CHECK )?ADD\s*(?:CONSTRAINT \[(.*)\])? PRIMARY KEY (?:CLUSTERED|NONCLUSTERED)?/ ) { my $schemaname=relabel_schemas($1); @@ -1685,13 +2117,12 @@ sub parse_dump if (defined $3) { $constraint->{NAME} = $3; - } + } - CONS: while (my $consline= read_and_clean($file)) { next if ($consline =~ /^\($/); - if ($consline =~ /^\t\[(.*)\] ASC,?$/) + if ($consline =~ /^\t\[(.*)\]\s+(?:ASC)?,?$/) { push @{$constraint->{COLS}}, ($1); } @@ -1707,7 +2138,7 @@ sub parse_dump } else { - die "Cannot understand $consline."; + croak "Cannot understand $consline."; } } } @@ -1748,13 +2179,13 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \((\(?(?:-)?\d+(?:\.\d+)?\))?\) FOR \[(.*)\]/ ) { - store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } elsif ($line =~ /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(('.*')\) FOR \[(.*)\]/ ) { - store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } # Yes, we also get default NULL (what for ? :) ), and sometimes with a different case @@ -1762,7 +2193,7 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(((?i)NULL)\) FOR \[(.*)\]/ ) { - store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } # And there are also constraints with functions and other strange code in them. Put them as unsure @@ -1770,13 +2201,13 @@ sub parse_dump /^ALTER TABLE \[(.*)\]\.\[(.*)\] ADD\s*(?:CONSTRAINT \[.*\])?\s*DEFAULT \(\(?(.*)\)?\) FOR \[(.*)\]/ ) { - store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value + store_default_value(relabel_schemas($1),$2,$4,$3,$line); # schema,table,col,value } # FK constraint. It's multi line, we have to look for references, and what to do on update, delete, etc (I have only seen delete cascade for now) # Constraint name is optionnal elsif ($line =~ - /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? FOREIGN KEY\((.*?)\)/ + /^ALTER TABLE \[(.*)\]\.\[(.*)\]\s+WITH (?:NO)?CHECK(?: NOT FOR REPLICATION)?\s+ADD(?:\s+CONSTRAINT \[(.*)\])? FOREIGN KEY\((.*?)\)/ ) { # This is a FK definition. We have the foreign table definition in next line. @@ -1825,20 +2256,24 @@ sub parse_dump { $constraint->{ON_UPD_CASC} = 1; } - elsif ($fk =~ /^NOT FOR REPLICATION$/) - { - next; # We don't care for this, it has no meaning for PostgreSQL - } + elsif ($fk =~ /^ON UPDATE SET NULL\s*$/) + { + $constraint->{ON_UPD_SET_NULL} = 1; + } + elsif ($fk =~ /^NOT FOR REPLICATION\s*$/) + { + next; # We don't care for this, it has no meaning for PostgreSQL + } else { - die "Cannot parse $fk $., in a FK. This is a bug"; + croak "Cannot parse $fk $., in a FK. This is a bug"; } } } # Check constraint. As it can be arbitrary code, we just get this code, and hope it will work on PG (it will be stored in a special script file) elsif ($line =~ - /ALTER TABLE \[(.*)\]\.\[(.*)\] WITH (?:NO)?CHECK ADD(?:\s+CONSTRAINT \[(.*)\])? CHECK \(\((.*)\)\)/ + /ALTER TABLE \[(.*)\]\.\[(.*)\]\s+(?:WITH? (?:NO)?CHECK? )?ADD(?:\s+CONSTRAINT (.*))?\s+CHECK(?: NOT FOR REPLICATION)?\s+\((\((.*)\)|(.*))\)/ ) { # Check constraint. We'll do what we can, syntax may be different. @@ -1847,13 +2282,12 @@ sub parse_dump my $constxt = $4; my $schema = relabel_schemas($1); $constraint->{TABLE} = $table; - if (defined $3) - { - $constraint->{NAME} = $3; + if (defined $3) { + my $constraint_name = $3; + $constraint_name = $1 if ($constraint_name =~ /\[(.*)\]/); + $constraint->{NAME} = $constraint_name; } $constraint->{TYPE} = 'CHECK'; - $constxt =~ - s/\[(\S+?)\]/$1/g; # We remove the []. And hope this will parse $constraint->{TEXT} = $constxt; push @{$objects->{SCHEMAS}->{$schema}->{'TABLES'}->{$table}->{CONSTRAINTS}}, ($constraint); @@ -1882,16 +2316,16 @@ sub parse_dump # First step: what kind is it ? we are only interested in comments for now $sqlproperty =~ /\@name=N'(.*?)'/ - or die + or croak "Cannot find a name for this extended property: $sqlproperty"; my $propertyname = $1; - if ($propertyname =~ /^(MS_DiagramPaneCount|MS_DiagramPane1|Display Name|Description|Example Values|Source System|Table Description|Table Type|ETL Rules|Display Folder|SCD Type|Source Datatype)$/) + if ($propertyname =~ /^(AggregateType|AllowZeroLength|AppendOnly|Attributes|CollatingOrder|ColumnHidden|ColumnOrder|ColumnWidth|DataUpdatable|DateCreated|DefaultValue|Description|Display Folder|Display Name|DisplayViewsOnSharePointSite|ETL Rules|Example Values|FilterOnLoad|GUID|HideNewField|LastUpdated|microsoft_database_tools_support|MS_DecimalPlaces|MS_DefaultView|MS_DiagramPane1|MS_DiagramPane2|MS_DiagramPaneCount|MS_DisplayControl|MS_Format|MS_Hyperlink|MS_IMEMode|MS_IMESentMode|MS_InputMask|MS_OrderByOn|MS_Orientation|Name|OrderByOnLoad|OrdinalPosition|RecordCount|Required|SCD Type|ShowDatePicker|Size|Source Datatype|Source System|SourceField|SourceTable|Table Description|Table Type|TextAlign|TextFormat|TotalsRow|Type|UnicodeCompression|Updatable)$/) { # We don't dump these. They are graphical descriptions of the GUI next; } - elsif ($propertyname eq 'MS_Description') + elsif ($propertyname =~ /^(MS_Description|MS_SSMA_SOURCE)$/) { # This is a comment. We parse it. @@ -1900,7 +2334,7 @@ sub parse_dump # I hope it will be sufficient (won't be if someone decides to end a comment with a quote) unless ($sqlproperty =~ - /^EXEC sys.sp_addextendedproperty \@name=N'(.*?)'\s*,\s*\@value=N'(.*)'\s*,\s*\@level0type=N'(.*?)'\s*,\s*\@level0name=N'(.*?)'\s*(?:,\s*\@level1type=N'(.*?)'\s*,\s*\@level1name=N'(.*?)')\s*?(?:,\s*\@level2type=N'(.*?)'\s*,\s*\@level2name=N'(.*?)')?/s) + /^EXEC sys.sp_addextendedproperty \@name=N'(.*?)'\s*(?:,\s*\@value=N'(.*?)'\s*)?(?:,\s*\@level0type=N'(.*?)'\s*)?(?:,\s*\@level0name=N'(.*?)'\s*)?(?:,\s*\@level1type=N'(.*?)'\s*,\s*\@level1name=N'(.*?)')?\s*?(?:,\s*\@level2type=N'(.*?)'\s*,\s*\@level2name=N'(.*?)')?/s) { # Not parsing a comment should not stop print STDERR "Could not parse <$sqlproperty>. Ignored.\n"; @@ -1909,7 +2343,11 @@ sub parse_dump my ($comment, $schema, $obj, $objname, $subobj, $subobjname) = ($2, $4, $5, $6, $7, $8); $schema=relabel_schemas($schema); - if ($obj eq 'TABLE' and not defined $subobj) + if (not defined $obj) + { + $objects->{SCHEMAS}->{$schema}->{COMMENT} = $comment; + } + elsif ($obj eq 'TABLE' and not defined $subobj) { $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{COMMENT} = $comment; @@ -1924,9 +2362,26 @@ sub parse_dump $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{COLS} ->{$subobjname}->{COMMENT} = $comment; } + elsif ($obj eq 'TABLE' and $subobj eq 'CONSTRAINT') + { + # It's a bit tedious: constraints are stored in an array for each table + # Find the one + foreach my $constraint (@{$objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{CONSTRAINTS}}) + { + if (defined ($constraint->{NAME}) and $constraint->{NAME} eq $subobjname) + { + $constraint->{COMMENT}=$comment; + } + } + } + elsif ($obj eq 'TABLE' and $subobj eq 'INDEX') + { + $objects->{SCHEMAS}->{$schema}->{TABLES}->{$objname}->{INDEXES} + ->{$subobjname}->{COMMENT} = $comment; + } else { - die "Cannot understand this comment: $sqlproperty"; + croak "Cannot understand this comment: $sqlproperty"; } } elsif ($propertyname eq 'Dictionary') @@ -1934,7 +2389,7 @@ sub parse_dump # It seems to be another way to declare table comments. I hope this is right $sqlproperty =~ /^EXEC sys.sp_addextendedproperty \@name=N'(.*?)'\s*,\s*\@value=N'(.*?)(? not understood. Please send a bug report\n"; + croak "Schema comment : <$comment> not understood. Please send a bug report\n"; } } else { - die + croak "Don't know what to do with this extendedproperty: $sqlproperty"; } } + # Save variable for future use + elsif ($line =~ /^:setvar\s+(\S+)\s+"(.*)"/) + { + my $varname = $1; + my $varvalue = $2; + $objects->{VARIABLES}->{$varname} = $varvalue; + next; + } + # Ignore USE, GO, and things that have no meaning for postgresql elsif ($line =~ /^USE\s|^GO\s*$|\/\*\*\*\*|^SET ANSI_NULLS (ON|OFF)|^SET QUOTED_IDENTIFIER|^SET ANSI_PADDING|CHECK CONSTRAINT|^BEGIN|^END/ @@ -1980,11 +2444,21 @@ sub parse_dump next; } + # Ignore grant statements + elsif ($line =~ /^GRANT ([^ ]+) ON (.+) TO \[[^ ]+\]/) + { + next; + } + elsif ($line =~ /^GRANT VIEW ([^ ]+) ON (.+) TO \[[^ ]+\]/) + { + next; + } + elsif ($line =~ /^ALTER (ROLE|USER)/) { next; } - + # Ignore xml schema collections since they are not supported in pg elsif ($line =~ /^CREATE XML SCHEMA COLLECTION/) { @@ -1994,50 +2468,63 @@ sub parse_dump elsif ($line =~ /^ALTER XML SCHEMA COLLECTION/) { next; - } + } # Ignore existence tests… how could the object already exist anyway ? For now, only seen for views - elsif ($line =~ /^IF NOT EXISTS/) + # Also ignore version tests + elsif ($line =~ /^IF EXISTS|^IF \(\@\@microsoftversion/i) { - next; + # just read until next go (or EOF) + while (defined $line and $line !~ /^GO$/) + { + $line =read_and_clean($file); + } + } + elsif ($line =~ /^IF NOT EXISTS/i) + { + # Just ignore the line + next; } - # Ignore CREATE DATABASE: we hope that we are given a single database as an option. It is multiline. + # Ignore CREATE DATABASE: we hope that we are given a single database as + # an option. It is multiline. # Ignore everything until next GO - # Ignore ALTER DATABASE for the same reason. The given parameters have no meaning in PG anyway - # Except for SET ARITHABORT OFF, for which we print a warning because it probably means the database contents are weird (10/0 = null) + # Ignore ALTER DATABASE for the same reason. The given parameters have no + # meaning in PG anyway + # Except for SET ARITHABORT OFF, for which we print a warning because it + # probably means the database contents are weird (10/0 = null) elsif ($line =~ - /^ALTER DATABASE.* SET ARITHABORT OFF/) + /^ALTER DATABASE.* SET ARITHABORT OFF/) { - print STDERR "WARNING: the source database is set as ARITHABORT OFF.\n"; - print STDERR " It means that for SQL Server, 10/0 = NULL.\n"; - print STDERR " You'll probably have problems porting that to PostgreSQL.\n"; - while ($line !~ /^GO$/) - { - $line =read_and_clean($file); - } - # We read everything in the CREATE DATABASE. Back to work ! - next; + print STDERR "WARNING: the source database is set as ARITHABORT OFF.\n"; + print STDERR " It means that for SQL Server, 10/0 = NULL.\n"; + print STDERR " You'll probably have problems porting that to PostgreSQL.\n"; + while ($line !~ /^GO$/) + { + $line =read_and_clean($file); + } + # We read everything in the CREATE DATABASE. Back to work ! + next; } # Sometimes, when there is a ALTER DATABASE SET ARITHABORT OFF, there are SET ARITHABORT ON. Just ignore them elsif ($line =~ /^SET ARITHABORT ON/) { - next; + next; } # Sometimes we meet this: SET CONCAT_NULL_YIELDS_NULL ON. That's the normal behaviour for a SQL database. Just ignore elsif ($line =~ /^SET CONCAT_NULL_YIELDS_NULL ON/) { - next; + next; } # Same more or less elsif ($line =~ /^SET ANSI_WARNINGS ON/) { - next; + next; } # What the hell does this do in a dump ??? elsif ($line =~ /^SET NUMERIC_ROUNDABORT OFF/) { - next; + next; } # Same for tests about full text search. @@ -2060,7 +2547,7 @@ sub parse_dump { $line =read_and_clean($file); } - + next; } @@ -2098,6 +2585,11 @@ sub generate_schema open BEFORE, ">:utf8", $before_file or die "Cannot open $before_file, $!"; open AFTER, ">:utf8", $after_file or die "Cannot open $after_file, $!"; open UNSURE, ">:utf8", $unsure_file or die "Cannot open $unsure_file, $!"; + if ($col_map_file) + { + open NAMEMAP, ">:utf8", $col_map_file or die "Cannot open $col_map_file, $!"; + } + print BEFORE "\\set ON_ERROR_STOP\n"; print BEFORE "\\set ECHO all\n"; print BEFORE "BEGIN;\n"; @@ -2107,6 +2599,15 @@ sub generate_schema print UNSURE "\\set ON_ERROR_STOP\n"; print AFTER "\\set ECHO all\n"; print UNSURE "BEGIN;\n"; + if ($col_map_file && $col_map_file_header) + { + print NAMEMAP "Source_schema" . $col_map_file_delimiter . + "Source_table" . $col_map_file_delimiter . + "Source_column" . $col_map_file_delimiter . + "Schema" . $col_map_file_delimiter . + "Table" . $col_map_file_delimiter . + "Column\n"; + } # Are we case insensitive ? We have to install citext then # Won't work on pre-9.1 database. But as this is a migration tool @@ -2115,7 +2616,7 @@ sub generate_schema { print BEFORE "CREATE EXTENSION IF NOT EXISTS citext;\n"; } - + # Do we require PostGIS ? if ($requires_postgis) { @@ -2138,11 +2639,17 @@ sub generate_schema } } + # Set psql variables in UNSURE + foreach my $varname (sort keys %{$objects->{VARIABLES}}) + { + print UNSURE "\\set $varname '$objects->{VARIABLES}->{$varname}'\n"; + } + # For the rest, we iterate over schemas, except for array types (no point in complicating this) # The tables, columns, etc... will be created in the before script, so there is no dependancy # problem with constraints, that will be in the after script, except foreign keys which depend on unique indexes # We have to do all domains and types before all tables - # Don't care for dependancy + # Don't care for dependancy while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { # The user-defined types (domains, etc) @@ -2172,6 +2679,8 @@ sub generate_schema # The tables foreach my $table (sort keys %{$refschema->{TABLES}}) { + my $origschema = $refschema->{TABLES}->{$table}->{origschema}; + my $newtablename = format_identifier($table); my @colsdef; foreach my $col ( sort { @@ -2182,14 +2691,20 @@ sub generate_schema { my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; - my $coldef = format_identifier($col) . " " . $colref->{TYPE}; + my $newcolname = format_identifier($col); + my $coldef = $newcolname . " " . $colref->{TYPE}; if ($colref->{NOT_NULL}) { $coldef .= ' NOT NULL'; } push @colsdef, ($coldef); + if ($col_map_file) + { + print NAMEMAP $origschema . $col_map_file_delimiter . $table . $col_map_file_delimiter . $col . $col_map_file_delimiter + . $schema . $col_map_file_delimiter . $newtablename . $col_map_file_delimiter . $newcolname . "\n"; + } } - print BEFORE "CREATE TABLE " . format_identifier($schema) . '.' . format_identifier($table) . "( \n\t" + print BEFORE "CREATE TABLE " . format_identifier($schema) . '.' . $newtablename . "( \n\t" . join(",\n\t", @colsdef) . ");\n\n"; } @@ -2203,35 +2718,63 @@ sub generate_schema foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) { my $seqref = $refschema->{SEQUENCES}->{$sequence}; - print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence); - if (defined $seqref->{STEP}) - { - print AFTER " INCREMENT BY ",$seqref->{STEP}; - } - if (defined $seqref->{MIN}) - { - print AFTER " MINVALUE ",$seqref->{MIN}; - } - if (defined $seqref->{MAX}) - { - print AFTER " MAXVALUE ",$seqref->{MAX}; - } - if (defined $seqref->{START}) - { - print AFTER " START WITH ",$seqref->{START}; - } - if (defined $seqref->{CACHE}) - { - print AFTER " CACHE ",$seqref->{CACHE}; - } - if (defined $seqref->{OWNERTABLE}) - { - print AFTER " OWNED BY ",format_identifier($seqref->{OWNERSCHEMA}), - '.',format_identifier($seqref->{OWNERTABLE}), - '.',format_identifier($seqref->{OWNERCOL}); - } - print AFTER ";\n"; - } + + if ($use_identity_column and defined $seqref->{OWNERTABLE}) + { + # Add a statement of the form + # ALTER TABLE "schema"."table_name" ALTER COLUMN "column_name" ADD GENERATED ALWAYS AS IDENTITY (start 1000); + + print AFTER "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($seqref->{OWNERTABLE}) . " "; + print AFTER "ALTER COLUMN " . format_identifier($seqref->{OWNERCOL}) . " ADD GENERATED ALWAYS AS IDENTITY"; + + if (defined $seqref->{START} or defined $seqref->{STEP}) + { + print AFTER " ("; + if (defined $seqref->{START}) + { + print AFTER " START WITH ",$seqref->{START}; + } + + if (defined $seqref->{STEP}) + { + print AFTER " INCREMENT BY ",$seqref->{STEP}; + } + print AFTER ")"; + } + } + else + { + print AFTER "CREATE SEQUENCE " . format_identifier($schema) . '.' . format_identifier($sequence); + if (defined $seqref->{STEP}) + { + print AFTER " INCREMENT BY ",$seqref->{STEP}; + } + if (defined $seqref->{MIN}) + { + print AFTER " MINVALUE ",$seqref->{MIN}; + } + if (defined $seqref->{MAX}) + { + print AFTER " MAXVALUE ",$seqref->{MAX}; + } + if (defined $seqref->{START}) + { + print AFTER " START WITH ",$seqref->{START}; + } + if (defined $seqref->{CACHE}) + { + print AFTER " CACHE ",$seqref->{CACHE}; + } + if (defined $seqref->{OWNERTABLE}) + { + print AFTER " OWNED BY ",format_identifier($seqref->{OWNERSCHEMA}), + '.',format_identifier($seqref->{OWNERTABLE}), + '.',format_identifier($seqref->{OWNERCOL}); + } + } + + print AFTER ";\n"; + } # Now PK. We have to go through all tables foreach my $table (sort keys %{$refschema->{TABLES}}) @@ -2247,7 +2790,7 @@ sub generate_schema next; } my $pkdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; - if (defined $refpk->{NAME}) + if (defined $refpk->{NAME} and is_constraint_name_valid($refpk->{NAME})) { $pkdef .= " CONSTRAINT " . format_identifier($refpk->{NAME}); } @@ -2265,11 +2808,11 @@ sub generate_schema foreach my $table (sort keys %{$refschema->{TABLES}}) { foreach my $constraint ( - @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) + @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) { next unless ($constraint->{TYPE} eq 'UNIQUE'); my $consdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; - if (defined $constraint->{NAME}) + if (defined $constraint->{NAME} and is_constraint_name_valid($constraint->{NAME})) { $consdef .= " CONSTRAINT " . format_identifier($constraint->{NAME}); } @@ -2287,22 +2830,73 @@ sub generate_schema # They don't have a schema qualifier. But their table has, and they are in the same schema as their table foreach my $table (sort keys %{$refschema->{TABLES}}) { - foreach my $index ( - sort keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) + foreach + my $index ( + sort keys %{$refschema->{TABLES}->{$table}->{INDEXES}}) { + my $index_created = 0; my $idxref = $refschema->{TABLES}->{$table}->{INDEXES}->{$index}; - my $idxdef = "CREATE"; + my $idxdef .= ""; + if ($idxref->{DISABLE}) + { + $idxdef .= "-- "; + } + $idxdef .= "CREATE"; if ($idxref->{UNIQUE}) { $idxdef .= " UNIQUE"; } - $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" - . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ");\n"; - print AFTER $idxdef; - } - } - } + if (defined $idxref->{COLS}) + { + $idxdef .= " INDEX " . format_identifier($index) . " ON " . format_identifier($schema) . '.' . format_identifier($table) . " (" + . join(",", map{format_identifier_cols_index($_)} @{$idxref->{COLS}}) . ")"; + + if (defined $idxref->{INCLUDE}) { + $idxdef .= " INCLUDE (" . + join(",", map{format_identifier_cols_index($_)} @{$idxref->{INCLUDE}}) + . ")"; + } + + if (not defined $idxref->{WHERE} and not defined $idxref->{DISABLE}) { + $idxdef .= ";\n"; + print AFTER $idxdef; + # the possible comment would go to after file + $index_created = 1; + } + else { + + # this is either a disabled index or an index with a where declaration + if (defined $idxref->{WHERE}) { + print STDERR "Warning: index $schema.$index contains a where clause. It goes to unsure file\n"; + if ($idxref->{DISABLE}) { + # if disabled, will be on the same line + $idxdef .= " "; + } else { + # otherwise, write condition on a new line + $idxdef .= "\n"; + } + $idxdef .= "WHERE (" . convert_transactsql_code($idxref->{WHERE}) . ")"; + } + $idxdef .= ";\n"; + print UNSURE $idxdef; + # the possible comment would go to unsure file + $index_created = 2; + } + + # Produce the comments for indexes + if (defined $idxref->{COMMENT}) { + my $idxcomment = "COMMENT ON INDEX ". format_identifier($schema) . '.' . format_identifier($index) . " IS '" . $idxref->{COMMENT} . "';\n"; + if ($index_created == 1) { + print AFTER $idxcomment; + } elsif ($index_created == 2) { + print UNSURE $idxcomment; + } + } + } + } + } + } # Other constraints while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { @@ -2315,12 +2909,11 @@ sub generate_schema { next if ($constraint->{TYPE} =~ /^UNIQUE|PK$/); my $consdef = "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ADD"; - if (defined $constraint->{NAME}) + if (defined $constraint->{NAME} and is_constraint_name_valid($constraint->{NAME})) { $consdef .= " CONSTRAINT " . format_identifier($constraint->{NAME}); } - if ($constraint->{TYPE} eq - 'FK') # COLS are already a comma separated list + if ($constraint->{TYPE} eq 'FK') # COLS are already a comma separated list { # We need to convert the column list to protected names my @localcollist=map{format_identifier($_)} @{$constraint->{LOCAL_COLS}}; @@ -2347,41 +2940,69 @@ sub generate_schema { $consdef .= " ON UPDATE CASCADE"; } + if (defined $constraint->{ON_UPD_SET_NULL} + and $constraint->{ON_UPD_SET_NULL}) + { + $consdef .= " ON UPDATE SET NULL"; + } # We need a name on the constraint to be able to validate it later. Maybe it would be better to generate one # FIXME: we'll see later if a generator is needed (probably) - if ($constraint->{TYPE} eq 'FK' and ($validate_constraints =~ /^after|no$/) and defined($constraint->{NAME})) + if (($validate_constraints =~ /^after|no$/) and defined($constraint->{NAME}) and is_constraint_name_valid($constraint->{NAME})) { $consdef .= " NOT VALID"; } $consdef .= ";\n"; print AFTER $consdef; - if ($constraint->{TYPE} eq 'FK' and $validate_constraints eq 'after' and defined $constraint->{NAME}) + if ($validate_constraints eq 'after' and defined $constraint->{NAME} and is_constraint_name_valid($constraint->{NAME})) { print UNSURE "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " VALIDATE CONSTRAINT " . format_identifier($constraint->{NAME}) . ";\n"; } } elsif ($constraint->{TYPE} eq 'CHECK') - { - $consdef .= " CHECK (" . $constraint->{TEXT} . ");\n"; - print UNSURE $consdef - ; # Check constraints are SQL, so cannot be sure - } + { + $consdef .= " CHECK (" . convert_transactsql_code($constraint->{TEXT}) . ");\n"; + print UNSURE $consdef; # Check constraints are SQL, so cannot be sure + } elsif ($constraint->{TYPE} eq 'CHECK_CITEXT') { - # These have been generated here, for citext mostly. So we know their syntax is ok $consdef .= " CHECK (" . $constraint->{TEXT} . ");\n"; print BEFORE $consdef ; # These are for citext. So they should be checked asap } + elsif ($constraint->{TYPE} eq 'CHECK_BINARY_LENGTH') + { + # These have been generated here. Their syntax are ok. + $consdef .= " CHECK (" . $constraint->{TEXT} . ");\n"; + print BEFORE $consdef + ; # These are for bytea length, checked them asap + } else { # Shouldn't get there. it would mean I have forgotten a type of constraint - die "I couldn't translate a constraint. This is a bug"; + croak "I couldn't translate a constraint. This is a bug"; } } } } + # Another pass at constraints. This time we want to produce the comments + while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) + { + + # We have all we need for FKs now. We can put all other constraints (except PK of course) + foreach my $table (sort keys %{$refschema->{TABLES}}) + { + foreach my $constraint ( + @{$refschema->{TABLES}->{$table}->{CONSTRAINTS}}) + { + next unless defined ($constraint->{COMMENT}); + print UNSURE "COMMENT ON CONSTRAINT " + . format_identifier($constraint->{NAME}) . " ON " + . format_identifier($schema) . '.' . format_identifier($table) + . " IS '" . $constraint->{COMMENT} . "';\n"; + } + } + } # Default values while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) @@ -2394,16 +3015,31 @@ sub generate_schema { my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; next unless (defined $colref->{DEFAULT}); + my $default_value = $colref->{DEFAULT}->{VALUE}; + if ($default_value =~ /\(\$\((\S+)\)\)/) + { + $default_value = ":$1"; + } my $definition = - "ALTER TABLE " . format_identifier($schema) . '.' . format_identifier($table) . " ALTER COLUMN " . format_identifier($col) . " SET DEFAULT " - . $colref->{DEFAULT}->{VALUE} . ";\n"; + "ALTER TABLE " . format_identifier($schema) . '.' + . format_identifier($table) + . " ALTER COLUMN " . format_identifier($col) + . " SET DEFAULT " . $default_value . ";\n"; if ($colref->{DEFAULT}->{UNSURE}) - { + { print UNSURE $definition; } else { - print AFTER $definition; + if ($use_identity_column and ($definition =~ /nextval.+_seq/i)) + { + # Skip this set default item + } + else + { + print AFTER $definition; + } + } } } @@ -2414,18 +3050,28 @@ sub generate_schema { foreach my $sequence (sort keys %{$refschema->{SEQUENCES}}) { - my $seqref = $refschema->{SEQUENCES}->{$sequence}; - # This may not be an identity. Skip it then - next unless defined ($seqref->{OWNERCOL}); - - print AFTER "select setval('" . format_identifier($schema) . '.' . format_identifier($sequence) . "',(select max(". format_identifier($seqref->{OWNERCOL}) .") from " . format_identifier($seqref->{OWNERSCHEMA}) . '.'. format_identifier($seqref->{OWNERTABLE}) . ")::bigint);\n"; + my $seqref = $refschema->{SEQUENCES}->{$sequence}; + # This may not be an identity. Skip it then + next unless defined ($seqref->{OWNERCOL}); + next if defined ($use_identity_column); + print AFTER "select setval('" . format_identifier($schema) . '.' + . format_identifier($sequence) . "',(select " . ($seqref->{STEP} > 0 ? "max" : "min") . "(" + . format_identifier($seqref->{OWNERCOL}) .") from " + . format_identifier($seqref->{OWNERSCHEMA}) . '.' + . format_identifier($seqref->{OWNERTABLE}) . ")::bigint);\n"; } } - # Comments on tables and columns while (my ($schema, $refschema) = each %{$objects->{SCHEMAS}}) { + # Comments on schemas + if (defined($refschema->{COMMENT})) + { + print AFTER "COMMENT ON SCHEMA " . format_identifier($schema) . " IS '" + . $refschema->{COMMENT} . "';\n"; + } + # Comments on tables foreach my $table (sort keys %{$refschema->{TABLES}}) { @@ -2440,8 +3086,10 @@ sub generate_schema my $colref = $refschema->{TABLES}->{$table}->{COLS}->{$col}; if (defined($colref->{COMMENT})) { - print AFTER "COMMENT ON COLUMN " . format_identifier($schema) . '.' . format_identifier($table) . '.' . format_identifier($col) . " IS '" - . $colref->{COMMENT} . "';\n"; + print AFTER "COMMENT ON COLUMN " . format_identifier($schema) + . '.' . format_identifier($table) . '.' + . format_identifier($col) . " IS '" + . $colref->{COMMENT} . "';\n"; } } } @@ -2452,11 +3100,22 @@ sub generate_schema { my ($schema,$view)=@$viewref; my $refschema=$objects->{SCHEMAS}->{$schema}; - print UNSURE $refschema->{VIEWS}->{$view}->{SQL}, ";\n"; + print UNSURE "CREATE VIEW " + . format_identifier($schema) . '.' . format_identifier($view) . " "; + if (not defined $refschema->{VIEWS}->{$view}->{SQL}) { + my $view_columns = $refschema->{VIEWS}->{$view}->{COLS}; + my $query_columns = $refschema->{VIEWS}->{$view}->{QUERYCOLS}; + my $query = $refschema->{VIEWS}->{$view}->{QUERY}; + print UNSURE "($view_columns) AS\n\tSELECT\n\t\t" + . $query_columns . "\n\t" . "FROM $query;\n\n"; + } + else { + print UNSURE $refschema->{VIEWS}->{$view}->{SQL} . ";\n\n"; + } if (defined $refschema->{VIEWS}->{$view}->{COMMENT}) { - print UNSURE "COMMENT ON VIEW $schema.$view IS '" - . $refschema->{VIEWS}->{$view}->{COMMENT} . "';\n"; + print UNSURE "COMMENT ON VIEW $schema.$view IS '" + . $refschema->{VIEWS}->{$view}->{COMMENT} . "';\n"; } } # Trigger functions @@ -2507,6 +3166,7 @@ sub generate_schema close BEFORE; close AFTER; close UNSURE; + close NAMEMAP if ($col_map_file); } @@ -2564,7 +3224,6 @@ sub resolve_name_conflicts } $known_names{format_identifier($domain."2pgd")}=1; } - } # Then we scan all indexes @@ -2610,33 +3269,47 @@ sub resolve_name_conflicts # Parse command line my $help = 0; -my $options = GetOptions("k=s" => \$kettle, - "p=i" => \$parallelism, - "b=s" => \$before_file, - "a=s" => \$after_file, - "u=s" => \$unsure_file, - "h" => \$help, - "conf=s" => \$conf_file, - "sd=s" => \$sd, - "sh=s" => \$sh, - "si=s" => \$si, - "sp=s" => \$sp, - "su=s" => \$su, - "sw=s" => \$sw, - "pd=s" => \$pd, - "ph=s" => \$ph, - "pp=s" => \$pp, - "pu=s" => \$pu, - "pw=s" => \$pw, - "f=s" => \$filename, - "i" => \$case_insensitive, - "nr" => \$norelabel_dbo, - "num" => \$convert_numeric_to_int, - "relabel_schemas=s" => \$relabel_schemas, - "keep_identifier_case" =>\$keep_identifier_case, - "validate_constraints=s" =>\$validate_constraints, - "sort_size=i" =>\$sort_size, - "use_pk_if_possible=s" =>\$use_pk_if_possible,); +my $options = GetOptions( + "k=s" => \$kettle, + "pi=i" => \$parallelism_in, + "po=i" => \$parallelism_out, + "b=s" => \$before_file, + "a=s" => \$after_file, + "u=s" => \$unsure_file, + "h" => \$help, + "conf=s" => \$conf_file, + "sd=s" => \$sd, + "sh=s" => \$sh, + "si=s" => \$si, + "sp=s" => \$sp, + "su=s" => \$su, + "sw=s" => \$sw, + "pd=s" => \$pd, + "ph=s" => \$ph, + "pp=s" => \$pp, + "pu=s" => \$pu, + "pw=s" => \$pw, + "f=s" => \$filename, + "i" => \$case_insensitive, + "nr" => \$norelabel_dbo, + "num" => \$convert_numeric_to_int, + "col_map_file=s" => \$col_map_file, + "col_map_file_header" => \$col_map_file_header, + "col_map_file_delimiter=s" => \$col_map_file_delimiter, + "drop_rowversion" => \$drop_rowversion, + "relabel_schemas=s" => \$relabel_schemas, + "keep_identifier_case" => \$keep_identifier_case, + "camel_to_snake" => \$camel_to_snake, + "validate_constraints=s" => \$validate_constraints, + "sort_size=i" => \$sort_size, + "use_pk_if_possible=s" => \$use_pk_if_possible, + "ignore_errors" => \$ignore_errors, + "sforce_ssl" => \$sforce_ssl, + "pforce_ssl" => \$pforce_ssl, + "stringtype_unspecified" => \$stringtype_unspecified, + "skip_citext_length_check" => \$skip_citext_length_check, + "use_identity_column" => \$use_identity_column +); # We don't understand command line or have been asked for usage if (not $options or $help) @@ -2655,46 +3328,12 @@ sub resolve_name_conflicts # Set default values for anything not set yet set_default_conf_values(); -# We have no before, after, or unsure -if ( not $before_file - or not $after_file - or not $unsure_file - or not $filename) -{ - usage(); - exit 1; -} - -if ($validate_constraints !~ '^(yes|after|no)$') -{ - die "validate_constraints should be yes, after or no (default yes)\n"; -} - -# We have been asked for kettle, but the compulsory parameters are not there -if ($kettle - and ( not $sd - or not $sh - or not $sp - or not $su - or not defined($sw) # password can be empty, it just has to be defined - or not $pd - or not $ph - or not $pp - or not $pu - or not defined($pw) # password can be empty, it just has to be defined - ) -) -{ - usage(); - print - "You have to provide all connection information, if using -k or kettle directory set in configuration file\n"; - exit 1; -} +# Perform checks on parameters +process_check_parameters(); # We need to build %relabel_schemas from $relabel_schemas build_relabel_schemas(); - # Read SQL Server's dump file parse_dump(); @@ -2803,6 +3442,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -2828,6 +3468,7 @@ BEGIN EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -2837,16 +3478,99 @@ BEGIN USE_POOLINGN SQL_CONNECTset synchronous_commit to off; PRESERVE_RESERVED_WORD_CASEY + __stringtype_unspecified__ - Table inputUser Defined Java ClassY User Defined Java ClassTable outputY + + User Defined Java Class + Table output + Y + + + Table input + Block this step until steps finish + Y + + + Block this step until steps finish + User Defined Java Class + Y + + + + Execute SQL script + ExecSQL + + Y + + 1 + + none + + + __postgres_db__ + N + N + N + N + truncate table __postgres_schema_name__.__postgres_table_name__ + N + + + + + + + + + + + + + + + 64 + 160 + Y + + + + Block this step until steps finish + BlockUntilStepsFinish + + Y + + 1 + + none + + + + + Execute SQL script + 0 + + + + + + + + + + + 240 + 240 + Y + + User Defined Java Class UserDefinedJavaClass Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -2947,7 +3671,7 @@ BEGIN N - 122 + 182 160 Y @@ -2958,7 +3682,7 @@ BEGIN TableOutput Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -2967,7 +3691,7 @@ BEGIN __postgres_schema_name__ __postgres_table_name__
100 - Y + N N Y N @@ -3081,6 +3805,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -3105,6 +3830,8 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3114,16 +3841,99 @@ BEGIN USE_POOLINGN SQL_CONNECTset synchronous_commit to off; PRESERVE_RESERVED_WORD_CASEY + __stringtype_unspecified__ - Table inputUser Defined Java ClassY User Defined Java ClassTable outputY + + User Defined Java Class + Table output + Y + + + Table input + Block this step until steps finish + Y + + + Block this step until steps finish + User Defined Java Class + Y + + + + Execute SQL script + ExecSQL + + Y + + 1 + + none + + + __postgres_db__ + N + N + N + N + truncate table __postgres_schema_name__.__postgres_table_name__ + N + + + + + + + + + + + + + + + 64 + 160 + Y + + + + Block this step until steps finish + BlockUntilStepsFinish + + Y + + 1 + + none + + + + + Execute SQL script + 0 + + + + + + + + + + + 240 + 240 + Y + + User Defined Java Class UserDefinedJavaClass Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -3209,7 +4019,7 @@ BEGIN TableInput Y - __PARALLELISM__ + __PARALLELISM_IN__ none @@ -3223,7 +4033,7 @@ BEGIN N - 122 + 182 160 Y @@ -3234,7 +4044,7 @@ BEGIN TableOutput Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -3243,7 +4053,7 @@ BEGIN __postgres_schema_name__ __postgres_table_name__
100 - Y + N N Y N @@ -3302,6 +4112,8 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3311,6 +4123,7 @@ BEGIN USE_POOLINGN SQL_CONNECTset synchronous_commit to off; PRESERVE_RESERVED_WORD_CASEY + __stringtype_unspecified__ @@ -3456,28 +4269,28 @@ BEGIN $incremental_template= < - migration__sqlserver_table_name__ - - - - Normal - 0 - / - - - - - - - - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD - - -
- - + migration__sqlserver_table_name__ + + + + Normal + 0 + / + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD + + +
+ + ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS @@ -3536,6 +4349,8 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3544,6 +4359,7 @@ BEGIN SQL_CONNECTset synchronous_commit to off; SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN + __stringtype_unspecified__ @@ -3559,6 +4375,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -3604,7 +4421,7 @@ BEGIN UserDefinedJavaClass Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -3715,7 +4532,7 @@ BEGIN SortRows Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -3743,7 +4560,7 @@ BEGIN SortRows Y - __PARALLELISM__ + __PARALLELISM_OUT__ none @@ -3813,13 +4630,13 @@ BEGIN SynchronizeAfterMerge Y - __PARALLELISM__ + __PARALLELISM_OUT__ none __postgres_db__ - 100 + 100 N N @@ -3883,28 +4700,28 @@ BEGIN $incremental_template_sortable_pk= < - migration__sqlserver_table_name__ - - - - Normal - 0 - / - - - - - -
- - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD - - -
- - + migration__sqlserver_table_name__ + + + + Normal + 0 + / + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELD + + +
+ + ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS @@ -3963,6 +4780,8 @@ BEGIN + EXTRA_OPTION_POSTGRESQL.reWriteBatchedInsertstrue + __pforce_ssl__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN IS_CLUSTEREDN @@ -3971,6 +4790,7 @@ BEGIN SQL_CONNECTset synchronous_commit to off; SUPPORTS_BOOLEAN_DATA_TYPEY USE_POOLINGN + __stringtype_unspecified__ @@ -3986,6 +4806,7 @@ BEGIN + __sforce_ssl__ EXTRA_OPTION_MSSQL.instance__sqlserver_instance__ FORCE_IDENTIFIERS_TO_LOWERCASEN FORCE_IDENTIFIERS_TO_UPPERCASEN @@ -4142,13 +4963,13 @@ BEGIN SynchronizeAfterMerge Y - __PARALLELISM__ + __PARALLELISM_OUT__ none __postgres_db__ - 100 + 100 N N diff --git a/t/lint-kettle.t b/t/lint-kettle.t new file mode 100644 index 0000000..8ce0f0f --- /dev/null +++ b/t/lint-kettle.t @@ -0,0 +1,5 @@ +#!/usr/bin/env bats + +@test "PERL syntax check" { + perl -c kettle_report.pl +} diff --git a/t/lint-sqlserver2pgsql.t b/t/lint-sqlserver2pgsql.t new file mode 100644 index 0000000..56dac54 --- /dev/null +++ b/t/lint-sqlserver2pgsql.t @@ -0,0 +1,5 @@ +#!/usr/bin/env bats + +@test "PERL syntax check" { + perl -c sqlserver2pgsql.pl +} diff --git a/t/test_convert.t b/t/test_convert.t new file mode 100644 index 0000000..61013c0 --- /dev/null +++ b/t/test_convert.t @@ -0,0 +1,62 @@ +#!/usr/bin/env bats + +@test "schema conversion test" { + WORK_DIR="/tmp/tests" + + # we must add the KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y in kettle conf file + if [ ! -d ~/.kettle ]; then + mkdir ~/.kettle + fi + if [ ! -f ~/.kettle/kettle.properties ]; then + touch ~/.kettle/kettle.properties + fi + if [ -z $(grep "KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y" ~/.kettle/kettle.properties) ]; then + echo -e "\n# This line was added for sqlserver2pgsql tests" >> ~/.kettle/kettle.properties + echo -e "KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL=Y" >> ~/.kettle/kettle.properties + fi + + # create the array of command to try + options_to_try=( "-i" "-nr" "-num" "-validate_constraints=after" ) + declare -a many_combinations=("blank ") + + for option_to_try in ${options_to_try[@]} ; do + for existing_combination in ${many_combinations[@]}; do + all_combinations+="${option_to_try}_sep_${existing_combination} " + done + done + declare -a all_combinations=many_combinations + for existing_combination in ${all_combinations[@]}; do + all_combinations+="-keep_identifier_case_sep_${existing_combination} " + done + + # clear the possible leftovers + if [ -d $WORK_DIR ]; then + rm -rf $WORK_DIR + fi + mkdir -p $WORK_DIR + + # run the sqlserver2pgsql script for all files and confs + declare -i test_nb=1 + for reg_file in regression/*.sql ; do + for option in ${all_combinations[@]} ; do + CURRENT_DIR=$WORK_DIR/$test_nb + mkdir -p $CURRENT_DIR + real_option=$(echo $option | sed 's/blank//' | sed 's/_sep_/ /g') + command_line="./sqlserver2pgsql.pl -f $reg_file -b $CURRENT_DIR/before.sql -a $CURRENT_DIR/after.sql -u $CURRENT_DIR/unsure.sql -k $CURRENT_DIR/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 $real_option" + echo $command_line > $CURRENT_DIR/command_line + eval $command_line > $CURRENT_DIR/command_output 2>&1 + test_nb+=1 + done + done + for reg_file in regression/basic_test/*.sql ; do + for option in ${many_combinations[@]} ; do + CURRENT_DIR=$WORK_DIR/$test_nb + mkdir -p $CURRENT_DIR + real_option=$(echo $option | sed 's/blank//' | sed 's/_sep_/ /g') + command_line="./sqlserver2pgsql.pl -f $reg_file -b $CURRENT_DIR/before.sql -a $CURRENT_DIR/after.sql -u $CURRENT_DIR/unsure.sql -k $CURRENT_DIR/kettle -sd 1 -sh 1 -sp 1 -su 1 -sw 1 -pd 1 -ph 1 -pp 1 -pu 1 -pw 2 $real_option" + echo $command_line > $CURRENT_DIR/command_line + eval $command_line > $CURRENT_DIR/command_output 2>&1 + test_nb+=1 + done + done +} diff --git a/t/test_db_creation.t b/t/test_db_creation.t new file mode 100644 index 0000000..2b24be5 --- /dev/null +++ b/t/test_db_creation.t @@ -0,0 +1,16 @@ +#!/usr/bin/env bats + +@test "PostgreSQL database creation test" { + WORK_DIR="/tmp/tests" + cd $WORK_DIR + for test_dir in * ; do + cd $test_dir + cat command_line + createdb reg + psql reg < before.sql + psql reg < after.sql + psql reg < unsure.sql + dropdb reg + cd .. + done +}