diff --git a/docker/bin/trino_configure_and_start.sh b/docker/bin/trino_configure_and_start.sh index 4fd2fe4..7d12648 100755 --- a/docker/bin/trino_configure_and_start.sh +++ b/docker/bin/trino_configure_and_start.sh @@ -35,4 +35,4 @@ sed -i \ echo "[Trino] Starting server..." CATALOG_MANAGEMENT=static \ - ./trino/bin/run.sh >/dev/null 2> >(sed 's/^/[Trino] /') + ./trino/bin/run.sh 2>&1 | sed 's/^/[Trino] /' diff --git a/iceberg-test-optimized-manifests/00006-d43945ca-a30e-4182-aeb5-99bd766b328b.metadata.json b/iceberg-test-optimized-manifests/00006-d43945ca-a30e-4182-aeb5-99bd766b328b.metadata.json new file mode 100644 index 0000000..4a62728 --- /dev/null +++ b/iceberg-test-optimized-manifests/00006-d43945ca-a30e-4182-aeb5-99bd766b328b.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"2412fcf6-bba7-4564-82b7-12a18c0d277c","location":"s3://bemidb-us-west-1--dev/iceberg/postgres/test_table-bemidb-syncing-c35f2feac9664ab380ca3ae7a4fcefe2","last-sequence-number":4,"last-updated-ms":1755616365273,"last-column-id":44,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"bit_column","required":false,"type":"int"},{"id":3,"name":"bool_column","required":false,"type":"boolean"},{"id":4,"name":"bpchar_column","required":false,"type":"string"},{"id":5,"name":"varchar_column","required":false,"type":"string"},{"id":6,"name":"text_column","required":false,"type":"string"},{"id":7,"name":"int2_column","required":false,"type":"int"},{"id":8,"name":"int4_column","required":false,"type":"int"},{"id":9,"name":"int8_column","required":false,"type":"decimal(20, 0)"},{"id":10,"name":"hugeint_column","required":false,"type":"decimal(20, 0)"},{"id":11,"name":"xid_column","required":false,"type":"long"},{"id":12,"name":"xid8_column","required":false,"type":"decimal(20, 0)"},{"id":13,"name":"float4_column","required":false,"type":"float"},{"id":14,"name":"float8_column","required":false,"type":"double"},{"id":15,"name":"numeric_column","required":false,"type":"decimal(38, 2)"},{"id":16,"name":"numeric_column_without_precision","required":false,"type":"decimal(38, 19)"},{"id":17,"name":"date_column","required":false,"type":"date"},{"id":18,"name":"time_column","required":false,"type":"time"},{"id":19,"name":"timemscolumn","required":false,"type":"time"},{"id":20,"name":"timetz_column","required":false,"type":"time"},{"id":21,"name":"timetz_ms_column","required":false,"type":"time"},{"id":22,"name":"timestamp_column","required":false,"type":"timestamp"},{"id":23,"name":"timestamp_ms_column","required":false,"type":"timestamp"},{"id":24,"name":"timestamptz_column","required":false,"type":"timestamptz"},{"id":25,"name":"timestamptz_ms_column","required":false,"type":"timestamptz"},{"id":26,"name":"uuid_column","required":false,"type":"string"},{"id":27,"name":"bytea_column","required":false,"type":"binary"},{"id":28,"name":"interval_column","required":false,"type":"string"},{"id":29,"name":"point_column","required":false,"type":"string"},{"id":30,"name":"inet_column","required":false,"type":"string"},{"id":31,"name":"json_column","required":false,"type":"string"},{"id":32,"name":"jsonb_column","required":false,"type":"string"},{"id":33,"name":"tsvector_column","required":false,"type":"string"},{"id":34,"name":"xml_column","required":false,"type":"string"},{"id":35,"name":"pg_snapshot_column","required":false,"type":"string"},{"id":36,"name":"array_text_column","required":false,"type":{"type":"list","element-id":41,"element":"string","element-required":false}},{"id":37,"name":"array_int_column","required":false,"type":{"type":"list","element-id":42,"element":"int","element-required":false}},{"id":38,"name":"array_jsonb_column","required":false,"type":{"type":"list","element-id":43,"element":"string","element-required":false}},{"id":39,"name":"array_ltree_column","required":false,"type":{"type":"list","element-id":44,"element":"string","element-required":false}},{"id":40,"name":"user_defined_column","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"write.format.default":"PARQUET","write.parquet.compression-codec":"zstd","commit.retry.num-retries":"4"},"current-snapshot-id":4117982741288853649,"refs":{"main":{"snapshot-id":4117982741288853649,"type":"branch"}},"snapshots":[{"sequence-number":4,"snapshot-id":4117982741288853649,"parent-snapshot-id":4750718983626705147,"timestamp-ms":1755616364353,"summary":{"operation":"replace","manifests-created":"1","manifests-kept":"0","manifests-replaced":"2","entries-processed":"2","changed-partition-count":"0","total-records":"2","total-files-size":"13197","total-data-files":"2","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"476","engine-name":"trino","iceberg-version":"Apache Iceberg 1.9.1 (commit f40208ae6fb2f33e578c2637d3dea1db18739f31)"},"manifest-list":"s3://bemidb-us-west-1--dev/iceberg/postgres/test_table-bemidb-syncing-c35f2feac9664ab380ca3ae7a4fcefe2/metadata/snap-4117982741288853649-1-ace50a17-644b-4801-98a3-b0491e620eda.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1755616364353,"snapshot-id":4117982741288853649}],"metadata-log":[{"timestamp-ms":1755616355823,"metadata-file":"s3://bemidb-us-west-1--dev/iceberg/postgres/test_table-bemidb-syncing-c35f2feac9664ab380ca3ae7a4fcefe2/metadata/00000-aa858843-a1b3-430a-83e0-b4a7d77dbbd4.metadata.json"},{"timestamp-ms":1755616358632,"metadata-file":"s3://bemidb-us-west-1--dev/iceberg/postgres/test_table-bemidb-syncing-c35f2feac9664ab380ca3ae7a4fcefe2/metadata/00001-88a52bfd-ce85-4946-b390-c65e0066792c.metadata.json"},{"timestamp-ms":1755616359770,"metadata-file":"s3://bemidb-us-west-1--dev/iceberg/postgres/test_table-bemidb-syncing-c35f2feac9664ab380ca3ae7a4fcefe2/metadata/00002-2908d54d-380e-4b84-a2d5-94477e846177.metadata.json"},{"timestamp-ms":1755616361297,"metadata-file":"s3://bemidb-us-west-1--dev/iceberg/postgres/test_table-bemidb-syncing-c35f2feac9664ab380ca3ae7a4fcefe2/metadata/00003-61f616c1-cb87-4250-ad28-104d83224d3e.metadata.json"},{"timestamp-ms":1755616362704,"metadata-file":"s3://bemidb-us-west-1--dev/iceberg/postgres/test_table-bemidb-syncing-c35f2feac9664ab380ca3ae7a4fcefe2/metadata/00004-ffa0adab-ca4b-4c48-8c2b-2c05f7b39fbb.metadata.json"},{"timestamp-ms":1755616364353,"metadata-file":"s3://bemidb-us-west-1--dev/iceberg/postgres/test_table-bemidb-syncing-c35f2feac9664ab380ca3ae7a4fcefe2/metadata/00005-3401e52a-280a-4807-9147-8014c6f8b0eb.metadata.json"}]} \ No newline at end of file diff --git a/iceberg-test-optimized-manifests/20250819_151236_00003_e5m2c-0ff4ea89-f153-46e5-901e-f2004ac5134d.parquet b/iceberg-test-optimized-manifests/20250819_151236_00003_e5m2c-0ff4ea89-f153-46e5-901e-f2004ac5134d.parquet new file mode 100644 index 0000000..37bcb49 Binary files /dev/null and b/iceberg-test-optimized-manifests/20250819_151236_00003_e5m2c-0ff4ea89-f153-46e5-901e-f2004ac5134d.parquet differ diff --git a/iceberg-test-optimized-manifests/20250819_151240_00004_e5m2c-c85ae7de-4c8a-4da3-92fd-f0831d679ee0.parquet b/iceberg-test-optimized-manifests/20250819_151240_00004_e5m2c-c85ae7de-4c8a-4da3-92fd-f0831d679ee0.parquet new file mode 100644 index 0000000..698e6f3 Binary files /dev/null and b/iceberg-test-optimized-manifests/20250819_151240_00004_e5m2c-c85ae7de-4c8a-4da3-92fd-f0831d679ee0.parquet differ diff --git a/iceberg-test-optimized-manifests/ace50a17-644b-4801-98a3-b0491e620eda-m0.avro b/iceberg-test-optimized-manifests/ace50a17-644b-4801-98a3-b0491e620eda-m0.avro new file mode 100644 index 0000000..371ee08 --- /dev/null +++ b/iceberg-test-optimized-manifests/ace50a17-644b-4801-98a3-b0491e620eda-m0.avro @@ -0,0 +1,8 @@ +Obj schemaø.{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"bit_column","required":false,"type":"int"},{"id":3,"name":"bool_column","required":false,"type":"boolean"},{"id":4,"name":"bpchar_column","required":false,"type":"string"},{"id":5,"name":"varchar_column","required":false,"type":"string"},{"id":6,"name":"text_column","required":false,"type":"string"},{"id":7,"name":"int2_column","required":false,"type":"int"},{"id":8,"name":"int4_column","required":false,"type":"int"},{"id":9,"name":"int8_column","required":false,"type":"decimal(20, 0)"},{"id":10,"name":"hugeint_column","required":false,"type":"decimal(20, 0)"},{"id":11,"name":"xid_column","required":false,"type":"long"},{"id":12,"name":"xid8_column","required":false,"type":"decimal(20, 0)"},{"id":13,"name":"float4_column","required":false,"type":"float"},{"id":14,"name":"float8_column","required":false,"type":"double"},{"id":15,"name":"numeric_column","required":false,"type":"decimal(38, 2)"},{"id":16,"name":"numeric_column_without_precision","required":false,"type":"decimal(38, 19)"},{"id":17,"name":"date_column","required":false,"type":"date"},{"id":18,"name":"time_column","required":false,"type":"time"},{"id":19,"name":"timemscolumn","required":false,"type":"time"},{"id":20,"name":"timetz_column","required":false,"type":"time"},{"id":21,"name":"timetz_ms_column","required":false,"type":"time"},{"id":22,"name":"timestamp_column","required":false,"type":"timestamp"},{"id":23,"name":"timestamp_ms_column","required":false,"type":"timestamp"},{"id":24,"name":"timestamptz_column","required":false,"type":"timestamptz"},{"id":25,"name":"timestamptz_ms_column","required":false,"type":"timestamptz"},{"id":26,"name":"uuid_column","required":false,"type":"string"},{"id":27,"name":"bytea_column","required":false,"type":"binary"},{"id":28,"name":"interval_column","required":false,"type":"string"},{"id":29,"name":"point_column","required":false,"type":"string"},{"id":30,"name":"inet_column","required":false,"type":"string"},{"id":31,"name":"json_column","required":false,"type":"string"},{"id":32,"name":"jsonb_column","required":false,"type":"string"},{"id":33,"name":"tsvector_column","required":false,"type":"string"},{"id":34,"name":"xml_column","required":false,"type":"string"},{"id":35,"name":"pg_snapshot_column","required":false,"type":"string"},{"id":36,"name":"array_text_column","required":false,"type":{"type":"list","element-id":41,"element":"string","element-required":false}},{"id":37,"name":"array_int_column","required":false,"type":{"type":"list","element-id":42,"element":"int","element-required":false}},{"id":38,"name":"array_jsonb_column","required":false,"type":{"type":"list","element-id":43,"element":"string","element-required":false}},{"id":39,"name":"array_ltree_column","required":false,"type":{"type":"list","element-id":44,"element":"string","element-required":false}},{"id":40,"name":"user_defined_column","required":false,"type":"string"}]}avro.schema€8{"type":"record","name":"manifest_entry","fields":[{"name":"status","type":"int","field-id":0},{"name":"snapshot_id","type":["null","long"],"default":null,"field-id":1},{"name":"sequence_number","type":["null","long"],"default":null,"field-id":3},{"name":"file_sequence_number","type":["null","long"],"default":null,"field-id":4},{"name":"data_file","type":{"type":"record","name":"r2","fields":[{"name":"content","type":"int","doc":"Contents of the file: 0=data, 1=position deletes, 2=equality deletes","field-id":134},{"name":"file_path","type":"string","doc":"Location URI with FS scheme","field-id":100},{"name":"file_format","type":"string","doc":"File format name: avro, orc, or parquet","field-id":101},{"name":"partition","type":{"type":"record","name":"r102","fields":[]},"doc":"Partition data tuple, schema based on the partition spec","field-id":102},{"name":"record_count","type":"long","doc":"Number of records in the file","field-id":103},{"name":"file_size_in_bytes","type":"long","doc":"Total file size in bytes","field-id":104},{"name":"column_sizes","type":["null",{"type":"array","items":{"type":"record","name":"k117_v118","fields":[{"name":"key","type":"int","field-id":117},{"name":"value","type":"long","field-id":118}]},"logicalType":"map"}],"doc":"Map of column id to total size on disk","default":null,"field-id":108},{"name":"value_counts","type":["null",{"type":"array","items":{"type":"record","name":"k119_v120","fields":[{"name":"key","type":"int","field-id":119},{"name":"value","type":"long","field-id":120}]},"logicalType":"map"}],"doc":"Map of column id to total count, including null and NaN","default":null,"field-id":109},{"name":"null_value_counts","type":["null",{"type":"array","items":{"type":"record","name":"k121_v122","fields":[{"name":"key","type":"int","field-id":121},{"name":"value","type":"long","field-id":122}]},"logicalType":"map"}],"doc":"Map of column id to null value count","default":null,"field-id":110},{"name":"nan_value_counts","type":["null",{"type":"array","items":{"type":"record","name":"k138_v139","fields":[{"name":"key","type":"int","field-id":138},{"name":"value","type":"long","field-id":139}]},"logicalType":"map"}],"doc":"Map of column id to number of NaN values in the column","default":null,"field-id":137},{"name":"lower_bounds","type":["null",{"type":"array","items":{"type":"record","name":"k126_v127","fields":[{"name":"key","type":"int","field-id":126},{"name":"value","type":"bytes","field-id":127}]},"logicalType":"map"}],"doc":"Map of column id to lower bound","default":null,"field-id":125},{"name":"upper_bounds","type":["null",{"type":"array","items":{"type":"record","name":"k129_v130","fields":[{"name":"key","type":"int","field-id":129},{"name":"value","type":"bytes","field-id":130}]},"logicalType":"map"}],"doc":"Map of column id to upper bound","default":null,"field-id":128},{"name":"key_metadata","type":["null","bytes"],"doc":"Encryption key metadata blob","default":null,"field-id":131},{"name":"split_offsets","type":["null",{"type":"array","items":"long","element-id":133}],"doc":"Splittable offsets","default":null,"field-id":132},{"name":"equality_ids","type":["null",{"type":"array","items":"int","element-id":136}],"doc":"Equality comparison field IDs","default":null,"field-id":135},{"name":"sort_order_id","type":["null","int"],"doc":"Sort order ID","default":null,"field-id":140},{"name":"referenced_data_file","type":["null","string"],"doc":"Fully qualified location (URI with FS scheme) of a data file that all deletes reference","default":null,"field-id":143}]},"field-id":2}]}avro.codecdeflateformat-version2"partition-spec-id0iceberg.schemaž.{"type":"struct","schema-id":0,"fields":[{"id":0,"name":"status","required":true,"type":"int"},{"id":1,"name":"snapshot_id","required":false,"type":"long"},{"id":3,"name":"sequence_number","required":false,"type":"long"},{"id":4,"name":"file_sequence_number","required":false,"type":"long"},{"id":2,"name":"data_file","required":true,"type":{"type":"struct","fields":[{"id":134,"name":"content","required":true,"type":"int","doc":"Contents of the file: 0=data, 1=position deletes, 2=equality deletes"},{"id":100,"name":"file_path","required":true,"type":"string","doc":"Location URI with FS scheme"},{"id":101,"name":"file_format","required":true,"type":"string","doc":"File format name: avro, orc, or parquet"},{"id":102,"name":"partition","required":true,"type":{"type":"struct","fields":[]},"doc":"Partition data tuple, schema based on the partition spec"},{"id":103,"name":"record_count","required":true,"type":"long","doc":"Number of records in the file"},{"id":104,"name":"file_size_in_bytes","required":true,"type":"long","doc":"Total file size in bytes"},{"id":108,"name":"column_sizes","required":false,"type":{"type":"map","key-id":117,"key":"int","value-id":118,"value":"long","value-required":true},"doc":"Map of column id to total size on disk"},{"id":109,"name":"value_counts","required":false,"type":{"type":"map","key-id":119,"key":"int","value-id":120,"value":"long","value-required":true},"doc":"Map of column id to total count, including null and NaN"},{"id":110,"name":"null_value_counts","required":false,"type":{"type":"map","key-id":121,"key":"int","value-id":122,"value":"long","value-required":true},"doc":"Map of column id to null value count"},{"id":137,"name":"nan_value_counts","required":false,"type":{"type":"map","key-id":138,"key":"int","value-id":139,"value":"long","value-required":true},"doc":"Map of column id to number of NaN values in the column"},{"id":125,"name":"lower_bounds","required":false,"type":{"type":"map","key-id":126,"key":"int","value-id":127,"value":"binary","value-required":true},"doc":"Map of column id to lower bound"},{"id":128,"name":"upper_bounds","required":false,"type":{"type":"map","key-id":129,"key":"int","value-id":130,"value":"binary","value-required":true},"doc":"Map of column id to upper bound"},{"id":131,"name":"key_metadata","required":false,"type":"binary","doc":"Encryption key metadata blob"},{"id":132,"name":"split_offsets","required":false,"type":{"type":"list","element-id":133,"element":"long","element-required":true},"doc":"Splittable offsets"},{"id":135,"name":"equality_ids","required":false,"type":{"type":"list","element-id":136,"element":"int","element-required":true},"doc":"Equality comparison field IDs"},{"id":140,"name":"sort_order_id","required":false,"type":"int","doc":"Sort order ID"},{"id":143,"name":"referenced_data_file","required":false,"type":"string","doc":"Fully qualified location (URI with FS scheme) of a data file that all deletes reference"}]}}]}partition-spec[]contentdataà‘à�@˜3%e´ù–lÂgkÈå”Ák#UÇó#†1ÃB¡Ä1–6[2ÉÌdf: !>CŒ9hC EÊv'“7mµmb’vSÊÒEAWOËž<,ø(«Þ<,(]ðâ +Eö°Èº¨ìÁªeÁC|™„U,„=øæñ�÷æ}gx¿ßïøîǟܾsïÆ§GúÐw±Ÿ+d³-ºµÑnI;}éí$E’Út7»áÐí­e»�þ`­GûÙ[\Ø­M*Mwô÷¶��í5ÉÉé®êRÛɆf·r¦ìØ9›.ÚšëP—ªÙ¶=°³ª¬ê²©äW]QsƪÌ"·Jõ-Õ‘d×Õ¨mæ%WÑs’fP]ÊË +•\U–5ÛÑ•œÖÎtíÞk;t²ž®¿øÒ3 <<‡Ö|5™ošA+Tjáz¤µbõxmÆJ4ijÉÚ¬5g¥¬+me,ÙR-ígœ5ÝÂû\ñLi�\æÊor•‹\uÅz¹Þm¸Í‹Üò+ÀŒÙ ù‘Ç1„†1‚QŒag0�"&qç0… ˜Æ ʨ¢†šXÀ"–�`+Xe6uÃßô-ûÆ–à?ð€ „@€0D +1ˆÃ $@„$̤`Ò�TÐÀ +P„(Cª`AЄe¬!Ï€ÏýÈñÁV×Y·{�ЮÝ‹ ? ÃAˆüh4: £IDÂðÎõ'}o߈ +ã æ±0L—GqþóßjdFˆI•ÆO '0m')") - PanicIfError(trino.Config, err) - - _, err = trino.ExecContext(ctx, "ALTER TABLE "+quotedTrinoTablePath+" EXECUTE optimize_manifests") - PanicIfError(trino.Config, err) - - _, err = trino.ExecContext(ctx, "ALTER TABLE "+quotedTrinoTablePath+" EXECUTE expire_snapshots(retention_threshold => '0m')") - PanicIfError(trino.Config, err) + // ctx := context.Background() + + // _, err := trino.ExecContext(ctx, "ALTER TABLE "+quotedTrinoTablePath+" EXECUTE optimize") + // PanicIfError(trino.Config, err) + // + // _, err = trino.ExecContext(ctx, "ALTER TABLE "+quotedTrinoTablePath+" EXECUTE remove_orphan_files(retention_threshold => '0m')") + // PanicIfError(trino.Config, err) + + // _, err := trino.ExecContext(ctx, "ALTER TABLE "+quotedTrinoTablePath+" EXECUTE optimize_manifests") + // PanicIfError(trino.Config, err) + // + // _, err = trino.ExecContext(ctx, "ALTER TABLE "+quotedTrinoTablePath+" EXECUTE expire_snapshots(retention_threshold => '0m')") + // PanicIfError(trino.Config, err) } func (trino *Trino) CreateSchemaIfNotExists() { diff --git a/src/syncer-postgres/syncer_utils.go b/src/syncer-postgres/syncer_utils.go index 2aa5370..dbb897f 100644 --- a/src/syncer-postgres/syncer_utils.go +++ b/src/syncer-postgres/syncer_utils.go @@ -109,7 +109,7 @@ func (utils *SyncerUtils) InsertFromCappedBuffer(trino *common.Trino, quotedTrin rowValuesStatement := "(" + strings.Join(rowValues, ",") + ")" currentRowCount++ - if len(currentSql)+len(rowValuesStatement)+1 < common.TRINO_MAX_QUERY_LENGTH { // +1 for the comma + if currentRowCount == 1 { if currentSql != insertSqlPrefix { currentSql += "," }