diff --git a/.github/workflows/test-java.yml b/.github/workflows/test-java.yml index b6f0c21d8dc..d474919b63a 100644 --- a/.github/workflows/test-java.yml +++ b/.github/workflows/test-java.yml @@ -62,13 +62,13 @@ jobs: run: gradle --stop - name: Run mvn test - run: mvn test --no-transfer-progress -q -B -pl SQL-compiler -Dsurefire.failIfNoSpecifiedTests=false + run: mvn test --no-transfer-progress -B -pl SQL-compiler -Dsurefire.failIfNoSpecifiedTests=false working-directory: ./sql-to-dbsp-compiler if: ${{ vars.CI_DRY_RUN != 'true' }} - name: Run one quick SLT test if: ${{ vars.CI_DRY_RUN != 'true' }} - run: mvn test --no-transfer-progress -q -B -Dsurefire.failIfNoSpecifiedTests=false -Dtest=RotateTests#quick + run: mvn test --no-transfer-progress -B -Dsurefire.failIfNoSpecifiedTests=false -Dtest=RotateTests#quick working-directory: ./sql-to-dbsp-compiler - name: Print sccache stats diff --git a/crates/adapters/src/integrated.rs b/crates/adapters/src/integrated.rs index 7ea87c6be76..1e42e089e0e 100644 --- a/crates/adapters/src/integrated.rs +++ b/crates/adapters/src/integrated.rs @@ -36,7 +36,7 @@ where } /// Create an instance of an integrated output endpoint given its config -/// and output relation schema.p +/// and output relation schema. #[allow(unused)] pub fn create_integrated_output_endpoint( endpoint_id: EndpointId, diff --git a/crates/sqllib/src/source.rs b/crates/sqllib/src/source.rs index 255feacb498..4419f226b9e 100644 --- a/crates/sqllib/src/source.rs +++ b/crates/sqllib/src/source.rs @@ -69,7 +69,7 @@ impl fmt::Display for SourcePositionRange { /// Maps "keys" to source line position information #[doc(hidden)] -#[derive(Default)] +#[derive(Debug, Default)] pub struct SourceMap { map: BTreeMap<(&'static str, u32), SourcePosition>, } diff --git a/docs.feldera.com/docs/connectors/sources/debezium.md b/docs.feldera.com/docs/connectors/sources/debezium.md index 8d70f658d0f..d25eb09e5be 100644 --- a/docs.feldera.com/docs/connectors/sources/debezium.md +++ b/docs.feldera.com/docs/connectors/sources/debezium.md @@ -194,7 +194,7 @@ CREATE TABLE my_table ( ) ``` -::: warning +:::warning Notice that the data format provided by the connector contains data and metadata columns; for example, a metadata column is the "operation", which diff --git a/docs.feldera.com/docs/connectors/unique_keys.md b/docs.feldera.com/docs/connectors/unique_keys.md index fcee0791517..668244e61be 100644 --- a/docs.feldera.com/docs/connectors/unique_keys.md +++ b/docs.feldera.com/docs/connectors/unique_keys.md @@ -19,7 +19,7 @@ sources and data delivery to external sinks in two key ways: from tables *without* primary keys. Deleting a non-existent record for such tables will produce unexpected results or even pipeline crashes. -::: danger +:::danger Deleting records that do not exist from a table without a primary key is an undefined operation, and can produce incorrect results or crashes. diff --git a/docs.feldera.com/docs/sql/grammar.md b/docs.feldera.com/docs/sql/grammar.md index 2cade6f1709..80e3492dbff 100644 --- a/docs.feldera.com/docs/sql/grammar.md +++ b/docs.feldera.com/docs/sql/grammar.md @@ -347,7 +347,7 @@ values : { VALUES | VALUE } expression [, expression ]* select - : SELECT [ ALL | DISTINCT ] + : SELECT [ hintComment ] [ ALL | DISTINCT ] { projectItem [, projectItem ]* } FROM tableExpression [ WHERE booleanExpression ] @@ -360,7 +360,7 @@ select ``` tablePrimary : tableName '(' TABLE tableName ')' - | tablePrimary '(' columnDecl [, columnDecl ]* ')' + | tablePrimary [ hintComment ] '(' columnDecl [, columnDecl ]* ')' | [ LATERAL ] '(' query ')' | UNNEST '(' expression ')' [ WITH ORDINALITY ] | TABLE '(' functionName '(' expression [, expression ]* ')' ')' @@ -442,6 +442,67 @@ the column names are *not* used to reorder columns. In `orderItem`, if expression is a positive integer n, it denotes the nth item in the `SELECT` clause. +## SQL hints + +A hint is an instruction to the optimizer. When writing SQL, you may +know information about the data unknown to the optimizer. Hints +enable you to make decisions normally made by the optimizer. + +We support hints in two locations: + +- Query Hint: right after the `SELECT` keyword; +- Table Hint: right after the referenced table or view name. + +``` +SELECT /*+ broadcast(S), shard(T) */ +FROM + T /*+ size(5) */ +JOIN + S +``` + +The syntax of hints is: + +``` +hintComment + : '/*+' hint [, hint ]* '*/' + +hint: + hintName + | hintName '(' optionKey '=' optionVal [, optionKey '=' optionVal ]* ')' + | hintName '(' hintOption [, hintOption ]* ')' + +optionKey + : simpleIdentifier + | stringLiteral + +optionVal + : simpleIdentifier + | stringLiteral + +hintOption + : simpleIdentifier + | numericLiteral + | stringLiteral +``` + +### Supported hints and their impact on query implementation + +:::warning + +These hints are considered still experimental, and they may change + +::: + +- `broadcast(`*table*`)`: Indicates that the following `JOIN` should be implemented using + a broadcast-join strategy by broadcasting the input with alias *table* +- `shard(`*table*`)`: Indicates that the following `JOIN` should be implemented using + a hash-join strategy by sharding the input with alias *table* +- `balance(`*table*`)`: Indicates that the following `JOIN` should be implemented using + a balanced strategy by hashing on all fields the input with alias *table* + +Note: specifying hints may inhibit some compiler optimizations. + ## Creating indexes Feldera supports the `CREATE INDEX` SQL statement with the following diff --git a/docs.feldera.com/docs/sql/identifiers.md b/docs.feldera.com/docs/sql/identifiers.md index ad4b793f0ab..77363300255 100644 --- a/docs.feldera.com/docs/sql/identifiers.md +++ b/docs.feldera.com/docs/sql/identifiers.md @@ -107,7 +107,7 @@ GROUP BY x+1 HAVING x+1 > 0; ``` -::: danger +:::danger In order to preserve the semantics of standard SQL programs unchanged, a new column alias is used only if there is no column with the same diff --git a/js-packages/profiler-lib/src/profile.ts b/js-packages/profiler-lib/src/profile.ts index 57b176c63b6..6bc32d8d7bf 100644 --- a/js-packages/profiler-lib/src/profile.ts +++ b/js-packages/profiler-lib/src/profile.ts @@ -26,7 +26,7 @@ interface JsonMetricBase { interface MetricValue { }; // Base types -interface PersistentIdMetricValue extends MetricValue { +interface StringMetricValue extends MetricValue { type: "string"; value: string; } @@ -54,6 +54,11 @@ interface BytesMetricValue { value: number; } +interface BoolMetricValue { + type: "bool"; + value: boolean; +} + interface PercentMetricValue { type: "percent"; value: { @@ -110,12 +115,13 @@ interface CacheStatsValue extends MetricValue { } type JsonMetricValue = + BoolMetricValue | CountMetricValue | BytesMetricValue | PercentMetricValue | DurationMetricValue | StatsMetricValue | - PersistentIdMetricValue | + StringMetricValue | DistributionValue | RetainmentValue | IntMetricValue | @@ -375,6 +381,10 @@ class BooleanValue extends PropertyValue { this.value = id; } + static fromBoolMetric(value: BoolMetricValue): BooleanValue { + return new BooleanValue(value.value); + } + getNumericValue(): Option { return this.value ? Option.some(0) : Option.some(1); } @@ -721,12 +731,17 @@ export class Measurement { let metric_id = metric.metric_id; let kind = metric_id.split("_").pop(); // count, bytes, etc if (metric.labels) { - let labels = metric.labels.map(e => e.join(":")).join("."); - if (metric.labels.length !== 0) { - metric_id = metric_id + "." + labels; - } + let labels = metric.labels.map(e => e.join(":")).join("."); + if (metric.labels.length !== 0) { + metric_id = metric_id + "." + labels; + } } switch (kind) { + case "bool": { + let m = metric.value as BoolMetricValue; + let value = BooleanValue.fromBoolMetric(m); + return [new Measurement(metric_id, Option.some(value))]; + } case "bytes": { let m = metric.value as BytesMetricValue; let bytes = BytesValue.fromBytesMetric(m); @@ -767,7 +782,7 @@ export class Measurement { ]; } case "id": { // persistent_id - let s = metric.value as PersistentIdMetricValue; + let s = metric.value as StringMetricValue; let str = StringValue.fromString(s.value); return [new Measurement(metric_id, Option.some(str))]; } @@ -845,6 +860,11 @@ export class Measurement { new Measurement(metric_id + ".steps", Option.some(steps)), ]; } + case "policy": { + let s = metric.value as StringMetricValue; + let str = StringValue.fromString(s.value); + return [new Measurement(metric_id, Option.some(str))]; + } } return []; } diff --git a/python/tests/runtime_aggtest/arithmetic_tests/test_date.py b/python/tests/runtime_aggtest/arithmetic_tests/test_date.py index c751b0f8346..a181e7c89e1 100644 --- a/python/tests/runtime_aggtest/arithmetic_tests/test_date.py +++ b/python/tests/runtime_aggtest/arithmetic_tests/test_date.py @@ -335,7 +335,7 @@ def __init__(self): "ytm_str": "+10-01", "dth_str": "+3683 00", "dtm_str": "+3683 00:00", - "dts_str": "+3683 00:00:00.000000", + "dts_str": "+3683 00:00:00", "htm_str": "+88392:00", "hts_str": "+88392:00:00.000000", "mts_str": "+5303520:00.000000", @@ -345,7 +345,7 @@ def __init__(self): "ytm_str": "-2-08", "dth_str": "-980 00", "dtm_str": "-980 00:00", - "dts_str": "-980 00:00:00.000000", + "dts_str": "-980 00:00:00", "htm_str": "-23520:00", "hts_str": "-23520:00:00.000000", "mts_str": "-1411200:00.000000", @@ -355,7 +355,7 @@ def __init__(self): "ytm_str": "+20-06", "dth_str": "+7506 00", "dtm_str": "+7506 00:00", - "dts_str": "+7506 00:00:00.000000", + "dts_str": "+7506 00:00:00", "htm_str": "+180144:00", "hts_str": "+180144:00:00.000000", "mts_str": "+10808640:00.000000", @@ -646,7 +646,7 @@ def __init__(self): "ytm_neg": "-10-01", "dth_neg": "-3683 00", "dtm_neg": "-3683 00:00", - "dts_neg": "-3683 00:00:00.000000", + "dts_neg": "-3683 00:00:00", "htm_neg": "-88392:00", "hts_neg": "-88392:00:00.000000", "mts_neg": "-5303520:00.000000", @@ -656,7 +656,7 @@ def __init__(self): "ytm_neg": "+2-08", "dth_neg": "+980 00", "dtm_neg": "+980 00:00", - "dts_neg": "+980 00:00:00.000000", + "dts_neg": "+980 00:00:00", "htm_neg": "+23520:00", "hts_neg": "+23520:00:00.000000", "mts_neg": "+1411200:00.000000", @@ -666,7 +666,7 @@ def __init__(self): "ytm_neg": "-20-06", "dth_neg": "-7506 00", "dtm_neg": "-7506 00:00", - "dts_neg": "-7506 00:00:00.000000", + "dts_neg": "-7506 00:00:00", "htm_neg": "-180144:00", "hts_neg": "-180144:00:00.000000", "mts_neg": "-10808640:00.000000", diff --git a/python/tests/runtime_aggtest/arithmetic_tests/test_interval.py b/python/tests/runtime_aggtest/arithmetic_tests/test_interval.py index b4d8adeae74..74701952f71 100644 --- a/python/tests/runtime_aggtest/arithmetic_tests/test_interval.py +++ b/python/tests/runtime_aggtest/arithmetic_tests/test_interval.py @@ -350,7 +350,7 @@ def __init__(self): "dth_days": "+1705 19", "dth_dth": "+1704 21", "dtm_dtm": "+1704 20:47", - "dts_dts": "+1704 20:46:46.000000", + "dts_dts": "+1704 20:46:46", "htm_hr": "+40939:42", "htm_htm": "+40939:29", "hts_hts": "+40939:28:20.000000", @@ -364,7 +364,7 @@ def __init__(self): "dth_days": "-1130 04", "dth_dth": "-1131 02", "dtm_dtm": "-1131 02:55", - "dts_dts": "-1131 02:55:14.000000", + "dts_dts": "-1131 02:55:14", "htm_hr": "-27124:00", "htm_htm": "-27124:13", "hts_hts": "-27124:13:40.000000", @@ -378,7 +378,7 @@ def __init__(self): "dth_days": "+3703 02", "dth_dth": "+3702 04", "dtm_dtm": "+3702 03:22", - "dts_dts": "+3702 03:21:46.000000", + "dts_dts": "+3702 03:21:46", "htm_hr": "+88874:17", "htm_htm": "+88874:04", "hts_hts": "+88874:03:20.000000", @@ -413,7 +413,7 @@ def __init__(self): "dth_days": "+2005 19", "dth_dth": "+2006 17", "dtm_dtm": "+2006 18:37", - "dts_dts": "+2006 18:37:14.000000", + "dts_dts": "+2006 18:37:14", "htm_hr": "+48139:42", "htm_htm": "+48139:55", "hts_hts": "+48139:55:40.000000", @@ -427,7 +427,7 @@ def __init__(self): "dth_days": "-830 04", "dth_dth": "-829 06", "dtm_dtm": "-829 05:05", - "dts_dts": "-829 05:04:46.000000", + "dts_dts": "-829 05:04:46", "htm_hr": "-19924:00", "htm_htm": "-19923:47", "hts_hts": "-19923:46:20.000000", @@ -441,7 +441,7 @@ def __init__(self): "dth_days": "+4003 02", "dth_dth": "+4004 00", "dtm_dtm": "+4004 01:12", - "dts_dts": "+4004 01:12:14.000000", + "dts_dts": "+4004 01:12:14", "htm_hr": "+96074:17", "htm_htm": "+96074:30", "hts_hts": "+96074:30:40.000000", @@ -476,7 +476,7 @@ def __init__(self): "const_dth_days": "+1705 19", "const_dth_dth": "+1704 21", "const_dtm_dtm": "+1704 20:47", - "const_dts_dts": "+1704 20:46:46.000000", + "const_dts_dts": "+1704 20:46:46", "const_htm_hr": "+40939:42", "const_htm_htm": "+40939:29", "const_hts_hts": "+40939:28:20.000000", @@ -508,7 +508,7 @@ def __init__(self): "const_dth_days": "+2005 19", "const_dth_dth": "+2006 17", "const_dtm_dtm": "+2006 18:37", - "const_dts_dts": "+2006 18:37:14.000000", + "const_dts_dts": "+2006 18:37:14", "const_htm_hr": "+48139:42", "const_htm_htm": "+48139:55", "const_hts_hts": "+48139:55:40.000000", @@ -597,7 +597,7 @@ def __init__(self): self.data = [ { "id": 0, - "seconds": "+414165762.000000", + "seconds": "+414165762.360000", "minutes": "+6902762", "hours": "+115046", "days": "+4793", @@ -606,7 +606,7 @@ def __init__(self): }, { "id": 1, - "seconds": "-218744971.000000", + "seconds": "-218744971.200000", "minutes": "-3645749", "hours": "-60762", "days": "-2531", @@ -615,7 +615,7 @@ def __init__(self): }, { "id": 2, - "seconds": "+859899865.000000", + "seconds": "+859899865.860000", "minutes": "+14331664", "hours": "+238861", "days": "+9952", @@ -640,7 +640,7 @@ def __init__(self): self.data = [ { "id": 0, - "seconds": "+62076236.000000", + "seconds": "+62076236.933797", "minutes": "+1034603", "hours": "+17243", "days": "+718", @@ -649,7 +649,7 @@ def __init__(self): }, { "id": 1, - "seconds": "-32786062.000000", + "seconds": "-32786062.717770", "minutes": "-546434", "hours": "-9107", "days": "-379", @@ -658,7 +658,7 @@ def __init__(self): }, { "id": 2, - "seconds": "+128884018.000000", + "seconds": "+128884018.583042", "minutes": "+2148066", "hours": "+35801", "days": "+1491", @@ -727,7 +727,7 @@ def __init__(self): self.data = [ { "id": 0, - "sec_str": "+414165762.000000", + "sec_str": "+414165762.360000", "min_str": "+6902762", "hrs_str": "+115044", "days_str": "+4791", @@ -736,7 +736,7 @@ def __init__(self): }, { "id": 1, - "sec_str": "-218744971.000000", + "sec_str": "-218744971.200000", "min_str": "-3645749", "hrs_str": "-60762", "days_str": "-2531", @@ -745,7 +745,7 @@ def __init__(self): }, { "id": 2, - "sec_str": "+859899865.000000", + "sec_str": "+859899865.860000", "min_str": "+14331664", "hrs_str": "+238860", "days_str": "+9952", @@ -770,7 +770,7 @@ def __init__(self): self.data = [ { "id": 0, - "sec_str": "+62076236.000000", + "sec_str": "+62076236.933797", "min_str": "+1034603", "hrs_str": "+17243", "days_str": "+718", @@ -779,7 +779,7 @@ def __init__(self): }, { "id": 1, - "sec_str": "-32786062.000000", + "sec_str": "-32786062.717770", "min_str": "-546434", "hrs_str": "-9107", "days_str": "-379", @@ -788,7 +788,7 @@ def __init__(self): }, { "id": 2, - "sec_str": "+128884018.000000", + "sec_str": "+128884018.583042", "min_str": "+2148066", "hrs_str": "+35801", "days_str": "+1491", @@ -829,7 +829,7 @@ def __init__(self): "ytm_neg": "-5-00", "dth_neg": "-1855 19", "dtm_neg": "-1855 19:42", - "dts_neg": "-1855 19:42:00.000000", + "dts_neg": "-1855 19:42:00", "htm_neg": "-44539:42", "hts_neg": "-44539:42:00.000000", "mts_neg": "-2672382:00.000000", @@ -839,7 +839,7 @@ def __init__(self): "ytm_neg": "+2-08", "dth_neg": "+980 04", "dtm_neg": "+980 04:00", - "dts_neg": "+980 04:00:00.000000", + "dts_neg": "+980 04:00:00", "htm_neg": "+23524:00", "hts_neg": "+23524:00:00.000000", "mts_neg": "+1411440:00.000000", @@ -849,7 +849,7 @@ def __init__(self): "ytm_neg": "-10-06", "dth_neg": "-3853 02", "dtm_neg": "-3853 02:17", - "dts_neg": "-3853 02:17:00.000000", + "dts_neg": "-3853 02:17:00", "htm_neg": "-92474:17", "hts_neg": "-92474:17:00.000000", "mts_neg": "-5548457:00.000000", @@ -876,30 +876,30 @@ def __init__(self): "ytm": "+12-10", "dth": "+4793 14", "dtm": "+4793 14:02", - "dts": "+4793 14:02:42.000000", + "dts": "+4793 14:02:42.360000", "htm": "+115046:02", - "hts": "+115046:02:42.000000", - "mts": "+6902762:42.000000", + "hts": "+115046:02:42.360000", + "mts": "+6902762:42.360000", }, { "id": 1, "ytm": "-6-10", "dth": "-2531 18", "dtm": "-2531 18:29", - "dts": "-2531 18:29:31.000000", + "dts": "-2531 18:29:31.200000", "htm": "-60762:29", - "hts": "-60762:29:31.000000", - "mts": "-3645749:31.000000", + "hts": "-60762:29:31.200000", + "mts": "-3645749:31.200000", }, { "id": 2, "ytm": "+27-01", "dth": "+9952 13", "dtm": "+9952 13:04", - "dts": "+9952 13:04:25.000000", + "dts": "+9952 13:04:25.860000", "htm": "+238861:04", - "hts": "+238861:04:25.000000", - "mts": "+14331664:25.000000", + "hts": "+238861:04:25.860000", + "mts": "+14331664:25.860000", }, ] self.sql = """CREATE MATERIALIZED VIEW iinterval_mul_double AS SELECT @@ -923,30 +923,30 @@ def __init__(self): "ytm": "+1-11", "dth": "+718 11", "dtm": "+718 11:23", - "dts": "+718 11:23:56.000000", + "dts": "+718 11:23:56.933797", "htm": "+17243:23", - "hts": "+17243:23:56.000000", - "mts": "+1034603:56.000000", + "hts": "+17243:23:56.933797", + "mts": "+1034603:56.933797", }, { "id": 1, "ytm": "-1-00", "dth": "-379 11", "dtm": "-379 11:14", - "dts": "-379 11:14:22.000000", + "dts": "-379 11:14:22.717770", "htm": "-9107:14", - "hts": "-9107:14:22.000000", - "mts": "-546434:22.000000", + "hts": "-9107:14:22.717770", + "mts": "-546434:22.717770", }, { "id": 2, "ytm": "+4-00", "dth": "+1491 17", "dtm": "+1491 17:06", - "dts": "+1491 17:06:58.000000", + "dts": "+1491 17:06:58.583042", "htm": "+35801:06", - "hts": "+35801:06:58.000000", - "mts": "+2148066:58.000000", + "hts": "+35801:06:58.583042", + "mts": "+2148066:58.583042", }, ] self.sql = """CREATE MATERIALIZED VIEW iinterval_div_double AS SELECT @@ -988,7 +988,7 @@ def __init__(self): "const_ytm_neg": "+2-08", "const_dth_neg": "+980 04", "const_dtm_neg": "+980 04:00", - "const_dts_neg": "+980 04:00:00.000000", + "const_dts_neg": "+980 04:00:00", "const_htm_neg": "+23524:00", "const_hts_neg": "+23524:00:00.000000", "const_mts_neg": "+1411440:00.000000", @@ -1012,10 +1012,10 @@ def __init__(self): "const_ytm": "-6-10", "const_dth": "-2531 18", "const_dtm": "-2531 18:29", - "const_dts": "-2531 18:29:31.000000", + "const_dts": "-2531 18:29:31.200000", "const_htm": "-60762:29", - "const_hts": "-60762:29:31.000000", - "const_mts": "-3645749:31.000000", + "const_hts": "-60762:29:31.200000", + "const_mts": "-3645749:31.200000", } ] self.sql = """CREATE MATERIALIZED VIEW const_interval_mul_double AS SELECT @@ -1036,10 +1036,10 @@ def __init__(self): "const_ytm": "-1-00", "const_dth": "-379 11", "const_dtm": "-379 11:14", - "const_dts": "-379 11:14:22.000000", + "const_dts": "-379 11:14:22.717771", "const_htm": "-9107:14", - "const_hts": "-9107:14:22.000000", - "const_mts": "-546434:22.000000", + "const_hts": "-9107:14:22.717771", + "const_mts": "-546434:22.717771", } ] self.sql = """CREATE MATERIALIZED VIEW const_interval_div_double AS SELECT diff --git a/python/tests/runtime_aggtest/arithmetic_tests/test_timestamp.py b/python/tests/runtime_aggtest/arithmetic_tests/test_timestamp.py index 42ad84e4efe..2c7ff990ad6 100644 --- a/python/tests/runtime_aggtest/arithmetic_tests/test_timestamp.py +++ b/python/tests/runtime_aggtest/arithmetic_tests/test_timestamp.py @@ -384,7 +384,7 @@ def __init__(self): "ytm_str": "+5-00", "dth_str": "+1855 19", "dtm_str": "+1855 19:42", - "dts_str": "+1855 19:42:00.000000", + "dts_str": "+1855 19:42:00", "htm_str": "+44539:42", "hts_str": "+44539:42:00.000000", "mts_str": "+2672382:00.000000", @@ -394,7 +394,7 @@ def __init__(self): "ytm_str": "-2-08", "dth_str": "-980 04", "dtm_str": "-980 04:00", - "dts_str": "-980 04:00:00.000000", + "dts_str": "-980 04:00:00", "htm_str": "-23524:00", "hts_str": "-23524:00:00.000000", "mts_str": "-1411440:00.000000", @@ -404,7 +404,7 @@ def __init__(self): "ytm_str": "+10-06", "dth_str": "+3853 02", "dtm_str": "+3853 02:17", - "dts_str": "+3853 02:17:00.000000", + "dts_str": "+3853 02:17:00", "htm_str": "+92474:17", "hts_str": "+92474:17:00.000000", "mts_str": "+5548457:00.000000", diff --git a/python/tests/runtime_aggtest/illarg_tests/test_cast.py b/python/tests/runtime_aggtest/illarg_tests/test_cast.py index ef18f661c44..c1384ba893a 100644 --- a/python/tests/runtime_aggtest/illarg_tests/test_cast.py +++ b/python/tests/runtime_aggtest/illarg_tests/test_cast.py @@ -174,7 +174,7 @@ def __init__(self): "to_decimal1": None, "to_real": Decimal("1.5927495E+12"), "to_double": Decimal("1592749424123"), - "to_varchar": "2020-06-21 14:23:44", + "to_varchar": "2020-06-21 14:23:44.123654", "to_date": "2020-06-21", "to_time": "14:23:44.123654", } @@ -209,7 +209,10 @@ def __init__(self): class illarg_cast_time_legal(TstView): def __init__(self): self.data = [ - {"to_varchar": "14:23:44", "to_timestamp": "1970-01-01T14:23:44.456"} + { + "to_varchar": "14:23:44.456000000", + "to_timestamp": "1970-01-01T14:23:44.456", + } ] self.sql = """CREATE MATERIALIZED VIEW cast_time_legal AS SELECT CAST(tme AS VARCHAR) AS to_varchar, diff --git a/python/tests/runtime_aggtest/illarg_tests/test_grammar_tbl_fn.py b/python/tests/runtime_aggtest/illarg_tests/test_grammar_tbl_fn.py index f9ba7f238f6..4cf5559102a 100644 --- a/python/tests/runtime_aggtest/illarg_tests/test_grammar_tbl_fn.py +++ b/python/tests/runtime_aggtest/illarg_tests/test_grammar_tbl_fn.py @@ -515,19 +515,6 @@ def __init__(self): ROW_NUMBER() OVER (ORDER BY intt DESC) <= 2""" -# Negative Test -class illarg_qualify_top_k_illegal(TstView): - def __init__(self): - self.sql = """CREATE MATERIALIZED VIEW qualify_top_k_illegal AS SELECT - id, intt - FROM illegal_tbl - QUALIFY - ROW_NUMBER() OVER (ORDER BY intt DESC) <= 2 OR - RANK() OVER (ORDER BY intt DESC) <= 2 OR - DENSE_RANK() OVER (ORDER BY intt DESC) <= 2""" - self.expected_error = "Not yet implemented" - - # IS TRUE/IS FALSE/IS NOT TRUE/IS NOT FALSE class illarg_is_true_false_legal(TstView): def __init__(self): diff --git a/python/tests/runtime_aggtest/illarg_tests/test_str_bin_type_fn.py b/python/tests/runtime_aggtest/illarg_tests/test_str_bin_type_fn.py index c417916a7df..6c32f42e4d2 100644 --- a/python/tests/runtime_aggtest/illarg_tests/test_str_bin_type_fn.py +++ b/python/tests/runtime_aggtest/illarg_tests/test_str_bin_type_fn.py @@ -54,7 +54,7 @@ def __init__(self): "reall": "-57681.18-57681.18", "dbl": "-38.2711234601246-38.2711234601246", "booll": "TRUETRUE", - "tmestmp": "2020-06-21 14:23:442020-06-21 14:23:44", + "tmestmp": "2020-06-21 14:23:44.1236542020-06-21 14:23:44.123654", "uuidd": "42b8fec7-c7a3-4531-9611-4bde80f9cb4c42b8fec7-c7a3-4531-9611-4bde80f9cb4c", } ] @@ -160,7 +160,7 @@ def __init__(self): class illarg_right_cast_legal(TstView): def __init__(self): # checked manually - self.data = [{"tmestmp": "44", "uuidd": "4c"}] + self.data = [{"tmestmp": "54", "uuidd": "4c"}] self.sql = """CREATE MATERIALIZED VIEW right_cast_legal AS SELECT RIGHT(tmestmp, 2) AS tmestmp, RIGHT(uuidd, 2) AS uuidd diff --git a/python/tests/runtime_aggtest/illarg_tests2/test_window_agg.py b/python/tests/runtime_aggtest/illarg_tests2/test_window_agg.py index 5151a0ac7b4..4bdcb8ca82a 100644 --- a/python/tests/runtime_aggtest/illarg_tests2/test_window_agg.py +++ b/python/tests/runtime_aggtest/illarg_tests2/test_window_agg.py @@ -87,13 +87,14 @@ def __init__(self): # Negative Test -class illarg_window_dense_rank_illegal(TstView): - def __init__(self): - # checked manually - self.sql = """CREATE MATERIALIZED VIEW illarg_window_dense_rank_illegal AS SELECT - id, DENSE_RANK() OVER (ORDER BY 1) AS count - FROM illegal_tbl""" - self.expected_error = "DENSE_RANK only supported in a TopK pattern" +# TODO: fix this test, it now should pass +# class illarg_window_dense_rank_illegal(TstView): +# def __init__(self): +# # checked manually +# self.sql = """CREATE MATERIALIZED VIEW illarg_window_dense_rank_illegal AS SELECT +# id, DENSE_RANK() OVER (ORDER BY 1) AS count +# FROM illegal_tbl""" +# self.expected_error = "DENSE_RANK only supported in a TopK pattern" # LAG @@ -155,13 +156,14 @@ def __init__(self): # Negative Test -class illarg_window_rank_illegal(TstView): - def __init__(self): - # checked manually - self.sql = """CREATE MATERIALIZED VIEW illarg_window_rank_illegal AS SELECT - id, RANK() OVER (ORDER BY NULL) AS count - FROM illegal_tbl""" - self.expected_error = "RANK only supported in a TopK pattern" +# TODO: fix this test, it now should pass +# class illarg_window_rank_illegal(TstView): +# def __init__(self): +# # checked manually +# self.sql = """CREATE MATERIALIZED VIEW illarg_window_rank_illegal AS SELECT +# id, RANK() OVER (ORDER BY NULL) AS count +# FROM illegal_tbl""" +# self.expected_error = "RANK only supported in a TopK pattern" # MAX @@ -219,13 +221,14 @@ def __init__(self): # Negative Test -class illarg_window_row_num_illegal(TstView): - def __init__(self): - # checked manually - self.sql = """CREATE MATERIALIZED VIEW illarg_window_row_num_illegal AS SELECT - id, ROW_NUMBER() OVER (ORDER BY NULL) AS count - FROM illegal_tbl""" - self.expected_error = "ROW_NUMBER only supported in a TopK pattern" +# TODO: fix this test, it now should pass +# class illarg_window_row_num_illegal(TstView): +# def __init__(self): +# # checked manually +# self.sql = """CREATE MATERIALIZED VIEW illarg_window_row_num_illegal AS SELECT +# id, ROW_NUMBER() OVER (ORDER BY NULL) AS count +# FROM illegal_tbl""" +# self.expected_error = "ROW_NUMBER only supported in a TopK pattern" # SUM diff --git a/sql-to-dbsp-compiler/SQL-compiler/pom.xml b/sql-to-dbsp-compiler/SQL-compiler/pom.xml index ad7503c8545..4381b142979 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/pom.xml +++ b/sql-to-dbsp-compiler/SQL-compiler/pom.xml @@ -73,6 +73,7 @@ maven-surefire-plugin 3.1.2 + alphabetical -ea diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/annotation/JoinStrategy.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/annotation/JoinStrategy.java new file mode 100644 index 00000000000..27c9fd15957 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/annotation/JoinStrategy.java @@ -0,0 +1,70 @@ +package org.dbsp.sqlCompiler.circuit.annotation; + +import com.fasterxml.jackson.databind.JsonNode; +import org.dbsp.sqlCompiler.compiler.errors.SourcePositionRange; +import org.dbsp.util.JsonStream; +import org.dbsp.util.Utilities; + +public class JoinStrategy extends Annotation { + public final SourcePositionRange position; + public final Strategy strategy; + /** Which input to apply the strategy to */ + public final int input; + /** Original input name; used for error reporting only */ + public final String inputName; + + public enum Strategy { + /** Broadcast */ + Broadcast, + /** Shard */ + Shard, + /** Dynamic join balancing by hashing entire record */ + Balance, + } + + public JoinStrategy(SourcePositionRange range, Strategy strategy, int input, String inputName) { + this.position = range; + this.strategy = strategy; + this.input = input; + this.inputName = inputName; + } + + @Override + public void asJson(JsonStream stream) { + stream.beginObject() + .appendClass(this) + .label("strategy") + .append(this.strategy.name()) + .label("input") + .append(this.input) + .label("inputName") + .append(this.inputName) + .endObject(); + } + + public SourcePositionRange getPosition() { + return this.position; + } + + public boolean compatible(JoinStrategy other) { + return this.strategy == other.strategy && this.input == other.input; + } + + @Override + public String toString() { + return this.strategy.name() + "(" + this.inputName + ")"; + } + + public String toRust() { + // Does not contain input; the strategy is actually applied to the corresponding input stream + String policy = this.strategy.name(); + return "BalancerHint::Policy(Some(PartitioningPolicy::" + policy + "))"; + } + + public static JoinStrategy fromJson(JsonNode node) { + Strategy strategy = Strategy.valueOf(Utilities.getStringProperty(node, "strategy")); + int input = Utilities.getIntProperty(node, "input"); + String inputName = Utilities.getStringProperty(node, "inputName"); + return new JoinStrategy(SourcePositionRange.INVALID, strategy, input, inputName); + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInputMapWithWaterlineOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInputMapWithWaterlineOperator.java index b0b58f15209..a162d9a83a0 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInputMapWithWaterlineOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInputMapWithWaterlineOperator.java @@ -98,7 +98,7 @@ public DBSPType outputType(int outputNo) { case 0 -> this.outputType; case 1 -> this.errorType; case 2 -> new DBSPTypeTypedBox(this.lub.getResultType(), false); - default -> throw new InternalCompilerError("Unpexected output " + outputNo); + default -> throw new InternalCompilerError("Unexpected output " + outputNo); }; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java index 2c02ea6ec8e..58d249c8af7 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java @@ -84,7 +84,7 @@ import org.dbsp.util.IndentStream; import org.dbsp.util.Linq; import org.dbsp.util.Logger; -import org.dbsp.util.RelJsonWriter; +import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.RelJsonWriter; import org.dbsp.util.Utilities; import javax.annotation.Nullable; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/BaseRustCodeGenerator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/BaseRustCodeGenerator.java index b34217c1fc3..a886b5aca8a 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/BaseRustCodeGenerator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/BaseRustCodeGenerator.java @@ -25,7 +25,6 @@ public abstract class BaseRustCodeGenerator implements ICodeGenerator { boolean generateUdfInclude = true; boolean generateMalloc = true; boolean generateTuples = true; - boolean declareSourceMap = false; protected BaseRustCodeGenerator() { this.id = crdId++; @@ -39,11 +38,6 @@ public BaseRustCodeGenerator withGenerateTuples(boolean generate) { return this; } - public BaseRustCodeGenerator withDeclareSourceMap(boolean declare) { - this.declareSourceMap = declare; - return this; - } - public BaseRustCodeGenerator withUdf(boolean udf) { this.generateUdfInclude = udf; return this; @@ -100,7 +94,8 @@ public void addDependency(String crate) { static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; #[allow(non_upper_case_globals)] #[export_name = "malloc_conf"] - pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\\0";"""; + pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\\0"; + """; public static final String STANDARD_PREAMBLE = """ use dbsp::{ @@ -108,10 +103,19 @@ public void addDependency(String crate) { UnimplementedSemigroup, DefaultSemigroup, HasOne, HasZero, AddByRef, NegByRef, AddAssignByRef, }, - circuit::{checkpointer::Checkpoint, Circuit, CircuitConfig, NestedCircuit, RootCircuit, Stream, StepSize}, + circuit::{ + checkpointer::Checkpoint, + circuit_builder::CircuitBase, + Circuit, CircuitConfig, + NestedCircuit, + RootCircuit, + Stream, + StepSize + }, operator::{ apply_n, dynamic::aggregate::{ArgMinSome, Max, Min, MinSome1, Postprocess}, + dynamic::balance::*, ConstantGenerator, Generator, Fold, diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustFileWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustFileWriter.java index 58150323bf0..345e1a42c1e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustFileWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustFileWriter.java @@ -20,7 +20,6 @@ public class RustFileWriter extends RustWriter { StructuresUsed used = new StructuresUsed(); boolean findUsed = true; boolean slt = false; - boolean test = false; final LateMaterializations materializations; public RustFileWriter(LateMaterializations materializations) { @@ -42,12 +41,6 @@ String rustPreamble() { /** Special support for running the SLT tests */ public RustFileWriter forSlt() { this.slt = true; - this.test = true; - return this; - } - - public RustFileWriter withTest(boolean test) { - this.test = test; return this; } @@ -119,17 +112,13 @@ public void write(DBSPCompiler compiler) { this.outputBuilder.append(BaseRustCodeGenerator.ALLOC_PREAMBLE); if (this.generateUdfInclude) this.generateUdfInclude(); - if (this.test) + if (compiler.options.ioOptions.testing) this.builder().append(""" #[cfg(test)] use readers::*;""").newline(); for (String dep : this.dependencies) - this.builder().append("use ").append(dep).append("::*;"); - - if (this.declareSourceMap) { - SourcePositionResource.generateDeclaration(this.outputBuilder); - } + this.builder().append("use ").append(dep).append("::*;").newline(); ToRustInnerVisitor innerVisitor = new ToRustInnerVisitor(compiler, this.builder(), null, false); ProjectDeclarations declarationsDone = new ProjectDeclarations(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/SourcePositionResource.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/SourcePositionResource.java index 35e462f894f..568cbd62b9d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/SourcePositionResource.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/SourcePositionResource.java @@ -57,22 +57,26 @@ HashAndIndex getKey(@Nullable DBSPOperator operator, DBSPHandleErrorExpression e return new HashAndIndex(name, expression.index); } - public static final String STATIC_NAME = "SOURCE_MAP_STATIC"; + static final String STATIC_NAME_PREFIX = "SOURCE_MAP_STATIC"; public boolean isEmpty() { return this.keyToPosition.isEmpty(); } - public static void generateDeclaration(IIndentStream builder) { + private static String staticName(String name) { + return STATIC_NAME_PREFIX + "_" + name; + } + + public static void generateDeclaration(IIndentStream builder, String name) { builder.append("pub static ") - .append(STATIC_NAME) + .append(staticName(name)) .append(": ") .append("OnceLock = OnceLock::new();") .newline(); } - public void generateInitializer(IIndentStream builder) { - builder.append(STATIC_NAME) + public void generateInitializer(IIndentStream builder, String name) { + builder.append(staticName(name)) .append(".get_or_init(|| {") .increase() .append("let mut m = SourceMap::new();") @@ -93,11 +97,11 @@ public void generateInitializer(IIndentStream builder) { .newline(); } - public static void generateReference(IIndentStream builder, String sourceMapName) { + public static void generateReference(IIndentStream builder, String sourceMapName, String staticName) { builder.append("let ") .append(sourceMapName) .append(" = ") - .append(STATIC_NAME) + .append(staticName(staticName)) .append(".get().unwrap();") .newline(); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java index 4654f2f84de..02a2a8b6484 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java @@ -732,6 +732,17 @@ else if (Float.isInfinite(value)) { return VisitDecision.STOP; } + @Override + public VisitDecision preorder(DBSPWindowBoundExpression bound) { + String beforeAfter = bound.isPreceding ? "Before" : "After"; + this.builder.append("RelOffset::") + .append(beforeAfter) + .append("("); + bound.representation.accept(this); + this.builder.append(")"); + return VisitDecision.STOP; + } + @Override public VisitDecision preorder(DBSPComment comment) { if (comment.comment.contains("\n")) { @@ -2012,7 +2023,9 @@ public VisitDecision preorder(DBSPFunction function) { NeedsSourceMap finder = new NeedsSourceMap(this.compiler); finder.apply(function.body); if (finder.found) { - SourcePositionResource.generateReference(this.builder, CircuitWriter.SOURCE_MAP_VARIABLE_NAME); + SourcePositionResource.generateReference( + this.builder, CircuitWriter.SOURCE_MAP_VARIABLE_NAME, + Objects.requireNonNull(this.circuitContext).name); } function.body.accept(this); this.builder.decrease() diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java index c0a6ce0d964..8e379a192a9 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; +import org.dbsp.sqlCompiler.circuit.annotation.JoinStrategy; import org.dbsp.sqlCompiler.circuit.operator.*; import org.dbsp.sqlCompiler.circuit.OutputPort; import org.dbsp.sqlCompiler.circuit.annotation.OperatorHash; @@ -129,6 +130,7 @@ public class ToRustVisitor extends CircuitVisitor { * @param builder Emit the output here. * @param metadata Program metadata for the program compiled. * @param projectDeclarations Information about global per-circuit structures. + * @param materializations Data structures materialized at the end. */ public ToRustVisitor(DBSPCompiler compiler, IIndentStream builder, ProgramMetadata metadata, ProjectDeclarations projectDeclarations, LateMaterializations materializations) { @@ -154,6 +156,10 @@ public ToRustVisitor withPreferHash(boolean preferHash) { return this; } + String getCircuitName() { + return this.getCircuit().name; + } + void processNode(IDBSPNode node) { DBSPOperator op = node.as(DBSPOperator.class); if (op != null) @@ -242,6 +248,11 @@ public static void registerPreprocessors(DBSPCompiler compiler, IIndentStream bu } } + void emitBalancerHints() { + for (var hint: this.materializations.balancerHints) + hint.emit(this.builder); + } + @Override public VisitDecision preorder(DBSPCircuit circuit) { IndentStream signature = new IndentStreamBuilder(); @@ -290,7 +301,7 @@ public VisitDecision preorder(DBSPCircuit circuit) { .newline(); } this.builder - .append("let (circuit, streams) = Runtime::init_circuit(cconf, |circuit| {") + .append("let (mut circuit, streams) = Runtime::init_circuit(cconf, |circuit| {") .increase(); if (!this.useHandles) this.builder.append("let mut catalog = Catalog::new();").newline(); @@ -300,9 +311,10 @@ public VisitDecision preorder(DBSPCircuit circuit) { .getCircuitVisitor(true); collector.apply(circuit); if (!this.sourcePositionResource.isEmpty()) { - SourcePositionResource.generateDeclaration(this.builder); - this.sourcePositionResource.generateInitializer(this.builder); - SourcePositionResource.generateReference(this.builder, CircuitWriter.SOURCE_MAP_VARIABLE_NAME); + SourcePositionResource.generateDeclaration(this.builder, this.getCircuitName()); + this.sourcePositionResource.generateInitializer(this.builder, this.getCircuitName()); + SourcePositionResource.generateReference( + this.builder, CircuitWriter.SOURCE_MAP_VARIABLE_NAME, this.getCircuitName()); } registerPreprocessors(this.compiler, this.builder); @@ -339,6 +351,9 @@ public VisitDecision preorder(DBSPCircuit circuit) { .append("})?;") .newline(); + // Balancer hints must be invoked after circuit construction is completed. + this.emitBalancerHints(); + this.builder .append("Ok((circuit, streams))") .newline() @@ -1022,7 +1037,7 @@ public VisitDecision preorder(DBSPSinkOperator operator) { this.builder.append(",").newline(); raw.tupFields[1].accept(this.innerVisitor); this.builder.decrease().append(">"); - this.builder.append("(hash, ").newline() + this.builder.append("(hash,").newline() .append(this.getInputName(operator, 0)) .append(".clone()") .append(", &SqlIdentifier::from(\""); @@ -1145,7 +1160,7 @@ public VisitDecision preorder(DBSPChainAggregateOperator operator) { this.operationCall(operator); this.builder.increase(); operator.init.accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.getFunction().accept(this.innerVisitor); this.innerVisitor.setOperatorContext(null); this.builder.newline().decrease().append(")") @@ -1331,9 +1346,9 @@ public VisitDecision preorder(DBSPConcreteAsofJoinOperator operator) { .append(", ") .newline(); operator.getFunction().accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.leftTimestamp.accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.rightTimestamp.accept(this.innerVisitor); this.builder.newline() .decrease() @@ -1376,6 +1391,8 @@ VisitDecision processJoinIndexOperator(DBSPJoinBaseOperator operator) { .append(this.markDistinct(operator)) .append(";"); this.tagStream(operator); + this.builder.newline(); + this.emitBalancerHints(operator); this.innerVisitor.setOperatorContext(null); return VisitDecision.STOP; } @@ -1413,9 +1430,9 @@ public VisitDecision preorder(DBSPLagOperator operator) { .increase(); DBSPISizeLiteral offset = new DBSPISizeLiteral(operator.offset); offset.accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.projection.accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.getFunction().accept(this.innerVisitor); this.builder.newline() .decrease() @@ -1429,21 +1446,12 @@ public VisitDecision preorder(DBSPLagOperator operator) { void emitWindowBounds(DBSPWindowBoundExpression lower, DBSPWindowBoundExpression upper) { this.builder.append("RelRange::new(").increase(); - this.emitWindowBound(lower); + lower.accept(this.innerVisitor); this.builder.append(",").newline(); - this.emitWindowBound(upper); + upper.accept(this.innerVisitor); this.builder.newline().decrease().append(")"); } - void emitWindowBound(DBSPWindowBoundExpression bound) { - String beforeAfter = bound.isPreceding ? "Before" : "After"; - this.builder.append("RelOffset::") - .append(beforeAfter) - .append("("); - bound.representation.accept(this.innerVisitor); - this.builder.append(")"); - } - boolean inOuterCircuit() { return this.getParent().is(DBSPCircuit.class); } @@ -1464,9 +1472,9 @@ public VisitDecision preorder(DBSPPartitionedRollingAggregateOperator operator) this.operationCall(operator); this.builder.increase(); operator.partitioningFunction.accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.getFunction().accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); this.emitWindowBounds(operator.lower, operator.upper); this.builder .decrease() @@ -1497,9 +1505,9 @@ public VisitDecision preorder(DBSPPartitionedRollingAggregateWithWaterlineOperat .append(this.getInputName(operator, 1)) .append(", "); operator.partitioningFunction.accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.getFunction().accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); this.emitWindowBounds(operator.lower, operator.upper); this.builder .decrease() @@ -1553,7 +1561,7 @@ public VisitDecision preorder(DBSPAggregateLinearPostprocessOperator operator) { this.operationCall(operator); this.builder.increase(); operator.getFunction().accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.postProcess.accept(this.innerVisitor); this.builder.newline() .decrease() @@ -1583,11 +1591,11 @@ public VisitDecision preorder(DBSPAggregateLinearPostprocessRetainKeysOperator o .append(this.getInputName(operator, 1)) // FIXME: temporary workaround until the compiler learns about TypedBox .append(".apply(|bound| TypedBox::<_, DynData>::new(bound.clone()))") - .append(", ").newline(); + .append(",").newline(); operator.retainKeysFunction.accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.getFunction().accept(this.innerVisitor); - this.builder.append(", ").newline(); + this.builder.append(",").newline(); operator.postProcess.accept(this.innerVisitor); this.builder.newline() .decrease() @@ -1689,6 +1697,23 @@ IIndentStream writeComments(DBSPOperator operator) { (more ? (operator.comment != null ? "\n" + operator.comment : "") : "")); } + /** Generate hints for the dynamic join balancer based on user-supplied annotations */ + void emitBalancerHints(DBSPBinaryOperator operator) { + var strategies = operator.annotations.get(JoinStrategy.class); + for (var strategy: strategies) { + switch (strategy.strategy) { + case Shard: + case Balance: + case Broadcast: { + this.materializations.recordHint(operator, strategy); + break; + } + default: + break; + } + } + } + VisitDecision processJoinBase(DBSPJoinBaseOperator operator) { this.computeHash(operator); this.innerVisitor.setOperatorContext(operator); @@ -1711,10 +1736,17 @@ VisitDecision processJoinBase(DBSPJoinBaseOperator operator) { .append(this.markDistinct(operator)) .append(";"); this.tagStream(operator); + this.builder.newline(); + this.emitBalancerHints(operator); this.innerVisitor.setOperatorContext(null); return VisitDecision.STOP; } + @Override + public VisitDecision preorder(DBSPStreamJoinOperator operator) { + return this.processJoinBase(operator); + } + @Override public VisitDecision preorder(DBSPJoinOperator operator) { return this.processJoinBase(operator); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CircuitWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CircuitWriter.java index 40afcc375f1..fb5b7530f27 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CircuitWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CircuitWriter.java @@ -3,6 +3,7 @@ import org.dbsp.sqlCompiler.circuit.DBSPCircuit; import org.dbsp.sqlCompiler.circuit.annotation.OperatorHash; import org.dbsp.sqlCompiler.circuit.operator.DBSPControlledKeyFilterOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPInputMapWithWaterlineOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPNestedOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; @@ -60,6 +61,9 @@ private void processChild(DBSPOperator node, boolean useHandles) { .append(","); } } + if (useHandles && node.is(DBSPInputMapWithWaterlineOperator.class)) { + this.builder().append("handle_").append(name); + } this.builder().append(")"); } this.builder().append(" = "); @@ -82,14 +86,14 @@ private void processChild(DBSPOperator node, boolean useHandles) { name = input.getName(false); this.builder().append("&") .append(name) - .append(","); + .append(".clone(), "); } if (node.is(DBSPControlledKeyFilterOperator.class)) { DBSPControlledKeyFilterOperator filter = node.to(DBSPControlledKeyFilterOperator.class); if (this.materializations.hasRight(filter)) { DBSPSourceMultisetOperator source = this.materializations.getLeft(filter); String sourceName = source.getNodeName(false); - this.builder().append("handle_").append(sourceName); + this.builder().append("handle_").append(sourceName).append(".clone()"); } } this.builder().append(");").newline(); @@ -118,6 +122,7 @@ void writeCircuit(DBSPCompiler compiler, DBSPCircuit circuit) { signature.decrease().append(")"); } + SourcePositionResource.generateDeclaration(this.builder(), circuit.name); this.builder().append("pub fn ") .append(circuit.getName()); @@ -132,7 +137,7 @@ void writeCircuit(DBSPCompiler compiler, DBSPCircuit circuit) { .append("let cconf = cconf.with_step_size(StepSize::FullSteps);") .newline(); } - this.builder().append("let (circuit, streams) = "); + this.builder().append("let (mut circuit, streams) = "); if (!useHandles) this.builder().append("dbsp_adapters::server::init_circuit(cconf, Box::new("); else @@ -148,8 +153,9 @@ void writeCircuit(DBSPCompiler compiler, DBSPCircuit circuit) { CircuitVisitor collector = new CollectSourcePositions(compiler, sourcePositionResource) .getCircuitVisitor(true); collector.apply(circuit); - sourcePositionResource.generateInitializer(this.builder()); - SourcePositionResource.generateReference(this.builder(), CircuitWriter.SOURCE_MAP_VARIABLE_NAME); + sourcePositionResource.generateInitializer(this.builder(), circuit.name); + SourcePositionResource.generateReference( + this.builder(), CircuitWriter.SOURCE_MAP_VARIABLE_NAME, circuit.name); // Process sources first for (DBSPOperator node : circuit.getAllOperators()) @@ -166,11 +172,11 @@ void writeCircuit(DBSPCompiler compiler, DBSPCircuit circuit) { this.builder().append("Ok(("); for (IInputOperator operator: circuit.sourceOperators.values()) { String name = operator.asOperator().getNodeName(false); - this.builder().append("handle_").append(name).append(", "); + this.builder().append("handle_").append(name).append(".clone(), "); } for (DBSPSinkOperator operator: circuit.sinkOperators.values()) { String name = operator.getNodeName(false); - this.builder().append(name).append(", "); + this.builder().append(name).append(".clone(), "); } this.builder().append("))"); } @@ -180,6 +186,7 @@ void writeCircuit(DBSPCompiler compiler, DBSPCircuit circuit) { .append(!useHandles ? ")" : "") .append("?;") .newline(); + this.emitBalancerHints(); this.builder() .append("Ok((circuit, streams))") .newline() @@ -188,6 +195,11 @@ void writeCircuit(DBSPCompiler compiler, DBSPCircuit circuit) { .newline(); } + void emitBalancerHints() { + for (var hint: this.materializations.balancerHints) + hint.emit(this.builder()); + } + @Override public void write(DBSPCompiler compiler) { this.builder().append(COMMON_PREAMBLE); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CrateGenerator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CrateGenerator.java index 592ee6f59dd..66755201e8a 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CrateGenerator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CrateGenerator.java @@ -25,6 +25,7 @@ public final class CrateGenerator { public final String crateName; public final boolean enterprise; public final boolean lib; + public final boolean testing; /** Cargo file name */ public static final String CARGO = "Cargo.toml"; @@ -38,7 +39,7 @@ public final class CrateGenerator { final ICodeGenerator codeGenerator; public CrateGenerator(File baseDirectory, String directory, String crateName, ICodeGenerator codeGenerator, - boolean enterprise, boolean lib) { + boolean enterprise, boolean lib, boolean testing) { this.crateName = crateName; this.directory = directory; this.baseDirectory = baseDirectory; @@ -46,6 +47,7 @@ public CrateGenerator(File baseDirectory, String directory, String crateName, IC this.codeGenerator = codeGenerator; this.enterprise = enterprise; this.lib = lib; + this.testing = testing; } @Override @@ -55,7 +57,8 @@ public boolean equals(Object o) { CrateGenerator that = (CrateGenerator) o; return this.baseDirectory.equals(that.baseDirectory) && this.crateName.equals(that.crateName) && - this.directory.equals(that.directory); + this.directory.equals(that.directory) && + this.testing == that.testing; } @Override @@ -63,6 +66,7 @@ public int hashCode() { int result = this.baseDirectory.hashCode(); result = 31 * result + this.crateName.hashCode(); result = 31 * result + this.directory.hashCode(); + result = 31 * result + Boolean.hashCode(this.testing); return result; } @@ -85,10 +89,16 @@ void generateCargo(PrintStream stream) { stream.println(""" version = "0.1.0" edition = "2021" - publish = false - + publish = false"""); + if (this.testing) { + stream.println(""" [dev-dependencies] - uuid = { version = "1.6.1" }"""); + # Only used in tests + readers = { workspace = true } + uuid = { workspace = true } + metrics = { workspace = true } + metrics-util = { workspace = true }"""); + } stream.println(); if (this.lib) { stream.println("[lib]"); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/MultiCrates.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/MultiCrates.java index 3606eb7c663..6b67917c715 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/MultiCrates.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/MultiCrates.java @@ -32,8 +32,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** Data structure representing the crates generated for a program * when compiled using multiple crates. */ @@ -78,6 +80,13 @@ public boolean enterprise() { this.materializations = materializations; boolean enterprise = this.enterprise(); + CircuitWriter mainWriter = new CircuitWriter(this.materializations); + BaseRustCodeGenerator globalsWriter = new RustFileWriter(this.materializations) + .withUdf(true).withMalloc(false).withGenerateTuples(false); + // Main crate contains the circuit + this.main = new CrateGenerator(this.rootDirectory, CRATES_DIRECTORY, this.getMainName(), mainWriter, + enterprise, !compiler.options.generateMultiCrateMain(), compiler.options.ioOptions.testing); + // One crate for each tuple size used for (int i : used.tupleSizesUsed) { if (used.isPredefined(i)) continue; @@ -86,8 +95,10 @@ public boolean enterprise() { BaseRustCodeGenerator tWriter = new RustFileWriter(this.materializations) .setUsed(t).withUdf(false).withMalloc(false); CrateGenerator tuple = new CrateGenerator( - this.rootDirectory, CRATES_DIRECTORY, FILE_PREFIX + "tuple" + i, tWriter, enterprise, true); + this.rootDirectory, CRATES_DIRECTORY, FILE_PREFIX + "tuple" + i, tWriter, + enterprise, true, compiler.options.ioOptions.testing); Utilities.putNew(this.tupleCrates, i, tuple); + this.main.addDependency(tuple); } // One crate for each semigroup size used @@ -97,7 +108,8 @@ public boolean enterprise() { BaseRustCodeGenerator tWriter = new RustFileWriter(this.materializations) .setUsed(t).withUdf(false).withMalloc(false); CrateGenerator semi = new CrateGenerator( - this.rootDirectory, CRATES_DIRECTORY, FILE_PREFIX + "semi" + i, tWriter, enterprise, true); + this.rootDirectory, CRATES_DIRECTORY, FILE_PREFIX + "semi" + i, tWriter, + enterprise, true, compiler.options.ioOptions.testing); if (!used.isPredefined(i)) { CrateGenerator tuple = Utilities.getExists(this.tupleCrates, i); semi.addDependency(tuple); @@ -105,21 +117,17 @@ public boolean enterprise() { Utilities.putNew(this.semiCrates, i, semi); } - CircuitWriter mainWriter = new CircuitWriter(this.materializations); - BaseRustCodeGenerator globalsWriter = new RustFileWriter(this.materializations) - .withUdf(true).withMalloc(false).withGenerateTuples(false).withDeclareSourceMap(true); - // Main crate contains the circuit - this.main = new CrateGenerator(this.rootDirectory, CRATES_DIRECTORY, this.getMainName(), - mainWriter, enterprise, !compiler.options.generateMultiCrateMain()); // Crate with global variables - this.globals = new CrateGenerator(this.rootDirectory, CRATES_DIRECTORY, this.getGlobalsName(), globalsWriter, enterprise, true); + this.globals = new CrateGenerator(this.rootDirectory, CRATES_DIRECTORY, this.getGlobalsName(), + globalsWriter, enterprise, true, compiler.options.ioOptions.testing); this.main.addDependency(this.globals); } CrateGenerator createOperatorCrate(DBSPCircuit circuit, DBSPOperator operator, ICircuit parent, boolean enterprise) { String name = FILE_PREFIX + operator.getNodeName(true); SingleOperatorWriter single = new SingleOperatorWriter(operator, circuit, parent, this.materializations); - return new CrateGenerator(this.rootDirectory, CRATES_DIRECTORY, name, single, enterprise, true); + return new CrateGenerator(this.rootDirectory, CRATES_DIRECTORY, name, single, + enterprise, true, compiler.options.ioOptions.testing); } static class UsesComparator extends InnerVisitor { @@ -216,6 +224,7 @@ void addNodes(List nodes) { RustWriter.StructuresUsed locallyUsed = new RustWriter.StructuresUsed(); RustWriter.FindInnerResources finder = new RustWriter.FindInnerResources(compiler, locallyUsed); + Set globalDeclarations = new HashSet<>(); for (IDBSPNode node: nodes) { if (node.is(IDBSPInnerNode.class)) this.globals.add(node); @@ -223,7 +232,11 @@ void addNodes(List nodes) { DBSPCircuit circuit = node.to(DBSPCircuit.class); this.main.add(circuit); for (DBSPDeclaration decl: circuit.declarations) { - this.globals.add(decl.item); + // The same declaration may be repeated in multiple circuits + if (!globalDeclarations.contains(decl.item.getName())) { + this.globals.add(decl.item); + globalDeclarations.add(decl.item.getName()); + } } this.declarationMap = circuit.declarationMap; for (DBSPOperator operator: circuit.allOperators) { @@ -232,7 +245,8 @@ void addNodes(List nodes) { DBSPNestedOperator nested = operator.to(DBSPNestedOperator.class); NestedOperatorWriter writer = new NestedOperatorWriter(nested, circuit, this.materializations); String name = FILE_PREFIX + operator.getNodeName(true); - op = new CrateGenerator(this.rootDirectory, CRATES_DIRECTORY, name, writer, this.enterprise(), true); + op = new CrateGenerator(this.rootDirectory, CRATES_DIRECTORY, name, writer, + this.enterprise(), true, compiler.options.ioOptions.testing); op.add(nested); for (DBSPOperator inside: nested.getAllOperators()) { if (inside.is(DBSPViewDeclarationOperator.class)) @@ -275,13 +289,12 @@ void addNodes(List nodes) { } void write() throws IOException { - this.globals.write(this.compiler); File file = new File(new File( new File(new File(globals.baseDirectory, MultiCrates.CRATES_DIRECTORY), globals.crateName), "src"), DBSPCompiler.UDF_FILE_NAME); - if (!file.exists()) + if (!file.exists()) { Utilities.createEmptyFile(file.toPath()); - this.main.write(compiler); + } for (CrateGenerator gen: this.semiCrates.values()) gen.write(this.compiler); @@ -300,5 +313,10 @@ void write() throws IOException { written.put(op, op); op.write(this.compiler); } + + // Emit after processing the operators. + // This gives them a chance to accumulate information in LateMaterializations. + this.globals.write(this.compiler); + this.main.write(compiler); } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/MultiCratesWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/MultiCratesWriter.java index c8ced545f81..4f2bf59e67b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/MultiCratesWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/MultiCratesWriter.java @@ -123,6 +123,15 @@ public void write(List generators, CompilerOptions options) thro """.replace("$ROOT", relativePath)); } + if (options.ioOptions.testing) { + cargoStream.println(""" + # Only used in tests + readers = { path = "../lib/readers" } + uuid = { version = "1.17.0" } + metrics = { version = "0.23.0" } + metrics-util = { version = "0.17.0" }"""); + } + cargoStream.close(); } @@ -144,7 +153,7 @@ public void write(DBSPCompiler compiler) throws IOException { .withUdf(false).withMalloc(false).withGenerateTuples(false); CrateGenerator test = new CrateGenerator( this.rootDirectory, MultiCrates.CRATES_DIRECTORY, MultiCratesWriter.getTestName(), writer, - crates.enterprise(), true); + crates.enterprise(), true, compiler.options.ioOptions.testing); RustWriter.StructuresUsed locallyUsed = new RustWriter.StructuresUsed(); FindInnerResources finder = new FindInnerResources(compiler, locallyUsed); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java index 235c9bb3d81..ba939bb6abc 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java @@ -6,6 +6,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPAtomicSumOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPControlledKeyFilterOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPDeltaOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPInputMapWithWaterlineOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPInternOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPNestedOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; @@ -20,15 +21,18 @@ import org.dbsp.sqlCompiler.compiler.backend.rust.BaseRustCodeGenerator; import org.dbsp.sqlCompiler.compiler.backend.rust.RustWriter; import org.dbsp.sqlCompiler.compiler.backend.rust.ToRustVisitor; +import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.ProgramIdentifier; import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision; import org.dbsp.sqlCompiler.compiler.visitors.inner.InnerVisitor; import org.dbsp.sqlCompiler.compiler.visitors.outer.LateMaterializations; import org.dbsp.sqlCompiler.ir.expression.DBSPStaticExpression; import org.dbsp.sqlCompiler.ir.statement.DBSPStaticItem; import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeStruct; +import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeIndexedZSet; -import java.util.HashMap; -import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; /** Writes the implementation of a single operator as a function that instantiates * the operator in circuit. */ @@ -61,7 +65,8 @@ public SingleOperatorWriter(DBSPOperator operator, DBSPCircuit circuit, /** Collects all {@link DBSPStaticExpression}s that appear in an expression */ static class FindStatics extends InnerVisitor { - final Map found = new HashMap<>(); + // Use a sorted map for a deterministic iterator + final SortedMap found = new TreeMap<>(); public FindStatics(DBSPCompiler compiler) { super(compiler); @@ -102,9 +107,15 @@ public void write(DBSPCompiler compiler) { this.builder() .append(RustWriter.COMMON_PREAMBLE) .append(RustWriter.STANDARD_PREAMBLE); + if (compiler.options.ioOptions.testing) { + this.builder().append(""" + #[cfg(test)] + use readers::*;""").newline(); + } ToRustVisitor visitor = new ToRustVisitor( compiler, this.builder(), this.circuit.metadata, new ProjectDeclarations(), this.materializations) .withPreferHash(true); + visitor.innerVisitor.setOperatorContext(this.operator); final String name = this.operator.getNodeName(true); this.builder().newline(); for (String dep: this.dependencies) @@ -179,6 +190,19 @@ public void write(DBSPCompiler compiler) { streamType.accept(visitor.innerVisitor); this.builder().append(","); } + if (useHandles && operator.is(DBSPInputMapWithWaterlineOperator.class)) { + DBSPInputMapWithWaterlineOperator wop = operator.to(DBSPInputMapWithWaterlineOperator.class); + DBSPTypeIndexedZSet ix = wop.getOutputIndexedZSetType(); + this.builder().append("MapHandle<"); + ix.keyType.accept(visitor.innerVisitor); + this.builder().append(", "); + ix.elementType.accept(visitor.innerVisitor); + this.builder().append(", "); + DBSPTypeStruct upsertStruct = wop.getStructUpsertType( + new ProgramIdentifier(wop.getOriginalRowType().hashName + "_upsert", false)); + upsertStruct.toTupleDeep().accept(visitor.innerVisitor); + this.builder().append(">"); + } this.builder().append(")"); } } @@ -252,6 +276,10 @@ public void write(DBSPCompiler compiler) { OutputPort port = operator.getOutput(i); this.builder().append(port.getName(true)); this.builder().append(", "); + + } + if (useHandles && operator.is(DBSPInputMapWithWaterlineOperator.class)) { + this.builder().append("handle"); } this.builder().append(")"); } else if (operator.is(DBSPSourceBaseOperator.class)) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java index 6abf0f89b5a..264f670f960 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java @@ -42,6 +42,7 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Uncollect; import org.apache.calcite.rel.core.Window; +import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalAsofJoin; import org.apache.calcite.rel.logical.LogicalCorrelate; @@ -74,6 +75,8 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; +import org.dbsp.sqlCompiler.circuit.annotation.Annotations; +import org.dbsp.sqlCompiler.circuit.annotation.JoinStrategy; import org.dbsp.sqlCompiler.circuit.operator.DBSPAsofJoinOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPAggregateZeroOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPConstantOperator; @@ -1520,12 +1523,30 @@ DBSPSimpleOperator insertCastMap(CalciteObject conditionNode, DBSPSimpleOperator TypeCompiler.makeZSet(resultType), operator.outputPort()); } + JoinStrategy fromJoinHint(RelHint hint) { + JoinStrategy.Strategy strat = JoinStrategy.Strategy.valueOf(Utilities.titleCase(hint.hintName)); + Utilities.enforce(hint.listOptions.size() == 2); + int input = Integer.parseInt(hint.listOptions.get(0)); + String table = hint.listOptions.get(1); + return new JoinStrategy(new SourcePositionRange(hint.pos), strat, input, table); + } + private void visitJoin(LogicalJoin join) { final CalciteObject conditionNode = CalciteObject.create(join, join.getCondition()); final IntermediateRel node = CalciteObject.create(join, conditionNode.getPositionRange()); // The result is the sum of all these operators. final List sumInputs = new ArrayList<>(); + final Annotations strategies = new Annotations(); + for (var hint: join.getHints()) { + // We only consider hints with an empty inherit path. + // We assume the other hints were inserted by the optimizer during rewrites + if (SqlToRelCompiler.isJoinHint(hint) && hint.inheritPath.isEmpty()) { + var strategy = this.fromJoinHint(hint); + strategies.add(strategy); + } + } + JoinRelType joinType = join.getJoinType(); if (joinType == JoinRelType.ANTI || joinType == JoinRelType.SEMI) throw new UnimplementedException("JOIN of type " + joinType + " not yet implemented", node); @@ -1680,6 +1701,7 @@ private void visitJoin(LogicalJoin join) { joinResult = new DBSPStreamJoinOperator(node, TypeCompiler.makeZSet(lr.getType()), makeTuple, left.isMultiset || right.isMultiset, leftNonNullIndex.outputPort(), rightNonNullIndex.outputPort(), false); + joinResult.addAnnotations(strategies, DBSPSimpleOperator.class); inner = joinResult; if (joinType == JoinRelType.LEFT && leftPulled == left && !hasFilter) { @@ -1724,6 +1746,7 @@ private void visitJoin(LogicalJoin join) { node, TypeCompiler.makeZSet(makeLeftTuple.getResultType()), makeLeftTuple, left.isMultiset || right.isMultiset, leftDiff.outputPort(), rightDiff.outputPort(), false); + lj.addAnnotations(strategies, DBSPSimpleOperator.class); this.addOperator(lj); final DBSPIntegrateOperator integrate = new DBSPIntegrateOperator(node, lj.outputPort()); // Additional casts may be needed to fix key field types on the left side diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/TableData.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/TableData.java index 73ec06492ea..6d1a0d7452b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/TableData.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/TableData.java @@ -116,8 +116,10 @@ public VisitDecision preorder(DBSPTypeDecimal node) { } /** Create an expression that will evaluate to the contents of this table. - * If necessary and possible, write the data to a file and generate code to read it. */ - public DBSPExpression createOutput(DBSPCompiler compiler, String name, String directory) throws IOException { + * If necessary and possible, write the data to a file and generate code to read it. + * * @param executionDirectory Relative directory where executable will run */ + public DBSPExpression createOutput(DBSPCompiler compiler, String name, + String directory, String executionDirectory) throws IOException { HasDecimalOrDate hd = new HasDecimalOrDate(compiler); hd.apply(this.data); if (this.data().size() > 30 && !hd.found) { @@ -129,7 +131,7 @@ public DBSPExpression createOutput(DBSPCompiler compiler, String name, String di ToCsvVisitor.toCsv(compiler, file, this.data()); return new DBSPApplyExpression(CalciteObject.EMPTY, "read_csv", this.data().getType(), - new DBSPStrLiteral(fileName, false)); + new DBSPStrLiteral(executionDirectory + "/" + fileName, false)); } else { return this.data(); } @@ -140,9 +142,11 @@ public DBSPExpression createOutput(DBSPCompiler compiler, String name, String di * @param name Name for the created function. * @param tables Values that produce the input * @param directory Directory where temporary files can be written + * @param executionDirectory Relative directory where executable will run */ - public static DBSPFunction createInputFunction(DBSPCompiler compiler, String name, - TableData[] tables, String directory) throws IOException { + public static DBSPFunction createInputFunction( + DBSPCompiler compiler, String name, + TableData[] tables, String directory, String executionDirectory) throws IOException { DBSPExpression[] fields = new DBSPExpression[tables.length]; Set seen = new HashSet<>(); for (int i = 0; i < tables.length; i++) { @@ -168,7 +172,7 @@ public static DBSPFunction createInputFunction(DBSPCompiler compiler, String nam ToCsvVisitor.toCsv(compiler, file, tables[i].data); fields[i] = new DBSPApplyExpression(CalciteObject.EMPTY, "read_csv", tables[i].data.getType(), - new DBSPStrLiteral(fileName, false)); + new DBSPStrLiteral(executionDirectory + "/" + fileName, false)); } DBSPRawTupleExpression result = new DBSPRawTupleExpression(fields); return new DBSPFunction(CalciteObject.EMPTY, name, new ArrayList<>(), diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/FelderaSqlToRelConverter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/FelderaSqlToRelConverter.java new file mode 100644 index 00000000000..1c7dfeaaf79 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/FelderaSqlToRelConverter.java @@ -0,0 +1,52 @@ +package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.prepare.Prepare; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.hint.Hintable; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql2rel.SqlRexConvertletTable; +import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Collections; +import java.util.List; + +/** Extension to Calcite's {@link SqlToRelConverter}'s class. + * We do exactly the same thing, but also attach ALIAS annotations to FROM nodes that have an AS alias, + * so we can carry some alias information to the Rel representation. */ +public class FelderaSqlToRelConverter extends SqlToRelConverter { + public FelderaSqlToRelConverter( + RelOptTable.ViewExpander viewExpander, @Nullable SqlValidator validator, + Prepare.CatalogReader catalogReader, RelOptCluster cluster, + SqlRexConvertletTable convertletTable, Config config) { + super(viewExpander, validator, catalogReader, cluster, convertletTable, config); + } + + @Override + protected void convertFrom(Blackboard bb, @Nullable SqlNode from, @Nullable List fieldNames) { + super.convertFrom(bb, from, fieldNames); + if (from != null && from.getKind() == SqlKind.AS) { + SqlCall call = (SqlCall) from; + RelNode root = bb.root; + + // Attach the original AS alias as a HINT_ALIAS to the rel node. + if (root instanceof Hintable hintable) { + RelHint alias = RelHint.builder(SqlToRelCompiler.HINT_ALIAS) + .hintOption(call.operand(1).toString()) + .position(call.getParserPosition()) + .build(); + RelNode newRoot = hintable.attachHints(Collections.singletonList(alias)); + boolean isLeaf = this.leaves.containsKey(root); + if (isLeaf) + leaves.remove(root); + bb.setRoot(newRoot, isLeaf); + } + } + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/RelJsonWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/RelJsonWriter.java similarity index 98% rename from sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/RelJsonWriter.java rename to sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/RelJsonWriter.java index 1229f98251f..1340b4cd2b1 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/RelJsonWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/RelJsonWriter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.dbsp.util; +package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -30,6 +30,7 @@ import org.apache.calcite.util.JsonBuilder; import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; +import org.dbsp.util.Utilities; /** Reimplementation of RelJsonWriter from Calcite which exposes the RelIdMap. */ public class RelJsonWriter implements RelWriter { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/RewriteJoinAnnotations.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/RewriteJoinAnnotations.java new file mode 100644 index 00000000000..a218023a962 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/RewriteJoinAnnotations.java @@ -0,0 +1,185 @@ +package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttleImpl; +import org.apache.calcite.rel.hint.Hintable; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.logical.LogicalProject; +import org.dbsp.sqlCompiler.compiler.IErrorReporter; +import org.dbsp.sqlCompiler.compiler.errors.SourcePosition; +import org.dbsp.sqlCompiler.compiler.errors.SourcePositionRange; +import org.dbsp.util.Utilities; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Rewrites a Join annotation into a lower-level form. + * Must be run before optimizations. + * + *

Initially join annotations specify table names: e.g. + * SELECT /*+ broadcast(T) * / T JOIN S ON ... ; + * These are carried as Broadcast hints in the Rel plan. + * Also, when converting SqlToRel, the two inputs of a join will be tagged with ALIAS annotations + * that will contain the table names: e.g., + * LogicalJoin(condition=[=($0, $1)], joinType=[inner])[[broadcast inheritPath:[0, 0] options:[t]] + * LogicalTableScan(table=[[schema, t]])[[alias inheritPath:[] options:[t]]] + * LogicalTableScan(table=[[schema, s]])[[alias inheritPath:[] options:[s]]] + * The optimizer later will move things around, and the ALIAS annotations may vanish if they are on subqueries. + * So before optimizing we replace BROADCAST(S) with BROADCAST(inputNo, S): + * LogicalJoin(condition=[=($0, $1)], joinType=[inner])[[broadcast inheritPath:[0, 0] options:[0, T]] + * + *

Note that all annotations inserted here will have an inheritPath: [] (empty). + * This will be useful later: all annotations with a non-empty path were inserted by the optimizer. + */ +public class RewriteJoinAnnotations extends RelShuttleImpl { + private final IErrorReporter reporter; + + static class HintAndCount { + public final RelHint hint; + public List from; + + HintAndCount(RelHint hint) { + this.hint = hint; + this.from = new ArrayList<>(); + } + + void increment(SourcePositionRange referenced) { + if (referenced.isValid()) + this.from.add(referenced); + } + } + + final Map hintsUsed; + + RewriteJoinAnnotations(IErrorReporter reporter) { + this.reporter = reporter; + this.hintsUsed = new HashMap<>(); + } + + @Override + public RelNode visit(LogicalProject project) { + // Remove join hints from projects, where they are inserted initially. + // This will prevent them from being propagated by other query optimization passes + project = (LogicalProject) super.visit(project); + List newHints = new ArrayList<>(); + + boolean changes = false; + for (var hint: project.getHints()) { + var newHint = hint; + if (SqlToRelCompiler.isJoinHint(hint)) + newHint = null; + changes = changes || (newHint != hint); + if (newHint != null) + newHints.add(newHint); + } + + if (changes) + return project.withHints(newHints); + return project; + } + + /** Called after the visit of a Rel tree is completed. + * Reports errors for hints not used or used multiple times. */ + public void done() { + for (var entry: this.hintsUsed.entrySet()) { + var hc = entry.getValue(); + if (hc.from.size() == 1) continue; + var hint = hc.hint; + var tableName = hint.listOptions.get(0); + if (hc.from.isEmpty()) { + this.reporter.reportError(new SourcePositionRange(hint.pos), "Cannot apply hint", + "Hint " + Utilities.singleQuote(hint.hintName + "(" + tableName + ")") + + " specifies an input which cannot be identified among the join inputs"); + } else { + this.reporter.reportError(new SourcePositionRange(hint.pos), "Ambiguous hint", + "Hint " + Utilities.singleQuote(hint.hintName + "(" + tableName + ")") + + " matches " + hc.from.size() + " different collections"); + for (SourcePositionRange pos: hc.from) { + this.reporter.reportError(pos, "Ambiguous hint", "Matching table", true); + } + } + } + } + + /** Called when a join hint is used in some table */ + void referenced(RelHint hint, SourcePositionRange reference) { + SourcePositionRange range = new SourcePositionRange(hint.pos); + if (!range.isValid()) + return; + if (!this.hintsUsed.containsKey(range)) + this.hintsUsed.put(range, new HintAndCount(hint)); + this.hintsUsed.get(range).increment(reference); + } + + /** Given an alias, figure out which side of the join it's coming from; rewrite the hint. + * Return null if this is a join hint which cannot be resolved. + * Report an error in this case. + * Return the original hint if it is not a join hint. */ + @Nullable + RelHint rewriteJoinInputHint(RelHint hint, LogicalJoin join) { + if (!SqlToRelCompiler.isJoinHint(hint)) + return hint; + + // Validation should ensure that the exists. + String tableName = hint.listOptions.get(0); + var inputs = join.getInputs(); + for (int i = 0; i < inputs.size(); i++) { + RelNode node = inputs.get(i); + if (node instanceof Hintable hintable) { + var inputHints = hintable.getHints(); + for (var inputHint: inputHints) { + if (inputHint.hintName.equals(SqlToRelCompiler.HINT_ALIAS)) { + String collectionAlias = inputHint.listOptions.get(0); + if (collectionAlias.equals(tableName)) { + // Notice that we discard the inheritPath of the original hint + this.referenced(hint, new SourcePositionRange(inputHint.pos)); + return RelHint.builder(hint.hintName) + .hintOption(Integer.toString(i)) + .hintOption(tableName) + .position(hint.pos) + .build(); + } + } + } + } + } + // The hint is not referenced by this join, but maybe there is another one which uses it. + this.referenced(hint, SourcePositionRange.INVALID); + return null; + } + + @Override + public RelNode visit(LogicalJoin join) { + join = (LogicalJoin) super.visit(join); + // Use a set to eliminate duplicates + // Sometimes Calcite will create multiple versions of a hint with different inheritPaths + Set seen = new HashSet<>(); + List newHints = new ArrayList<>(); + + boolean changes = false; + for (var hint: join.getHints()) { + SourcePositionRange range = new SourcePositionRange(hint.pos); + if (range.isValid() && seen.contains(range)) { + // Duplicate version of the same hint + changes = true; + continue; + } + seen.add(range); + RelHint newHint = this.rewriteJoinInputHint(hint, join); + changes = changes || (newHint != hint); + if (newHint != null) + newHints.add(newHint); + } + + if (changes) + return join.withHints(newHints); + return join; + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java index d830975c393..ab1fecaf900 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java @@ -35,7 +35,6 @@ import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgramBuilder; import org.apache.calcite.prepare.CalciteCatalogReader; @@ -43,7 +42,13 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.externalize.RelWriterImpl; +import org.apache.calcite.rel.hint.HintPredicates; +import org.apache.calcite.rel.hint.HintStrategy; +import org.apache.calcite.rel.hint.HintStrategyTable; +import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalTableScan; import org.apache.calcite.rel.logical.LogicalValues; @@ -78,7 +83,6 @@ import org.apache.calcite.sql.SqlCharStringLiteral; import org.apache.calcite.sql.SqlCollectionTypeNameSpec; import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlExplainFormat; import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlIdentifier; @@ -123,6 +127,7 @@ import org.apache.calcite.sql2rel.ConvertToChecked; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.Litmus; import org.dbsp.generated.parser.DbspParserImpl; import org.dbsp.sqlCompiler.compiler.CompilerOptions; import org.dbsp.sqlCompiler.compiler.IErrorReporter; @@ -179,6 +184,8 @@ import org.dbsp.util.Utilities; import javax.annotation.Nullable; +import java.io.PrintWriter; +import java.io.StringWriter; import java.nio.charset.Charset; import java.util.AbstractList; import java.util.ArrayList; @@ -221,7 +228,7 @@ public class SqlToRelCompiler implements IWritesLogs { @Nullable private SqlValidator validator; @Nullable - private SqlToRelConverter converter; + private FelderaSqlToRelConverter converter; private final ExtraValidation extraValidator; private final CalciteConnectionConfig connectionConfig; private final IErrorReporter errorReporter; @@ -262,6 +269,34 @@ public class SqlToRelCompiler implements IWritesLogs { DEFAULT_TYPE_ALIASES.put("BOOL", SqlTypeName.BOOLEAN); } + // Three Join hints. In the SqlNode stage these hints hold a tableName argument, + // in the RelNode stage they hold a list with 2 elements (inputNo, tableName). + public static final String HINT_BROADCAST = "broadcast"; + public static final String HINT_BALANCE = "balance"; + public static final String HINT_SHARD = "shard"; + // Hint holding the name of an aliased sub-relation; inserted by SqlToRelConverter. + public static final String HINT_ALIAS = "alias"; + + static final Set JOIN_HINTS = new HashSet<>() {{ + add(HINT_BALANCE); + add(HINT_BROADCAST); + add(HINT_SHARD); + }}; + + public static boolean isJoinHint(RelHint hint) { + return JOIN_HINTS.contains(hint.hintName); + } + + boolean checkHintHasArgument(RelHint hint, Litmus unused) { + if (hint.listOptions.isEmpty()) { + this.errorReporter.reportError(new SourcePositionRange(hint.pos), + "Invalid hint", + "Hint " + Utilities.singleQuote(hint.hintName) + " requires a list of arguments"); + return false; + } + return true; + } + public SqlToRelCompiler(CompilerOptions options, IErrorReporter errorReporter) { this.options = options; this.errorReporter = errorReporter; @@ -269,10 +304,10 @@ public SqlToRelCompiler(CompilerOptions options, IErrorReporter errorReporter) { this.definedViews = new HashSet<>(); this.extraValidator = new ExtraValidation(errorReporter); + java.util.Properties connConfigProp = new java.util.Properties(); // This influences function name lookup. // We want that to be case-insensitive. // Notice that this does NOT affect the parser, only the validator. - java.util.Properties connConfigProp = new java.util.Properties(); connConfigProp.put(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), String.valueOf(true)); connConfigProp.put(CalciteConnectionProperty.DEFAULT_NULL_COLLATION.camelName(), NULL_COLLATION); this.udt = new HashMap<>(); @@ -295,7 +330,22 @@ public SqlToRelCompiler(CompilerOptions options, IErrorReporter errorReporter) { // We use a series of planner stages later to perform the real optimizations. RelOptPlanner planner = new HepPlanner(new HepProgramBuilder().build()); planner.setExecutor(RexUtil.EXECUTOR); + HintStrategyTable hints = HintStrategyTable.builder() + .hintStrategy(HINT_BROADCAST, HintStrategy.builder(HintPredicates.JOIN) + .optionChecker(this::checkHintHasArgument) + .build()) + .hintStrategy(HINT_SHARD, HintStrategy.builder(HintPredicates.JOIN) + .optionChecker(this::checkHintHasArgument) + .build()) + .hintStrategy(HINT_BALANCE, HintStrategy.builder(HintPredicates.JOIN) + .optionChecker(this::checkHintHasArgument) + .build()) + // Alias is an internal hint used by the compiler, not exposed to users + // It is really an annotation attached to a rel node remembering the original AS alias + .hintStrategy(HINT_ALIAS, HintStrategy.builder(HintPredicates.JOIN).build()) + .build(); this.cluster = RelOptCluster.create(planner, new RexBuilder(typeFactory)); + this.cluster.setHintStrategies(hints); var metadataProvider = ChainedRelMetadataProvider.of(List.of(RelMdRowCount.SOURCE, DefaultRelMetadataProvider.INSTANCE)); this.cluster.setMetadataProvider(metadataProvider); @@ -306,13 +356,14 @@ public SqlToRelCompiler(CompilerOptions options, IErrorReporter errorReporter) { .withExpand(true) .withTrimUnusedFields(true) .withRelBuilderConfigTransform(t -> t.withSimplify(false)) + .withHintStrategyTable(hints) ; this.validator = null; this.converter = null; this.usedViewDeclarations = new HashSet<>(); this.declaredViews = new HashMap<>(); this.relBuilder = this.converterConfig.getRelBuilderFactory() - .create(cluster, null) + .create(this.cluster, null) .transform(this.converterConfig.getRelBuilderConfigTransform()); SqlOperatorTable operatorTable = this.createOperatorTable(); @@ -640,7 +691,7 @@ public void addOperatorTable(SqlOperatorTable operatorTable) { this.typeFactory, validatorConfig ); - this.converter = new SqlToRelConverter( + this.converter = new FelderaSqlToRelConverter( (type, query, schema, path) -> null, this.validator, catalogReader, @@ -670,9 +721,7 @@ public boolean functionExists(String identifier) { /** aka EXPLAIN */ public static String getPlan(RelNode rel) { - return RelOptUtil.dumpPlan("[Logical plan]", rel, - SqlExplainFormat.TEXT, - SqlExplainLevel.NON_COST_ATTRIBUTES); + return toString(rel, SqlExplainLevel.ALL_ATTRIBUTES, true); } public RexBuilder getRexBuilder() { @@ -2054,6 +2103,14 @@ void validateTableProperty(ProgramIdentifier table, SqlFragment key, SqlFragment } } + /** Convert a plan to a human-readable string. */ + public static String toString(final RelNode rel, SqlExplainLevel detailLevel, boolean expand) { + final StringWriter sw = new StringWriter(); + final RelWriter planWriter = new RelWriterImpl(new PrintWriter(sw), detailLevel, false, expand); + rel.explain(planWriter); + return sw.toString(); + } + @Nullable public CreateViewStatement compileCreateView( ParsedStatement node, Map lateness, @@ -2126,6 +2183,12 @@ public CreateViewStatement compileCreateView( props = new Properties(viewProperties); } + // System.out.println(getPlan(relRoot.rel)); + RewriteJoinAnnotations joinConverter = new RewriteJoinAnnotations(this.errorReporter); + RelNode converted = joinConverter.visit(relRoot.rel); + joinConverter.done(); + relRoot = relRoot.withRel(converted); + // Convert plan to use checked arithmetic // Must be done before optimizations. ConvertToChecked checkedConverter = new ConvertToChecked(this.getRexBuilder(), true); @@ -2286,6 +2349,7 @@ else if (name.equalsIgnoreCase("off")) value = SqlLiteral.createBoolean(false, value.getParserPosition()); } } + Utilities.enforce(value != null); RexNode expression = this.getConverter().convertExpression(value); this.errorReporter.setErrorContext(SourcePositionRange.INVALID); return new SetOptionStatement(node, ProgramIdentifier.fromSqlId(id), expression); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/optimizer/CalciteOptimizer.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/optimizer/CalciteOptimizer.java index 0119d89d46c..f7788eb3ccf 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/optimizer/CalciteOptimizer.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/optimizer/CalciteOptimizer.java @@ -9,6 +9,8 @@ import org.apache.calcite.rel.RelVisitor; import org.apache.calcite.rel.core.Correlate; import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.hint.Hintable; +import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.rel.rules.PruneEmptyRules; import org.apache.calcite.rel.rules.SingleValuesOptimizationRules; @@ -23,6 +25,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Predicate; /** Optimizer using the Calcite program rewrite rules */ public class CalciteOptimizer implements IWritesLogs { @@ -50,6 +53,38 @@ public static boolean containsCorrelate(RelNode node) { return false; } + /** True if 'node' has hints that pass the specified predicate */ + static boolean containsHints(RelNode node, Predicate predicate) { + if (node instanceof Hintable ha) { + if (!ha.getHints().isEmpty()) { + for (var hint : ha.getHints()) { + if (predicate.test(hint)) + return true; + } + } + } + for (RelNode input : node.getInputs()) { + if (containsHints(input, predicate)) + return true; + } + return false; + } + + /** True if the specified node has join-related hints */ + static boolean containsJoinHints(RelNode node) { + return containsHints(node, SqlToRelCompiler::isJoinHint); + } + + static int joinCount(RelNode node) { + int recurse = 0; + for (RelNode input : node.getInputs()) { + recurse += joinCount(input); + } + if (node instanceof Join) + recurse += 1; + return recurse; + } + public interface CalciteOptimizerStep { /** Name of the optimizer step */ String getName(); @@ -325,8 +360,10 @@ public String toString() { )); var hyper = new BaseOptimizerStep("Hypergraph", 0) { HepProgram getProgram(RelNode node, int level) { - // only call this if there are no Correlates - if (!containsCorrelate(node)) { + // only call this if there are no Correlates or join hints + if (!containsCorrelate(node) && + !containsJoinHints(node) && + (joinCount(node) > 1)) { this.addRules(level, CoreRules.JOIN_TO_HYPER_GRAPH, CoreRules.HYPER_GRAPH_OPTIMIZE); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteObject/RelAnd.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteObject/RelAnd.java index d2e89c9b419..7ee925c9f61 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteObject/RelAnd.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteObject/RelAnd.java @@ -8,7 +8,10 @@ import org.dbsp.util.Linq; import org.dbsp.util.Utilities; +import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -39,7 +42,7 @@ public IIndentStream asJson(IIndentStream stream, Map idRemap) .appendJsonLabelAndColon("and") .append("[").increase(); boolean first = true; - for (CalciteRelNode node : this.nodes) { + for (CalciteRelNode node : this.sort()) { if (!first) stream.append(","); first = false; @@ -79,9 +82,16 @@ public CalciteRelNode intermediate() { throw new UnimplementedException("intermediate " + this); } + List sort() { + List list = new ArrayList<>(this.nodes); + list.sort(Comparator.comparingLong(LastRel::getId)); + return list; + } + @Override public String toString() { - return this.getId() + " And(" + String.join(",", Linq.map(this.nodes, CalciteObject::toString)) + ")"; + // Sort for a deterministic order + return this.getId() + " And(" + String.join(",", Linq.map(this.sort(), CalciteObject::toString)) + ")"; } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ImplementStatics.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ImplementStatics.java index 13acc720787..93990021583 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ImplementStatics.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ImplementStatics.java @@ -130,7 +130,7 @@ public VisitDecision preorder(DBSPExpression expression) { DBSPExpression result; DBSPExpression renumbered = this.renumber.apply(expression).to(DBSPExpression.class); - String name = DBSPStaticExpression.generateName(renumbered, this.compiler); + String name = DBSPStaticExpression.generateName(renumbered, this.operatorContext, this.compiler); DBSPStaticExpression stat = new DBSPStaticExpression(expression.getNode(), renumbered, name); if (this.declare) { DBSPStaticItem item = new DBSPStaticItem(stat); @@ -154,7 +154,7 @@ VisitDecision replaceLiteral(DBSPLiteral expression) { } if (canonical == null) { - String name = DBSPStaticExpression.generateName(expression, this.compiler); + String name = DBSPStaticExpression.generateName(expression, this.operatorContext, this.compiler); DBSPStaticExpression stat = new DBSPStaticExpression(expression.getNode(), expression, name); if (this.declare) { DBSPStaticItem item = new DBSPStaticItem(stat); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerVisitor.java index 22ad5dce790..404a7b91402 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerVisitor.java @@ -23,6 +23,7 @@ package org.dbsp.sqlCompiler.compiler.visitors.inner; +import org.dbsp.sqlCompiler.circuit.DBSPCircuit; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.ICompilerComponent; @@ -156,12 +157,17 @@ public abstract class InnerVisitor implements IRTransform, IWritesLogs, IHasId, public final DBSPCompiler compiler; protected final List context; @Nullable protected DBSPOperator operatorContext; + @Nullable protected DBSPCircuit circuitContext; @Override public void setOperatorContext(@Nullable DBSPOperator operatorContext) { this.operatorContext = operatorContext; } + public void setCircuitContext(@Nullable DBSPCircuit circuit) { + this.circuitContext = circuit; + } + public InnerVisitor(DBSPCompiler compiler) { this.id = crtId++; this.compiler = compiler; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/BalancedJoins.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/BalancedJoins.java index 727c1d606b3..aaa71c22d7e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/BalancedJoins.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/BalancedJoins.java @@ -11,6 +11,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPLeftJoinOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPNestedOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; import org.dbsp.sqlCompiler.circuit.operator.IGCOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.util.Linq; @@ -56,8 +57,9 @@ public void postorder(DBSPLeftJoinOperator operator) { OutputPort left = this.mapped(operator.left()); OutputPort right = this.mapped(operator.right()); - DBSPJoinBaseOperator result = new DBSPLeftJoinOperator(operator.getRelNode(), operator.getOutputZSetType(), - operator.getFunction(), operator.isMultiset, left, right, true); + DBSPSimpleOperator result = new DBSPLeftJoinOperator(operator.getRelNode(), operator.getOutputZSetType(), + operator.getFunction(), operator.isMultiset, left, right, true) + .copyAnnotations(operator); this.map(operator, result); } @@ -70,8 +72,9 @@ public void postorder(DBSPJoinOperator operator) { OutputPort left = this.mapped(operator.left()); OutputPort right = this.mapped(operator.right()); - DBSPJoinBaseOperator result = new DBSPJoinOperator(operator.getRelNode(), operator.getOutputZSetType(), - operator.getFunction(), operator.isMultiset, left, right, true); + DBSPSimpleOperator result = new DBSPJoinOperator(operator.getRelNode(), operator.getOutputZSetType(), + operator.getFunction(), operator.isMultiset, left, right, true) + .copyAnnotations(operator); this.map(operator, result); } @@ -84,9 +87,10 @@ public void postorder(DBSPJoinIndexOperator operator) { OutputPort left = this.mapped(operator.left()); OutputPort right = this.mapped(operator.right()); - DBSPJoinBaseOperator result = new DBSPJoinIndexOperator( + DBSPSimpleOperator result = new DBSPJoinIndexOperator( operator.getRelNode(), operator.getOutputIndexedZSetType(), - operator.getFunction(), operator.isMultiset, left, right, true); + operator.getFunction(), operator.isMultiset, left, right, true) + .copyAnnotations(operator); this.map(operator, result); } @@ -99,9 +103,10 @@ public void postorder(DBSPLeftJoinIndexOperator operator) { OutputPort left = this.mapped(operator.left()); OutputPort right = this.mapped(operator.right()); - DBSPJoinBaseOperator result = new DBSPLeftJoinIndexOperator( + DBSPSimpleOperator result = new DBSPLeftJoinIndexOperator( operator.getRelNode(), operator.getOutputIndexedZSetType(), - operator.getFunction(), operator.isMultiset, left, right, true); + operator.getFunction(), operator.isMultiset, left, right, true) + .copyAnnotations(operator); this.map(operator, result); } @@ -114,10 +119,11 @@ public void postorder(DBSPJoinFilterMapOperator operator) { OutputPort left = this.mapped(operator.left()); OutputPort right = this.mapped(operator.right()); - DBSPJoinBaseOperator result = new DBSPJoinFilterMapOperator( + DBSPSimpleOperator result = new DBSPJoinFilterMapOperator( operator.getRelNode(), operator.getOutputZSetType(), operator.getFunction(), operator.filter, operator.map, - operator.isMultiset, left, right, true); + operator.isMultiset, left, right, true) + .copyAnnotations(operator); this.map(operator, result); } @@ -130,10 +136,11 @@ public void postorder(DBSPLeftJoinFilterMapOperator operator) { OutputPort left = this.mapped(operator.left()); OutputPort right = this.mapped(operator.right()); - DBSPJoinBaseOperator result = new DBSPLeftJoinFilterMapOperator( + DBSPSimpleOperator result = new DBSPLeftJoinFilterMapOperator( operator.getRelNode(), operator.getOutputZSetType(), operator.getFunction(), operator.filter, operator.map, - operator.isMultiset, left, right, true); + operator.isMultiset, left, right, true) + .copyAnnotations(operator); this.map(operator, result); } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CheckHints.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CheckHints.java new file mode 100644 index 00000000000..c25c49d8b84 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CheckHints.java @@ -0,0 +1,54 @@ +package org.dbsp.sqlCompiler.compiler.visitors.outer; + +import org.dbsp.sqlCompiler.circuit.OutputPort; +import org.dbsp.sqlCompiler.circuit.annotation.JoinStrategy; +import org.dbsp.sqlCompiler.circuit.operator.DBSPJoinBaseOperator; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Check that there are no conflicting hints. */ +public class CheckHints extends CircuitVisitor { + final Map> portHints; + + void addHint(OutputPort port, JoinStrategy hint) { + if (!this.portHints.containsKey(port)) { + this.portHints.put(port, new ArrayList<>()); + } else { + for (var existing: this.portHints.get(port)) { + if (!existing.compatible(hint)) { + this.compiler.reportError(hint.getPosition(), + "Incompatible hints", + "Found two conflicting hints for the same collection: " + hint); + this.compiler.reportError(existing.getPosition(), + "Incompatible hints", + "Previous hint: " + existing, + true); + } + } + } + this.portHints.get(port).add(hint); + } + + public CheckHints(DBSPCompiler compiler) { + super(compiler); + this.portHints = new HashMap<>(); + } + + @Override + public void postorder(DBSPJoinBaseOperator join) { + var hints = join.annotations.get(JoinStrategy.class); + int inputNo = 0; + for (var input: join.inputs) { + for (var hint: hints) { + if (hint.input == inputNo) { + this.addHint(input, hint); + } + } + inputNo++; + } + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java index 41fac942386..b6fcdf3e8e1 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java @@ -167,6 +167,7 @@ void createOptimizer() { // From now on we cannot really change the graph anymore. // this.add(new TestSerialize(compiler)); + this.add(new CheckHints(compiler)); this.add(new ComparatorDeclarations(compiler, new DeclareComparators(compiler))); this.add(new CompactNames(compiler)); this.add(new MerkleOuter(compiler, true)); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LateMaterializations.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LateMaterializations.java index ea8a0e1c0e3..da0b41779ca 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LateMaterializations.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LateMaterializations.java @@ -1,19 +1,43 @@ package org.dbsp.sqlCompiler.compiler.visitors.outer; +import org.dbsp.sqlCompiler.circuit.annotation.JoinStrategy; +import org.dbsp.sqlCompiler.circuit.annotation.OperatorHash; +import org.dbsp.sqlCompiler.circuit.operator.DBSPBinaryOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPControlledKeyFilterOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceMultisetOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.util.Bijection; +import org.dbsp.util.IIndentStream; +import org.dbsp.util.Utilities; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; /** - * Discover input nodes that immediately feed the left input of a controlled_key_filter operator. - * This pattern is generated by inputs with LATENESS annotations. - * If the input table is materialized, the materialized stream has - * to be the output of the controlled_key_filter operator. */ + * Data structure holding per-circuit information which is generated after all nodes have been processed. + */ public class LateMaterializations extends CircuitVisitor { // Keeps track of tables that have to be materialized at a different point in the circuit. + // Discover input nodes that immediately feed the left input of a controlled_key_filter operator. + // This pattern is generated by inputs with LATENESS annotations. + // If the input table is materialized, the materialized stream has + // to be the output of the controlled_key_filter operator. */ final Bijection materialization = new Bijection<>(); + public final List balancerHints = new ArrayList<>(); + + public record BalancerHint(DBSPBinaryOperator operator, JoinStrategy strategy) { + public void emit(IIndentStream builder) { + builder.append("circuit.set_balancer_hint(").increase(); + var input = this.strategy.input == 0 ? this.operator.left() : operator.right(); + var inputHash = OperatorHash.getHash(input.node(), true); + builder.append(Utilities.doubleQuote(Objects.requireNonNull(inputHash).toString(), true)) + .append(",").newline(); + builder.append(this.strategy.toRust()).newline(); + builder.decrease().append(")?;").newline(); + } + } public LateMaterializations(DBSPCompiler compiler) { super(compiler); @@ -39,4 +63,9 @@ public boolean hasRight(DBSPControlledKeyFilterOperator dest) { public DBSPSourceMultisetOperator getLeft(DBSPControlledKeyFilterOperator dest) { return this.materialization.getLeft(dest); } + + public void recordHint(DBSPBinaryOperator operator, JoinStrategy strategy) { + BalancerHint hint = new BalancerHint(operator, strategy); + this.balancerHints.add(hint); + } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerAsof.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerAsof.java index 25c534a243e..2c8fe07d85f 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerAsof.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerAsof.java @@ -291,7 +291,9 @@ public void postorder(DBSPAsofJoinOperator join) { DBSPConcreteAsofJoinOperator result = new DBSPConcreteAsofJoinOperator(node, join.getOutputZSetType(), function, leftTimestamp, rightTimestamp, join.comparator, join.isMultiset, join.isLeft, - leftIndex.outputPort(), rightIndex.outputPort()); + leftIndex.outputPort(), rightIndex.outputPort()) + .copyAnnotations(join) + .to(DBSPConcreteAsofJoinOperator.class); this.map(join, result); // If there is an IntegrateTraceRetainValues operator on the left input, it needs to be moved. diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeIncrementalVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeIncrementalVisitor.java index 0de7e7116d1..6948667493f 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeIncrementalVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeIncrementalVisitor.java @@ -69,6 +69,7 @@ public void linear(DBSPUnaryOperator operator) { if (source.node().is(DBSPIntegrateOperator.class)) { DBSPSimpleOperator replace = operator .withInputs(source.node().inputs, true) + .copyAnnotations(operator) .to(DBSPSimpleOperator.class); this.addOperator(replace); DBSPIntegrateOperator integral = new DBSPIntegrateOperator( @@ -129,7 +130,8 @@ public void postorder(DBSPStreamJoinOperator operator) { DBSPSimpleOperator replace = new DBSPJoinOperator(operator.getRelNode(), operator.getOutputZSetType(), operator.getFunction(), operator.isMultiset, - sourceSource.get(0), sourceSource.get(1), operator.balanced); + sourceSource.get(0), sourceSource.get(1), operator.balanced) + .copyAnnotations(operator); this.addOperator(replace); DBSPIntegrateOperator integral = new DBSPIntegrateOperator( operator.getRelNode(), replace.outputPort()); @@ -147,7 +149,8 @@ public void postorder(DBSPStreamJoinIndexOperator operator) { DBSPSimpleOperator replace = new DBSPJoinIndexOperator(operator.getRelNode(), operator.getOutputIndexedZSetType(), operator.getFunction(), operator.isMultiset, - sourceSource.get(0), sourceSource.get(1), operator.balanced); + sourceSource.get(0), sourceSource.get(1), operator.balanced) + .copyAnnotations(operator); this.addOperator(replace); DBSPIntegrateOperator integral = new DBSPIntegrateOperator(operator.getRelNode(), replace.outputPort()); this.map(operator, integral); @@ -162,7 +165,8 @@ public void postorder(DBSPStreamAntiJoinOperator operator) { if (Linq.all(sources, s -> s.node().is(DBSPIntegrateOperator.class))) { List sourceSource = Linq.map(sources, s -> s.node().inputs.get(0)); DBSPSimpleOperator replace = new DBSPAntiJoinOperator(operator.getRelNode(), - sourceSource.get(0), sourceSource.get(1)); + sourceSource.get(0), sourceSource.get(1)) + .copyAnnotations(operator); this.addOperator(replace); DBSPIntegrateOperator integral = new DBSPIntegrateOperator(operator.getRelNode(), replace.outputPort()); this.map(operator, integral); @@ -177,7 +181,7 @@ public void postorder(DBSPSumOperator operator) { if (Linq.all(sources, s -> s.node().is(DBSPIntegrateOperator.class))) { List sourceSource = Linq.map(sources, s -> s.node().inputs.get(0)); DBSPSimpleOperator replace = new DBSPSumOperator(operator.getRelNode(), sourceSource) - .addAnnotations(operator.annotations, DBSPSimpleOperator.class); + .copyAnnotations(operator); this.addOperator(replace); DBSPIntegrateOperator integral = new DBSPIntegrateOperator(operator.getRelNode(), replace.outputPort()); this.map(operator, integral); @@ -192,7 +196,7 @@ public void postorder(DBSPAtomicSumOperator operator) { if (Linq.all(sources, s -> s.node().is(DBSPIntegrateOperator.class))) { List sourceSource = Linq.map(sources, s -> s.node().inputs.get(0)); DBSPSimpleOperator replace = new DBSPAtomicSumOperator(operator.getRelNode(), sourceSource) - .addAnnotations(operator.annotations, DBSPSimpleOperator.class); + .copyAnnotations(operator); this.addOperator(replace); DBSPIntegrateOperator integral = new DBSPIntegrateOperator(operator.getRelNode(), replace.outputPort()); this.map(operator, integral); @@ -206,7 +210,8 @@ public void postorder(DBSPSubtractOperator operator) { List sources = Linq.map(operator.inputs, this::mapped); if (Linq.all(sources, s -> s.node().is(DBSPIntegrateOperator.class))) { List sourceSource = Linq.map(sources, s -> s.node().inputs.get(0)); - DBSPSimpleOperator replace = new DBSPSubtractOperator(operator.getRelNode(), sourceSource.get(0), sourceSource.get(1)); + DBSPSimpleOperator replace = new DBSPSubtractOperator(operator.getRelNode(), sourceSource.get(0), sourceSource.get(1)) + .copyAnnotations(operator); this.addOperator(replace); DBSPIntegrateOperator integral = new DBSPIntegrateOperator(operator.getRelNode(), replace.outputPort()); this.map(operator, integral); @@ -219,7 +224,8 @@ public void postorder(DBSPSubtractOperator operator) { public void postorder(DBSPStreamDistinctOperator operator) { OutputPort source = this.mapped(operator.input()); if (source.node().is(DBSPIntegrateOperator.class)) { - DBSPSimpleOperator replace = new DBSPDistinctOperator(operator.getRelNode(), source.node().inputs.get(0)); + DBSPSimpleOperator replace = new DBSPDistinctOperator(operator.getRelNode(), source.node().inputs.get(0)) + .copyAnnotations(operator); this.addOperator(replace); DBSPIntegrateOperator integral = new DBSPIntegrateOperator(operator.getRelNode(), replace.outputPort()); this.map(operator, integral); @@ -234,7 +240,8 @@ public void postorder(DBSPStreamAggregateOperator operator) { if (source.node().is(DBSPIntegrateOperator.class)) { DBSPSimpleOperator replace = new DBSPAggregateOperator( source.node().getRelNode(), operator.getOutputIndexedZSetType(), - operator.getAggregator(), operator.aggregateList, source.node().inputs.get(0)); + operator.getAggregator(), operator.aggregateList, source.node().inputs.get(0)) + .copyAnnotations(operator); this.addOperator(replace); DBSPIntegrateOperator integral = new DBSPIntegrateOperator(operator.getRelNode(), replace.outputPort()); this.map(operator, integral); @@ -253,6 +260,7 @@ public void postorder(DBSPDeltaOperator operator) { Utilities.enforce(source.node().is(DBSPIntegrateOperator.class)); DBSPSimpleOperator result = operator.withInputs( Linq.list(source.node().to(DBSPUnaryOperator.class).input()), true) + .copyAnnotations(operator) .to(DBSPSimpleOperator.class); this.map(operator, result); return; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjectionVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjectionVisitor.java index 430fe89f4da..2dec9cdc341 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjectionVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjectionVisitor.java @@ -10,6 +10,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPLeftJoinOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPMapIndexOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPMapOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPStarJoinIndexOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPStarJoinOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPStreamJoinIndexOperator; @@ -224,25 +225,29 @@ static DBSPSimpleOperator mapIndexAfterJoin( if (source.is(DBSPJoinOperator.class)) { Utilities.enforce(join != null); - DBSPJoinBaseOperator result = new DBSPJoinIndexOperator(node, operator.getOutputIndexedZSetType(), - newFunction, operator.isMultiset, join.left(), join.right(), join.balanced); + DBSPSimpleOperator result = new DBSPJoinIndexOperator(node, operator.getOutputIndexedZSetType(), + newFunction, operator.isMultiset, join.left(), join.right(), join.balanced) + .copyAnnotations(source); result.setDerivedFrom(source); return result; } else if (source.is(DBSPStreamJoinOperator.class)) { Utilities.enforce(join != null); - DBSPJoinBaseOperator result = new DBSPStreamJoinIndexOperator(node, operator.getOutputIndexedZSetType(), - newFunction, operator.isMultiset, join.left(), join.right(), join.balanced); + DBSPSimpleOperator result = new DBSPStreamJoinIndexOperator(node, operator.getOutputIndexedZSetType(), + newFunction, operator.isMultiset, join.left(), join.right(), join.balanced) + .copyAnnotations(source); result.setDerivedFrom(source); return result; } else if (source.is(DBSPLeftJoinOperator.class)) { Utilities.enforce(join != null); - DBSPJoinBaseOperator result = new DBSPLeftJoinIndexOperator(node, operator.getOutputIndexedZSetType(), - newFunction, operator.isMultiset, join.left(), join.right(), join.balanced); + DBSPSimpleOperator result = new DBSPLeftJoinIndexOperator(node, operator.getOutputIndexedZSetType(), + newFunction, operator.isMultiset, join.left(), join.right(), join.balanced) + .copyAnnotations(source); result.setDerivedFrom(source); return result; } else if (source.is(DBSPStarJoinOperator.class)) { DBSPSimpleOperator result = new DBSPStarJoinIndexOperator(node, operator.getOutputIndexedZSetType(), - newFunction, operator.isMultiset, source.inputs); + newFunction, operator.isMultiset, source.inputs) + .copyAnnotations(source); result.setDerivedFrom(source); return result; } else { @@ -266,25 +271,29 @@ static DBSPSimpleOperator mapIndexAfterJoinIndex( DBSPJoinBaseOperator join = source.as(DBSPJoinBaseOperator.class); if (source.is(DBSPJoinIndexOperator.class)) { Utilities.enforce(join != null); - DBSPJoinBaseOperator result = new DBSPJoinIndexOperator(node, operator.getOutputIndexedZSetType(), - newFunction, operator.isMultiset, join.left(), join.right(), join.balanced); + DBSPSimpleOperator result = new DBSPJoinIndexOperator(node, operator.getOutputIndexedZSetType(), + newFunction, operator.isMultiset, join.left(), join.right(), join.balanced) + .copyAnnotations(join); result.setDerivedFrom(source); return result; } else if (source.is(DBSPLeftJoinIndexOperator.class)) { Utilities.enforce(join != null); - DBSPJoinBaseOperator result = new DBSPLeftJoinIndexOperator(node, operator.getOutputIndexedZSetType(), - newFunction, operator.isMultiset, join.left(), join.right(), join.balanced); + DBSPSimpleOperator result = new DBSPLeftJoinIndexOperator(node, operator.getOutputIndexedZSetType(), + newFunction, operator.isMultiset, join.left(), join.right(), join.balanced) + .copyAnnotations(join); result.setDerivedFrom(source); return result; } else if (source.is(DBSPStarJoinIndexOperator.class)) { DBSPSimpleOperator result = new DBSPStarJoinIndexOperator(node, operator.getOutputIndexedZSetType(), - newFunction, operator.isMultiset, source.inputs); + newFunction, operator.isMultiset, source.inputs) + .copyAnnotations(source); result.setDerivedFrom(source); return result; } else if (source.is(DBSPStreamJoinIndexOperator.class)) { Utilities.enforce(join != null); - DBSPJoinBaseOperator result = new DBSPStreamJoinIndexOperator(node, operator.getOutputIndexedZSetType(), - newFunction, operator.isMultiset, join.left(), join.right(), join.balanced); + DBSPSimpleOperator result = new DBSPStreamJoinIndexOperator(node, operator.getOutputIndexedZSetType(), + newFunction, operator.isMultiset, join.left(), join.right(), join.balanced) + .copyAnnotations(source); result.setDerivedFrom(source); return result; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPStaticExpression.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPStaticExpression.java index 9ce2a8b002f..e00bc7d0787 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPStaticExpression.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPStaticExpression.java @@ -1,6 +1,7 @@ package org.dbsp.sqlCompiler.ir.expression; import com.fasterxml.jackson.databind.JsonNode; +import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.backend.JsonDecoder; import org.dbsp.sqlCompiler.compiler.backend.MerkleInner; @@ -15,6 +16,8 @@ import org.dbsp.util.IndentStreamBuilder; import org.dbsp.util.Utilities; +import javax.annotation.Nullable; + /** Represents an expression that is compiled into a * Lazy declaration and a reference to the lazy lock value. * The name of the cell is not represented explicitly. */ @@ -30,9 +33,11 @@ public DBSPStaticExpression(CalciteObject node, DBSPExpression initializer, Stri this.name = name; } - public static String generateName(DBSPExpression initializer, DBSPCompiler compiler) { + public static String generateName( + DBSPExpression initializer, @Nullable DBSPOperator operatorContext, DBSPCompiler compiler) { IndentStream stream = new IndentStreamBuilder(); ToRustInnerVisitor toRust = new ToRustInnerVisitor(compiler, stream, null, false); + toRust.setOperatorContext(operatorContext); initializer.accept(toRust); stream.append(":"); initializer.type.accept(toRust); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java index 6ca9f2d6594..9cb800c0d8a 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java @@ -62,6 +62,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -308,6 +309,8 @@ public static VE putNew(Map map, K key, VE value) { return value; } + /** Print a ResultSet obtained from some database. */ + @SuppressWarnings("unused") public static void showResultSet(ResultSet result, PrintStream out) throws SQLException { int columnCount = result.getMetaData().getColumnCount(); @@ -428,10 +431,16 @@ public static void runProcess(String directory, Map environment, public static void createEmptyFile(Path path) { try { + File parent = path.getParent().toFile(); + if (!parent.exists()) { + Files.createDirectories(path.getParent()); + } PrintStream outputStream = new PrintStream(Files.newOutputStream(path)); outputStream.println(); outputStream.close(); - } catch (IOException ignored) {} + } catch (IOException e) { + throw new RuntimeException("Cannot create file " + path + ": " + e.getMessage()); + } } public static void runProcess(String directory, String... commands) throws IOException, InterruptedException { @@ -650,4 +659,12 @@ public static List expandBraces(String pattern) { public static boolean inCI() { return System.getenv("CI") != null; } + + /** Convert str to title case: capital followed by lowercase */ + public static String titleCase(String str) { + if (str.isEmpty()) + return str; + return str.substring(0, 1).toUpperCase(Locale.ENGLISH) + + str.substring(1).toLowerCase(Locale.ENGLISH); + } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java index ab23f9bdbfb..cff672b1f71 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java @@ -4,9 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.calcite.adapter.jdbc.JdbcSchema; -import org.apache.calcite.jdbc.CalciteConnection; -import org.apache.calcite.schema.SchemaPlus; import org.dbsp.sqlCompiler.CompilerMain; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; import org.dbsp.sqlCompiler.circuit.operator.IInputOperator; @@ -23,15 +20,13 @@ import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeIndexedZSet; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeZSet; +import org.dbsp.util.NullPrintStream; import org.dbsp.util.Utilities; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; -import javax.sql.DataSource; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.PrintStream; @@ -41,10 +36,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; -import java.sql.Statement; import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -1393,8 +1385,11 @@ public void generateDFRecursiveTest() throws IOException, SQLException { );"""; File file = createInputScript(sql); File json = this.createTempJsonFile(); + PrintStream savedOut = System.out; + System.setOut(NullPrintStream.INSTANCE); CompilerMessages msg = CompilerMain.execute( "-v", "1", "--dataflow", json.getPath(), "--noRust", file.getPath()); + System.setOut(savedOut); Assert.assertEquals(0, msg.exitCode); String jsonContents = Utilities.readFile(json.toPath()); String expected = TestUtil.readStringFromResourceFile("metadataTests-generateDFRecursive.json"); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java index b0a23f9d509..c1beb975fd8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java @@ -454,4 +454,13 @@ public void issue6150() throws IOException, SQLException, InterruptedException { File file = createInputScript(builder.toString()); compileToMultiCrate(file.getAbsolutePath(), true, true); } + + @Test + public void testHints() throws SQLException, IOException, InterruptedException { + String sql = """ + CREATE TABLE T(x INT); + CREATE TABLE S(x INT); + CREATE VIEW V AS SELECT /*+ broadcast(T), shard(S) */ * FROM T JOIN S USING (x);"""; + compileProgramToMultiCrate(sql, true); + } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java index d4bd89a808f..7062e387a9d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java @@ -224,7 +224,7 @@ public void rustCsvTest() throws IOException, InterruptedException { DBSPTypeVoid.INSTANCE, body, Linq.list("#[test]")); PrintStream stream = new PrintStream(BaseSQLTests.TEST_FILE_PATH); - RustFileWriter writer = new RustFileWriter(new LateMaterializations(compiler)).withTest(true); + RustFileWriter writer = new RustFileWriter(new LateMaterializations(compiler)); writer.setOutputBuilder(new IndentStream(stream)); writer.add(tester); writer.write(compiler); @@ -257,7 +257,7 @@ public void rustCsvTest2() throws IOException, InterruptedException { DBSPTypeVoid.INSTANCE, body, Linq.list("#[test]")); PrintStream outputStream = new PrintStream(BaseSQLTests.TEST_FILE_PATH, StandardCharsets.UTF_8); - RustFileWriter writer = new RustFileWriter(new LateMaterializations(compiler)).withTest(true); + RustFileWriter writer = new RustFileWriter(new LateMaterializations(compiler)); writer.setOutputBuilder(new IndentStream(outputStream)); writer.add(tester); writer.write(compiler); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ParserTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ParserTests.java index 690722daa2c..b3924153912 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ParserTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ParserTests.java @@ -23,11 +23,16 @@ package org.dbsp.sqlCompiler.compiler.sql; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlSetOption; import org.apache.calcite.sql.parser.SqlParseException; import org.dbsp.sqlCompiler.compiler.CompilerOptions; import org.dbsp.sqlCompiler.compiler.StderrErrorReporter; +import org.dbsp.sqlCompiler.compiler.errors.SourceFileContents; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.ParsedStatement; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.SqlToRelCompiler; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteRelNode; @@ -38,6 +43,8 @@ import org.dbsp.sqlCompiler.compiler.frontend.parser.SqlExtendedColumnDeclaration; import org.dbsp.sqlCompiler.compiler.frontend.parser.SqlCreateTable; import org.dbsp.sqlCompiler.compiler.frontend.parser.SqlForeignKey; +import org.dbsp.sqlCompiler.compiler.frontend.statements.CreateViewStatement; +import org.dbsp.sqlCompiler.compiler.frontend.statements.RelStatement; import org.junit.Assert; import org.junit.Test; @@ -96,6 +103,47 @@ public void testSetOption() throws SqlParseException { Assert.assertTrue(node.get(0).statement() instanceof SqlSetOption); } + @Test + public void hintTest() throws SqlParseException { + SqlToRelCompiler compiler = this.getCompiler(); + List node = compiler.parseStatements(""" + CREATE TABLE T(x INT); + CREATE VIEW V AS SELECT /*+ broadcast(T) */ * FROM T JOIN T AS S USING (x);"""); + Assert.assertNotNull(node); + Assert.assertEquals(2, node.size()); + Assert.assertTrue(node.get(1).statement() instanceof SqlCreateView); + SqlNode query = ((SqlCreateView) node.get(1).statement()).query; + Assert.assertTrue(query instanceof SqlSelect); + Assert.assertNotNull(((SqlSelect) query).getHints()); + Assert.assertEquals(1, ((SqlSelect) query).getHints().size()); + + var sources = new SourceFileContents(); + compiler.compile(node.get(0), sources); + RelStatement q = compiler.compile(node.get(1), sources); + Assert.assertTrue(q instanceof CreateViewStatement); + RelNode rel = q.to(CreateViewStatement.class).getRel(); + Assert.assertTrue(rel instanceof LogicalProject); + LogicalProject proj = (LogicalProject) rel; + Assert.assertEquals(1, proj.getHints().size()); + Assert.assertTrue(proj.getInput() instanceof LogicalJoin); + LogicalJoin join = (LogicalJoin) proj.getInput(); + Assert.assertEquals(1, join.getHints().size()); + Assert.assertEquals("broadcast", join.getHints().get(0).hintName); + } + + @Test + public void hintGrammarExampleTests() throws SqlParseException { + // Test the example from the grammar documentation + SqlToRelCompiler compiler = this.getCompiler(); + List node = compiler.parseStatements(""" + CREATE TABLE T(x INT); + CREATE TABLE S(x INT); + CREATE VIEW V AS SELECT /*+ hint1, hint2(a='1', b='2') */ * + FROM T /*+ hint3(5, 'x') */ + JOIN S /*+ hint4(c='id'), hint5 */ on T.x = S.x;"""); + Assert.assertNotNull(node); + } + @Test public void parseRecursiveTest() throws SqlParseException { SqlToRelCompiler compiler = this.getCompiler(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java index a650cc0f472..963d5b59b36 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java @@ -7,7 +7,6 @@ import org.junit.Test; import java.io.File; -import java.io.IOException; import java.sql.SQLException; /** Compile the SQL programs in the feldera-qa repository, and rust-compile the results. */ diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/IncrementalRegression2Tests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/IncrementalRegression2Tests.java index 48751128937..2fc853fa659 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/IncrementalRegression2Tests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/IncrementalRegression2Tests.java @@ -1,6 +1,5 @@ package org.dbsp.sqlCompiler.compiler.sql.simple; -import org.checkerframework.checker.nullness.qual.Nullable; import org.dbsp.sqlCompiler.circuit.operator.DBSPFlatMapIndexOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPMapIndexOperator; import org.dbsp.sqlCompiler.compiler.CompilerOptions; @@ -15,9 +14,11 @@ import org.dbsp.sqlCompiler.ir.type.DBSPTypeCode; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; +import org.dbsp.util.NullPrintStream; import org.junit.Assert; import org.junit.Test; +import java.io.PrintStream; import java.sql.SQLException; import java.util.ArrayList; import java.util.Random; @@ -470,6 +471,8 @@ public void endVisit() { @Test public void issue5935() { + PrintStream savedErr = System.err; + System.setErr(NullPrintStream.INSTANCE); var ccs = this.getCCS(""" CREATE TABLE orders ( order_id INT NOT NULL PRIMARY KEY, @@ -497,6 +500,7 @@ customer_name VARCHAR(200) NOT NULL, LEFT JOIN customers AS sc ON o.shipping_customer_id = sc.customer_id ORDER BY o.order_id;"""); + System.setErr(savedErr); // Validated on Postgres ccs.stepWeightOne(""" INSERT INTO CUSTOMERS VALUES diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/IncrementalRegressionTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/IncrementalRegressionTests.java index e2c6986cf33..7bbbf5b6549 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/IncrementalRegressionTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/IncrementalRegressionTests.java @@ -922,6 +922,23 @@ public void postorder(DBSPOperator operator) { return result; } + @Test + public void testMerkleHints() { + // Hints should not change hashes + var cc0 = this.getCC(""" + CREATE TABLE T(x INT); + CREATE VIEW V AS SELECT /*+ broadcast(T) */ * FROM T JOIN T AS S USING (x);"""); + Assert.assertEquals(0, cc0.compiler.messages.exitCode); + Map hash0 = this.collectHashes(cc0); + + var cc1 = this.getCC(""" + CREATE TABLE T(x INT); + CREATE VIEW V AS SELECT * FROM T JOIN T AS S USING (x);"""); + Assert.assertEquals(0, cc1.compiler.messages.exitCode); + Map hash1 = this.collectHashes(cc1); + Assert.assertEquals(hash0, hash1); + } + @Test public void testHashes() { var cc0 = this.getCC(""" diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/Regression2Tests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/Regression2Tests.java index c110d71292d..e7bfc67a26e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/Regression2Tests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/Regression2Tests.java @@ -1,6 +1,7 @@ package org.dbsp.sqlCompiler.compiler.sql.simple; import org.dbsp.sqlCompiler.circuit.DBSPDeclaration; +import org.dbsp.sqlCompiler.circuit.annotation.JoinStrategy; import org.dbsp.sqlCompiler.circuit.annotation.OperatorHash; import org.dbsp.sqlCompiler.circuit.operator.DBSPAggregateLinearPostprocessOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPAggregateOperatorBase; @@ -8,6 +9,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPPartitionedRollingAggregateOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPStreamAggregateOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPStreamJoinOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.circuit.operator.DBSPWindowOperator; import org.dbsp.sqlCompiler.compiler.TestUtil; @@ -436,11 +438,14 @@ public void endVisit() { public void issue5541c() { DBSPCompiler compiler = this.testCompiler(); compiler.options.ioOptions.quiet = false; + PrintStream saved = System.err; + System.setErr(NullPrintStream.INSTANCE); compiler.submitStatementsForCompilation(""" CREATE TABLE T(x INT, z INT, y INT); CREATE LOCAL VIEW V AS SELECT SUM(x) AS sum, AVG(z) AS max, y FROM T GROUP BY y; CREATE VIEW W AS SELECT sum, y FROM V;"""); var ccs = this.getCCS(compiler); + System.setErr(saved); ccs.visit(new CircuitVisitor(ccs.compiler) { int aggregates; @@ -519,7 +524,8 @@ public void postorder(DBSPDeclaration decl) { } }); - Assert.assertTrue(found[0]); + // When generating code for multi crates the statics are generated differently + Assert.assertTrue(found[0] || cc.compiler.options.ioOptions.multiCrates()); } @Test @@ -1180,4 +1186,86 @@ public void issue6181() { NULL (1 row)"""); } + + @Test + public void missingHintArgument() { + this.statementsFailingInCompilation(""" + CREATE TABLE T(x INT); + CREATE TABLE S(x INT); + CREATE VIEW V AS SELECT /*+ broadcast */ * + FROM T JOIN S on T.x = S.x;""", "Invalid hint: Hint 'broadcast' requires a list of arguments"); + } + + @Test + public void testHints() { + var ccs = this.getCCS(""" + CREATE TABLE T(x INT); + CREATE TABLE S(x INT); + CREATE VIEW V AS SELECT /*+ broadcast(T), shard(S) */ * FROM T JOIN S USING (x);"""); + ccs.visit(new CircuitVisitor(ccs.compiler) { + boolean joinFound = false; + + @Override + public void postorder(DBSPStreamJoinOperator join) { + this.joinFound = true; + Assert.assertEquals("[Broadcast(t), Shard(s)]", join.annotations.get(JoinStrategy.class).toString()); + } + + @Override + public void endVisit() { + Assert.assertTrue(this.joinFound); + } + }); + } + + @Test + public void testThreeWayJoinHints() { + var ccs = this.getCCS(""" + CREATE TABLE T(x INT); + CREATE TABLE S(x INT); + CREATE TABLE U(x INT); + CREATE VIEW V AS SELECT /*+ broadcast(T), shard(S), balance(U) */ * FROM T JOIN S USING (x) JOIN U USING (x);"""); + ccs.visit(new CircuitVisitor(ccs.compiler) { + int joins = 0; + + @Override + public void postorder(DBSPStreamJoinOperator join) { + String strat = join.annotations.get(JoinStrategy.class).toString(); + this.joins++; + Assert.assertTrue(strat.equals("[Broadcast(t), Shard(s)]") || strat.equals("[Balance(u)]")); + } + + @Override + public void endVisit() { + Assert.assertEquals(2, this.joins); + } + }); + } + + @Test + public void testNoMatchingTableHint() { + this.statementsFailingInCompilation(""" + CREATE TABLE T(x INT); + CREATE VIEW V AS SELECT /*+ broadcast(X), shard(S) */ * FROM T JOIN T AS S USING (x);""", + "Cannot apply hint: Hint 'broadcast(x)' specifies an input which cannot be identified among the join inputs"); + } + + @Test + public void testConflictingHints() { + this.statementsFailingInCompilation(""" + CREATE TABLE T(x INT); + CREATE VIEW V AS SELECT /*+ broadcast(T), shard(S) */ * FROM T JOIN T AS S USING (x);""", + "Incompatible hints: Found two conflicting hints for the same collection: Shard(s)"); + } + + @Test + public void testAmbiguousHint() { + this.statementsFailingInCompilation(""" + CREATE TABLE T(x INT); + CREATE TABLE S(x INT); + CREATE VIEW V AS WITH + TMP AS (SELECT * FROM T JOIN S USING(x)) + SELECT /*+ broadcast(T) */ * FROM T JOIN TMP USING (X); + """, "Ambiguous hint: Hint 'broadcast(t)' matches 2 different collections"); + } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java index 5911d2ce361..aa2cca2b52d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java @@ -29,6 +29,7 @@ import org.dbsp.sqlCompiler.compiler.backend.rust.RustFileWriter; import org.dbsp.sqlCompiler.compiler.backend.rust.RustWriter; import org.dbsp.sqlCompiler.compiler.backend.rust.StubsWriter; +import org.dbsp.sqlCompiler.compiler.backend.rust.multi.MultiCrates; import org.dbsp.sqlCompiler.compiler.backend.rust.multi.MultiCratesWriter; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.SqlToRelCompiler; import org.dbsp.sqlCompiler.compiler.sql.MultiCrateTests; @@ -318,24 +319,33 @@ static int runTests(List toRun, int testNumber, DBSPCompiler compiler, HashMap inputFunctions = new HashMap<>(); PrintStream outputStream = null; RustWriter writer; - String directory; + final String directory; + final String executionDirectory; Path stubsDir; String testCrate = ""; if (!multiCrates) { outputStream = new PrintStream(Files.newOutputStream(getTestFilePath())); // We do not need to run the LateMaterializations - writer = new RustFileWriter(new LateMaterializations(compiler)).withTest(true); + writer = new RustFileWriter(new LateMaterializations(compiler)); writer.setOutputBuilder(new IndentStream(outputStream)); directory = RUST_DIRECTORY; + executionDirectory = "."; stubsDir = Paths.get(directory); createEmptyStubs(); } else { directory = RUST_MULTI_DIRECTORY; + executionDirectory = "../.."; MultiCratesWriter multiWriter = new MultiCratesWriter(directory, "x", true); testCrate = MultiCratesWriter.getTestName(); writer = multiWriter; stubsDir = multiWriter.stubsDirectory(); + + // Make sure there is no stray udf.rs file from a previous test + Path path = Paths.get(BaseSQLTests.RUST_MULTI_DIRECTORY + "/crates", + MultiCrates.FILE_PREFIX + "x_globals", "src", DBSPCompiler.UDF_FILE_NAME); + File udfFile = path.toFile(); + Utilities.deleteFile(udfFile, true); } StubsWriter stubsWriter = new StubsWriter(stubsDir.resolve(DBSPCompiler.STUBS_FILE_NAME)); @@ -347,7 +357,7 @@ static int runTests(List toRun, int testNumber, DBSPCompiler compiler, " are not compiled with the same options: " + test.ccs.compiler.options.diff(compiler.options)); test.ccs.circuit.setName("circuit" + testNumber); - List testers = test.createTesterCode(testNumber, RUST_DIRECTORY, inputFunctions); + List testers = test.createTesterCode(testNumber, directory, executionDirectory, inputFunctions); writer.add(test.ccs.circuit); if (multiCrates) { stubsWriter.add(test.ccs.circuit); @@ -362,7 +372,7 @@ static int runTests(List toRun, int testNumber, DBSPCompiler compiler, testNumber++; } writer.write(compiler); - if (outputStream !=null) + if (outputStream != null) outputStream.close(); stubsWriter.write(compiler); if (!skipRust) { @@ -408,11 +418,15 @@ protected void addFailingRustTestCase(String message, CompilerCircuitStream ccs) testsToRun.add(test); } - public CompilerOptions testOptions() { - CompilerOptions options = new CompilerOptions(); + void ciOptions(CompilerOptions options) { if (Utilities.inCI()) // Set to compile to multiple crates options.ioOptions.crates = "x"; + } + + public CompilerOptions testOptions() { + CompilerOptions options = new CompilerOptions(); + this.ciOptions(options); options.ioOptions.testing = true; options.languageOptions.throwOnError = true; options.languageOptions.generateInputForEveryTable = true; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/SqlIoTest.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/SqlIoTest.java index 8cb12916c14..b69774e7228 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/SqlIoTest.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/SqlIoTest.java @@ -52,6 +52,7 @@ public void setOptimize(boolean optimize) { public CompilerOptions testOptions() { CompilerOptions options = new CompilerOptions(); + this.ciOptions(options); options.ioOptions.testing = true; options.ioOptions.quiet = true; options.ioOptions.emitHandles = true; @@ -60,7 +61,9 @@ public CompilerOptions testOptions() { options.languageOptions.generateInputForEveryTable = true; options.languageOptions.incrementalize = false; options.languageOptions.unrestrictedIOTypes = true; - options.ioOptions.verbosity = 2; + if (!options.ioOptions.multiCrates()) + // Comments interfere with hashes + options.ioOptions.verbosity = 2; options.languageOptions.ignoreOrderBy = true; return options; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TestCase.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TestCase.java index 64ffebe117a..2f438fc65b9 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TestCase.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TestCase.java @@ -65,12 +65,13 @@ boolean hasData() { * Generates a Rust function which tests a {@link DBSPCircuit}. * @param changeFunction Maps a Change id to a function that generates the contents of the change. * Used to share changes between many tests. + * @param executionDirectory Relative directory where executable will run. * * @return The code for a function that runs the circuit with the specified * input and tests the produced output, plus the code for functions that generate * new inputs. */ List createTesterCode( - int testNumber, @SuppressWarnings("SameParameterValue") String codeDirectory, + int testNumber, String codeDirectory, String executionDirectory, Map changeFunction) throws IOException { List result = new ArrayList<>(); @@ -114,7 +115,7 @@ List createTesterCode( tableData[i] = inputs.getSet(i); String functionName = "input" + changeId; DBSPFunction inputFunction = TableData.createInputFunction( - this.ccs.compiler, functionName, tableData, codeDirectory); + this.ccs.compiler, functionName, tableData, codeDirectory, executionDirectory); if (!changeFunction.containsKey(changeId)) { result.add(inputFunction); changeFunction.put(changeId, inputFunction); @@ -191,7 +192,8 @@ List createTesterCode( // Currently we don't have any tests with this case. TableData expected = outputs.getSet(i); - DBSPExpression viewContents = expected.createOutput(this.ccs.compiler, "out" + testNumber, codeDirectory); + DBSPExpression viewContents = expected.createOutput(this.ccs.compiler, "out" + testNumber, + codeDirectory, executionDirectory); DBSPExpression actual = new DBSPApplyExpression("read_output_spine", DBSPTypeAny.getDefault(), streams.getVarReference().field(changes.inputs.getSetCount() + i + skippedOutputs).borrow()); var produced = new DBSPLetStatement("produced_result", actual); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generateDFRecursive.json b/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generateDFRecursive.json index 558d99a2642..863aefbef72 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generateDFRecursive.json +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generateDFRecursive.json @@ -77,39 +77,6 @@ }, { "id": 3, - "relOp": "LogicalFilter", - "condition": { - "pos": { - "line": 20, - "column": 27, - "end_line": 20, - "end_column": 37 - }, - "op": { - "name": "<", - "kind": "LESS_THAN", - "syntax": "BINARY" - }, - "operands": [ - { - "input": 0, - "name": "$0" - }, - { - "literal": 10, - "type": { - "type": "INTEGER", - "nullable": false - } - } - ] - }, - "inputs": [ - 2 - ] - }, - { - "id": 4, "relOp": "LogicalProject", "fields": [ "n", @@ -157,7 +124,7 @@ ] }, { - "id": 5, + "id": 4, "relOp": "LogicalFilter", "condition": { "pos": { @@ -186,22 +153,31 @@ ] }, "inputs": [ - 4 + 3 ] }, + { + "id": 5, + "relOp": "LogicalTableScan", + "table": [ + "schema", + "fibonacci-decl" + ], + "inputs": [] + }, { "id": 6, - "relOp": "LogicalJoin", + "relOp": "LogicalFilter", "condition": { "pos": { - "line": 19, - "column": 8, - "end_line": 19, - "end_column": 26 + "line": 20, + "column": 27, + "end_line": 20, + "end_column": 37 }, "op": { - "name": "=", - "kind": "EQUALS", + "name": "<", + "kind": "LESS_THAN", "syntax": "BINARY" }, "operands": [ @@ -210,50 +186,47 @@ "name": "$0" }, { - "input": 4, - "name": "$4" + "literal": 10, + "type": { + "type": "INTEGER", + "nullable": false + } } ] }, - "joinType": "inner", "inputs": [ - 3, 5 ] }, { "id": 7, - "relOp": "LogicalProject", - "fields": [ - "n0", - "value0", - "$f2", - "n", - "value" - ], - "exprs": [ - { - "input": 2, - "name": "$2" - }, - { - "input": 3, - "name": "$3" - }, - { - "input": 4, - "name": "$4" + "relOp": "LogicalJoin", + "condition": { + "pos": { + "line": 19, + "column": 8, + "end_line": 19, + "end_column": 26 }, - { - "input": 0, - "name": "$0" + "op": { + "name": "=", + "kind": "EQUALS", + "syntax": "BINARY" }, - { - "input": 1, - "name": "$1" - } - ], + "operands": [ + { + "input": 3, + "name": "$3" + }, + { + "input": 2, + "name": "$2" + } + ] + }, + "joinType": "inner", "inputs": [ + 4, 6 ] }, @@ -382,7 +355,12 @@ "table": "fibonacci-decl", "backedges": [{ "node": "s7", "output": 0 }], "calcite": { - "final": 2 + "and": [ + { + "final": 2 + },{ + "final": 5 + }] }, "positions": [], "persistent_id": "1cad9022fb9400950c4b4a4d9aaf96ca4433f989b896a5147b0e3335d900fe21" @@ -395,9 +373,9 @@ "calcite": { "seq": [ { - "final": 3 + "final": 6 },{ - "partial": 6 + "partial": 7 }] }, "positions": [ @@ -414,11 +392,11 @@ "calcite": { "seq": [ { - "final": 4 + "final": 3 },{ - "final": 5 + "final": 4 },{ - "partial": 6 + "partial": 7 }] }, "positions": [ @@ -441,18 +419,18 @@ "s6": { "operation": "join", "inputs": [ - { "node": "s3", "output": 0 }, - { "node": "s4", "output": 0 } + { "node": "s4", "output": 0 }, + { "node": "s3", "output": 0 } ], "calcite": { - "partial": 6 + "partial": 7 }, "positions": [ {"start_line_number":15,"start_column":9,"end_line_number":15,"end_column":18}, {"start_line_number":16,"start_column":9,"end_line_number":16,"end_column":33}, {"start_line_number":19,"start_column":8,"end_line_number":19,"end_column":26} ], - "persistent_id": "81b49d948c1c4a6d4e863722062a57fc85c42a70bbb799f2d4a00dc990d03e18" + "persistent_id": "41c15145ffd7f94a9c7335f8c82ca5f9fcd14d9d5c62c8aeea8616a7eb7433f4" }, "s7": { "operation": "sum", @@ -464,7 +442,7 @@ "final": 10 }, "positions": [], - "persistent_id": "34a1b540135f85b90260d5a595eac0de35e34b0bef4ca4e4467cf721a09b62b9" + "persistent_id": "f9e9f0eed2184cdd3cacd079404f8ff466e148cfafe969034ddacc0f747a8d17" } }, "s8": { "operation": "inspect", @@ -479,7 +457,7 @@ {"start_line_number":4,"start_column":1,"end_line_number":21,"end_column":1}, {"start_line_number":4,"start_column":1,"end_line_number":21,"end_column":1} ], - "persistent_id": "0ad92ba0ff858d83b7356b3a91040ce921e4a81ec92647aad112549d5d6620d8" + "persistent_id": "c79ff4ab927d459ed213c8a0e70e93c7fdc0ed93011443b0ae9f1404fc33eca1" }, "s9": { "operation": "constant", "inputs": [], diff --git a/sql-to-dbsp-compiler/calcite_version.env b/sql-to-dbsp-compiler/calcite_version.env index 396209b7ca2..aa76e7329cc 100644 --- a/sql-to-dbsp-compiler/calcite_version.env +++ b/sql-to-dbsp-compiler/calcite_version.env @@ -3,5 +3,5 @@ CALCITE_REPO="https://github.com/apache/calcite.git" #CALCITE_REPO="https://github.com/mihaibudiu/calcite" CALCITE_BRANCH="main" CALCITE_CURRENT="1.41.0" -CALCITE_NEXT_COMMIT="b2e1ab3e46643f30c7263c1520564c11c0e40053" +CALCITE_NEXT_COMMIT="b4a9b28799eceb1d23e5c5791d37a2d7f2afa607" CALCITE_NEXT="1.42.0"