diff --git a/sql-to-dbsp-compiler/README.md b/sql-to-dbsp-compiler/README.md index 8bec46c8bdf..759d976dc2c 100644 --- a/sql-to-dbsp-compiler/README.md +++ b/sql-to-dbsp-compiler/README.md @@ -15,7 +15,7 @@ Java dependencies. The code generated by the compiler is Rust. To run it you need a working installation of Rust (we recommend using rustup to install: ). The compiler uses the -current version of the DBSP library sources from github +current version of the DBSP library sources from Github (). If you want to generate images of the query plans you need diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java index c91e91c18f8..e1238ad064f 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java @@ -70,11 +70,9 @@ public class CompilerMain { } void usage(JCommander commander) { - /* // JCommander mistakenly prints this as default value // if it manages to parse it partially. - this.options.ioOptions.loggingLevel.clear(); - */ + // this.options.ioOptions.loggingLevel.clear(); commander.usage(); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/DBSPCircuit.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/DBSPCircuit.java index 61b07ac18a8..efa77a0af46 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/DBSPCircuit.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/DBSPCircuit.java @@ -26,8 +26,6 @@ import com.fasterxml.jackson.databind.JsonNode; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSinkOperator; -import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceBaseOperator; -import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceTableOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPViewDeclarationOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPViewOperator; import org.dbsp.sqlCompiler.circuit.operator.IInputOperator; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateZeroOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateZeroOperator.java index 401d332563a..f7c32dc34b1 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateZeroOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateZeroOperator.java @@ -28,7 +28,7 @@ * For this we synthesize the following graph: * | * {}/{c->1}------------------------ - * | map (|x| x -> z} | + * | map (|x| x -> z) | * {}/{z->1} | * | - | * {} {z->-1} {z->1} (constant) | diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java index 4a4d987127f..9f01a3fe13c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java @@ -11,7 +11,6 @@ import org.dbsp.sqlCompiler.compiler.visitors.monotone.PartiallyMonotoneTuple; import org.dbsp.sqlCompiler.compiler.visitors.outer.CircuitVisitor; import org.dbsp.sqlCompiler.ir.DBSPParameter; -import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPOpcode; import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; @@ -35,7 +34,7 @@ public DBSPIntegrateTraceRetainKeysOperator( data.outputType(), data.isMultiset(), data, control, false); } - /** Create a operator to retain keys and returns it. May return null if the keys contain no fields. */ + /** Create an operator to retain keys and returns it. May return null if the keys contain no fields. */ @Nullable public static DBSPIntegrateTraceRetainKeysOperator create( CalciteRelNode node, OutputPort data, IMaybeMonotoneType dataProjection, OutputPort control) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java index 9da5f1d95e3..416a2132ef0 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java @@ -36,12 +36,14 @@ public DBSPIntegrateTraceRetainValuesOperator( public static DBSPIntegrateTraceRetainValuesOperator create( CalciteRelNode node, OutputPort data, IMaybeMonotoneType dataProjection, OutputPort control) { DBSPType controlType = control.outputType(); - Utilities.enforce(controlType.is(DBSPTypeTupleBase.class), "Control type is not a tuple: " + controlType); + Utilities.enforce(controlType.is(DBSPTypeTupleBase.class), + "Control type is not a tuple: " + controlType); DBSPTypeTupleBase controlTuple = controlType.to(DBSPTypeTupleBase.class); Utilities.enforce(controlTuple.size() == 2); DBSPVariablePath controlArg = controlType.ref().var(); - Utilities.enforce(data.outputType().is(DBSPTypeIndexedZSet.class)); + Utilities.enforce(data.outputType().is(DBSPTypeIndexedZSet.class), + "Data is not indexed: " + data.outputType()); DBSPType valueType = data.getOutputIndexedZSetType().elementType; DBSPVariablePath dataArg = valueType.ref().var(); DBSPParameter param = new DBSPParameter(dataArg.variable, dataArg.getType()); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNestedOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNestedOperator.java index 26190c36b35..be6f9265974 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNestedOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNestedOperator.java @@ -87,7 +87,7 @@ public void addOperator(DBSPOperator operator) { /** Add an output for this nested operator. The port may be null if the corresponding output * has actually been deleted. * @param view View corresponding to output. - * @param port Output port corresponding to the view (may be in a differentiator). + * @param port Output port corresponding to the view (could be in a differentiator). * @return A port of this operator that corresponds */ public OutputPort addOutput(ProgramIdentifier view, @Nullable OutputPort port) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPOperatorWithError.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPOperatorWithError.java index 942fdbca56e..be08de690c4 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPOperatorWithError.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPOperatorWithError.java @@ -8,8 +8,6 @@ import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; import org.dbsp.sqlCompiler.ir.type.DBSPType; -import java.util.List; - /** These operators have *two* outputs: a regular stream output * and an error output */ public abstract class DBSPOperatorWithError diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSimpleOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSimpleOperator.java index 2686efbd7b4..1ea7c1cb2af 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSimpleOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSimpleOperator.java @@ -46,7 +46,7 @@ public abstract class DBSPSimpleOperator extends DBSPOperator /** Type of output produced. */ public final DBSPType outputType; /** True if the output of the operator is a multiset. Conservative approximation; - * if this is 'false', it is surely false. It if is true, the output may still be a set. */ + * if this is 'false', it is surely false. If it is true, the output may still be a set. */ public final boolean isMultiset; /** True if the operator contains an integrator */ public final boolean containsIntegrator; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPViewOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPViewOperator.java index ead58edafd3..ebcf0490f00 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPViewOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPViewOperator.java @@ -15,7 +15,6 @@ import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; import org.dbsp.sqlCompiler.ir.type.DBSPType; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeStruct; -import org.dbsp.util.Linq; import org.dbsp.util.Utilities; import javax.annotation.Nullable; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/InputColumnMetadata.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/InputColumnMetadata.java index b5f719268fd..e525cea9ddc 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/InputColumnMetadata.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/InputColumnMetadata.java @@ -9,7 +9,6 @@ import org.dbsp.sqlCompiler.ir.DBSPNode; import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; import org.dbsp.sqlCompiler.ir.type.DBSPType; -import org.dbsp.sqlCompiler.ir.type.IHasType; import org.dbsp.util.IJson; import org.dbsp.util.Utilities; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/SourcePositionResource.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/SourcePositionResource.java index 8fc2095c5a4..6af18220dca 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/SourcePositionResource.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/SourcePositionResource.java @@ -2,11 +2,9 @@ import org.dbsp.sqlCompiler.circuit.annotation.OperatorHash; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; -import org.dbsp.sqlCompiler.compiler.backend.rust.multi.CircuitWriter; import org.dbsp.sqlCompiler.compiler.errors.SourcePosition; import org.dbsp.sqlCompiler.compiler.errors.SourcePositionRange; import org.dbsp.sqlCompiler.ir.expression.DBSPHandleErrorExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPStaticExpression; import org.dbsp.util.HashString; import org.dbsp.util.IIndentStream; import org.dbsp.util.Utilities; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java index ecdf245661b..2cf05a5930c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java @@ -48,7 +48,6 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPDistinctOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPJoinBaseOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; -import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceTableOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPStreamJoinIndexOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPIndexedTopKOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPIntegrateTraceRetainKeysOperator; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CircuitWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CircuitWriter.java index 87187c67f53..68f8527d036 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CircuitWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/CircuitWriter.java @@ -7,7 +7,6 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSinkOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceBaseOperator; -import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceTableOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPViewBaseOperator; import org.dbsp.sqlCompiler.circuit.operator.IInputOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java index eebbdd9a1c1..82981d62f4e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java @@ -34,7 +34,8 @@ public final class SingleOperatorWriter extends BaseRustCodeGenerator { /* Example output: * ... preamble ... - * pub fn create_xxx(circuit: &RootCircuit, hash: Option<&'static str>, sourceMap: &'static SourceMap, catalog: &mut Catalog, + * pub fn create_xxx(circuit: &RootCircuit, hash: Option<&'static str>, + * sourceMap: &'static SourceMap, catalog: &mut Catalog, * i0: &Stream>>>, * i1: &Stream>>>, * i2: &Stream>>>, ) -> diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java index 1d9d3b16ebb..e9bee779515 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java @@ -449,34 +449,36 @@ UnimplementedException decorrelateError(CalciteObject node) { } void visitCorrelate(LogicalCorrelate correlate) { - // We decorrelate queries using Calcite's optimizer, which doesn't always work. - // In particular, it won't decorrelate queries with unnest. - // Here we check for unnest-type queries. We assume that unnest queries - // have a restricted plan of this form: - // LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{...}]) - // LeftSubquery (arbitrary) - // Uncollect - // LogicalProject(COL=[$cor0.ARRAY]) // uncollectInput - // LogicalValues(tuples=[[{ 0 }]]) - // or - // LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{...}]) - // LeftSubquery - // LogicalProject // rightProject - // Uncollect - // LogicalProject(COL=[$cor0.ARRAY]) // uncollectInput - // LogicalValues(tuples=[[{ 0 }]]) - // Instead of projecting and joining again we directly apply flatmap. - // The translation for this is: - // stream.flat_map({ - // move |x: &Tuple2, Option>, | -> _ { - // let xA: Vec = x.0.clone(); - // let xB: x.1.clone(); - // x.0.clone().into_iter().map({ - // move |e: i32, | -> Tuple3, Option, i32> { - // Tuple3::new(xA.clone(), xB.clone(), e) - // } - // }) - // }); + /* + We decorrelate queries using Calcite's optimizer, which doesn't always work. + In particular, it won't decorrelate queries with unnest. + Here we check for unnest-type queries. We assume that unnest queries + have a restricted plan of this form: + LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{...}]) + LeftSubquery (arbitrary) + Uncollect + LogicalProject(COL=[$cor0.ARRAY]) // uncollectInput + LogicalValues(tuples=[[{ 0 }]]) + or + LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{...}]) + LeftSubquery + LogicalProject // rightProject + Uncollect + LogicalProject(COL=[$cor0.ARRAY]) // uncollectInput + LogicalValues(tuples=[[{ 0 }]]) + Instead of projecting and joining again we directly apply flatmap. + The translation for this is: + stream.flat_map({ + move |x: &Tuple2, Option>, | -> _ { + let xA: Vec = x.0.clone(); + let xB: x.1.clone(); + x.0.clone().into_iter().map({ + move |e: i32, | -> Tuple3, Option, i32> { + Tuple3::new(xA.clone(), xB.clone(), e) + } + }) + }); + */ CalciteObject node = CalciteObject.create(correlate); DBSPTypeTuple type = this.convertType(correlate.getRowType(), false).to(DBSPTypeTuple.class); /* @@ -485,7 +487,6 @@ void visitCorrelate(LogicalCorrelate correlate) { if (correlate.getJoinType().isOuterJoin()) throw this.decorrelateError(node); */ - this.visit(correlate.getLeft(), 0, correlate); DBSPSimpleOperator left = this.getInputAs(correlate.getLeft(), true); DBSPTypeTuple leftElementType = left.getOutputZSetElementType().to(DBSPTypeTuple.class); @@ -2664,7 +2665,7 @@ DBSPSimpleOperator implement(DBSPSimpleOperator unusedInput, DBSPSimpleOperator } // This operator is always incremental, so create the non-incremental version - // of it by adding a D and an I around it. + // of it by adding a Differentiator and an Integrator around it. DBSPDifferentiateOperator diff = new DBSPDifferentiateOperator(this.node, inputIndexed); this.compiler.getCircuit().addOperator(diff); @@ -3091,7 +3092,7 @@ DBSPSimpleOperator implement(DBSPSimpleOperator input, DBSPSimpleOperator lastOp // Compute the window aggregate // This operator is always incremental, so create the non-incremental version - // of it by adding a D and an I around it. + // of it by adding a Differentiator and an Integrator around it. DBSPDifferentiateOperator diff = new DBSPDifferentiateOperator(node, mapIndex.outputPort()); this.compiler.getCircuit().addOperator(diff); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java index d9773e9308a..4e0df34ff20 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java @@ -803,7 +803,7 @@ static CompilationError operandCountError(CalciteObject node, String name, int o @Nullable DBSPExpression makeRegex(DBSPStringLiteral lit) { DBSPTypeUser user = new DBSPTypeUser(CalciteObject.EMPTY, USER, "Regex", true); - // Here we lie about the type: new does not return an Regex, but a Result. + // Here we lie about the type: new does not return a Regex, but a Result. // We lie again that ok returns an unchanged type. These two lies cancel out. DBSPExpression init = user.constructor("new", lit.toStr()); init = init.applyMethod("ok", init.getType()); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteOptimizer.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteOptimizer.java index 9bcab3a168a..ce08211f242 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteOptimizer.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteOptimizer.java @@ -55,7 +55,7 @@ public interface CalciteOptimizerStep { RelNode optimize(RelNode rel, int level); } - /** Base class for optimizations that use a Hep optimizer */ + /** Base class for optimizations that use a HEP optimizer */ public static abstract class HepOptimizerStep implements CalciteOptimizerStep { /** The program that performs the optimization for the specified optimization level */ abstract HepProgram getProgram(RelNode node, int level); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SetopOptimizerRule.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SetopOptimizerRule.java index d16bea2f8e4..fc9d46bcdc2 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SetopOptimizerRule.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SetopOptimizerRule.java @@ -85,7 +85,7 @@ private void match(RelOptRuleCall call) { Pair first = extractSourceAndCond(inputs.get(0).stripped().getInput(0).stripped()); // Groups conditions by their source relational node and input position. - // - Key: Pair of (sourceRelNode, inputPosition) + // - Key: A pair of (sourceRelNode, inputPosition) // - inputPosition is null for mergeable conditions // - inputPosition contains original index for non-mergeable inputs // - Value: List of conditions diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/statements/IHasSchema.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/statements/IHasSchema.java index a49dfb28ebd..f7703c8c53b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/statements/IHasSchema.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/statements/IHasSchema.java @@ -124,7 +124,7 @@ static RelColumnMetadata relMetaFromJson(JsonNode node, int index, RelDataTypeFa String json = Utilities.deterministicObjectMapper().writeValueAsString(jsonType); RelDataType type = RelJsonReader.readType(typeFactory, json); RelDataTypeField field = new RelDataTypeFieldImpl(name, index, type); - // Do we need lateness, watermark, etc? + // Do we need lateness, watermark, etc.? boolean interned = node.has("interned"); return new RelColumnMetadata(CalciteObject.EMPTY, field, isPrimaryKey, caseSensitive, null, null, null, SourcePositionRange.INVALID, interned); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ValueNumbering.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ValueNumbering.java index 92b984b7d8f..d6761501c72 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ValueNumbering.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ValueNumbering.java @@ -339,7 +339,7 @@ void checkRepresentation(DBSPExpression expression, Representation repr, boolean @Override public void startVisit(IDBSPInnerNode node) { // This node may not be a DBSPClosureExpression. - // It can be a e.g., Fold constructor. + // It can be e.g., Fold constructor. Logger.INSTANCE.belowLevel(this, 1) .append("CSE analyzing ") .appendSupplier(node::toString) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/AdjustSqlIndex.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/AdjustSqlIndex.java index a51b50b470d..b7b72322b6b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/AdjustSqlIndex.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/AdjustSqlIndex.java @@ -11,7 +11,7 @@ import org.dbsp.sqlCompiler.ir.type.IsNumericType; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeISize; -/** Converts a[index] into a[(isize)(index - 1] */ +/** Converts a[index] into a[(isize)(index - 1)] */ public class AdjustSqlIndex extends ExpressionTranslator { public AdjustSqlIndex(DBSPCompiler compiler) { super(compiler); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CSE.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CSE.java index 5a205eae19a..d38e702711d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CSE.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CSE.java @@ -8,7 +8,6 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPIntegrateTraceRetainValuesOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPNestedOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; -import org.dbsp.sqlCompiler.circuit.operator.DBSPOperatorWithError; import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; import org.dbsp.sqlCompiler.circuit.OutputPort; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ImplementNow.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ImplementNow.java index ba7dc7bef69..3959ba85027 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ImplementNow.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ImplementNow.java @@ -78,7 +78,7 @@ /** Implements the "now" operator. * This requires: * - using the input stream called now - * - rewriting map operators that have calls to "now()' into a join followed by a map + * - rewriting map operators that have calls to "now()" into a join followed by a map * - rewriting the invocations to the now() function in the map function to references to the input variable */ public class ImplementNow extends Passes { /** Discovers whether an expression contains a call to the now() function */ @@ -215,7 +215,7 @@ public VisitDecision preorder(DBSPApplyExpression expression) { /** Replace map operators that contain now() as an expression with * map operators that take an extra field and use that instead of the now() call. - * Insert a join prior to such operators (which also requires a MapIndex operator. + * Insert a join prior to such operators (which also requires a MapIndex operator). * Also inserts a MapIndex operator to index the 'NOW' built-in table. * Same for filter operators. */ static class RewriteNow extends CircuitCloneVisitor { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeMaps.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeMaps.java index 431652d31be..0a1fe65b969 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeMaps.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeMaps.java @@ -186,7 +186,7 @@ public void postorder(DBSPMapIndexOperator operator) { DBSPSimpleOperator newJoin = join .withInputs(Linq.list(leftPort, rightPort), false) - .to(DBSPSimpleOperator.class);; + .to(DBSPSimpleOperator.class); if (newJoin.outputType.sameType(join.outputType)) { super.postorder(operator); return; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjectionVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjectionVisitor.java index 51ea2307d36..606fff2d1c6 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjectionVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjectionVisitor.java @@ -67,7 +67,7 @@ public void postorder(DBSPMapOperator operator) { sourceFunction.rightProjections, sourceFunction.ordinalityIndexType, shuffle); DBSPSimpleOperator result = source.simpleNode() .withFunction(newFunction, operator.outputType) - .to(DBSPSimpleOperator.class);; + .to(DBSPSimpleOperator.class); this.map(operator, result); return; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/StrayGC.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/StrayGC.java index 39166b55cb6..ac8d0ce6393 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/StrayGC.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/StrayGC.java @@ -4,6 +4,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPAggregateOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPBinaryOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPChainAggregateOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPInputMapWithWaterlineOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPIntegrateTraceRetainKeysOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPIntegrateTraceRetainValuesOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPLagOperator; @@ -22,6 +23,8 @@ public StrayGC(DBSPCompiler compiler, CircuitGraphs g) { void check(DBSPBinaryOperator operator) { // At least one sibling on the left input must contain an integral var left = operator.left(); + if (left.operator.is(DBSPInputMapWithWaterlineOperator.class)) + return; CircuitGraph graph = this.getGraph(); for (Port sibling: graph.getSuccessors(left.operator)) { DBSPOperator so = sibling.node(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/intern/Intern.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/intern/Intern.java index ca78e53e6e8..6a53c7f15b8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/intern/Intern.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/intern/Intern.java @@ -39,7 +39,7 @@ public boolean contains(int i) { public String toString() { return this.columns.toString(); } - }; + } final Map internedInputs; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/InsertLimiters.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/InsertLimiters.java index e74cafc02b9..1d13eebf5b2 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/InsertLimiters.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/InsertLimiters.java @@ -91,7 +91,6 @@ import org.dbsp.sqlCompiler.ir.expression.literal.DBSPStringLiteral; import org.dbsp.sqlCompiler.ir.expression.DBSPZSetExpression; import org.dbsp.sqlCompiler.ir.type.DBSPType; -import org.dbsp.sqlCompiler.ir.type.IsNumericType; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeRawTuple; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTupleBase; @@ -100,7 +99,6 @@ import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBaseType; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeString; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeTimestamp; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeIndexedZSet; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeTypedBox; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeWeight; @@ -1359,8 +1357,8 @@ DBSPExpression eq(DBSPExpression left, DBSPExpression right) { * {@link DBSPInputMapWithWaterlineOperator}*/ DBSPOperator processLateness( ProgramIdentifier viewOrTable, DBSPSimpleOperator operator, DBSPSimpleOperator expansion) { - MonotoneExpression expression = this.expansionMonotoneValues.get(expansion); - if (expression == null) { + MonotoneExpression me = this.expansionMonotoneValues.get(expansion); + if (me == null) { this.nonMonotone(expansion); return operator; } @@ -1421,8 +1419,9 @@ DBSPOperator processLateness( DBSPControlledKeyFilterOperator filter = this.createControlledKeyFilter( operator.getRelNode(), viewOrTable, replacement.outputPort(), - Monotonicity.getBodyType(expression), delay.outputPort()); + Monotonicity.getBodyType(me), delay.outputPort()); DBSPOperator result = filter; + DBSPInputMapWithWaterlineOperator newSource = null; if (replaceIndexedInput) { List keyFields = IndexedInputs.getKeyFields(multisetInput); @@ -1435,15 +1434,15 @@ DBSPOperator processLateness( Utilities.enforce(filter.error.parameters.length == 4); DBSPClosureExpression error = filter.error.body.closure( filter.error.parameters[0], k.asParameter(), filter.error.parameters[1], filter.error.parameters[3]); - result = new DBSPInputMapWithWaterlineOperator( + newSource = new DBSPInputMapWithWaterlineOperator( multisetInput.getRelNode(), multisetInput.sourceName, keyFields, indexedOutputType, multisetInput.originalRowType, multisetInput.metadata, multisetInput.tableName, min.closure(), timestamp.closure(k, t), max, ff, error); - this.errorStreams.add(result.getOutput(1)); - waterlineOutputPort = result.getOutput(2); - this.addOperator(result); + this.errorStreams.add(newSource.getOutput(1)); + waterlineOutputPort = newSource.getOutput(2); + this.addOperator(newSource); - result = new DBSPDeindexOperator(multisetInput.getRelNode(), result.getOutput(0)); + result = new DBSPDeindexOperator(multisetInput.getRelNode(), newSource.getOutput(0)); } else { OutputPort errorPort = result.getOutput(1); if (!this.reachableFromError.contains(result.inputs.get(0).operator)) { @@ -1471,6 +1470,17 @@ DBSPOperator processLateness( if (operator != expansion) this.markBound(expansion.outputPort(), extend.outputPort()); + if (INSERT_RETAIN_VALUES && replaceIndexedInput) { + IMaybeMonotoneType projection = me.getMonotoneType().to(MonotoneClosureType.class).getBodyType(); + // The new input operator produces an indexed Zset, need to adjust the projection + projection = new PartiallyMonotoneTuple( + List.of(NonMonotoneType.nonMonotone(indexedOutputType.keyType), projection), true, false); + DBSPSimpleOperator retain = DBSPIntegrateTraceRetainValuesOperator.create( + operator.getRelNode(), newSource.getOutput(0), + projection, extend.outputPort()); + this.addOperator(retain); + } + return result; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/unusedFields/ReplaceCommonProjections.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/unusedFields/ReplaceCommonProjections.java index b1da96c7a13..1250bec55e9 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/unusedFields/ReplaceCommonProjections.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/unusedFields/ReplaceCommonProjections.java @@ -5,7 +5,6 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPFilterOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPMapIndexOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPMapOperator; -import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceMapOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceMultisetOperator; @@ -16,11 +15,8 @@ import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeRawTuple; import org.dbsp.util.Linq; -import org.dbsp.util.Utilities; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class ReplaceCommonProjections extends CircuitCloneVisitor { final FindCommonProjections fcp; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPHandleErrorExpression.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPHandleErrorExpression.java index 2edee7bdd5b..ca3947d61ed 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPHandleErrorExpression.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPHandleErrorExpression.java @@ -53,7 +53,7 @@ public enum RuntimeBehavior { PanicWithSource, /** On error return None; this is the behavior of e.g., safe_cast in SQL */ ReturnNone - }; + } public final RuntimeBehavior runtimeBehavior; /** Index in the table with error messages; used for PanicWithSource; 0 if unused. */ diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/user/DBSPTypeMap.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/user/DBSPTypeMap.java index 32085d5ba9d..8e0916710ec 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/user/DBSPTypeMap.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/user/DBSPTypeMap.java @@ -9,7 +9,6 @@ import org.dbsp.sqlCompiler.ir.type.DBSPType; import org.dbsp.sqlCompiler.ir.type.ICollectionType; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeRawTuple; -import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; import org.dbsp.util.Linq; import org.dbsp.util.Utilities; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java index 3d7be80763e..cab73754e23 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java @@ -498,7 +498,7 @@ static void toDepth(JsonNode node, int depth, IIndentStream stream) { } } - /** Serialize as String a object to the specified depth */ + /** Serialize as String an object to the specified depth */ public static String toDepth(JsonNode node, int depth) { IndentStreamBuilder builder = new IndentStreamBuilder(); toDepth(node, depth, builder); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/CatalogTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/CatalogTests.java index eae3b49da46..fe2a217a920 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/CatalogTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/CatalogTests.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; import org.dbsp.sqlCompiler.circuit.operator.DBSPInputMapWithWaterlineOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPIntegrateTraceRetainValuesOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSinkOperator; import org.dbsp.sqlCompiler.compiler.CompilerOptions; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; @@ -194,8 +195,16 @@ PARTITION BY customer_id, DATE_TRUNC(timestamp_column, MONTH) FROM table_name; """); + // Note: we compile without -i, so there are fewer GC operators than expected ccs.visit(new CircuitVisitor(ccs.compiler) { int imww = 0; + int retain = 0; + + // Check that InputMapWithWaterline is GC-ed + @Override + public void postorder(DBSPIntegrateTraceRetainValuesOperator node) { + this.retain++; + } @Override public void postorder(DBSPInputMapWithWaterlineOperator operator) { @@ -205,6 +214,7 @@ public void postorder(DBSPInputMapWithWaterlineOperator operator) { @Override public void endVisit() { Assert.assertEquals(1, this.imww); + Assert.assertEquals(1, this.retain); } }); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java index 668335afa6b..7804f34e927 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java @@ -1457,7 +1457,7 @@ CREATE TABLE S ( } @Test - public void issue4466() throws IOException, InterruptedException, SQLException { + public void issue4466() throws IOException, SQLException { File file = createInputScript(""" CREATE TABLE tbl(arr1 INT ARRAY); CREATE MATERIALIZED VIEW v AS SELECT diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/functions/FunctionsTest.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/functions/FunctionsTest.java index 3fc544b3236..08c7a100527 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/functions/FunctionsTest.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/functions/FunctionsTest.java @@ -2287,7 +2287,7 @@ public void issue4559() { this.getCC(""" CREATE TABLE str_tbl(id INT, str VARCHAR); CREATE VIEW V0 AS SELECT POSITION('🐍' in str), POSITION('い' in str), POSITION('√ ' in str) FROM str_tbl; - CREATE VIEW V1 AS SELECT REGEXP_REPLACE(str, '([🐍い]|/¯|√\s*)+$', 'i') FROM str_tbl; + CREATE VIEW V1 AS SELECT REGEXP_REPLACE(str, '([🐍い]|/¯|√\\s*)+$', 'i') FROM str_tbl; CREATE VIEW V2 AS SELECT RLIKE(str, '🐍.'), RLIKE(str, '..い.'), RLIKE(str, '....√ ') FROM str_tbl; CREATE VIEW V3 AS SELECT SPLIT(str, '🐍.'), SPLIT(str, '..い.'), SPLIT(str, '....√ ') FROM str_tbl; CREATE VIEW V4 AS SELECT SPLIT_PART(str, '🐍.', 2), SPLIT_PART(str, '..い.', 2), SPLIT_PART(str, '....√ ', 2) FROM str_tbl"""); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresNumericTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresNumericTests.java index c9d0488ea31..645b28d459d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresNumericTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresNumericTests.java @@ -855,30 +855,6 @@ SELECT x, round(x), round(x,1) as round1, truncate(x), truncate(x,1) as trunc1 -7.777 | -8 | -7.8 | -7 | -7.7"""); } - // -- the large values fall into the numeric abbreviation code's maximal classes - //WITH v(x) AS - // (VALUES('0'::numeric),('1'),('-1'),('4.2'),('-7.777'),('1e340'),('-1e340'), - // ('inf'),('-inf'),('nan'), - // ('inf'),('-inf'),('nan')) - //SELECT substring(x::text, 1, 32) - //FROM v ORDER BY x; - // substring - //---------------------------------- - // -Infinity - // -Infinity - // -1000000000000000000000000000000 - // -7.777 - // -1 - // 0 - // 1 - // 4.2 - // 10000000000000000000000000000000 - // Infinity - // Infinity - // NaN - // NaN - //(13 rows) - @Test public void testSqrt() { // Removed 'inf' and 'nan'. diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresTimestampTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresTimestampTests.java index 7d5e62faead..5143d88f71f 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresTimestampTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresTimestampTests.java @@ -100,7 +100,7 @@ public void prepareInputs(DBSPCompiler compiler) { // -- this fails (even though TZ is a no-op, we still look it up) // INSERT INTO TIMESTAMP_TBL VALUES ('19970710 173201 America/Does_not_exist'); // ERROR: time zone "america/does_not_exist" not recognized - // LINE 1: INSERT INTO TIMESTAMP_TBL VALUES ('19970710 173201 America/D... + // LINE 1: INSERT INTO TIMESTAMP_TBL VALUES ('19970710 173201 America/D...) // -- Check date conversion and date arithmetic "INSERT INTO TIMESTAMP_TBL VALUES ('1997-06-10 18:32:01');\n" + // INSERT INTO TIMESTAMP_TBL VALUES ('1997-06-10 18:32:01 PDT'); "INSERT INTO TIMESTAMP_TBL VALUES ('1997-02-10 17:32:01');\n" + // INSERT INTO TIMESTAMP_TBL VALUES ('Feb 10 17:32:01 1997'); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/quidem/HRWinAggTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/quidem/HRWinAggTests.java index b7285a48697..6c10ec841dc 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/quidem/HRWinAggTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/quidem/HRWinAggTests.java @@ -3,7 +3,7 @@ import org.junit.Ignore; import org.junit.Test; -// From from winagg.iq that use the HR database +// From winagg.iq that use the HR database public class HRWinAggTests extends HrBaseTests { @Test @Ignore("ORDER BY strings not supported https://github.com/feldera/feldera/issues/457, first_value") diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/recursive/IncrementalRecursiveTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/recursive/IncrementalRecursiveTests.java index 00f7997f8d3..6b71a20becd 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/recursive/IncrementalRecursiveTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/recursive/IncrementalRecursiveTests.java @@ -2,7 +2,6 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPNestedOperator; import org.dbsp.sqlCompiler.compiler.CompilerOptions; -import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.sql.tools.BaseSQLTests; import org.dbsp.sqlCompiler.compiler.visitors.outer.CircuitVisitor; import org.junit.Assert; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/ArrayFunctionsTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/ArrayFunctionsTests.java index 05c3181c0d3..0e6e91e4d07 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/ArrayFunctionsTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/ArrayFunctionsTests.java @@ -1124,22 +1124,22 @@ public void testArrayInsert() { {1, 2, null, 3} (1 row)"""); - // select array_insert(array[array[1,2]], 1, array[1])", + // select array_insert(array[array[1,2]], 1, array[1]); // r // --- - // [[1], [1, 2]]", "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); - // select array_insert(array[array[1,2]], -1, array[1])", + // [[1], [1, 2]]" + // select array_insert(array[array[1,2]], -1, array[1]); // r // --- - // "[[1, 2], [1]]", "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); - // select array_insert(array[map[1, 'a']], 1, map[2, 'b'])", "[{2=b}, {1=a}]", + // [[1, 2], [1]] + // select array_insert(array[map[1, 'a']], 1, map[2, 'b']); // r // --- - // "(INTEGER NOT NULL, CHAR(1) NOT NULL) MAP NOT NULL ARRAY NOT NULL"); - // select array_insert(array[map[1, 'a']], -1, map[2, 'b'])", "[{1=a}, {2=b}]", + // [{2=b}, {1=a}] + // select array_insert(array[map[1, 'a']], -1, map[2, 'b']); // r // --- - // "(INTEGER NOT NULL, CHAR(1) NOT NULL) MAP NOT NULL ARRAY NOT NULL"); + // [{1=a}, {2=b}] } @Test diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/NaiveIncrementalTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/NaiveIncrementalTests.java index da7b68e7cdf..ad1786b5872 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/NaiveIncrementalTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/NaiveIncrementalTests.java @@ -24,7 +24,6 @@ package org.dbsp.sqlCompiler.compiler.sql.simple; import org.dbsp.sqlCompiler.compiler.CompilerOptions; -import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.sql.tools.Change; import org.dbsp.sqlCompiler.compiler.sql.tools.InputOutputChangeStream; import org.dbsp.sqlCompiler.ir.expression.DBSPZSetExpression; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/OptimizedIncrementalTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/OptimizedIncrementalTests.java index 2a1b0a3d700..6c93ba07e70 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/OptimizedIncrementalTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/OptimizedIncrementalTests.java @@ -24,7 +24,6 @@ package org.dbsp.sqlCompiler.compiler.sql.simple; import org.dbsp.sqlCompiler.compiler.CompilerOptions; -import org.dbsp.sqlCompiler.compiler.DBSPCompiler; public class OptimizedIncrementalTests extends NaiveIncrementalTests { @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/StructTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/StructTests.java index f486e7fa4ff..1a2cbb6902c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/StructTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/StructTests.java @@ -1,6 +1,5 @@ package org.dbsp.sqlCompiler.compiler.sql.simple; -import org.dbsp.sqlCompiler.compiler.CompilerOptions; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; import org.dbsp.sqlCompiler.compiler.sql.tools.Change; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/streaming/StreamingTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/streaming/StreamingTests.java index d2a885fea66..2ad4c0a03ba 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/streaming/StreamingTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/streaming/StreamingTests.java @@ -1049,7 +1049,7 @@ public void endVisit() { Assert.assertEquals(this.expectedWaterline, this.waterline); Assert.assertEquals(this.expectedChain, this.chain); } - }; + } @Test public void testNow5() { @@ -1938,11 +1938,6 @@ public void errorStreamQueryTest() { DBSPTypeString.varchar(false) ); DBSPType e = new DBSPTypeTuple(DBSPTypeString.varchar(false)); - /* - DBSPTypeMap map = new DBSPTypeMap( - new DBSPTypeVariant(false), - new DBSPTypeVariant(false), false); - */ String sql = """ CREATE TABLE series ( diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java index 64b7ade9a40..c5e2897462a 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java @@ -66,7 +66,7 @@ public class BaseSQLTests { // Debugging: set to true to only compile SQL public static final boolean skipRust = false; // Debugging: set to only accept some tests - public static Predicate acceptTest = x -> true; + public static final Predicate acceptTest = x -> true; /** Override this method to prepare the tables on * which the tests are built. */ diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TableParser.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TableParser.java index eeed43472f9..bfc867b9bca 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TableParser.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TableParser.java @@ -36,7 +36,6 @@ import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeArray; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeZSet; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeString; import org.dbsp.util.Linq; import org.dbsp.util.Utilities; diff --git a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/Circuit.java b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/Circuit.java index 83facae73ee..7b9fc9635f0 100644 --- a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/Circuit.java +++ b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/Circuit.java @@ -1,6 +1,5 @@ package org.dbsp.simulator; -import com.fasterxml.jackson.databind.ser.Serializers; import org.dbsp.simulator.operators.BaseOperator; import org.dbsp.simulator.operators.InputOperator; import org.dbsp.simulator.operators.OutputOperator; diff --git a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/IndexOperator.java b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/IndexOperator.java index 3d4e13ea398..0a83ba6f718 100644 --- a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/IndexOperator.java +++ b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/IndexOperator.java @@ -1,17 +1,12 @@ package org.dbsp.simulator.operators; import org.dbsp.simulator.types.CollectionType; -import org.dbsp.simulator.types.SqlType; -import org.dbsp.simulator.values.DynamicSqlValue; import org.dbsp.simulator.values.RuntimeFunction; import org.dbsp.simulator.collections.BaseCollection; import org.dbsp.simulator.collections.IndexedZSet; import org.dbsp.simulator.collections.ZSet; -import org.dbsp.simulator.types.DataType; import org.dbsp.simulator.values.SqlTuple; -import java.util.function.Function; - public class IndexOperator extends UnaryOperator { public final RuntimeFunction keyFunction; @@ -21,6 +16,7 @@ public IndexOperator(CollectionType outputType, this.keyFunction = keyFunction; } + @SuppressWarnings("unchecked") @Override public void step() { BaseCollection input = this.input().getCurrentValue(); diff --git a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/JoinOperator.java b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/JoinOperator.java index 7e15ac4486d..554107794a4 100644 --- a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/JoinOperator.java +++ b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/JoinOperator.java @@ -3,9 +3,6 @@ import org.dbsp.simulator.collections.BaseCollection; import org.dbsp.simulator.collections.IndexedZSet; import org.dbsp.simulator.types.CollectionType; -import org.dbsp.simulator.types.DataType; -import org.dbsp.simulator.types.SqlType; -import org.dbsp.simulator.types.WeightType; import org.dbsp.simulator.values.SqlTuple; import java.util.function.BiFunction; @@ -21,6 +18,7 @@ protected JoinOperator(BiFunction combiner, this.combiner = combiner; } + @SuppressWarnings("unchecked") @Override public void step() { BaseCollection left = this.inputs[0].getCurrentValue(); diff --git a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/OutputOperator.java b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/OutputOperator.java index be8a178d225..cadeb5f387b 100644 --- a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/OutputOperator.java +++ b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/OutputOperator.java @@ -1,8 +1,6 @@ package org.dbsp.simulator.operators; import org.dbsp.simulator.collections.BaseCollection; -import org.dbsp.simulator.types.DataType; -import org.dbsp.simulator.types.SqlType; public class OutputOperator extends UnaryOperator { final String name; diff --git a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/SelectOperator.java b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/SelectOperator.java index 9167133ec55..63d2f17ac53 100644 --- a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/SelectOperator.java +++ b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/operators/SelectOperator.java @@ -16,6 +16,7 @@ public SelectOperator(CollectionType outputType, this.tupleTransform = tupleTransform; } + @SuppressWarnings("unchecked") @Override public void step() { ZSet input = (ZSet) this.input().getCurrentValue(); diff --git a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/types/IntegerWeightType.java b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/types/IntegerWeightType.java index 21f03c5cc35..1b6cfafe949 100644 --- a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/types/IntegerWeightType.java +++ b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/types/IntegerWeightType.java @@ -1,7 +1,5 @@ package org.dbsp.simulator.types; -import org.dbsp.simulator.util.ICastable; - /** * Weights represented as integers. * Throws on overflow. diff --git a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/values/BooleanSqlValue.java b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/values/BooleanSqlValue.java index 3ca2c7bd506..4c73e5d48a3 100644 --- a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/values/BooleanSqlValue.java +++ b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/values/BooleanSqlValue.java @@ -1,7 +1,6 @@ package org.dbsp.simulator.values; import org.dbsp.simulator.types.BooleanSqlType; -import org.dbsp.simulator.types.IntegerSqlType; import org.jetbrains.annotations.NotNull; import javax.annotation.Nullable; diff --git a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/values/SqlTuple.java b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/values/SqlTuple.java index ae599ec704e..364c2be7c2c 100644 --- a/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/values/SqlTuple.java +++ b/sql-to-dbsp-compiler/simulator/src/main/java/org/dbsp/simulator/values/SqlTuple.java @@ -5,7 +5,6 @@ import org.dbsp.util.Linq; import org.jetbrains.annotations.NotNull; -import javax.management.DynamicMBean; import java.util.ArrayList; import java.util.List; import java.util.Objects; diff --git a/sql-to-dbsp-compiler/simulator/src/test/java/IntegratorTests.java b/sql-to-dbsp-compiler/simulator/src/test/java/IntegratorTests.java index bb70f3a6f58..95b2b8227d0 100644 --- a/sql-to-dbsp-compiler/simulator/src/test/java/IntegratorTests.java +++ b/sql-to-dbsp-compiler/simulator/src/test/java/IntegratorTests.java @@ -4,8 +4,8 @@ public class IntegratorTests { int sum(int[] data) { int sum = 0; - for (int i = 0; i < data.length; i++) { - sum += data[i]; + for (int datum : data) { + sum += datum; } return sum; } diff --git a/sql-to-dbsp-compiler/simulator/src/test/java/org/dbsp/simulator/CircuitTests.java b/sql-to-dbsp-compiler/simulator/src/test/java/org/dbsp/simulator/CircuitTests.java index 80b5f985963..6bf649da509 100644 --- a/sql-to-dbsp-compiler/simulator/src/test/java/org/dbsp/simulator/CircuitTests.java +++ b/sql-to-dbsp-compiler/simulator/src/test/java/org/dbsp/simulator/CircuitTests.java @@ -20,6 +20,7 @@ import org.junit.Test; import java.util.List; +import java.util.Objects; import java.util.function.Function; public class CircuitTests { @@ -31,8 +32,9 @@ public void circuit() { InputOperator input = new InputOperator("T", new ZSetType(new TupleSqlType(List.of(IntegerSqlType.INSTANCE)), w)); circuit.addOperator(input); Function func = x -> - new SqlTuple(new IntegerSqlValue(x.to(SqlTuple.class).get(0) - .to(IntegerSqlValue.class).getValue() + 1)); + new SqlTuple(new IntegerSqlValue( + Objects.requireNonNull(x.to(SqlTuple.class).get(0) + .to(IntegerSqlValue.class).getValue()) + 1)); SelectOperator select = new SelectOperator(new ZSetType(new TupleSqlType(IntegerSqlType.INSTANCE), w), new RuntimeFunction<>(new FunctionType(IntegerSqlType.INSTANCE, IntegerSqlType.INSTANCE), func), input.getOutput()); @@ -59,9 +61,6 @@ public void integratorTest() { Circuit circuit = new Circuit(); InputOperator input = new InputOperator("T", new ZSetType(new TupleSqlType(List.of(IntegerSqlType.INSTANCE)), w)); circuit.addOperator(input); - Function func = x -> - new SqlTuple(new IntegerSqlValue(x.to(SqlTuple.class).get(0) - .to(IntegerSqlValue.class).getValue() + 1)); IntegrateOperator select = new IntegrateOperator(input.getOutput()); circuit.addOperator(select); OutputOperator output = new OutputOperator("V", select.getOutput()); diff --git a/sql-to-dbsp-compiler/simulator/src/test/java/org/dbsp/simulator/SimulatorTests.java b/sql-to-dbsp-compiler/simulator/src/test/java/org/dbsp/simulator/SimulatorTests.java index d4b4a5fa702..b17e23bd121 100644 --- a/sql-to-dbsp-compiler/simulator/src/test/java/org/dbsp/simulator/SimulatorTests.java +++ b/sql-to-dbsp-compiler/simulator/src/test/java/org/dbsp/simulator/SimulatorTests.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import org.apache.commons.math3.analysis.function.Add; import org.dbsp.simulator.collections.IndexedZSet; import org.dbsp.simulator.collections.ZSet; import org.dbsp.simulator.types.IntegerSqlType; @@ -106,7 +105,7 @@ public String getName() { @Override public SqlType getType() { - return null; + return new TupleSqlType(); } @Override @@ -447,7 +446,7 @@ public void testMultiply() { public void testIndex() { ZSet input = getPersons(); IndexedZSet index = input.index( - new RuntimeFunction(p -> new StringSqlValue(p.name != null ? p.name.substring(0, 1) : null))); + new RuntimeFunction<>(p -> new StringSqlValue(p.name != null ? p.name.substring(0, 1) : null))); Assert.assertEquals(2, index.groupCount()); } diff --git a/sql-to-dbsp-compiler/using.md b/sql-to-dbsp-compiler/using.md index e37869d1463..3f0857a7463 100644 --- a/sql-to-dbsp-compiler/using.md +++ b/sql-to-dbsp-compiler/using.md @@ -149,7 +149,7 @@ Here is a description of the non-obvious command-line options: --lenient: Some SQL queries generate output views having multiple columns with the same name. Such views can cause problems with other tools - that interface with the compiler outputs. By default the compiler will + that interface with the compiler outputs. By default, the compiler will emit an error when given such views. For example, the following definition: `CREATE VIEW V AS SELECT T.COL2, S.COL2 from T, S`