From fa62d5d82b24a3afc2f3b998aed8488d73c165d6 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 26 May 2026 20:26:40 -0700 Subject: [PATCH] [SQL] Context-sensitive conditionals optimization Signed-off-by: Mihai Budiu --- .../aggregate_tests/test_float_tbl.py | 2 +- .../DBSPIntegrateTraceRetainKeysOperator.java | 6 +- ...BSPIntegrateTraceRetainValuesOperator.java | 4 +- .../compiler/visitors/VisitorProfiles.java | 2 +- .../compiler/visitors/inner/IRTransform.java | 1 + .../visitors/inner/ResolveReferences.java | 9 +- .../compiler/visitors/inner/Simplify.java | 3 + .../visitors/inner/SimplifyConditionals.java | 371 ++++++++++++++++++ .../visitors/inner/TranslateVisitor.java | 2 +- .../visitors/outer/CircuitOptimizer.java | 2 + .../outer/monotonicity/InsertLimiters.java | 6 +- .../sqlCompiler/ir/aggregate/DBSPMinMax.java | 5 +- .../ir/expression/DBSPFlatmap.java | 2 +- .../ir/expression/DBSPVariablePath.java | 1 + .../compiler/ir/TestSimplifyConditionals.java | 170 ++++++++ .../sqlCompiler/compiler/sql/QATests.java | 2 +- 16 files changed, 569 insertions(+), 19 deletions(-) create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/SimplifyConditionals.java create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/TestSimplifyConditionals.java diff --git a/python/tests/runtime_aggtest/aggregate_tests/test_float_tbl.py b/python/tests/runtime_aggtest/aggregate_tests/test_float_tbl.py index 4ed0f883c4a..4ee3a8b8fe3 100644 --- a/python/tests/runtime_aggtest/aggregate_tests/test_float_tbl.py +++ b/python/tests/runtime_aggtest/aggregate_tests/test_float_tbl.py @@ -8,7 +8,7 @@ def __init__(self): self.sql = """CREATE TABLE real_tbl( id INT, c1 REAL, - c2 REAL NOT NULL);""" + c2 REAL NOT NULL)""" self.data = [ {"id": 0, "c1": None, "c2": 2231.791}, {"id": 0, "c1": 57681.18, "c2": -38.27112}, diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java index 95df2418c44..5525dfd6c9c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java @@ -54,7 +54,7 @@ public static DBSPIntegrateTraceRetainKeysOperator create( DBSPParameter param; DBSPExpression compare; DBSPVariablePath controlArg = controlType.ref().var(); - DBSPExpression compare0 = controlArg.deref().field(0).not(); + DBSPExpression compare0 = controlArg.deepCopy().deref().field(0).not(); if (data.outputType().is(DBSPTypeIndexedZSet.class)) { DBSPType keyType = data.getOutputIndexedZSetType().keyType; if (keyType.sameType(DBSPTypeTuple.EMPTY)) @@ -69,7 +69,7 @@ public static DBSPIntegrateTraceRetainKeysOperator create( DBSPExpression project = dataField0 .projectExpression(dataArg.deref()); compare = DBSPControlledKeyFilterOperator.generateTupleCompare( - project, controlArg.deref().field(1).field(0), DBSPOpcode.CONTROLLED_FILTER_GTE); + project, controlArg.deepCopy().deref().field(1).field(0), DBSPOpcode.CONTROLLED_FILTER_GTE); } else { DBSPType keyType = data.getOutputZSetElementType(); DBSPVariablePath dataArg = keyType.ref().var(); @@ -78,7 +78,7 @@ public static DBSPIntegrateTraceRetainKeysOperator create( return null; DBSPExpression project = dataProjection.projectExpression(dataArg.deref()); compare = DBSPControlledKeyFilterOperator.generateTupleCompare( - project, controlArg.deref().field(1), DBSPOpcode.CONTROLLED_FILTER_GTE); + project, controlArg.deepCopy().deref().field(1), DBSPOpcode.CONTROLLED_FILTER_GTE); } compare = ExpressionCompiler.makeBinaryExpression( node, compare.getType(), DBSPOpcode.OR, compare0, compare); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java index b5a11a48ada..9a8b082a5e7 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java @@ -60,9 +60,9 @@ public static DBSPIntegrateTraceRetainValuesOperator create( .to(PartiallyMonotoneTuple.class) .getField(1) .projectExpression(dataArg.deref()); - DBSPExpression compare0 = controlArg.deref().field(0).not(); + DBSPExpression compare0 = controlArg.deepCopy().deref().field(0).not(); DBSPExpression compare = DBSPControlledKeyFilterOperator.generateTupleCompare( - project, controlArg.deref().field(1), DBSPOpcode.CONTROLLED_FILTER_GTE); + project, controlArg.deepCopy().deref().field(1), DBSPOpcode.CONTROLLED_FILTER_GTE); compare = ExpressionCompiler.makeBinaryExpression( node, compare.getType(), DBSPOpcode.OR, compare0, compare); DBSPExpression closure = compare.closure(param, controlArg.asParameter()); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/VisitorProfiles.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/VisitorProfiles.java index 86e73b2136a..99013cc4879 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/VisitorProfiles.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/VisitorProfiles.java @@ -57,7 +57,7 @@ void start(String visitor) { void stop(String visitor) { long end = System.currentTimeMillis(); var pair = Utilities.removeLast(this.running); - Utilities.enforce(pair.left.equals(visitor)); + Utilities.enforce(pair.left.equals(visitor), () -> "Expected to finish " + pair.left + " but it is " + visitor); Long started = pair.right; Profile previous = this.profiles.getOrDefault(visitor, new Profile(0, 0)); this.profiles.put(visitor, previous.add(end - started)); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/IRTransform.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/IRTransform.java index cda75f9d8aa..f5c54858c16 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/IRTransform.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/IRTransform.java @@ -6,6 +6,7 @@ import java.util.function.Function; +/** A function that transforms an InnerNode into another one. */ public interface IRTransform extends Function, ICastable { /** The operator containing the inner node that is being transformed */ void setOperatorContext(DBSPOperator operator); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferences.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferences.java index 3d33b2a01cc..dbab98888d5 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferences.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferences.java @@ -69,14 +69,13 @@ public void postorder(DBSPLetStatement statement) { @Override public VisitDecision preorder(DBSPLetExpression expression) { + // Initializer variables resolved in outer context + expression.initializer.accept(this); this.substitutionContext.newContext(); this.substitutionContext.substitute(expression.variable.variable, expression); - return VisitDecision.CONTINUE; - } - - @Override - public void postorder(DBSPLetExpression expression) { + expression.consumer.accept(this); this.substitutionContext.popContext(); + return VisitDecision.STOP; } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Simplify.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Simplify.java index 753b970d442..438afa4766e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Simplify.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Simplify.java @@ -148,6 +148,9 @@ else if (source.is(DBSPIfExpression.class)) { ifExp.condition, ifExp.positive.is_null(), ifExp.negative.is_null()); + } else if (source.is(DBSPBaseTupleExpression.class)) { + // An explicit tuple constructor is never null + result = new DBSPBoolLiteral(false); } this.map(expression, result); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/SimplifyConditionals.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/SimplifyConditionals.java new file mode 100644 index 00000000000..8af51affbde --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/SimplifyConditionals.java @@ -0,0 +1,371 @@ +package org.dbsp.sqlCompiler.compiler.visitors.inner; + +import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; +import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError; +import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision; +import org.dbsp.sqlCompiler.ir.DBSPParameter; +import org.dbsp.sqlCompiler.ir.IDBSPDeclaration; +import org.dbsp.sqlCompiler.ir.IDBSPInnerNode; +import org.dbsp.sqlCompiler.ir.expression.DBSPApplyExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPApplyMethodExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPBinaryExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPBlockExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPCastExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPDerefExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPFieldExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPIfExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPIsNullExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPLetExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPPathExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPUnaryExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPUnwrapExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPBoolLiteral; +import org.dbsp.sqlCompiler.ir.statement.DBSPLetStatement; +import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool; +import org.dbsp.util.Logger; +import org.dbsp.util.Utilities; + +import java.util.HashMap; +import java.util.Map; + +/** Simplifies expressions based on their context within a conditional; + * if (a) { ... a ... } simplifies the second occurrence of 'a' to 'true'. */ +public class SimplifyConditionals implements IRTransform { + final ContainsIf containsIf; + final FindConstantExpressions findConstants; + final ReplaceConstantExpressions replaceConstants; + final DBSPCompiler compiler; + + public SimplifyConditionals(DBSPCompiler compiler) { + this.compiler = compiler; + this.containsIf = new ContainsIf(compiler); + this.findConstants = new FindConstantExpressions(compiler); + this.replaceConstants = new ReplaceConstantExpressions(compiler, this.findConstants.constants); + } + + @Override + public void setOperatorContext(DBSPOperator operator) { + this.containsIf.setOperatorContext(operator); + this.findConstants.setOperatorContext(operator); + this.replaceConstants.setOperatorContext(operator); + } + + @Override + public IDBSPInnerNode apply(IDBSPInnerNode node) { + if (!node.is(DBSPExpression.class)) + return node; + this.containsIf.apply(node); + if (this.containsIf.found()) { + var tree = node.to(DBSPExpression.class).ensureTree(this.compiler); + this.findConstants.apply(tree); + return this.replaceConstants.apply(tree); + } else { + return node; + } + } + + record ExpressionWithContext(DBSPExpression expression, Scopes scopes) {} + + /** Check if an expression contains a {@link DBSPIfExpression} subexpression. */ + static class ContainsIf extends InnerVisitor { + boolean found = false; + + ContainsIf(DBSPCompiler compiler) { + super(compiler); + } + + @Override + public VisitDecision preorder(DBSPType type) { + return VisitDecision.STOP; + } + + @Override + public VisitDecision preorder(DBSPIfExpression expression) { + this.found = true; + return VisitDecision.STOP; + } + + @Override + public void startVisit(IDBSPInnerNode node) { + super.startVisit(node); + this.found = false; + } + + public boolean found() { + return this.found; + } + } + + /** Symbolically evaluate a program and detect Boolean expressions that are constant + * because they are guarded by an if expression. E.g., in `if (a) { ... a ... }` + * the second `a` is `true`. Requires analyzed expressions to be trees. */ + static class FindConstantExpressions extends SymbolicInterpreter { + // Output result is assembled here: for each expression its actual value + final Map constants; + // Expressions that are currently known to be constant + final Map state; + // Scope of expression currently being analyzed + final Scopes substitutionContext; + // Maps each declaration to itself; needed by the EquivalenceContext + final Substitution identitySubstitution; + + public FindConstantExpressions(DBSPCompiler compiler) { + super(compiler); + this.constants = new HashMap<>(); + this.state = new HashMap<>(); + this.substitutionContext = new Scopes<>(); + this.identitySubstitution = new Substitution<>(); + } + + @Override + public VisitDecision preorder(DBSPIfExpression expression) { + expression.condition.accept(this); + ExpressionWithContext conditionValue = this.get(expression.condition); + Logger.INSTANCE.belowLevel(this, 2) + .append("Entering positive branch ") + .appendSupplier(expression.condition::toString); + this.state.put(conditionValue, true); + expression.positive.accept(this); + Logger.INSTANCE.belowLevel(this, 2) + .append("Leaving positive branch ") + .appendSupplier(expression.condition::toString); + this.state.remove(conditionValue); + + if (expression.negative != null) { + Logger.INSTANCE.belowLevel(this, 2) + .append("Entering negative branch ") + .appendSupplier(expression.condition::toString); + this.state.put(conditionValue, false); + expression.negative.accept(this); + Logger.INSTANCE.belowLevel(this, 2) + .append("Leaving negative branch ") + .appendSupplier(expression.condition::toString); + this.state.remove(conditionValue); + } + + if (this.maybeGet(expression) == null) { + ExpressionWithContext result = new ExpressionWithContext(expression, this.substitutionContext.clone()); + this.set(expression, result); + } + return VisitDecision.STOP; + } + + /** Check if an expression can be substituted with a constant */ + void checkIfConstant(DBSPExpression expression) { + if (this.constants.containsKey(expression)) + // We already know it's a constant + return; + if (expression.getType().is(DBSPTypeBool.class) && !expression.getType().mayBeNull) { + // Compare with all constant expressions in the current state + for (var exp: this.state.keySet()) { + EquivalenceContext context = new EquivalenceContext( + this.substitutionContext, exp.scopes, this.identitySubstitution); + if (context.equivalent(expression, exp.expression)) { + boolean value = this.state.get(exp); + Logger.INSTANCE.belowLevel(this, 1) + .append("Detected constant expression ") + .appendSupplier(expression::toString) + .append(" = ") + .append(value); + Utilities.putNew(this.constants, expression, value); + break; + } + } + } + } + + @Override + public void postorder(DBSPExpression expression) { + if (this.maybeGet(expression) != null) + // We already have a value for this expression + return; + this.checkIfConstant(expression); + ExpressionWithContext result = new ExpressionWithContext(expression, this.substitutionContext.clone()); + this.set(expression, result); + } + + // Most of the methods below are from ResolveReferences; Java has no multiple inheritance, + // so we need to reimplement them. + + @Override + public VisitDecision preorder(DBSPVariablePath variable) { + IDBSPDeclaration declaration = this.substitutionContext.get(variable.variable); + if (declaration == null) { + throw new InternalCompilerError("Could not resolve " + variable); + } + this.checkIfConstant(variable); + ExpressionWithContext result = new ExpressionWithContext(variable, this.substitutionContext.clone()); + this.set(variable, result); + return VisitDecision.STOP; + } + + @Override + public VisitDecision preorder(DBSPParameter parameter) { + this.substitutionContext.substitute(parameter.name, parameter); + Utilities.putNew(this.identitySubstitution, parameter, parameter); + return VisitDecision.CONTINUE; + } + + @Override + public VisitDecision preorder(DBSPBlockExpression block) { + this.substitutionContext.newContext(); + return VisitDecision.CONTINUE; + } + + @Override + public void postorder(DBSPBlockExpression block) { + this.substitutionContext.popContext(); + } + + @Override + public void postorder(DBSPLetStatement statement) { + this.substitutionContext.substitute(statement.variable, statement); + Utilities.putNew(this.identitySubstitution, statement, statement); + } + + @Override + public VisitDecision preorder(DBSPLetExpression expression) { + // The initialized is evaluated before the new declaration takes place + expression.initializer.accept(this); + this.substitutionContext.newContext(); + this.substitutionContext.substitute(expression.variable.variable, expression); + Utilities.putNew(this.identitySubstitution, expression, expression); + expression.consumer.accept(this); + this.substitutionContext.popContext(); + return VisitDecision.STOP; + } + + @Override + public VisitDecision preorder(DBSPClosureExpression expression) { + this.substitutionContext.newContext(); + return VisitDecision.CONTINUE; + } + + @Override + public void postorder(DBSPClosureExpression expression) { + this.substitutionContext.popContext(); + } + + @Override + public void startVisit(IDBSPInnerNode node) { + this.state.clear(); + this.constants.clear(); + this.identitySubstitution.clear(); + this.substitutionContext.clear(); + this.substitutionContext.newContext(); + super.startVisit(node); + } + + @Override + public void endVisit() { + this.substitutionContext.popContext(); + this.substitutionContext.mustBeEmpty(); + super.endVisit(); + } + } + + /** Rewrite expressions that are known to be constant booleans replacing them + * with the actual constant value. */ + static class ReplaceConstantExpressions extends ExpressionTranslator { + final Map constants; + + public ReplaceConstantExpressions(DBSPCompiler compiler, Map constants) { + super(compiler); + this.constants = constants; + } + + boolean process(DBSPExpression expression) { + if (this.constants.containsKey(expression)) { + var result = this.constants.get(expression); + var lit = new DBSPBoolLiteral(expression.getNode(), expression.getType(), result); + this.map(expression, lit); + return true; + } + return false; + } + + // We only need to handle expressions that can have a Boolean type + + @Override + public void postorder(DBSPVariablePath var) { + if (!this.process(var)) { + super.postorder(var); + } + } + + @Override + public void postorder(DBSPApplyExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + + @Override + public void postorder(DBSPApplyMethodExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + + @Override + public void postorder(DBSPBinaryExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + + @Override + public void postorder(DBSPCastExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + + @Override + public void postorder(DBSPDerefExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + + @Override + public void postorder(DBSPFieldExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + + @Override + public void postorder(DBSPIsNullExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + + @Override + public void postorder(DBSPPathExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + + @Override + public void postorder(DBSPUnaryExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + + @Override + public void postorder(DBSPUnwrapExpression node) { + if (!this.process(node)) { + super.postorder(node); + } + } + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/TranslateVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/TranslateVisitor.java index 6522786a465..f5d1e20063e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/TranslateVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/TranslateVisitor.java @@ -83,7 +83,7 @@ public TranslateVisitor(DBSPCompiler compiler) { protected void set(IDBSPInnerNode node, T translation) { if (this.translationMap.containsKey(node)) { T old = this.translationMap.get(node); - if (old != translation) + if (!old.equals(translation)) throw new InternalCompilerError("Changing value of " + node + " from\n" + old + " to\n" + translation, node.getNode()); return; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java index a2699458f99..525ead4d9f2 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java @@ -33,6 +33,7 @@ import org.dbsp.sqlCompiler.compiler.errors.CompilationError; import org.dbsp.sqlCompiler.compiler.visitors.inner.CanonicalForm; import org.dbsp.sqlCompiler.compiler.visitors.inner.EliminateDump; +import org.dbsp.sqlCompiler.compiler.visitors.inner.SimplifyConditionals; import org.dbsp.sqlCompiler.compiler.visitors.outer.expandCasts.ExpandCasts; import org.dbsp.sqlCompiler.compiler.visitors.inner.ExpandWriteLog; import org.dbsp.sqlCompiler.compiler.visitors.inner.ImplementStatics; @@ -153,6 +154,7 @@ void createOptimizer() { this.add(new OptimizeWithGraph(compiler, g -> new ChainVisitor(compiler, g))); this.add(new ImplementChains(compiler)); this.add(new ExpandCasts(compiler)); + this.add(new CircuitRewriter(compiler, new SimplifyConditionals(compiler), false)); this.add(new Simplify(compiler).getCircuitRewriter(true)); this.add(new ImplementJoins(compiler)); this.add(new RemoveViewOperators(compiler, true)); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/InsertLimiters.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/InsertLimiters.java index 83d8defd444..4041e5ef1f8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/InsertLimiters.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/InsertLimiters.java @@ -173,8 +173,8 @@ OutputPort createApply(OutputPort source, @Nullable DBSPSimpleOperator represent Utilities.enforce(this.circuit.contains(represented)); } DBSPVariablePath var = source.outputType().ref().var(); - DBSPExpression v0 = var.deref().field(0); - DBSPExpression v1 = var.deref().field(1); + DBSPExpression v0 = var.deepCopy().deref().field(0); + DBSPExpression v1 = var.deepCopy().deref().field(1); DBSPExpression min = function.getResultType().minimumValue(); DBSPExpression call = function.call(v1.borrow()).reduce(this.compiler); DBSPExpression cond = new DBSPTupleExpression(v0, @@ -1713,7 +1713,7 @@ DBSPOperator processLateness( // that is not 'minimum'. DBSPVariablePath var = waterlineOutputPort.outputType().ref().var(); // If the v is &TypedBox, v.deref().deref() has type T - DBSPExpression unwrapped = replaceIndexedInput ? var.deref().deref() : var.deref(); + DBSPExpression unwrapped = replaceIndexedInput ? var.deepCopy().deref().deref() : var.deepCopy().deref(); DBSPExpression eq = eq(minValue, unwrapped); DBSPSimpleOperator extend = new DBSPApplyOperator(operator.getRelNode(), new DBSPTupleExpression( diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/aggregate/DBSPMinMax.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/aggregate/DBSPMinMax.java index 745d47ccd00..132150b9f85 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/aggregate/DBSPMinMax.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/aggregate/DBSPMinMax.java @@ -40,7 +40,10 @@ public DBSPMinMax(CalciteObject node, DBSPType type, @Override public DBSPExpression deepCopy() { - return this; + if (this.postProcessing == null) + return this; + return new DBSPMinMax(this.node, this.type, + this.postProcessing.deepCopy().to(DBSPClosureExpression.class), this.aggregation); } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPFlatmap.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPFlatmap.java index 82e5657cb10..34d091280bd 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPFlatmap.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPFlatmap.java @@ -3,10 +3,10 @@ import com.fasterxml.jackson.databind.JsonNode; import org.dbsp.sqlCompiler.compiler.backend.JsonDecoder; import org.dbsp.sqlCompiler.compiler.errors.UnimplementedException; +import org.dbsp.sqlCompiler.compiler.visitors.inner.EquivalenceContext; import org.dbsp.sqlCompiler.compiler.visitors.outer.LowerCircuitVisitor; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision; -import org.dbsp.sqlCompiler.compiler.visitors.inner.EquivalenceContext; import org.dbsp.sqlCompiler.compiler.visitors.inner.InnerVisitor; import org.dbsp.sqlCompiler.ir.IDBSPInnerNode; import org.dbsp.sqlCompiler.ir.type.DBSPType; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPVariablePath.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPVariablePath.java index 09afcd8a5eb..7c3bad7af0b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPVariablePath.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPVariablePath.java @@ -106,6 +106,7 @@ public boolean equivalent(EquivalenceContext context, DBSPExpression other) { Utilities.enforce(rightDeclaration != null, () -> "Declaration for variable " + Utilities.singleQuote(otherExpression.variable) + " not found"); IDBSPDeclaration subst = context.leftToRight.get(leftDeclaration); + Utilities.enforce(subst != null); return subst.equals(rightDeclaration); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/TestSimplifyConditionals.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/TestSimplifyConditionals.java new file mode 100644 index 00000000000..1bbadef2c2e --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/TestSimplifyConditionals.java @@ -0,0 +1,170 @@ +package org.dbsp.sqlCompiler.compiler.ir; + +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; +import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; +import org.dbsp.sqlCompiler.compiler.sql.tools.BaseSQLTests; +import org.dbsp.sqlCompiler.compiler.visitors.inner.CanonicalForm; +import org.dbsp.sqlCompiler.compiler.visitors.inner.Simplify; +import org.dbsp.sqlCompiler.compiler.visitors.inner.SimplifyConditionals; +import org.dbsp.sqlCompiler.ir.expression.DBSPBinaryExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPIfExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPLetExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPOpcode; +import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntLiteral; +import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.DBSPTypeCode; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; +import org.junit.Assert; +import org.junit.Test; + +/** Unit tests for {@link SimplifyConditionals} */ +public class TestSimplifyConditionals extends BaseSQLTests { + @Test + public void testVariable() { + DBSPCompiler compiler = this.testCompiler(); + DBSPType b = DBSPTypeBool.INSTANCE; + DBSPVariablePath x = b.var(); + // inner = |x: boolean| if (x) { x } else { !x } + DBSPClosureExpression clo = new DBSPIfExpression( + CalciteObject.EMPTY, + x.deepCopy(), + x.deepCopy(), + x.deepCopy().not()).closure(x); + + SimplifyConditionals sc = new SimplifyConditionals(compiler); + var result = sc.apply(clo); + CanonicalForm cf = new CanonicalForm(compiler); + result = cf.apply(result); + + Assert.assertEquals(""" + (|p0: b| + (if p0 { + true + } else { + (! false) + }))""", result.toString()); + Simplify simplify = new Simplify(compiler); + result = simplify.apply(result); + result = cf.apply(result); + + Assert.assertEquals(""" + (|p0: b| + true)""", result.toString()); + } + + @Test + public void testComplexComparison() { + DBSPCompiler compiler = this.testCompiler(); + DBSPType i32 = DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT32, false); + DBSPType b = DBSPTypeBool.INSTANCE; + + DBSPVariablePath x = i32.var(); + DBSPIntLiteral two = new DBSPI32Literal(CalciteObject.EMPTY, i32, 2); + DBSPIntLiteral one = new DBSPI32Literal(CalciteObject.EMPTY, i32, 1); + DBSPIntLiteral zero = new DBSPI32Literal(CalciteObject.EMPTY, i32, 0); + DBSPExpression xPlusOne = new DBSPBinaryExpression(CalciteObject.EMPTY, i32, DBSPOpcode.ADD, x, one); + DBSPExpression lZ = new DBSPBinaryExpression(CalciteObject.EMPTY, b, DBSPOpcode.LT, xPlusOne, zero); + // |x: i32| { + // if ((x + 1) < 0) { + // if ((x + 1) < 0) { 0 } else { 1 } + // } else { + // 2 + // } + // } + var innerIf = new DBSPIfExpression( + CalciteObject.EMPTY, + lZ.deepCopy(), + zero.deepCopy(), + one.deepCopy()); + var clo = new DBSPIfExpression( + CalciteObject.EMPTY, + lZ.deepCopy(), + innerIf, + two).closure(x); + CanonicalForm cf = new CanonicalForm(compiler); + + SimplifyConditionals sc = new SimplifyConditionals(compiler); + var result = sc.apply(clo); + result = cf.apply(result); + Assert.assertEquals(""" + (|p0: i32| + (if ((p0 + 1) < 0) { + (if true { + 0 + } else { + 1 + }) + } else { + 2 + }))""", result.toString()); + + Simplify simplify = new Simplify(compiler); + result = simplify.apply(result); + result = cf.apply(result); + Assert.assertEquals(""" + (|p0: i32| + (if ((p0 + 1) < 0) { + 0 + } else { + 2 + }))""", result.toString()); + } + + @Test + public void testAliasedVariable() { + DBSPCompiler compiler = this.testCompiler(); + DBSPType i32 = DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT32, false); + DBSPType b = DBSPTypeBool.INSTANCE; + + // The compiler never reuses variable names, but this is supposed to work too. + DBSPVariablePath x = i32.var(); + DBSPIntLiteral two = new DBSPI32Literal(CalciteObject.EMPTY, i32, 2); + DBSPIntLiteral one = new DBSPI32Literal(CalciteObject.EMPTY, i32, 1); + DBSPIntLiteral zero = new DBSPI32Literal(CalciteObject.EMPTY, i32, 0); + DBSPExpression xPlusOne = new DBSPBinaryExpression(CalciteObject.EMPTY, i32, DBSPOpcode.ADD, x, one); + DBSPExpression lZ = new DBSPBinaryExpression(CalciteObject.EMPTY, b, DBSPOpcode.LT, xPlusOne, zero); + // inner = |x: i32| { + // if ((x + 1) < 0) { + // let x = x + 1; + // if ((x + 1) < 0) { 0 } else { 1 } + // } else { + // 2 + // } + var innerIf = new DBSPIfExpression( + CalciteObject.EMPTY, + lZ.deepCopy(), + zero.deepCopy(), + one.deepCopy()); + var let = new DBSPLetExpression(x, xPlusOne.deepCopy(), innerIf); + var clo = new DBSPIfExpression( + CalciteObject.EMPTY, + lZ.deepCopy(), + let, + two).closure(x); + CanonicalForm cf = new CanonicalForm(compiler); + var initial = cf.apply(clo); + Assert.assertEquals(""" + (|p0: i32| + (if ((p0 + 1) < 0) { + {let p1 = (p0 + 1); + (if ((p1 + 1) < 0) { + 0 + } else { + 1 + })} + } else { + 2 + }))""", initial.toString()); + + // This cannot be simplified + SimplifyConditionals sc = new SimplifyConditionals(compiler); + var result = sc.apply(clo); + result = cf.apply(result); + Assert.assertEquals(initial.toString(), result.toString()); + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java index 9ba3adc942f..4a2a95ddd64 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java @@ -91,7 +91,7 @@ static List getQATests() { } @Test - public void qaTests() throws SQLException { + public void qaTests() throws SQLException, IOException { // BaseSQLTests.showPlan(); for (File c : getQATests()) { // This program cannot be compiled because it contains a udf