Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,18 @@
import org.junit.runner.RunWith;

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.HashSet;
import java.util.Set;

import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest;
import static org.apache.iotdb.itbase.constant.TestConstant.DEVICE;
import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR;
import static org.apache.iotdb.itbase.constant.TestConstant.count;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

@RunWith(IoTDBTestRunner.class)
Expand All @@ -42,6 +51,8 @@ public class IoTDBPredicateConversionTreeIT {

private static final String DATABASE_NAME = "root.test_pred";
private static final String DEVICE_ID = DATABASE_NAME + ".d1";
private static final String NOT_DATABASE_NAME = "root.test_not_regression";
private static final String TEMPLATE_DATABASE_NAME = "root.test_not_template";

private static final String[] SQLs =
new String[] {
Expand All @@ -57,7 +68,31 @@ public class IoTDBPredicateConversionTreeIT {
"INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(7, null, null)",
"INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(8, 1000, 1000)",
"INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(9, -29, -29)",
"INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(10, -30, -30)"
"INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(10, -30, -30)",
"CREATE DATABASE " + NOT_DATABASE_NAME,
"CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
"CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d1.s2 WITH DATATYPE=INT32, ENCODING=PLAIN",
"CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d1.b WITH DATATYPE=BOOLEAN, ENCODING=PLAIN",
"INSERT INTO " + NOT_DATABASE_NAME + ".d1(time, s1, s2, b) VALUES(1, 1, 10, true)",
"INSERT INTO " + NOT_DATABASE_NAME + ".d1(time, s1, s2, b) VALUES(2, 6, 20, false)",
"INSERT INTO " + NOT_DATABASE_NAME + ".d1(time, s1, s2, b) VALUES(3, 3, 30, true)",
"INSERT INTO " + NOT_DATABASE_NAME + ".d1(time, s1, s2, b) VALUES(4, 8, 40, false)",
"CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d2.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
"INSERT INTO " + NOT_DATABASE_NAME + ".d2(time, s1) VALUES(1, 2)",
"INSERT INTO " + NOT_DATABASE_NAME + ".d2(time, s1) VALUES(2, 7)",
"CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d3.x WITH DATATYPE=DOUBLE, ENCODING=PLAIN",
"CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d3.y WITH DATATYPE=INT32, ENCODING=PLAIN",
"INSERT INTO " + NOT_DATABASE_NAME + ".d3(time, x, y) VALUES(1, NaN, 1)",
"INSERT INTO " + NOT_DATABASE_NAME + ".d3(time, x, y) VALUES(2, 3.0, 2)",
"INSERT INTO " + NOT_DATABASE_NAME + ".d3(time, x, y) VALUES(3, 10.0, 3)",
"CREATE DATABASE " + TEMPLATE_DATABASE_NAME,
"CREATE DEVICE TEMPLATE not_predicate_template ALIGNED (s1 INT64, b BOOLEAN)",
"SET DEVICE TEMPLATE not_predicate_template TO " + TEMPLATE_DATABASE_NAME + ".d1",
"SET DEVICE TEMPLATE not_predicate_template TO " + TEMPLATE_DATABASE_NAME + ".d2",
"INSERT INTO " + TEMPLATE_DATABASE_NAME + ".d1(time, s1, b) ALIGNED VALUES(1, 3, true)",
"INSERT INTO " + TEMPLATE_DATABASE_NAME + ".d1(time, s1, b) ALIGNED VALUES(2, 8, false)",
"INSERT INTO " + TEMPLATE_DATABASE_NAME + ".d2(time, s1, b) ALIGNED VALUES(1, 4, false)",
"INSERT INTO " + TEMPLATE_DATABASE_NAME + ".d2(time, s1, b) ALIGNED VALUES(2, 9, true)"
};

@BeforeClass
Expand Down Expand Up @@ -290,4 +325,198 @@ public void testDoubleLiteralOverflow() {
expectedHeaderInt64,
allInt64Rows);
}

@Test
public void testWhereAndHavingWithNot() {
String[] expectedWhereHeader = new String[] {TIMESTAMP_STR, DEVICE_ID + ".int32_col"};
resultSetEqualTest(
"SELECT int32_col FROM " + DEVICE_ID + " WHERE NOT (int32_col > 20)",
expectedWhereHeader,
new String[] {"1,20,", "9,-29,", "10,-30,"});
resultSetEqualTest(
"SELECT int32_col FROM " + DEVICE_ID + " WHERE NOT (int32_col > 20 AND int32_col < 40)",
expectedWhereHeader,
new String[] {"1,20,", "8,1000,", "9,-29,", "10,-30,"});

String[] expectedHavingHeader = new String[] {TIMESTAMP_STR, count(DEVICE_ID + ".int32_col")};
resultSetEqualTest(
"SELECT count(int32_col) FROM "
+ DEVICE_ID
+ " GROUP BY ([1,11),2ms) HAVING NOT (count(int32_col) > 1)",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both HAVING NOT (count(int32_col) > 1) cases (with and without ALIGN BY DEVICE) already pass on master, so they don't cover the HAVING change. The HAVING bug this PR actually fixes is a composite NOT on the non-ALIGN BY DEVICE and GROUP BY LEVEL paths (see the comment on analyzeHavingBase).

It would be good to add:

  • composite HAVING, e.g. HAVING NOT (count(...) > 1 AND sum(...) > 50), with and without GROUP BY LEVEL
  • non-invertible NOT: WHERE NOT bool_series, WHERE NOT bool_series OR s1 > 5, HAVING NOT last_value(bool_series)
  • nested negation: WHERE NOT (NOT (NOT (A AND B)))
  • a FLOAT/DOUBLE series containing NaN under NOT

Any of the last three would have caught the issues raised in the other comments. PredicateUtilsTest#testLogicNotCannotPushDownToSource covers only the checker, not the analyzer path.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added the requested coverage: composite HAVING with and without GROUP BY LEVEL, non-invertible boolean series/functions/CASE, triple negation, NaN, and aligned-template predicates.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The missing HAVING NOT last_value(bool_series) coverage is added now — that was the one item from your list I had missed. Added in 1b9ddcf:

  • testHavingNotWithBooleanAggregation: HAVING NOT last_value(b) with GROUP BY ([1,5),1ms), plus the same query with ALIGN BY DEVICE.
  • testTemplatedHavingNotWithBooleanAggregation: the aligned-template path, SELECT count(s1) FROM root.test_not_template.* GROUP BY ([1,3),1ms) HAVING NOT last_value(b) ALIGN BY DEVICE.

The other cases you listed were already in the PR: composite HAVING with and without GROUP BY LEVEL, WHERE NOT bool_series, WHERE NOT bool_series OR s1 > 5, NOT (CASE WHEN ...) OR ..., triple negation and the NaN cases.

Verified locally on the latest head: PredicateUtilsTest 13/13 and IoTDBPredicateConversionTreeIT 11/11 (including the two new tests).

expectedHavingHeader,
new String[] {"5,1,", "7,1,"});

String[] expectedWhereAlignByDeviceHeader = new String[] {TIMESTAMP_STR, DEVICE, "int32_col"};
resultSetEqualTest(
"SELECT int32_col FROM root.test_pred.* WHERE NOT (int32_col > 20 AND int32_col < 40) ALIGN BY DEVICE",
expectedWhereAlignByDeviceHeader,
new String[] {
"1,root.test_pred.d1,20,",
"8,root.test_pred.d1,1000,",
"9,root.test_pred.d1,-29,",
"10,root.test_pred.d1,-30,",
});

String[] expectedHavingAlignByDeviceHeader =
new String[] {TIMESTAMP_STR, DEVICE, count("int32_col")};
resultSetEqualTest(
"SELECT count(int32_col) FROM root.test_pred.* GROUP BY ([1,11),2ms) HAVING NOT (count(int32_col) > 1) ALIGN BY DEVICE",
expectedHavingAlignByDeviceHeader,
new String[] {"5,root.test_pred.d1,1,", "7,root.test_pred.d1,1,"});
}

@Test
public void testTemplatedWhereWithNot() {
String[] expectedHeader = new String[] {TIMESTAMP_STR, DEVICE, "s1"};
resultSetEqualTest(
"SELECT s1 FROM " + TEMPLATE_DATABASE_NAME + ".* WHERE NOT (s1 > 5) ALIGN BY DEVICE",
expectedHeader,
new String[] {
"1,root.test_not_template.d1,3,", "1,root.test_not_template.d2,4,",
});

String[] expectedBooleanHeader = new String[] {TIMESTAMP_STR, DEVICE, "s1", "b"};
resultSetEqualTest(
"SELECT s1, b FROM " + TEMPLATE_DATABASE_NAME + ".* WHERE NOT b ALIGN BY DEVICE",
expectedBooleanHeader,
new String[] {
"2,root.test_not_template.d1,8,false,", "1,root.test_not_template.d2,4,false,",
});
}

@Test
public void testUnsupportedNotPredicate() {
String[] expectedHeader =
new String[] {TIMESTAMP_STR, NOT_DATABASE_NAME + ".d1.s1", NOT_DATABASE_NAME + ".d1.b"};
resultSetEqualTest(
"SELECT s1, b FROM " + NOT_DATABASE_NAME + ".d1 WHERE NOT b OR s1 > 5",
expectedHeader,
new String[] {"2,6,false,", "4,8,false,"});
resultSetEqualTest(
"SELECT s1, b FROM " + NOT_DATABASE_NAME + ".d1 WHERE NOT b",
expectedHeader,
new String[] {"2,6,false,", "4,8,false,"});
resultSetEqualTest(
"SELECT s1, b FROM "
+ NOT_DATABASE_NAME
+ ".d1 WHERE NOT (CASE WHEN s1 > 5 THEN true ELSE false END) OR s2 > 35",
expectedHeader,
new String[] {"1,1,true,", "3,3,true,", "4,8,false,"});
}

@Test
public void testNotPredicateWithTimeAndMultipleSeries() {
resultSetEqualTest(
"SELECT s1, s2 FROM " + NOT_DATABASE_NAME + ".d1 WHERE NOT (time > 2 OR s1 > 5)",
new String[] {TIMESTAMP_STR, NOT_DATABASE_NAME + ".d1.s1", NOT_DATABASE_NAME + ".d1.s2"},
new String[] {"1,1,10,"});

resultSetEqualTest(
"SELECT s1, s2 FROM "
+ NOT_DATABASE_NAME
+ ".d1 WHERE NOT (time > 2 OR s1 > 5) ALIGN BY DEVICE",
new String[] {TIMESTAMP_STR, DEVICE, "s1", "s2"},
new String[] {"1," + NOT_DATABASE_NAME + ".d1,1,10,"});
}

@Test
public void testNestedNotPredicate() {
String[] expectedHeader =
new String[] {TIMESTAMP_STR, NOT_DATABASE_NAME + ".d1.s1", NOT_DATABASE_NAME + ".d1.s2"};
resultSetEqualTest(
"SELECT s1, s2 FROM "
+ NOT_DATABASE_NAME
+ ".d1 WHERE NOT (NOT (NOT (s1 > 5 AND s2 > 25)))",
expectedHeader,
new String[] {"1,1,10,", "2,6,20,", "3,3,30,"});
}

@Test
public void testNotPredicateWithNaN() throws Exception {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
try (ResultSet resultSet =
statement.executeQuery(
"SELECT x, y FROM " + NOT_DATABASE_NAME + ".d3 WHERE NOT (x > 5 OR y > 2)")) {
assertTrue(resultSet.next());
assertEquals(1, resultSet.getLong(1));
assertTrue(Double.isNaN(resultSet.getDouble(2)));
assertEquals(1, resultSet.getInt(3));
assertTrue(resultSet.next());
assertEquals(2, resultSet.getLong(1));
assertEquals(3.0, resultSet.getDouble(2), 0.0);
assertEquals(2, resultSet.getInt(3));
assertFalse(resultSet.next());
}

try (ResultSet resultSet =
statement.executeQuery(
"SELECT x, y FROM " + NOT_DATABASE_NAME + ".d3 WHERE NOT (x + y > 5)")) {
assertTrue(resultSet.next());
assertEquals(1, resultSet.getLong(1));
assertTrue(Double.isNaN(resultSet.getDouble(2)));
assertTrue(resultSet.next());
assertEquals(2, resultSet.getLong(1));
assertFalse(resultSet.next());
}
}
}

@Test
public void testCompositeHavingNot() throws Exception {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
try (ResultSet resultSet =
statement.executeQuery(
"SELECT count(s1), sum(s2) FROM "
+ NOT_DATABASE_NAME
+ ".d1 HAVING NOT (count(s1) > 5 AND sum(s2) > 50)")) {
assertTrue(resultSet.next());
assertEquals(4, resultSet.getLong(1));
assertEquals(100.0, resultSet.getDouble(2), 0.0);
assertFalse(resultSet.next());
}

try (ResultSet resultSet =
statement.executeQuery(
"SELECT count(s1) FROM "
+ NOT_DATABASE_NAME
+ ".* GROUP BY ([1,5),10ms), LEVEL=1 "
+ "HAVING NOT (count(s1) > 0 AND sum(s2) > 150)")) {
assertTrue(resultSet.next());
assertEquals(1, resultSet.getLong(1));
assertFalse(resultSet.next());
}
}
}

@Test
public void testHavingNotWithBooleanAggregation() {
resultSetEqualTest(
"SELECT count(s1) FROM "
+ NOT_DATABASE_NAME
+ ".d1 GROUP BY ([1,5),1ms) HAVING NOT last_value(b)",
new String[] {TIMESTAMP_STR, count(NOT_DATABASE_NAME + ".d1.s1")},
new String[] {"2,1,", "4,1,"});

resultSetEqualTest(
"SELECT count(s1) FROM "
+ NOT_DATABASE_NAME
+ ".d1 GROUP BY ([1,5),1ms) HAVING NOT last_value(b) ALIGN BY DEVICE",
new String[] {TIMESTAMP_STR, DEVICE, count("s1")},
new String[] {"2," + NOT_DATABASE_NAME + ".d1,1,", "4," + NOT_DATABASE_NAME + ".d1,1,"});
}

@Test
public void testTemplatedHavingNotWithBooleanAggregation() {
Set<String> expectedRows = new HashSet<>();
expectedRows.add("1," + TEMPLATE_DATABASE_NAME + ".d2,1,");
expectedRows.add("2," + TEMPLATE_DATABASE_NAME + ".d1,1,");
resultSetEqualTest(
"SELECT count(s1) FROM "
+ TEMPLATE_DATABASE_NAME
+ ".* GROUP BY ([1,3),1ms) HAVING NOT last_value(b) ALIGN BY DEVICE",
TIMESTAMP_STR + "," + DEVICE + "," + count("s1") + ",",
expectedRows);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -642,8 +642,6 @@ public final class DataNodeQueryMessages {
"TIMESTAMP IN-list literal can only be LongLiteral, DoubleLiteral, or GenericLiteral; actual type: %s";
public static final String IS_NULL_CAN_BE_PUSHED_DOWN =
"IS NULL can be pushed down";
public static final String GROUP_BY_TIME_CANNOT_BE_REVERSED =
"GROUP BY TIME cannot be reversed";

// --- Plan / Optimization ---

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -622,8 +622,6 @@ public final class DataNodeQueryMessages {
"TIMESTAMP 的 IN 列表字面量只能是 LongLiteral、DoubleLiteral 或 GenericLiteral,实际类型为 %s";
public static final String IS_NULL_CAN_BE_PUSHED_DOWN =
"IS NULL 可以下推";
public static final String GROUP_BY_TIME_CANNOT_BE_REVERSED =
"GROUP BY TIME 不能反转";

// --- Plan / Optimization ---

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1547,7 +1547,7 @@ private void analyzeDeviceToWhere(
while (deviceIterator.hasNext()) {
PartialPath devicePath = deviceIterator.next();
Expression whereExpression =
analyzeWhereSplitByDevice(queryStatement, devicePath, schemaTree, queryContext);
analyzeWhereSplitByDevice(analysis, queryStatement, devicePath, schemaTree, queryContext);
if (whereExpression.equals(ConstantOperand.FALSE)) {
deviceIterator.remove();
} else if (whereExpression.equals(ConstantOperand.TRUE)) {
Expand Down Expand Up @@ -1580,7 +1580,7 @@ private void analyzeWhere(
schemaTree,
true,
queryContext);
Expression whereExpression = convertConJunctionsToWhereExpression(conJunctions);
Expression whereExpression = convertConJunctionsToWhereExpression(analysis, conJunctions);
if (whereExpression.equals(ConstantOperand.TRUE)) {
analysis.setWhereExpression(null);
analysis.setHasValueFilter(false);
Expand All @@ -1595,6 +1595,7 @@ private void analyzeWhere(
}

private Expression analyzeWhereSplitByDevice(
final Analysis analysis,
final QueryStatement queryStatement,
final PartialPath devicePath,
final ISchemaTree schemaTree,
Expand All @@ -1606,16 +1607,25 @@ private Expression analyzeWhereSplitByDevice(
schemaTree,
true,
queryContext);
return convertConJunctionsToWhereExpression(conJunctions);
return convertConJunctionsToWhereExpression(analysis, conJunctions);
}

private Expression convertConJunctionsToWhereExpression(List<Expression> conJunctions) {
private Expression convertConJunctionsToWhereExpression(
Analysis analysis, List<Expression> conJunctions) {
Expression predicate =
PredicateUtils.combineConjuncts(
conJunctions.stream().distinct().collect(Collectors.toList()));
predicate = PredicateUtils.simplifyPredicate(predicate);
predicate = normalizeExpression(predicate);
return predicate;
analyzeExpressionType(analysis, predicate);
Expression predicateWithoutNot =
PredicateUtils.predicateRemoveNot(predicate, analysis::getType);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now that NOT is removed after binding, a negated disjunction that contains a time condition leaves a time-only conjunct in the residual WHERE predicate. For example, NOT (time > 2 OR s1 > 5) becomes time <= 2 AND s1 <= 5. The new De Morgan branch in extractGlobalTimePredicate copies time <= 2 into the global time filter, but the residual still contains it.

When the plan has a FullOuterTimeJoin, PredicatePushDown#extractPushDownConjunctsForEachChild calls PredicateUtils#extractPredicateSourceSymbol for every conjunct. That method assumes the conjunct references at least one series, so pathIterator.next() throws on the time-only conjunct.

Repro (same setup as my first review):

SELECT s1, s2 FROM root.rv.d1 WHERE NOT (time > 2 OR s1 > 5)
--   expected: (1, 1, 10)    this PR: 305 ... executeStatement failed. null
SELECT s1, s2 FROM root.rv.d1 WHERE NOT (time > 2 OR s1 > 5) ALIGN BY DEVICE
--   expected: (1, root.rv.d1, 1, 10)    this PR: 305 ... null
SELECT s1, s2 FROM root.rv.d1 WHERE NOT (time > 2 OR s1 > 5) AND s2 > 5
--   expected: (1, 1, 10)    this PR: 305 ... null
SELECT s1, s2 FROM root.rv.d1 WHERE NOT ((time > 2 AND time < 4) OR s1 > 5)
--   expected: (1, 1, 10)    this PR: 305 ... null
java.util.NoSuchElementException
	at java.base/java.util.HashMap$HashIterator.nextNode(HashMap.java:1607)
	at java.base/java.util.HashMap$KeyIterator.next(HashMap.java:1628)
	at ...PredicateUtils.extractPredicateSourceSymbol(PredicateUtils.java:487)
	at ...PredicatePushDown$Rewriter.extractPushDownConjunctsForEachChild(PredicatePushDown.java:201)
	at ...PredicatePushDown$Rewriter.visitFullOuterTimeJoin(PredicatePushDown.java:145)

These queries only work today in two cases:

  • The query reads a single series. The conjuncts then go through visitSeriesScanSource; for example, SELECT s1 FROM root.rv.d1 WHERE NOT (time > 2 OR s1 > 5) pushes Time <= 2 & root.rv.d1.s1 <= 5 into the scan.
  • The device is aligned.

On master these queries fail with 301 This predicate contains a not!, so this isn't a regression. However, NOT (time > ... OR ...) is exactly the shape that 34c39ad targets, and it still fails for multi-series queries.

The underlying bug already exists on master. A time-only conjunct that can't be extracted as a global time filter crashes the same way there, e.g. SELECT s1, s2 FROM root.rv.d1 WHERE time % 2 = 0 or ... WHERE time > 2.5. So I'd suggest fixing it in extractPredicateSourceSymbol: return null when the conjunct references no series, so it stays in the FilterNode. Pushing time-only conjuncts to every child would also work. Either way, both cases are covered. Optionally, after the rewrite, you could also drop time-only conjuncts that are already implied by the global time filter from the residual.

Please also add an IT that reads at least two non-aligned series, e.g. SELECT s1, s2 ... WHERE NOT (time > 2 OR s1 > 5), with and without ALIGN BY DEVICE.

if (predicateWithoutNot != predicate) {
predicateWithoutNot = PredicateUtils.simplifyPredicate(predicateWithoutNot);
predicateWithoutNot = normalizeExpression(predicateWithoutNot);
analyzeExpressionType(analysis, predicateWithoutNot);
}
return predicateWithoutNot;
}

private void analyzeDeviceViewOutput(Analysis analysis, QueryStatement queryStatement) {
Expand Down
Loading
Loading