Skip to content

Fix library UDF edge cases #17871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Caideyipi wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Fix library UDF edge cases #17871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7b1b673
Fix library UDF edge cases
Caideyipi Jun 9, 2026
5288816
Fix library UDF edge cases
Caideyipi Jun 9, 2026
00f0976
Fix library UDF edge cases
Caideyipi Jun 9, 2026
61792b4
Fix more library UDF edge cases
Caideyipi Jun 9, 2026
28919e9
Fix library UDF matching edge cases
Caideyipi Jun 9, 2026
8fe96a5
Fix K-Sigma default window size
Caideyipi Jun 9, 2026
1e8dc32
Merge branch 'master' of https://github.com/apache/iotdb into fix/lib…
Caideyipi Jun 23, 2026
9d71399
Merge branch 'master' of https://github.com/apache/iotdb into fix/lib…
Caideyipi Sep 23, 2026
4f1228d
Fix localized UDF validation messages and quantile ordering
Caideyipi Oct 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,5 +167,44 @@ public final class LibraryUdfMessages {
public static final String EXCEPTION_FAIL_TO_GET_DATA_TYPE_IN_ROW_8CD82629 =
"Fail to get data type in row ";

// Parameter validation and utility exceptions added for UDF edge cases.
public static final String EXCEPTION_PARAMETER_Q1_AND_Q3_SHOULD_BE_FINITE_AND_Q1_SHOULD_BE_SMALLER_THAN_Q3_EB122E6C = "parameter $q1$ and $q3$ should be finite, and $q1$ should be smaller than $q3$";
public static final String EXCEPTION_PARAMETER_K_SHOULD_BE_FINITE_AND_LARGER_THAN_0_ECE1B163 = "Parameter k should be finite and larger than 0.";
public static final String EXCEPTION_PARAMETER_K_SHOULD_BE_A_POSITIVE_INTEGER_62AC8458 = "Parameter k should be a positive integer.";
public static final String EXCEPTION_PARAMETER_WINDOW_SHOULD_BE_A_POSITIVE_INTEGER_54FB7DBB = "Parameter window should be a positive integer.";
public static final String EXCEPTION_METHOD_SHOULD_BE_DEFAULT_OR_SERIES_181EEECF = "Method should be default or series.";
public static final String EXCEPTION_PARAMETER_R_SHOULD_BE_FINITE_AND_NON_NEGATIVE_4FFDF663 = "Parameter r should be finite and non-negative.";
public static final String EXCEPTION_PARAMETER_W_SHOULD_BE_A_POSITIVE_INTEGER_C6C838D2 = "Parameter w should be a positive integer.";
public static final String EXCEPTION_PARAMETER_S_SHOULD_BE_A_POSITIVE_INTEGER_B941BE85 = "Parameter s should be a positive integer.";
public static final String EXCEPTION_PARAMETER_LOWER_BOUND_AND_UPPER_BOUND_SHOULD_BE_FINITE_AND_LOWER_BOUND_SHOULD_BE_SMALLER_THAN_UPPER_BOUND_06A9FCA8 = "parameter \"lower_bound\" and \"upper_bound\" should be finite, and \"lower_bound\" should be smaller than \"upper_bound\".";
public static final String EXCEPTION_PARAMETER_LEN_SHOULD_BE_FINITE_AND_LARGER_THAN_0_F93F1EC9 = "Parameter len should be finite and larger than 0.";
public static final String EXCEPTION_PARAMETER_THRESHOLD_SHOULD_BE_FINITE_AND_NON_NEGATIVE_93B393D7 = "Parameter threshold should be finite and non-negative.";
public static final String EXCEPTION_PARAMETER_P_SHOULD_BE_A_POSITIVE_INTEGER_D7914C8A = "Parameter p should be a positive integer.";
public static final String EXCEPTION_THRESHOLD_HAS_TO_BE_FINITE_AND_NON_NEGATIVE_45C26FD0 = "threshold has to be finite and non-negative.";
public static final String EXCEPTION_PARAMETER_END_SHOULD_BE_FINITE_AND_LARGER_THAN_START_A5FB7AEC = "parameter $end$ should be finite and larger than $start$";
public static final String EXCEPTION_PARAMETER_MIN_AND_MAX_SHOULD_BE_FINITE_AND_MIN_SHOULD_BE_SMALLER_THAN_MAX_1EDB33AF = "parameter $min$ and $max$ should be finite, and $min$ should be smaller than $max$.";
public static final String EXCEPTION_ERROR_BOUND_SHOULD_BE_FINITE_AND_NO_LESS_THAN_0_E2C80B2C = "Error bound should be finite and no less than 0.";
public static final String EXCEPTION_PARAMETER_POINTS_SHOULD_BE_AT_LEAST_2_A4F55048 = "Parameter points should be at least 2.";
public static final String EXCEPTION_PARAMETER_AVG_IS_ILLEGAL_IT_SHOULD_BE_FINITE_D4B6B4FF = "Parameter \"avg\" is illegal. It should be finite.";
public static final String EXCEPTION_PARAMETER_SD_IS_ILLEGAL_IT_SHOULD_BE_FINITE_AND_LARGER_THAN_0_CCE0B275 = "Parameter \"sd\" is illegal. It should be finite and larger than 0.";
public static final String EXCEPTION_PARAMETER_WINDOW_SHOULD_BE_A_POSITIVE_DURATION_OR_A_POSITIVE_INTEGER_41E4F4DB = "Parameter window should be a positive duration or a positive integer.";
public static final String EXCEPTION_WINDOW_SIZE_SHOULD_BE_IN_0_INTEGER_MAX_VALUE_52F02FF6 = "Window size should be in (0, Integer.MAX_VALUE].";
public static final String EXCEPTION_METHOD_SHOULD_BE_MEDIAN_MODE_OR_CLUSTER_7C3B0F56 = "Method should be Median, Mode, or Cluster.";
public static final String EXCEPTION_ILLEGAL_FILL_METHOD_45FB5556 = "Illegal fill method.";
public static final String EXCEPTION_PARAMETER_SIGMA_SHOULD_BE_FINITE_AND_LARGER_THAN_0_538FEDBA = "Parameter $sigma$ should be finite and larger than 0.";
public static final String EXCEPTION_PARAMETER_MINSPEED_AND_MAXSPEED_SHOULD_BE_FINITE_AND_MINSPEED_SHOULD_BE_SMALLER_THAN_MAXSPEED_5998EA69 = "parameter $minSpeed$ and $maxSpeed$ should be finite, and $minSpeed$ should be smaller than $maxSpeed$.";
public static final String EXCEPTION_PARAMETER_CENTER_SHOULD_BE_FINITE_21F289A3 = "Parameter $center$ should be finite.";
public static final String EXCEPTION_METHOD_SHOULD_BE_SCREEN_OR_LSGREEDY_B5975D7A = "Method should be screen or lsgreedy.";
public static final String EXCEPTION_COEF_SHOULD_BE_FINITE_DOUBLE_DOUBLE_WHEN_METHOD_IS_BLANK_F72B1092 = "coef should be finite double,double... when method is blank.";
public static final String EXCEPTION_WAVELET_COEFFICIENTS_SHOULD_NOT_BE_EMPTY_2C78B047 = "Wavelet coefficients should not be empty.";
public static final String EXCEPTION_WAVELET_COEFFICIENTS_SHOULD_BE_FINITE_D764C838 = "Wavelet coefficients should be finite.";
public static final String EXCEPTION_ILLEGAL_PARAMETER_PATTERN_MUST_BE_FINITE_DOUBLE_DOUBLE_612DE434 = "Illegal parameter, pattern must be finite double,double...";
public static final String EXCEPTION_ILLEGAL_PARAMETER_THRESHOLD_MUST_BE_A_FINITE_NON_NEGATIVE_NUMBER_1C438942 = "Illegal parameter, threshold must be a finite non-negative number.";
public static final String EXCEPTION_ILLEGAL_PARAMETER_VALUEPATTERN_VALUES_MUST_BE_FINITE_FFD05AC9 = "Illegal parameter, valuePattern values must be finite.";
public static final String EXCEPTION_REGEX_SHOULD_NOT_BE_EMPTY_AND_HAS_TO_BE_A_VALID_REGULAR_EXPRESSION_751072ED = "regex should not be empty and has to be a valid regular expression.";
public static final String EXCEPTION_CAPACITY_SHOULD_BE_LARGER_THAN_0_055372C7 = "Capacity should be larger than 0.";
public static final String EXCEPTION_TIME_VALUE_SHOULD_BE_FINITE_A661DDB0 = "Time value should be finite.";
public static final String EXCEPTION_INVALID_DATETIME_FORMAT_A3DC435A = "Invalid datetime format.";

private LibraryUdfMessages() {}
}
39 changes: 39 additions & 0 deletions library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,5 +153,44 @@ public final class LibraryUdfMessages {
public static final String EXCEPTION_FAIL_TO_GET_DATA_TYPE_IN_ROW_8CD82629 =
"获取该行数据类型失败，行时间戳为 ";

// Parameter validation and utility exceptions added for UDF edge cases.
public static final String EXCEPTION_PARAMETER_Q1_AND_Q3_SHOULD_BE_FINITE_AND_Q1_SHOULD_BE_SMALLER_THAN_Q3_EB122E6C = "参数 $q1$ 和 $q3$ 必须为有限值，且 $q1$ 必须小于 $q3$";
public static final String EXCEPTION_PARAMETER_K_SHOULD_BE_FINITE_AND_LARGER_THAN_0_ECE1B163 = "参数 k 必须为有限且大于 0。";
public static final String EXCEPTION_PARAMETER_K_SHOULD_BE_A_POSITIVE_INTEGER_62AC8458 = "参数 k 必须为正整数。";
public static final String EXCEPTION_PARAMETER_WINDOW_SHOULD_BE_A_POSITIVE_INTEGER_54FB7DBB = "参数 window 必须为正整数。";
public static final String EXCEPTION_METHOD_SHOULD_BE_DEFAULT_OR_SERIES_181EEECF = "method 必须为 default 或 series。";
public static final String EXCEPTION_PARAMETER_R_SHOULD_BE_FINITE_AND_NON_NEGATIVE_4FFDF663 = "参数 r 必须为有限且非负。";
public static final String EXCEPTION_PARAMETER_W_SHOULD_BE_A_POSITIVE_INTEGER_C6C838D2 = "参数 w 必须为正整数。";
public static final String EXCEPTION_PARAMETER_S_SHOULD_BE_A_POSITIVE_INTEGER_B941BE85 = "参数 s 必须为正整数。";
public static final String EXCEPTION_PARAMETER_LOWER_BOUND_AND_UPPER_BOUND_SHOULD_BE_FINITE_AND_LOWER_BOUND_SHOULD_BE_SMALLER_THAN_UPPER_BOUND_06A9FCA8 = "参数 lower_bound 和 upper_bound 必须为有限值，且 lower_bound 必须小于 upper_bound。";
public static final String EXCEPTION_PARAMETER_LEN_SHOULD_BE_FINITE_AND_LARGER_THAN_0_F93F1EC9 = "参数 len 必须为有限且大于 0。";
public static final String EXCEPTION_PARAMETER_THRESHOLD_SHOULD_BE_FINITE_AND_NON_NEGATIVE_93B393D7 = "参数 threshold 必须为有限且非负。";
public static final String EXCEPTION_PARAMETER_P_SHOULD_BE_A_POSITIVE_INTEGER_D7914C8A = "参数 p 必须为正整数。";
public static final String EXCEPTION_THRESHOLD_HAS_TO_BE_FINITE_AND_NON_NEGATIVE_45C26FD0 = "threshold 必须为有限且非负。";
public static final String EXCEPTION_PARAMETER_END_SHOULD_BE_FINITE_AND_LARGER_THAN_START_A5FB7AEC = "参数 $end$ 必须为有限值且大于 $start$。";
public static final String EXCEPTION_PARAMETER_MIN_AND_MAX_SHOULD_BE_FINITE_AND_MIN_SHOULD_BE_SMALLER_THAN_MAX_1EDB33AF = "参数 $min$ 和 $max$ 必须为有限值，且 $min$ 必须小于 $max$。";
public static final String EXCEPTION_ERROR_BOUND_SHOULD_BE_FINITE_AND_NO_LESS_THAN_0_E2C80B2C = "误差界必须为有限且不小于 0。";
public static final String EXCEPTION_PARAMETER_POINTS_SHOULD_BE_AT_LEAST_2_A4F55048 = "参数 points 至少为 2。";
public static final String EXCEPTION_PARAMETER_AVG_IS_ILLEGAL_IT_SHOULD_BE_FINITE_D4B6B4FF = "参数 avg 无效，必须为有限值。";
public static final String EXCEPTION_PARAMETER_SD_IS_ILLEGAL_IT_SHOULD_BE_FINITE_AND_LARGER_THAN_0_CCE0B275 = "参数 sd 无效，必须为有限且大于 0。";
public static final String EXCEPTION_PARAMETER_WINDOW_SHOULD_BE_A_POSITIVE_DURATION_OR_A_POSITIVE_INTEGER_41E4F4DB = "参数 window 必须为正时长或正整数。";
public static final String EXCEPTION_WINDOW_SIZE_SHOULD_BE_IN_0_INTEGER_MAX_VALUE_52F02FF6 = "窗口大小必须在 (0, Integer.MAX_VALUE] 范围内。";
public static final String EXCEPTION_METHOD_SHOULD_BE_MEDIAN_MODE_OR_CLUSTER_7C3B0F56 = "method 必须为 Median、Mode 或 Cluster。";
public static final String EXCEPTION_ILLEGAL_FILL_METHOD_45FB5556 = "填充方法无效。";
public static final String EXCEPTION_PARAMETER_SIGMA_SHOULD_BE_FINITE_AND_LARGER_THAN_0_538FEDBA = "参数 $sigma$ 必须为有限且大于 0。";
public static final String EXCEPTION_PARAMETER_MINSPEED_AND_MAXSPEED_SHOULD_BE_FINITE_AND_MINSPEED_SHOULD_BE_SMALLER_THAN_MAXSPEED_5998EA69 = "参数 $minSpeed$ 和 $maxSpeed$ 必须为有限值，且 $minSpeed$ 必须小于 $maxSpeed$。";
public static final String EXCEPTION_PARAMETER_CENTER_SHOULD_BE_FINITE_21F289A3 = "参数 $center$ 必须为有限值。";
public static final String EXCEPTION_METHOD_SHOULD_BE_SCREEN_OR_LSGREEDY_B5975D7A = "method 必须为 screen 或 lsgreedy。";
public static final String EXCEPTION_COEF_SHOULD_BE_FINITE_DOUBLE_DOUBLE_WHEN_METHOD_IS_BLANK_F72B1092 = "method 为空时，coef 必须为有限的 double,double,...。";
public static final String EXCEPTION_WAVELET_COEFFICIENTS_SHOULD_NOT_BE_EMPTY_2C78B047 = "Wavelet 系数不能为空。";
public static final String EXCEPTION_WAVELET_COEFFICIENTS_SHOULD_BE_FINITE_D764C838 = "Wavelet 系数必须为有限值。";
public static final String EXCEPTION_ILLEGAL_PARAMETER_PATTERN_MUST_BE_FINITE_DOUBLE_DOUBLE_612DE434 = "非法参数，pattern 必须为有限的 double,double,...。";
public static final String EXCEPTION_ILLEGAL_PARAMETER_THRESHOLD_MUST_BE_A_FINITE_NON_NEGATIVE_NUMBER_1C438942 = "非法参数，threshold 必须为有限且非负数。";
public static final String EXCEPTION_ILLEGAL_PARAMETER_VALUEPATTERN_VALUES_MUST_BE_FINITE_FFD05AC9 = "非法参数，valuePattern 的值必须为有限数。";
public static final String EXCEPTION_REGEX_SHOULD_NOT_BE_EMPTY_AND_HAS_TO_BE_A_VALID_REGULAR_EXPRESSION_751072ED = "regex 不能为空且必须是有效的正则表达式。";
public static final String EXCEPTION_CAPACITY_SHOULD_BE_LARGER_THAN_0_055372C7 = "容量必须大于 0。";
public static final String EXCEPTION_TIME_VALUE_SHOULD_BE_FINITE_A661DDB0 = "时间值必须为有限值。";
public static final String EXCEPTION_INVALID_DATETIME_FORMAT_A3DC435A = "日期时间格式无效。";

private LibraryUdfMessages() {}
}
29 changes: 24 additions & 5 deletions library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.iotdb.library.anomaly;

import org.apache.iotdb.library.i18n.LibraryUdfMessages;
import org.apache.iotdb.library.util.Util;
import org.apache.iotdb.udf.api.UDTF;
import org.apache.iotdb.udf.api.access.Row;
Expand Down Expand Up @@ -59,10 +60,20 @@ public void validate(UDFParameterValidator validator) throws Exception {
"Parameter \"compute\" is illegal. Please use \"batch\" (for default) or \"stream\".",
validator.getParameters().getStringOrDefault("compute", BATCH_COMPUTE))
.validate(
params -> (double) params[0] < (double) params[1],
"parameter $q1$ should be smaller than $q3$",
params ->
Double.isFinite((double) params[0])
&& Double.isFinite((double) params[1])
&& (double) params[0] < (double) params[1],
LibraryUdfMessages
.EXCEPTION_PARAMETER_Q1_AND_Q3_SHOULD_BE_FINITE_AND_Q1_SHOULD_BE_SMALLER_THAN_Q3_EB122E6C,
validator.getParameters().getDoubleOrDefault("q1", -1),
validator.getParameters().getDoubleOrDefault("q3", 1));
if (validator
.getParameters()
.getStringOrDefault("compute", BATCH_COMPUTE)
.equalsIgnoreCase(STREAM_COMPUTE)) {
validator.validateRequiredAttribute("q1").validateRequiredAttribute("q3");
}
}

@Override
Expand All @@ -84,21 +95,29 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati

@Override
public void transform(Row row, PointCollector collector) throws Exception {
if (row.isNull(0)) {
return;
}
if (compute.equalsIgnoreCase(STREAM_COMPUTE) && q3 > q1) {
double v = Util.getValueAsDouble(row);
if (v < q1 - 1.5 * iqr || v > q3 + 1.5 * iqr) {
if (Double.isFinite(v) && (v < q1 - 1.5 * iqr || v > q3 + 1.5 * iqr)) {
collector.putDouble(row.getTime(), v);
}
} else if (compute.equalsIgnoreCase(BATCH_COMPUTE)) {
double v = Util.getValueAsDouble(row);
value.add(v);
timestamp.add(row.getTime());
if (Double.isFinite(v)) {
value.add(v);
timestamp.add(row.getTime());
}
}
}

@Override
public void terminate(PointCollector collector) throws Exception {
if (compute.equalsIgnoreCase(BATCH_COMPUTE)) {
if (value.isEmpty()) {
return;
}
q1 = Quantiles.quartiles().index(1).compute(value);
q3 = Quantiles.quartiles().index(3).compute(value);
iqr = q3 - q1;
Expand Down
18 changes: 14 additions & 4 deletions library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.iotdb.library.anomaly;

import org.apache.iotdb.library.i18n.LibraryUdfMessages;
import org.apache.iotdb.library.util.CircularQueue;
import org.apache.iotdb.library.util.LongCircularQueue;
import org.apache.iotdb.library.util.TypeServices;
Expand All @@ -33,6 +34,8 @@

/** This function detects outliers which lies over average +/- k * sigma. */
public class UDTFKSigma implements UDTF {
private static final int DEFAULT_WINDOW_SIZE = 10000;

private double mean = 0.0;
private double variance = 0.0;
private double sumX2 = 0.0;
Expand All @@ -54,10 +57,10 @@ public void validate(UDFParameterValidator validator) throws Exception {
.validate(
x -> (int) x > 0,
"Window size should be larger than 0.",
validator.getParameters().getIntOrDefault("window", 10))
validator.getParameters().getIntOrDefault("window", DEFAULT_WINDOW_SIZE))
.validate(
x -> (double) x > 0,
"Parameter k should be larger than 0.",
x -> Double.isFinite((double) x) && (double) x > 0,
LibraryUdfMessages.EXCEPTION_PARAMETER_K_SHOULD_BE_FINITE_AND_LARGER_THAN_0_ECE1B163,
validator.getParameters().getDoubleOrDefault("k", 3));
}

Expand All @@ -72,13 +75,20 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf
valueReader = TypeServices.ROW_VALUE_READER_SERVICE.call(TypeServices.toReadType(dataType));
numericReader = TypeServices.NUMERIC_ROW_READER_SERVICE.call(TypeServices.toReadType(dataType));
valueWriter = TypeServices.rowValueWriter(dataType);
this.windowSize = udfParameters.getIntOrDefault("window", 10000);
this.windowSize = udfParameters.getIntOrDefault("window", DEFAULT_WINDOW_SIZE);
this.mean = 0.0;
this.variance = 0.0;
this.sumX1 = 0.0;
this.sumX2 = 0.0;
this.v = new CircularQueue<>(windowSize);
this.t = new LongCircularQueue(windowSize);
}

@Override
public void transform(Row row, PointCollector collector) throws Exception {
if (row.isNull(0)) {
return;
}
double value = numericReader.read(row);
long timestamp = row.getTime();
if (Double.isFinite(value) && !Double.isNaN(value)) {
Expand Down
Loading
Loading