From f8d48e18ed87d3c064ec998ed856e1de4dc3dba4 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Jul 2023 21:49:32 +0800 Subject: [PATCH 1/3] fix tiflash overflow --- .../org/apache/spark/sql/IssueTestSuite.scala | 24 +++++++++++++++++++ .../tikv/columnar/TiBlockColumnVector.java | 6 +++++ 2 files changed, 30 insertions(+) diff --git a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala index 38bac47e56..9eda811110 100644 --- a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala @@ -22,6 +22,30 @@ import org.apache.spark.sql.functions.{col, sum} class IssueTestSuite extends BaseTiSparkTest { + test("test tiflash overflow in unsigned bigint") { + val dbTable = "tispark_test.tiflash_overflow" + tidbStmt.execute(s"drop table if exists $dbTable") + tidbStmt.execute( + s"CREATE TABLE $dbTable (`a` bigint(20) UNSIGNED NOT NULL,`b` bigint(20) UNSIGNED NOT NULL)") + tidbStmt.execute(s"insert into $dbTable values(16717361816800086255, 16717361816800086255)") + tidbStmt.execute(s"ALTER TABLE $dbTable SET TIFLASH REPLICA 1") + + Thread.sleep(5 * 1000) + + val prev = spark.conf.getOption(TiConfigConst.ISOLATION_READ_ENGINES) + try { + spark.conf + .set(TiConfigConst.ISOLATION_READ_ENGINES, TiConfigConst.TIFLASH_STORAGE_ENGINE) + val df = spark.sql(s"select * from $dbTable") + val row = Row(BigDecimal("16717361816800086255"), BigDecimal("16717361816800086255")) + checkAnswer(df, Seq(row)) + } finally { + spark.conf.set( + TiConfigConst.ISOLATION_READ_ENGINES, + prev.getOrElse(TiConfigConst.DEFAULT_STORAGE_ENGINES)) + } + } + test("test issue 2649") { val dbTable = "tispark_test.mutil_uniq" tidbStmt.execute(s"drop table if exists $dbTable") diff --git a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java index 12486e2728..6c60683fbb 100644 --- a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java +++ b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java @@ -18,6 +18,7 @@ import static com.pingcap.tikv.util.MemoryUtil.EMPTY_BYTE_BUFFER_DIRECT; +import com.google.common.primitives.UnsignedLong; import com.pingcap.tikv.codec.Codec.DateCodec; import com.pingcap.tikv.codec.Codec.DateTimeCodec; import com.pingcap.tikv.codec.ExtendedDateTime; @@ -25,6 +26,7 @@ import com.pingcap.tikv.types.AbstractDateTimeType; import com.pingcap.tikv.types.BytesType; import com.pingcap.tikv.types.DateType; +import com.pingcap.tikv.types.DecimalType; import com.pingcap.tikv.util.MemoryUtil; import java.math.BigDecimal; import java.nio.ByteBuffer; @@ -240,6 +242,10 @@ public double getDouble(int rowId) { @Override public BigDecimal getDecimal(int rowId, int precision, int scale) { long rowIdAddr = (long) rowId * fixedLength + dataAddr; + // avoid unsigned long overflow here + if (type == DecimalType.BIG_INT_DECIMAL) { + return new BigDecimal(UnsignedLong.fromLongBits(this.getLong(rowId)).bigIntegerValue()); + } if (fixedLength == 4) { return MemoryUtil.getDecimal32(rowIdAddr, scale); } else if (fixedLength == 8) { From 9fd50bd54f8dce43dc3f8974685ecb53e3197a32 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Jul 2023 22:45:46 +0800 Subject: [PATCH 2/3] fix tiflash overflow --- core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala index 9eda811110..7ac981396b 100644 --- a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala @@ -23,6 +23,9 @@ import org.apache.spark.sql.functions.{col, sum} class IssueTestSuite extends BaseTiSparkTest { test("test tiflash overflow in unsigned bigint") { + if (!enableTiFlashTest){ + cancel("tiflash test not enabled") + } val dbTable = "tispark_test.tiflash_overflow" tidbStmt.execute(s"drop table if exists $dbTable") tidbStmt.execute( From ffb8b68f24a1f03afd58164910d70584b3bc6b36 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Jul 2023 22:53:30 +0800 Subject: [PATCH 3/3] fmt --- core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala index 7ac981396b..2b6eb22adb 100644 --- a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala @@ -23,8 +23,8 @@ import org.apache.spark.sql.functions.{col, sum} class IssueTestSuite extends BaseTiSparkTest { test("test tiflash overflow in unsigned bigint") { - if (!enableTiFlashTest){ - cancel("tiflash test not enabled") + if (!enableTiFlashTest) { + cancel("tiflash test not enabled") } val dbTable = "tispark_test.tiflash_overflow" tidbStmt.execute(s"drop table if exists $dbTable")