From 1443b2276c85c3d1300e2cc6e2320022570bf018 Mon Sep 17 00:00:00 2001 From: Luca Date: Mon, 15 Feb 2021 00:54:20 +0100 Subject: [PATCH 1/4] initial route from cpp to rch --- src/result.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/result.cpp b/src/result.cpp index e5df40f..22d3247 100644 --- a/src/result.cpp +++ b/src/result.cpp @@ -61,6 +61,19 @@ void convertEntries(std::shared_ptr in, NullCol nullCol, RT &out, } } +template<> +void convertEntries(std::shared_ptr in, NullCol nullCol, Rcpp::StringVector &out, + size_t offset, size_t start, size_t end) { + for(size_t j = start; j < end; j++) { + // can't use the ternary operator here, since that would require explicit + // conversion from the Clickhouse storage type (which is far messier) + if(nullCol && nullCol->IsNull(j)) { + out[offset+j-start] = Rcpp::StringVector::get_na(); + } else { + out[offset+j-start] = (in->At(j).to_string()); + } + } +} template<> void convertEntries(std::shared_ptr in, NullCol nullCol, Rcpp::StringVector &out, @@ -278,6 +291,9 @@ std::unique_ptr Result::buildConverter(std::string name, ch::TypeRef case TC::UInt64: { return std::unique_ptr>(new ScalarConverter); } + case TC::Decimal: { + return std::unique_ptr>(new ScalarConverter); + } case TC::UUID: return std::unique_ptr>(new ScalarConverter); case TC::Float32: From c53836535dd3a3a1dd07b765f28e78f4545dc030 Mon Sep 17 00:00:00 2001 From: Luca Rauchenberger Date: Fri, 19 Mar 2021 17:06:32 +0100 Subject: [PATCH 2/4] added conversion function for Decimal-Result --- R/ClickhouseResult.R | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/R/ClickhouseResult.R b/R/ClickhouseResult.R index 9fadef2..f6c2bc6 100644 --- a/R/ClickhouseResult.R +++ b/R/ClickhouseResult.R @@ -27,6 +27,7 @@ setMethod("dbFetch", signature = "ClickhouseResult", definition = function(res, } ret <- fetch(res@ptr, n) ret <- convert_Int64(ret, res@Int64) + ret <- convert_Decimal(ret) if(res@toUTF8 == TRUE) ret <- encode_UTF(ret) @@ -50,6 +51,28 @@ convert_Int64 <- function(df, Int64) { } } +convert_Decimal <- function(df) { +# identify the columns by index that need changing + toConvert <- which(grepl("Decimal", attr(df, 'data.type'))) + +# get the scales + parse_scale <- function(str){ + x <- strsplit(str, ",")[[1]][[2]] + x <- substr(x,1,nchar(x)-1) + x <- as.numeric(x) + return(as.numeric(x)) + } + + for(column_i in toConvert) { + raw_type <- attr(df, 'data.type')[[column_i]] + scale <- parse_scale(raw_type) + # forces String to decimal representation + df[column_i] <- lapply(df[column_i], as.numeric) + df[column_i] <- df[column_i] / (10^scale) + } + return(df) +} + encode_UTF <- function(df){ toConvert <- which(attr(df, "data.type") %in% c("String", "FixedString", "Nullable(String)", "Nullable(FixedString)")) From a94860927fe42fa297a94f420f7edd0b889dc434 Mon Sep 17 00:00:00 2001 From: luca Date: Tue, 30 Mar 2021 14:14:32 +0200 Subject: [PATCH 3/4] initial prototype for inserting Decimal --- src/connection.cpp | 94 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 14 deletions(-) diff --git a/src/connection.cpp b/src/connection.cpp index 33fb408..8722a85 100644 --- a/src/connection.cpp +++ b/src/connection.cpp @@ -10,6 +10,8 @@ using namespace Rcpp; using namespace clickhouse; +ColumnRef convertDecimalColumn(SEXP v); + // [[Rcpp::export]] DataFrame fetch(XPtr res, ssize_t n) { return res->fetchFrame(n); @@ -165,10 +167,10 @@ std::shared_ptr vecToScalar(SEXP v, std::shared_ptr nullCol = n type_of_cor = Rf_inherits(v, "integer64") ? 99 : type_of; switch(type_of_cor) { - case 99: { - toColumnN(v, col, nullCol); - break; - } + case 99: { + toColumnN(v, col, nullCol); + break; + } case INTSXP: { // the lambda could be a default argument of toColumn, but that // appears to trigger a bug in GCC @@ -337,7 +339,12 @@ std::shared_ptr vecToEnum(SEXP v, TypeRef type, std::shared_ptr ColumnRef vecToColumn(TypeRef t, SEXP v, std::shared_ptr nullCol = nullptr) { using TC = Type::Code; + + printf("%i insert THIS !!!\n",t->GetCode()); + switch(t->GetCode()) { + // case TC::Decimal: + // return vecToScalar(v, nullCol); case TC::Int8: return vecToScalar(v, nullCol); case TC::Int16: @@ -405,10 +412,34 @@ ColumnRef vecToColumn(TypeRef t, SEXP v, std::shared_ptr nullCol = // [[Rcpp::export]] void insert(XPtr conn, String tableName, DataFrame df) { - StringVector names(df.names()); - std::vector colTypes; + // StringVector names(df.names()); + // std::vector colTypes; + // + // // determine actual column types + // conn->Select("SELECT * FROM "+std::string(tableName)+" LIMIT 0", [&colTypes] (const Block& block) { + // if(block.GetColumnCount() > 0 && colTypes.empty()) { + // for(ch::Block::Iterator bi(block); bi.IsValid(); bi.Next()) { + // colTypes.push_back(bi.Type()); + // } + // } + // }); + // + // if(colTypes.size() != static_cast(df.size())) { + // stop("input has "+std::to_string(df.size())+" columns, but table "+ + // std::string(tableName)+" has "+std::to_string(colTypes.size())); + // } + // + // Block block; + // for(size_t i = 0; i < colTypes.size(); i++) { + // ColumnRef ccol = vecToColumn(colTypes[i], df[i]); + // block.AppendColumn(std::string(names[i]), ccol); + // } + // conn->Insert(tableName, block); + // determine actual column types + StringVector names(df.names()); + std::vector colTypes; conn->Select("SELECT * FROM "+std::string(tableName)+" LIMIT 0", [&colTypes] (const Block& block) { if(block.GetColumnCount() > 0 && colTypes.empty()) { for(ch::Block::Iterator bi(block); bi.IsValid(); bi.Next()) { @@ -417,20 +448,55 @@ void insert(XPtr conn, String tableName, DataFrame df) { } }); - if(colTypes.size() != static_cast(df.size())) { - stop("input has "+std::to_string(df.size())+" columns, but table "+ - std::string(tableName)+" has "+std::to_string(colTypes.size())); - } + // prints out scale and nothing else... + // auto decimal = std::static_pointer_cast(colTypes[0]); + // std::cout << colTypes[0]->GetName() << "; - ) \n"; + // std::cout << decimal->GetScale() << "; - ) \n"; + // std::cout << decimal.GetPrecision() << "; - ) \n"; + + // creates table + // conn->Execute("CREATE TABLE IF NOT EXISTS default.decimalchcpp2_1 (d Decimal64(1)) ENGINE = Memory"); + // this will be inserted, sum of all columns! Block block; - for(size_t i = 0; i < colTypes.size(); i++) { - ColumnRef ccol = vecToColumn(colTypes[i], df[i]); - block.AppendColumn(std::string(names[i]), ccol); + + // // OLD + // auto d = std::make_shared(18, 1); + // d->Append(29.21111); + // // append and insert finished d column + // block.AppendColumn("d", d); + // conn->Insert("default.decimalchcpp2_1", block); + + // NEW + // for(size_t i = 0; i < colTypes.size(); i++) { + // ColumnRef ccol = vecToColumn(colTypes[i], df[i]); + // block.AppendColumn(std::string(names[i]), ccol); + // } + block.AppendColumn("d", convertDecimalColumn(df[0])); + + conn->Insert("default.decimalchcpp2_1", block); +} + + +ColumnRef convertDecimalColumn(SEXP v) { + // RESULT: creates sharedPtr to ColumnDecimal with (18,1) + std::shared_ptr col = std::make_shared(18, 1); + + + IntegerVector cv = Rcpp::as(v); + + for(typename IntegerVector::stored_type e : cv) { + col->Append(e); } - conn->Insert(tableName, block); + // d->Append(29); + + return col; } + + + // [[Rcpp::export]] bool validPtr(SEXP ptr) { return R_ExternalPtrAddr(ptr); From 96821c0b70c7664e9fdeaf01ede18fec8d53a777 Mon Sep 17 00:00:00 2001 From: luca Date: Tue, 30 Mar 2021 16:09:50 +0200 Subject: [PATCH 4/4] integrated decimal-prototype with other conversions --- src/connection.cpp | 78 +++++++++++----------------------------------- 1 file changed, 18 insertions(+), 60 deletions(-) diff --git a/src/connection.cpp b/src/connection.cpp index 8722a85..0d0bdbb 100644 --- a/src/connection.cpp +++ b/src/connection.cpp @@ -343,8 +343,8 @@ ColumnRef vecToColumn(TypeRef t, SEXP v, std::shared_ptr nullCol = printf("%i insert THIS !!!\n",t->GetCode()); switch(t->GetCode()) { - // case TC::Decimal: - // return vecToScalar(v, nullCol); + case TC::Decimal: + return convertDecimalColumn(v); case TC::Int8: return vecToScalar(v, nullCol); case TC::Int16: @@ -412,34 +412,10 @@ ColumnRef vecToColumn(TypeRef t, SEXP v, std::shared_ptr nullCol = // [[Rcpp::export]] void insert(XPtr conn, String tableName, DataFrame df) { - // StringVector names(df.names()); - // std::vector colTypes; - // - // // determine actual column types - // conn->Select("SELECT * FROM "+std::string(tableName)+" LIMIT 0", [&colTypes] (const Block& block) { - // if(block.GetColumnCount() > 0 && colTypes.empty()) { - // for(ch::Block::Iterator bi(block); bi.IsValid(); bi.Next()) { - // colTypes.push_back(bi.Type()); - // } - // } - // }); - // - // if(colTypes.size() != static_cast(df.size())) { - // stop("input has "+std::to_string(df.size())+" columns, but table "+ - // std::string(tableName)+" has "+std::to_string(colTypes.size())); - // } - // - // Block block; - // for(size_t i = 0; i < colTypes.size(); i++) { - // ColumnRef ccol = vecToColumn(colTypes[i], df[i]); - // block.AppendColumn(std::string(names[i]), ccol); - // } - // conn->Insert(tableName, block); - - - // determine actual column types StringVector names(df.names()); std::vector colTypes; + + // determine actual column types conn->Select("SELECT * FROM "+std::string(tableName)+" LIMIT 0", [&colTypes] (const Block& block) { if(block.GetColumnCount() > 0 && colTypes.empty()) { for(ch::Block::Iterator bi(block); bi.IsValid(); bi.Next()) { @@ -448,55 +424,37 @@ void insert(XPtr conn, String tableName, DataFrame df) { } }); - // prints out scale and nothing else... - // auto decimal = std::static_pointer_cast(colTypes[0]); - // std::cout << colTypes[0]->GetName() << "; - ) \n"; - // std::cout << decimal->GetScale() << "; - ) \n"; - // std::cout << decimal.GetPrecision() << "; - ) \n"; - - // creates table - // conn->Execute("CREATE TABLE IF NOT EXISTS default.decimalchcpp2_1 (d Decimal64(1)) ENGINE = Memory"); + if(colTypes.size() != static_cast(df.size())) { + stop("input has "+std::to_string(df.size())+" columns, but table "+ + std::string(tableName)+" has "+std::to_string(colTypes.size())); + } - // this will be inserted, sum of all columns! Block block; - - // // OLD - // auto d = std::make_shared(18, 1); - // d->Append(29.21111); - // // append and insert finished d column - // block.AppendColumn("d", d); - // conn->Insert("default.decimalchcpp2_1", block); - - // NEW - // for(size_t i = 0; i < colTypes.size(); i++) { - // ColumnRef ccol = vecToColumn(colTypes[i], df[i]); - // block.AppendColumn(std::string(names[i]), ccol); - // } - block.AppendColumn("d", convertDecimalColumn(df[0])); - - conn->Insert("default.decimalchcpp2_1", block); + for(size_t i = 0; i < colTypes.size(); i++) { + ColumnRef ccol = vecToColumn(colTypes[i], df[i]); + block.AppendColumn(std::string(names[i]), ccol); + } + conn->Insert(tableName, block); } - ColumnRef convertDecimalColumn(SEXP v) { - // RESULT: creates sharedPtr to ColumnDecimal with (18,1) - std::shared_ptr col = std::make_shared(18, 1); +// R_CH-column: Rcpp IntegerVector + IntegerVector cv = Rcpp::as(v); - IntegerVector cv = Rcpp::as(v); + // CH_CPP-column: sharedPtr to ColumnDecimal with (18,1) + std::shared_ptr col = std::make_shared(18, 1); + for(typename IntegerVector::stored_type e : cv) { col->Append(e); } - // d->Append(29); - return col; } - // [[Rcpp::export]] bool validPtr(SEXP ptr) { return R_ExternalPtrAddr(ptr);