From 86247d8ab22f4287caf0081a605d0e3da1bea567 Mon Sep 17 00:00:00 2001 From: Paxanator Date: Fri, 17 May 2019 23:15:43 -0500 Subject: [PATCH 01/11] fix(install): Added missing inst directory to install librdkafka on a new system --- Makefile | 9 +++------ fRanz/src/Makevars | 22 ++++++++++------------ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index a778cb6..ccd489e 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,5 @@ PACKAGE = fRanz -INSTALLDIR = $(HOME)/.$(PACKAGE)/local -OS = $(shell uname -s) - -.PHONY: install smoke test docs roxygen pdf version clean distclean cleanRcpp unlock +.PHONY: install test docs clean distclean cleanRcpp unlock install: unlock clean cleanRcpp # Install fRanz R package @@ -22,13 +19,13 @@ clean: distclean: clean cleanRcpp unlock: - # Remove 00LOCK-cpproll directory + # Remove 00LOCK directory for libpath in $$(Rscript -e "noquote(paste(.libPaths(), collapse = ' '))"); do \ echo "Unlocking $$libpath..." && \ rm -rf $$libpath/00LOCK-$(PACKAGE); \ done -docs roxygen: +docs: # Regenerate documentation with roxygen Rscript -e "roxygen2::roxygenize('$(PACKAGE)')" test: diff --git a/fRanz/src/Makevars b/fRanz/src/Makevars index e557180..1b1e235 100644 --- a/fRanz/src/Makevars +++ b/fRanz/src/Makevars @@ -1,23 +1,21 @@ -INSTALLDIR = $(HOME)/.fRanz/librdkafka -LIBRDKAFKADIR = $(PWD)/../inst/librdkafka-0.11.6 +INSTALLDIR = $(HOME)/.fRanz -PKG_LIBS = -L$(INSTALLDIR)/src-cpp -lrdkafka++ -PKG_CXXFLAGS = -std=c++11 -I$(INSTALLDIR)/src-cpp +PKG_LIBS = -L$(INSTALLDIR)/librdkafka/src-cpp -lrdkafka++ +PKG_CXXFLAGS = -std=c++11 -I$(INSTALLDIR)/librdkafka/src-cpp .PHONY: all install_librdkadka all: install_librdkadka install_librdkadka: - if [ ! -s $(INSTALLDIR)/src-cpp/librdkafka++.a ] ; \ + if [ ! -d $(INSTALLDIR)/librdkafka/src-cpp ] ; \ then \ - mkdir -p $(INSTALLDIR) && \ - cd $(shell dirname $(LIBRDKAFKADIR)) && \ - tar xzf $(LIBRDKAFKADIR).tar.gz && \ - cd $(LIBRDKAFKADIR) && \ + mkdir -p $(INSTALLDIR)/librdkafka && \ + cd ../inst && \ + tar xzf librdkafka-1.0.0.tar.gz && \ + cd librdkafka-1.0.0 && \ ./configure && \ $(MAKE) && \ - $(MAKE) install && \ - mv * $(INSTALLDIR) ; \ + $(MAKE) install && \ + mv * $(INSTALLDIR)/librdkafka ; \ fi - From ac1b58b2ed81f8e7277ace44d16c24c3d4352182 Mon Sep 17 00:00:00 2001 From: Paxanator Date: Sun, 19 May 2019 22:12:56 -0500 Subject: [PATCH 02/11] fix(build): Fixing install process to rely on system library as a temp measure --- .gitignore | 3 ++- .travis.yml | 9 ++++---- Makefile | 8 +++++++ README.md | 7 ++++++ fRanz/DESCRIPTION | 2 +- fRanz/R/KafkaBroker.R | 1 - fRanz/R/KafkaProducer.R | 5 ++++ fRanz/R/RcppExports.R | 15 ++++++++++-- fRanz/man/GetRdConsumer.Rd | 2 +- fRanz/man/GetRdProducer.Rd | 2 +- fRanz/man/{Kafka-Broker.Rd => KafkaBroker.Rd} | 3 +-- fRanz/man/KafkaConsume.Rd | 8 ++++++- fRanz/src/Makevars | 23 ++----------------- fRanz/src/RcppExports.cpp | 9 ++++---- fRanz/src/consumer.cpp | 17 ++++---------- fRanz/src/producer.cpp | 10 ++------ fRanz/src/utils.cpp | 2 +- fRanz/src/utils.h | 2 +- fRanz/tests/testthat/test-KafkaConsumer.R | 2 ++ 19 files changed, 69 insertions(+), 61 deletions(-) rename fRanz/man/{Kafka-Broker.Rd => KafkaBroker.Rd} (96%) diff --git a/.gitignore b/.gitignore index 05343b2..d1ca494 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ *.o *.so *.DS_Store -fRanz/inst/librdkafka* +franz/src/librdkafka +librdkafka* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 34c4509..88c8d13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,13 @@ services: before_install: - docker-compose up -d - sleep 20 + - cd fRanz language: r r: - release cache: packages - -os: - - linux - - osx +addons: + apt: + packages: + - librdkafka-dev diff --git a/Makefile b/Makefile index ccd489e..ff99bbf 100644 --- a/Makefile +++ b/Makefile @@ -32,3 +32,11 @@ test: # Run unit tests Rscript -e "devtools::test('$(PACKAGE)')" +librdkafka: + wget https://github.com/edenhill/librdkafka/archive/v1.0.0.tar.gz -O librdkafka-1.0.0.tar.gz && \ + tar xzf librdkafka-1.0.0.tar.gz && \ + cd librdkafka-1.0.0 && \ + ./configure && \ + $(MAKE) && \ + $(MAKE) install && \ + cd .. && rm -rf librdkafka-1.0.0; \ \ No newline at end of file diff --git a/README.md b/README.md index aa9a3dc..539c3ab 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,13 @@ fRanz is an open source R kafka client that allows users to read and write messages from kafka. It leverages the stability and performance of [librdkafka](https://github.com/edenhill/librdkafka) and implements ididiomatic R workflows ontop of it. +## Installation + +We're working on it. Currently you need librdkafka as a system available library in order to load the R package. In order to install from source you also need the headers. A make recipe in the top level `make librdkafka` should work for *nix systems. + + +No attempt has been made for windows compatability. + ## Example of sending and reading a message ```r diff --git a/fRanz/DESCRIPTION b/fRanz/DESCRIPTION index 932cb82..2f46aa7 100644 --- a/fRanz/DESCRIPTION +++ b/fRanz/DESCRIPTION @@ -1,6 +1,6 @@ Package: fRanz Type: Package -Title: An R Kafka Client +Title: An R Kafka Client. Version: 0.1.0 Date: 2019-05-13 Authors@R: c( diff --git a/fRanz/R/KafkaBroker.R b/fRanz/R/KafkaBroker.R index a920bc5..f6fa919 100644 --- a/fRanz/R/KafkaBroker.R +++ b/fRanz/R/KafkaBroker.R @@ -13,7 +13,6 @@ #' BROKER_HOST <- 'localhost' #' BROKER_PORT <- 9092 #' TOPIC_NAME <- 'myTestTopic' - #' # KafkaBroker #' broker <- KafkaBroker$new(host=BROKER_HOST, port=BROKER_PORT) #' diff --git a/fRanz/R/KafkaProducer.R b/fRanz/R/KafkaProducer.R index bb0b3d6..996e354 100644 --- a/fRanz/R/KafkaProducer.R +++ b/fRanz/R/KafkaProducer.R @@ -1,8 +1,13 @@ #' @title Kakfa Producer #' @name KafkaProducer +<<<<<<< HEAD #' @description A producer is an application that is responsible #' for publishing data to topics. #' @references \href{https://kafka.apache.org/documentation/#intro_producers}{Apache Kafka docs - Producers} +======= +#' @description TDB +#' @export +>>>>>>> fix(build): Fixing install process to rely on system library as a temp measure #' @importFrom R6 R6Class #' @export KafkaProducer <- R6::R6Class( diff --git a/fRanz/R/RcppExports.R b/fRanz/R/RcppExports.R index d058f7f..a0daa05 100644 --- a/fRanz/R/RcppExports.R +++ b/fRanz/R/RcppExports.R @@ -3,7 +3,7 @@ #' @title GetRdConsumer #' @name GetRdConsumer -#' @description Creates an Rcpp::XPtr. For more details on options \link{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md} +#' @description Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} #' @param keys a character vector indicating option keys to parameterize the RdKafka::Consumer #' @param values a character vector indicating option values to parameterize the RdKafka::Consumer. Must be of same length as keys. #' @return a Rcpp::XPtr @@ -24,18 +24,29 @@ RdSubscribe <- function(consumerPtr, Rtopics) { #' @title KafkaConsume #' @name KafkaConsume +<<<<<<< HEAD #' @description Consume a fixed number of results from whatever topic(s) #' the provided consumer is subscribed to. +======= +#' @description A method to that takes a consumer pointer and returns at most the specified number of results, unless the timeout is reached +>>>>>>> fix(build): Fixing install process to rely on system library as a temp measure #' @param consumerPtr a reference to a Rcpp::XPtr #' @param numResults how many results should be consumed before returning. Will return early if offset is at maximum +#' @param timeout the timeout in milliseconds. Default is 10000 #' @return a list of length numResults with values list(key=key,value=value) +<<<<<<< HEAD KafkaConsume <- function(consumerPtr, numResults) { .Call('_fRanz_KafkaConsume', PACKAGE = 'fRanz', consumerPtr, numResults) +======= +#' @export +KafkaConsume <- function(consumerPtr, numResults, timeout = 10000L) { + .Call('_fRanz_KafkaConsume', PACKAGE = 'fRanz', consumerPtr, numResults, timeout) +>>>>>>> fix(build): Fixing install process to rely on system library as a temp measure } #' @title GetRdProducer #' @name GetRdProducer -#' @description Creates an Rcpp::XPtr. For more details on options \link{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md} +#' @description Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} #' @param keys a character vector indicating option keys to parameterize the RdKafka::Producer #' @param values a character vector indicating option values to parameterize the RdKafka::Producer. Must be of same length as keys. #' @return a Rcpp::XPtr diff --git a/fRanz/man/GetRdConsumer.Rd b/fRanz/man/GetRdConsumer.Rd index 3abe841..8cb2e5c 100644 --- a/fRanz/man/GetRdConsumer.Rd +++ b/fRanz/man/GetRdConsumer.Rd @@ -15,5 +15,5 @@ GetRdConsumer(keys, values) a Rcpp::XPtr } \description{ -Creates an Rcpp::XPtr. For more details on options \link{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md} +Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} } diff --git a/fRanz/man/GetRdProducer.Rd b/fRanz/man/GetRdProducer.Rd index cf02b05..a7f11df 100644 --- a/fRanz/man/GetRdProducer.Rd +++ b/fRanz/man/GetRdProducer.Rd @@ -15,5 +15,5 @@ GetRdProducer(keys, values) a Rcpp::XPtr } \description{ -Creates an Rcpp::XPtr. For more details on options \link{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md} +Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} } diff --git a/fRanz/man/Kafka-Broker.Rd b/fRanz/man/KafkaBroker.Rd similarity index 96% rename from fRanz/man/Kafka-Broker.Rd rename to fRanz/man/KafkaBroker.Rd index 9fa2fd1..47833f9 100644 --- a/fRanz/man/Kafka-Broker.Rd +++ b/fRanz/man/KafkaBroker.Rd @@ -1,8 +1,7 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/KafkaBroker.R \docType{data} -\name{Kafka Broker} -\alias{Kafka Broker} +\name{KafkaBroker} \alias{KafkaBroker} \title{KafkaBroker} \format{An object of class \code{R6ClassGenerator} of length 24.} diff --git a/fRanz/man/KafkaConsume.Rd b/fRanz/man/KafkaConsume.Rd index b096c53..47f90b9 100644 --- a/fRanz/man/KafkaConsume.Rd +++ b/fRanz/man/KafkaConsume.Rd @@ -4,17 +4,23 @@ \alias{KafkaConsume} \title{KafkaConsume} \usage{ -KafkaConsume(consumerPtr, numResults) +KafkaConsume(consumerPtr, numResults, timeout = 10000L) } \arguments{ \item{consumerPtr}{a reference to a Rcpp::XPtr} \item{numResults}{how many results should be consumed before returning. Will return early if offset is at maximum} + +\item{timeout}{the timeout in milliseconds. Default is 10000} } \value{ a list of length numResults with values list(key=key,value=value) } \description{ +<<<<<<< HEAD Consume a fixed number of results from whatever topic(s) the provided consumer is subscribed to. +======= +A method to that takes a consumer pointer and returns at most the specified number of results, unless the timeout is reached +>>>>>>> fix(build): Fixing install process to rely on system library as a temp measure } diff --git a/fRanz/src/Makevars b/fRanz/src/Makevars index 1b1e235..1b93233 100644 --- a/fRanz/src/Makevars +++ b/fRanz/src/Makevars @@ -1,21 +1,2 @@ -INSTALLDIR = $(HOME)/.fRanz - -PKG_LIBS = -L$(INSTALLDIR)/librdkafka/src-cpp -lrdkafka++ -PKG_CXXFLAGS = -std=c++11 -I$(INSTALLDIR)/librdkafka/src-cpp - -.PHONY: all install_librdkadka - -all: install_librdkadka - -install_librdkadka: - if [ ! -d $(INSTALLDIR)/librdkafka/src-cpp ] ; \ - then \ - mkdir -p $(INSTALLDIR)/librdkafka && \ - cd ../inst && \ - tar xzf librdkafka-1.0.0.tar.gz && \ - cd librdkafka-1.0.0 && \ - ./configure && \ - $(MAKE) && \ - $(MAKE) install && \ - mv * $(INSTALLDIR)/librdkafka ; \ - fi +PKG_LIBS = -lrdkafka++ +CXX_STD=CXX11 \ No newline at end of file diff --git a/fRanz/src/RcppExports.cpp b/fRanz/src/RcppExports.cpp index 912569c..b608c23 100644 --- a/fRanz/src/RcppExports.cpp +++ b/fRanz/src/RcppExports.cpp @@ -30,14 +30,15 @@ BEGIN_RCPP END_RCPP } // KafkaConsume -Rcpp::List KafkaConsume(SEXP consumerPtr, int numResults); -RcppExport SEXP _fRanz_KafkaConsume(SEXP consumerPtrSEXP, SEXP numResultsSEXP) { +Rcpp::List KafkaConsume(SEXP consumerPtr, int numResults, int timeout); +RcppExport SEXP _fRanz_KafkaConsume(SEXP consumerPtrSEXP, SEXP numResultsSEXP, SEXP timeoutSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; Rcpp::traits::input_parameter< SEXP >::type consumerPtr(consumerPtrSEXP); Rcpp::traits::input_parameter< int >::type numResults(numResultsSEXP); - rcpp_result_gen = Rcpp::wrap(KafkaConsume(consumerPtr, numResults)); + Rcpp::traits::input_parameter< int >::type timeout(timeoutSEXP); + rcpp_result_gen = Rcpp::wrap(KafkaConsume(consumerPtr, numResults, timeout)); return rcpp_result_gen; END_RCPP } @@ -72,7 +73,7 @@ END_RCPP static const R_CallMethodDef CallEntries[] = { {"_fRanz_GetRdConsumer", (DL_FUNC) &_fRanz_GetRdConsumer, 2}, {"_fRanz_RdSubscribe", (DL_FUNC) &_fRanz_RdSubscribe, 2}, - {"_fRanz_KafkaConsume", (DL_FUNC) &_fRanz_KafkaConsume, 2}, + {"_fRanz_KafkaConsume", (DL_FUNC) &_fRanz_KafkaConsume, 3}, {"_fRanz_GetRdProducer", (DL_FUNC) &_fRanz_GetRdProducer, 2}, {"_fRanz_KafkaProduce", (DL_FUNC) &_fRanz_KafkaProduce, 5}, {NULL, NULL, 0} diff --git a/fRanz/src/consumer.cpp b/fRanz/src/consumer.cpp index 64ccaa7..0b872cc 100644 --- a/fRanz/src/consumer.cpp +++ b/fRanz/src/consumer.cpp @@ -1,19 +1,11 @@ -#include +#include #include #include "utils.h" -#include -#include -#include -#include -#include -#include -#include -#include //////////////////////////////////////////////////////////////////////////////////////// //' @title GetRdConsumer //' @name GetRdConsumer -//' @description Creates an Rcpp::XPtr. For more details on options \link{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md} +//' @description Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} //' @param keys a character vector indicating option keys to parameterize the RdKafka::Consumer //' @param values a character vector indicating option values to parameterize the RdKafka::Consumer. Must be of same length as keys. //' @return a Rcpp::XPtr @@ -54,14 +46,15 @@ int RdSubscribe(SEXP consumerPtr, const Rcpp::StringVector Rtopics) { //' the provided consumer is subscribed to. //' @param consumerPtr a reference to a Rcpp::XPtr //' @param numResults how many results should be consumed before returning. Will return early if offset is at maximum +//' @param timeout the timeout in milliseconds. Default is 10000 //' @return a list of length numResults with values list(key=key,value=value) // [[Rcpp::export]] -Rcpp::List KafkaConsume(SEXP consumerPtr, int numResults) { +Rcpp::List KafkaConsume(SEXP consumerPtr, int numResults, int timeout = 10000) { Rcpp::XPtr consumer(consumerPtr); Rcpp::List messages(numResults); for(int i = 0; i < numResults; i++) { - RdKafka::Message *msg = consumer->consume(10000); + RdKafka::Message *msg = consumer->consume(timeout); switch(msg->err()){ case RdKafka::ERR_NO_ERROR: { Rcpp::List message = Rcpp::List::create(Rcpp::Named("key") = *msg->key(), diff --git a/fRanz/src/producer.cpp b/fRanz/src/producer.cpp index 8929a07..69c73a3 100644 --- a/fRanz/src/producer.cpp +++ b/fRanz/src/producer.cpp @@ -1,16 +1,10 @@ -#include +#include #include #include "utils.h" -#include -#include -#include -#include -#include -#include //' @title GetRdProducer //' @name GetRdProducer -//' @description Creates an Rcpp::XPtr. For more details on options \link{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md} +//' @description Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} //' @param keys a character vector indicating option keys to parameterize the RdKafka::Producer //' @param values a character vector indicating option values to parameterize the RdKafka::Producer. Must be of same length as keys. //' @return a Rcpp::XPtr diff --git a/fRanz/src/utils.cpp b/fRanz/src/utils.cpp index 8144693..0ba3627 100644 --- a/fRanz/src/utils.cpp +++ b/fRanz/src/utils.cpp @@ -1,5 +1,5 @@ #include -#include +#include diff --git a/fRanz/src/utils.h b/fRanz/src/utils.h index c4b9156..16e9f93 100644 --- a/fRanz/src/utils.h +++ b/fRanz/src/utils.h @@ -1,4 +1,4 @@ #include -#include +#include RdKafka::Conf* MakeKafkaConfig(Rcpp::StringVector keys, Rcpp::StringVector values); \ No newline at end of file diff --git a/fRanz/tests/testthat/test-KafkaConsumer.R b/fRanz/tests/testthat/test-KafkaConsumer.R index 32a7db3..6299eea 100644 --- a/fRanz/tests/testthat/test-KafkaConsumer.R +++ b/fRanz/tests/testthat/test-KafkaConsumer.R @@ -2,6 +2,7 @@ context("KafkaConsumer and KafkaProducer") testthat::test_that("Testing KafkaConsumer and KafkaProducer work consuming 1 message",{ + testthat::skip_on_cran() ## Standard Set Up topic <- uuid::UUIDgenerate() group <- uuid::UUIDgenerate() @@ -25,6 +26,7 @@ testthat::test_that("Testing KafkaConsumer and KafkaProducer work consuming 1 me testthat::test_that("Testing KafkaConsumer and KafkaProducer work consuming random number of messages",{ + testthat::skip_on_cran() ## Standard Set Up topic <- uuid::UUIDgenerate() group <- uuid::UUIDgenerate() From 965637d9ca8cd47e374c22afd5015b483aa7acac Mon Sep 17 00:00:00 2001 From: Paxanator Date: Sun, 19 May 2019 22:33:11 -0500 Subject: [PATCH 03/11] ci: getting travis to work --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 88c8d13..6cfde74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ services: before_install: - docker-compose up -d - sleep 20 + - make librdkafka - cd fRanz language: r From 857021794bb9f5331a5756d51213f7ad30a88152 Mon Sep 17 00:00:00 2001 From: Paxanator Date: Sun, 19 May 2019 22:41:31 -0500 Subject: [PATCH 04/11] ci: getting travis to work --- .travis.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6cfde74..04b6864 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,16 +2,13 @@ services: - docker before_install: + - mkdir /usr/local/include /usr/local/lib + - make librdkafka - docker-compose up -d - sleep 20 - - make librdkafka - cd fRanz language: r r: - release -cache: packages -addons: - apt: - packages: - - librdkafka-dev +cache: packages \ No newline at end of file From 05689fed79b589a5d14a48f90146e85dd1540579 Mon Sep 17 00:00:00 2001 From: Paxanator Date: Sun, 19 May 2019 22:49:44 -0500 Subject: [PATCH 05/11] ci: getting travis to work --- .travis.yml | 5 +++-- Makefile | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 04b6864..73406ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,9 @@ services: - docker before_install: - - mkdir /usr/local/include /usr/local/lib - - make librdkafka + - mkdir dest + - make librdkafka LIBRDKAFKA_CONF=--prefix=dest + - export LD_LIBRARY_PATH=LD_LIBRARY_PATH:dest/lib - docker-compose up -d - sleep 20 - cd fRanz diff --git a/Makefile b/Makefile index ff99bbf..a082350 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ PACKAGE = fRanz .PHONY: install test docs clean distclean cleanRcpp unlock +LIBRDKAFKA_CONF='' install: unlock clean cleanRcpp # Install fRanz R package @@ -36,7 +37,7 @@ librdkafka: wget https://github.com/edenhill/librdkafka/archive/v1.0.0.tar.gz -O librdkafka-1.0.0.tar.gz && \ tar xzf librdkafka-1.0.0.tar.gz && \ cd librdkafka-1.0.0 && \ - ./configure && \ + ./configure $(LIBRDKAFKA_CONF)&& \ $(MAKE) && \ $(MAKE) install && \ cd .. && rm -rf librdkafka-1.0.0; \ \ No newline at end of file From 97f69a23c23d587995dddec658ce44d92cc88599 Mon Sep 17 00:00:00 2001 From: Paxanator Date: Mon, 27 May 2019 11:22:14 -0500 Subject: [PATCH 06/11] fix: travis build and R CMD Check --- .travis.yml | 5 +---- Makefile | 3 ++- fRanz/Dockerfile | 4 ++++ fRanz/cleanup | 4 ++++ fRanz/configure | 10 ++++++++++ fRanz/src/Makevars | 5 ++++- 6 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 fRanz/Dockerfile create mode 100755 fRanz/cleanup create mode 100755 fRanz/configure diff --git a/.travis.yml b/.travis.yml index 73406ae..7a990de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,7 @@ services: - docker -before_install: - - mkdir dest - - make librdkafka LIBRDKAFKA_CONF=--prefix=dest - - export LD_LIBRARY_PATH=LD_LIBRARY_PATH:dest/lib +before_install: - docker-compose up -d - sleep 20 - cd fRanz diff --git a/Makefile b/Makefile index a082350..ba23769 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ clean: # Remove cpp object files find $(PACKAGE)/src -name '*.o' -delete find $(PACKAGE)/src -name '*.so' -delete + rm -r $(PACKAGE)/src/librdkafka distclean: clean cleanRcpp @@ -40,4 +41,4 @@ librdkafka: ./configure $(LIBRDKAFKA_CONF)&& \ $(MAKE) && \ $(MAKE) install && \ - cd .. && rm -rf librdkafka-1.0.0; \ \ No newline at end of file + cd .. && rm -rf librdkafka-1.0.0; \ diff --git a/fRanz/Dockerfile b/fRanz/Dockerfile new file mode 100644 index 0000000..c269c40 --- /dev/null +++ b/fRanz/Dockerfile @@ -0,0 +1,4 @@ +FROM ubuntu:trusty + + +COPY ./ ./: diff --git a/fRanz/cleanup b/fRanz/cleanup new file mode 100755 index 0000000..c23a42f --- /dev/null +++ b/fRanz/cleanup @@ -0,0 +1,4 @@ +#!/bin/bash +set -eo + +rm -r librdkafka-1.0.0 librdkafka-1.0.0.tar.gz \ No newline at end of file diff --git a/fRanz/configure b/fRanz/configure new file mode 100755 index 0000000..c9b15ef --- /dev/null +++ b/fRanz/configure @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eo + +wget https://github.com/edenhill/librdkafka/archive/v1.0.0.tar.gz -O librdkafka-1.0.0.tar.gz && \ +tar xzf librdkafka-1.0.0.tar.gz && \ +cd librdkafka-1.0.0 && \ +./configure $(LIBRDKAFKA_CONF) --prefix="../src/librdkafka"&& \ +make && \ +make install; \ No newline at end of file diff --git a/fRanz/src/Makevars b/fRanz/src/Makevars index 1b93233..b4b6e54 100644 --- a/fRanz/src/Makevars +++ b/fRanz/src/Makevars @@ -1,2 +1,5 @@ -PKG_LIBS = -lrdkafka++ +LIBRDKAFKA_LOC = $(PWD)/librdkafka + +PKG_LIBS = -L$(LIBRDKAFKA_LOC)/lib -lrdkafka++ +PKG_CXXFLAGS = -I$(LIBRDKAFKA_LOC)/include CXX_STD=CXX11 \ No newline at end of file From fd5c1c22570d38176cb15aeb51fc82d0b2f17d41 Mon Sep 17 00:00:00 2001 From: Paxanator Date: Mon, 27 May 2019 11:48:07 -0500 Subject: [PATCH 07/11] fix(ci): Some stray merge conflicts made it through. Additionally spruced up Contributing and README. Removed stray Dockerfile --- CONTRIBUTING.md | 30 +++++++++++++++++++++--------- README.md | 9 ++------- fRanz/Dockerfile | 4 ---- fRanz/R/KafkaProducer.R | 5 ----- fRanz/R/RcppExports.R | 10 ---------- fRanz/man/GetRdConsumer.Rd | 19 ------------------- fRanz/man/GetRdProducer.Rd | 19 ------------------- fRanz/man/KafkaBroker.Rd | 2 +- fRanz/man/KafkaConsume.Rd | 26 -------------------------- fRanz/man/KafkaProduce.Rd | 25 ------------------------- fRanz/man/RdSubscribe.Rd | 20 -------------------- 11 files changed, 24 insertions(+), 145 deletions(-) delete mode 100644 fRanz/Dockerfile delete mode 100644 fRanz/man/GetRdConsumer.Rd delete mode 100644 fRanz/man/GetRdProducer.Rd delete mode 100644 fRanz/man/KafkaConsume.Rd delete mode 100644 fRanz/man/KafkaProduce.Rd delete mode 100644 fRanz/man/RdSubscribe.Rd diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0098e12..9183efb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,29 +56,41 @@ To submit a PR, please follow these steps: 1. Fork `fRanz` to your GitHub account 2. Create a branch on your fork and add your changes 3. If you are changing or adding to the R code in the package, add unit tests and integration tests confirming that your code works as expected -3. When you are ready, click "Compare & Pull Request". Open A PR comparing your branch to the `master` branch in this repo -4. In the description section on your PR, please indicate the following: +4. Any commits added should ideally follow [conventional commits](https://conventionalcommits.org). See the Conventional Commits section below for more detail +5. When you are ready, click "Compare & Pull Request". Open A PR comparing your branch to the `master` branch in this repo +6. In the description section on your PR, please indicate the following: - description of what the PR is trying to do and how it improves `fRanz` - links to any open [issues](https://github.com/UptakeOpenSource/fRanz/issues) that your PR is addressing We will try to review PRs promptly and get back to you within a few days. +### Conventional Commits + +We strive to follow conventional commits to make creating NEWS.md and other files easy to maintain. Additionally it provides ease to PR reviewers to semantically check which commit they want to review and sift through the changes. Though we are not terribly strict around this rule since it will cause lots of friction for first time contributors to have to re-author commits, and that is more important than havea pure git history, we do ask frequent commiters to follow this convention. + +In other words, this is a good useful convention but not at the expense of reducing interest or involvement in the project. This will come at the expense of automation, but that's ok. + +For more details see [conventional commits](https://conventionalcommits.org) + ## Running Tests Locally ### Development -> WIP: Document Local Development Setup +`NOTE TO DEVELOPERS: Please add tips as you find them` + +#### Speeding up development times -### Running Unit Tests -> WIP: Document How to Run Unit Tests +Currently `fRanz` installs the entire pulls and compiles the entire `librdkafka` source each time it is installed. This is good for portability but adds significant install time each time. In order to improve the experience developing, it's recommended the developer comments out the `configure` file after the first installation or other similar workarounds -### Running Integration Tests -> WIP: Document How to Run Integration Tests in Docker +### Running Unit and Integration Tests +> You must have `docker` and `docker-compose` configured correctly +> run `docker-compose up -d` from the root of the repository. This will start kafka running on your machine +> run `make test` or other R testing tools +> when done, make sure to call `docker-compose down` to shutdown any running docker instances ### Creating Releases -> WIP: Document How to Create Releases and Versioning +> When we figure this out we will document it! ## Package Versioning -> WIP: Document Package Versioning format ### Version Format We follow semantic versioning for `fRanz` releases, `MAJOR`.`MINOR`.`PATCH`: diff --git a/README.md b/README.md index 539c3ab..b72bb9c 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,6 @@ fRanz is an open source R kafka client that allows users to read and write messages from kafka. It leverages the stability and performance of [librdkafka](https://github.com/edenhill/librdkafka) and implements ididiomatic R workflows ontop of it. -## Installation - -We're working on it. Currently you need librdkafka as a system available library in order to load the R package. In order to install from source you also need the headers. A make recipe in the top level `make librdkafka` should work for *nix systems. - - -No attempt has been made for windows compatability. - ## Example of sending and reading a message ```r @@ -55,3 +48,5 @@ result # [[1]]$payload # [1] "My First Message" ``` + + diff --git a/fRanz/Dockerfile b/fRanz/Dockerfile deleted file mode 100644 index c269c40..0000000 --- a/fRanz/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM ubuntu:trusty - - -COPY ./ ./: diff --git a/fRanz/R/KafkaProducer.R b/fRanz/R/KafkaProducer.R index 996e354..bb0b3d6 100644 --- a/fRanz/R/KafkaProducer.R +++ b/fRanz/R/KafkaProducer.R @@ -1,13 +1,8 @@ #' @title Kakfa Producer #' @name KafkaProducer -<<<<<<< HEAD #' @description A producer is an application that is responsible #' for publishing data to topics. #' @references \href{https://kafka.apache.org/documentation/#intro_producers}{Apache Kafka docs - Producers} -======= -#' @description TDB -#' @export ->>>>>>> fix(build): Fixing install process to rely on system library as a temp measure #' @importFrom R6 R6Class #' @export KafkaProducer <- R6::R6Class( diff --git a/fRanz/R/RcppExports.R b/fRanz/R/RcppExports.R index a0daa05..71bc446 100644 --- a/fRanz/R/RcppExports.R +++ b/fRanz/R/RcppExports.R @@ -24,24 +24,14 @@ RdSubscribe <- function(consumerPtr, Rtopics) { #' @title KafkaConsume #' @name KafkaConsume -<<<<<<< HEAD #' @description Consume a fixed number of results from whatever topic(s) #' the provided consumer is subscribed to. -======= -#' @description A method to that takes a consumer pointer and returns at most the specified number of results, unless the timeout is reached ->>>>>>> fix(build): Fixing install process to rely on system library as a temp measure #' @param consumerPtr a reference to a Rcpp::XPtr #' @param numResults how many results should be consumed before returning. Will return early if offset is at maximum #' @param timeout the timeout in milliseconds. Default is 10000 #' @return a list of length numResults with values list(key=key,value=value) -<<<<<<< HEAD -KafkaConsume <- function(consumerPtr, numResults) { - .Call('_fRanz_KafkaConsume', PACKAGE = 'fRanz', consumerPtr, numResults) -======= -#' @export KafkaConsume <- function(consumerPtr, numResults, timeout = 10000L) { .Call('_fRanz_KafkaConsume', PACKAGE = 'fRanz', consumerPtr, numResults, timeout) ->>>>>>> fix(build): Fixing install process to rely on system library as a temp measure } #' @title GetRdProducer diff --git a/fRanz/man/GetRdConsumer.Rd b/fRanz/man/GetRdConsumer.Rd deleted file mode 100644 index 8cb2e5c..0000000 --- a/fRanz/man/GetRdConsumer.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/RcppExports.R -\name{GetRdConsumer} -\alias{GetRdConsumer} -\title{GetRdConsumer} -\usage{ -GetRdConsumer(keys, values) -} -\arguments{ -\item{keys}{a character vector indicating option keys to parameterize the RdKafka::Consumer} - -\item{values}{a character vector indicating option values to parameterize the RdKafka::Consumer. Must be of same length as keys.} -} -\value{ -a Rcpp::XPtr -} -\description{ -Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} -} diff --git a/fRanz/man/GetRdProducer.Rd b/fRanz/man/GetRdProducer.Rd deleted file mode 100644 index a7f11df..0000000 --- a/fRanz/man/GetRdProducer.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/RcppExports.R -\name{GetRdProducer} -\alias{GetRdProducer} -\title{GetRdProducer} -\usage{ -GetRdProducer(keys, values) -} -\arguments{ -\item{keys}{a character vector indicating option keys to parameterize the RdKafka::Producer} - -\item{values}{a character vector indicating option values to parameterize the RdKafka::Producer. Must be of same length as keys.} -} -\value{ -a Rcpp::XPtr -} -\description{ -Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} -} diff --git a/fRanz/man/KafkaBroker.Rd b/fRanz/man/KafkaBroker.Rd index 47833f9..e59f754 100644 --- a/fRanz/man/KafkaBroker.Rd +++ b/fRanz/man/KafkaBroker.Rd @@ -3,7 +3,7 @@ \docType{data} \name{KafkaBroker} \alias{KafkaBroker} -\title{KafkaBroker} +\title{Kafka Broker} \format{An object of class \code{R6ClassGenerator} of length 24.} \usage{ KafkaBroker diff --git a/fRanz/man/KafkaConsume.Rd b/fRanz/man/KafkaConsume.Rd deleted file mode 100644 index 47f90b9..0000000 --- a/fRanz/man/KafkaConsume.Rd +++ /dev/null @@ -1,26 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/RcppExports.R -\name{KafkaConsume} -\alias{KafkaConsume} -\title{KafkaConsume} -\usage{ -KafkaConsume(consumerPtr, numResults, timeout = 10000L) -} -\arguments{ -\item{consumerPtr}{a reference to a Rcpp::XPtr} - -\item{numResults}{how many results should be consumed before returning. Will return early if offset is at maximum} - -\item{timeout}{the timeout in milliseconds. Default is 10000} -} -\value{ -a list of length numResults with values list(key=key,value=value) -} -\description{ -<<<<<<< HEAD -Consume a fixed number of results from whatever topic(s) - the provided consumer is subscribed to. -======= -A method to that takes a consumer pointer and returns at most the specified number of results, unless the timeout is reached ->>>>>>> fix(build): Fixing install process to rely on system library as a temp measure -} diff --git a/fRanz/man/KafkaProduce.Rd b/fRanz/man/KafkaProduce.Rd deleted file mode 100644 index aa34306..0000000 --- a/fRanz/man/KafkaProduce.Rd +++ /dev/null @@ -1,25 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/RcppExports.R -\name{KafkaProduce} -\alias{KafkaProduce} -\title{KafkaProduce} -\usage{ -KafkaProduce(producer_pointer, topic, partition, keys, values) -} -\arguments{ -\item{producer_pointer}{a Rcpp::XPtr} - -\item{topic}{a string indicating the topic to produce to} - -\item{partition}{an integer indicating the partition to produce to} - -\item{keys}{a character vector for all the keys for the messages} - -\item{values}{a character vector for all the values for the messages. Must be of same length as keys} -} -\value{ -returns the number of messages succesfully sent -} -\description{ -Produces key/values to a particular topic on a particular partition -} diff --git a/fRanz/man/RdSubscribe.Rd b/fRanz/man/RdSubscribe.Rd deleted file mode 100644 index 2f6b355..0000000 --- a/fRanz/man/RdSubscribe.Rd +++ /dev/null @@ -1,20 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/RcppExports.R -\name{RdSubscribe} -\alias{RdSubscribe} -\title{RdSubscribe} -\usage{ -RdSubscribe(consumerPtr, Rtopics) -} -\arguments{ -\item{consumerPtr}{a reference to a Rcpp::XPtr} - -\item{Rtopics}{a character vector listing the topics to subscribe to} -} -\value{ -the int representation of the librdkafka error code of the response to subscribe. 0 is good -} -\description{ -A method to register a consumer with a set amount of topics as consumers. -This is important so the broker can track offsets and register it in a consumer group -} From d4c298592a6e56c82910fc03723f9921cbaa0803 Mon Sep 17 00:00:00 2001 From: Paxanator Date: Mon, 27 May 2019 12:03:18 -0500 Subject: [PATCH 08/11] fix(ci): Removing fq path in Makevars to attempt to rectify R CMD CHECK's ** testing if installed package can be loaded from temporary location --- fRanz/src/Makevars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fRanz/src/Makevars b/fRanz/src/Makevars index b4b6e54..534435d 100644 --- a/fRanz/src/Makevars +++ b/fRanz/src/Makevars @@ -1,4 +1,4 @@ -LIBRDKAFKA_LOC = $(PWD)/librdkafka +LIBRDKAFKA_LOC = librdkafka PKG_LIBS = -L$(LIBRDKAFKA_LOC)/lib -lrdkafka++ PKG_CXXFLAGS = -I$(LIBRDKAFKA_LOC)/include From c55435a11ffc3f2ab005469cac94a1c09674dbfc Mon Sep 17 00:00:00 2001 From: Paxanator Date: Mon, 27 May 2019 13:51:45 -0500 Subject: [PATCH 09/11] fix(check): Fixed all NOTES/WARNINGS on MAC for R CMD CHECK --- CONTRIBUTING.md | 3 ++- Makefile | 18 ++++++------------ fRanz/DESCRIPTION | 9 ++++----- fRanz/R/KafkaBroker.R | 4 +++- fRanz/R/KafkaConsumer.R | 6 ++++-- fRanz/configure | 22 ++++++++++++++++------ fRanz/man/GetRdConsumer.Rd | 19 +++++++++++++++++++ fRanz/man/GetRdProducer.Rd | 19 +++++++++++++++++++ fRanz/man/KafkaBroker.Rd | 4 +++- fRanz/man/KafkaConsume.Rd | 22 ++++++++++++++++++++++ fRanz/man/KafkaConsumer.Rd | 4 +++- fRanz/man/KafkaProduce.Rd | 25 +++++++++++++++++++++++++ fRanz/man/RdSubscribe.Rd | 20 ++++++++++++++++++++ fRanz/src/consumer.cpp | 6 +++--- fRanz/src/producer.cpp | 3 +-- 15 files changed, 150 insertions(+), 34 deletions(-) create mode 100644 fRanz/man/GetRdConsumer.Rd create mode 100644 fRanz/man/GetRdProducer.Rd create mode 100644 fRanz/man/KafkaConsume.Rd create mode 100644 fRanz/man/KafkaProduce.Rd create mode 100644 fRanz/man/RdSubscribe.Rd diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9183efb..387d75a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ For more details see [conventional commits](https://conventionalcommits.org) #### Speeding up development times -Currently `fRanz` installs the entire pulls and compiles the entire `librdkafka` source each time it is installed. This is good for portability but adds significant install time each time. In order to improve the experience developing, it's recommended the developer comments out the `configure` file after the first installation or other similar workarounds +Currently `fRanz` installs the entire pulls and compiles the entire `librdkafka` source each time it is installed. This is good for portability but adds significant install time each time. In order to improve the experience developing a guard is put in ./configure which might cause caching issues. Use `make clean` to purge your install. ### Running Unit and Integration Tests > You must have `docker` and `docker-compose` configured correctly @@ -88,6 +88,7 @@ Currently `fRanz` installs the entire pulls and compiles the entire `librdkafka` > when done, make sure to call `docker-compose down` to shutdown any running docker instances ### Creating Releases +> Currently the best practice known is under `make check` > When we figure this out we will document it! ## Package Versioning diff --git a/Makefile b/Makefile index ba23769..32ae951 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ PACKAGE = fRanz -.PHONY: install test docs clean distclean cleanRcpp unlock -LIBRDKAFKA_CONF='' +.PHONY: install test docs clean distclean cleanRcpp unlock build check install: unlock clean cleanRcpp # Install fRanz R package @@ -9,6 +8,10 @@ install: unlock clean cleanRcpp Rscript -e 'if (!"devtools" %in% rownames(installed.packages())) {install.packages("devtools", repos = "https://cran.rstudio.com")}' && \ Rscript -e "devtools::install(file.path(getwd(),'$(PACKAGE)'), force = TRUE, upgrade = FALSE)" +check: docs clean + R CMD build fRanz + R CMD check --as-cran `ls | grep fRanz*.tar.gz` + cleanRcpp: rm -f fRanz/R/RcppExport.R fRanz/src/RcppExport.cpp @@ -32,13 +35,4 @@ docs: Rscript -e "roxygen2::roxygenize('$(PACKAGE)')" test: # Run unit tests - Rscript -e "devtools::test('$(PACKAGE)')" - -librdkafka: - wget https://github.com/edenhill/librdkafka/archive/v1.0.0.tar.gz -O librdkafka-1.0.0.tar.gz && \ - tar xzf librdkafka-1.0.0.tar.gz && \ - cd librdkafka-1.0.0 && \ - ./configure $(LIBRDKAFKA_CONF)&& \ - $(MAKE) && \ - $(MAKE) install && \ - cd .. && rm -rf librdkafka-1.0.0; \ + Rscript -e "devtools::test('$(PACKAGE)')" \ No newline at end of file diff --git a/fRanz/DESCRIPTION b/fRanz/DESCRIPTION index 2f46aa7..77986c5 100644 --- a/fRanz/DESCRIPTION +++ b/fRanz/DESCRIPTION @@ -1,17 +1,16 @@ Package: fRanz Type: Package -Title: An R Kafka Client. +Title: An R Kafka Client +Description: This is an R package for interacting with Kafka. It provides a wrapper around librdkafka with idiomatic R classes to consume and produce to Kafka topics. Version: 0.1.0 Date: 2019-05-13 Authors@R: c( - person("Patrick", "Boueri", email = "patrick.boueri@uptake.com", role = c("aut")), - person("Mike", "Jermann", email = "mike.jermann@uptake.com", role = c("cre")) + person("Patrick", "Boueri", email = "patrick.boueri@uptake.com", role = c("cre")), + person("Mike", "Jermann", email = "mike.jermann@uptake.com", role = c("aut")) ) Maintainer: Patrick Boueri -Description: An R Kafka Client License: BSD_3_clause + file LICENSE Imports: - jsonlite, Rcpp, R6, methods diff --git a/fRanz/R/KafkaBroker.R b/fRanz/R/KafkaBroker.R index f6fa919..3d9bb06 100644 --- a/fRanz/R/KafkaBroker.R +++ b/fRanz/R/KafkaBroker.R @@ -26,7 +26,9 @@ #' #' #' # KafkaConsumer -#' consumer <- KafkaConsumer$new(brokers = list(broker), groupId = "test", extraOptions=list(`auto.offset.reset`="earliest")) +#' consumer <- KafkaConsumer$new(brokers = list(broker), +#' groupId = "test", +#' extraOptions=list(`auto.offset.reset`="earliest")) #' consumer$subscribe(topics = c(TOPIC_NAME)) #' result <- consumer$consume(topic=TOPIC_NAME) #' diff --git a/fRanz/R/KafkaConsumer.R b/fRanz/R/KafkaConsumer.R index 93de516..9f5b9d0 100644 --- a/fRanz/R/KafkaConsumer.R +++ b/fRanz/R/KafkaConsumer.R @@ -35,7 +35,9 @@ #' #' #' # KafkaConsumer -#' consumer <- KafkaConsumer$new(brokers = list(broker), groupId = "test", extraOptions=list(`auto.offset.reset`="earliest")) +#' consumer <- KafkaConsumer$new(brokers = list(broker), +#' groupId = "test", +#' extraOptions=list(`auto.offset.reset`="earliest")) #' consumer$subscribe(topics = c(TOPIC_NAME)) #' result <- consumer$consume(topic=TOPIC_NAME) #' @@ -71,7 +73,7 @@ KafkaConsumer <- R6::R6Class( } , consume = function(topic, numResults=100) { - Filter(function(msg) !is.null(msg), KafkaConsume(private$consumerPtr, numResults)) + return(KafkaConsume(private$consumerPtr, numResults)) } , getTopics = function() { diff --git a/fRanz/configure b/fRanz/configure index c9b15ef..0ddbdf8 100755 --- a/fRanz/configure +++ b/fRanz/configure @@ -2,9 +2,19 @@ set -eo -wget https://github.com/edenhill/librdkafka/archive/v1.0.0.tar.gz -O librdkafka-1.0.0.tar.gz && \ -tar xzf librdkafka-1.0.0.tar.gz && \ -cd librdkafka-1.0.0 && \ -./configure $(LIBRDKAFKA_CONF) --prefix="../src/librdkafka"&& \ -make && \ -make install; \ No newline at end of file + +### Install librdkafka. Check simply if the library exists if need be +if [ ! -d "src/librdkafka" ]; then + wget https://github.com/edenhill/librdkafka/archive/v1.0.0.tar.gz -O librdkafka-1.0.0.tar.gz && \ + tar xzf librdkafka-1.0.0.tar.gz && \ + INSTALL_PATH="$PWD/src/librdkafka" && \ + cd librdkafka-1.0.0 && \ + ./configure $(LIBRDKAFKA_CONF) --prefix="$INSTALL_PATH" && \ + make && \ + make install && \ + cd .. && \ + rm -r librdkafka-1.0.0 && \ + rm librdkafka-1.0.0.tar.gz; +else + echo "Detected librdkafka in src" +fi \ No newline at end of file diff --git a/fRanz/man/GetRdConsumer.Rd b/fRanz/man/GetRdConsumer.Rd new file mode 100644 index 0000000..8cb2e5c --- /dev/null +++ b/fRanz/man/GetRdConsumer.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/RcppExports.R +\name{GetRdConsumer} +\alias{GetRdConsumer} +\title{GetRdConsumer} +\usage{ +GetRdConsumer(keys, values) +} +\arguments{ +\item{keys}{a character vector indicating option keys to parameterize the RdKafka::Consumer} + +\item{values}{a character vector indicating option values to parameterize the RdKafka::Consumer. Must be of same length as keys.} +} +\value{ +a Rcpp::XPtr +} +\description{ +Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} +} diff --git a/fRanz/man/GetRdProducer.Rd b/fRanz/man/GetRdProducer.Rd new file mode 100644 index 0000000..a7f11df --- /dev/null +++ b/fRanz/man/GetRdProducer.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/RcppExports.R +\name{GetRdProducer} +\alias{GetRdProducer} +\title{GetRdProducer} +\usage{ +GetRdProducer(keys, values) +} +\arguments{ +\item{keys}{a character vector indicating option keys to parameterize the RdKafka::Producer} + +\item{values}{a character vector indicating option values to parameterize the RdKafka::Producer. Must be of same length as keys.} +} +\value{ +a Rcpp::XPtr +} +\description{ +Creates an Rcpp::XPtr. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration} +} diff --git a/fRanz/man/KafkaBroker.Rd b/fRanz/man/KafkaBroker.Rd index e59f754..8fc081e 100644 --- a/fRanz/man/KafkaBroker.Rd +++ b/fRanz/man/KafkaBroker.Rd @@ -34,7 +34,9 @@ producer$produce(topic = TOPIC_NAME, # KafkaConsumer -consumer <- KafkaConsumer$new(brokers = list(broker), groupId = "test", extraOptions=list(`auto.offset.reset`="earliest")) +consumer <- KafkaConsumer$new(brokers = list(broker), + groupId = "test", + extraOptions=list(`auto.offset.reset`="earliest")) consumer$subscribe(topics = c(TOPIC_NAME)) result <- consumer$consume(topic=TOPIC_NAME) diff --git a/fRanz/man/KafkaConsume.Rd b/fRanz/man/KafkaConsume.Rd new file mode 100644 index 0000000..a54f838 --- /dev/null +++ b/fRanz/man/KafkaConsume.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/RcppExports.R +\name{KafkaConsume} +\alias{KafkaConsume} +\title{KafkaConsume} +\usage{ +KafkaConsume(consumerPtr, numResults, timeout = 10000L) +} +\arguments{ +\item{consumerPtr}{a reference to a Rcpp::XPtr} + +\item{numResults}{how many results should be consumed before returning. Will return early if offset is at maximum} + +\item{timeout}{the timeout in milliseconds. Default is 10000} +} +\value{ +a list of length numResults with values list(key=key,value=value) +} +\description{ +Consume a fixed number of results from whatever topic(s) + the provided consumer is subscribed to. +} diff --git a/fRanz/man/KafkaConsumer.Rd b/fRanz/man/KafkaConsumer.Rd index 05787d5..a90c030 100644 --- a/fRanz/man/KafkaConsumer.Rd +++ b/fRanz/man/KafkaConsumer.Rd @@ -41,7 +41,9 @@ producer$produce(topic = TOPIC_NAME, # KafkaConsumer -consumer <- KafkaConsumer$new(brokers = list(broker), groupId = "test", extraOptions=list(`auto.offset.reset`="earliest")) +consumer <- KafkaConsumer$new(brokers = list(broker), + groupId = "test", + extraOptions=list(`auto.offset.reset`="earliest")) consumer$subscribe(topics = c(TOPIC_NAME)) result <- consumer$consume(topic=TOPIC_NAME) diff --git a/fRanz/man/KafkaProduce.Rd b/fRanz/man/KafkaProduce.Rd new file mode 100644 index 0000000..aa34306 --- /dev/null +++ b/fRanz/man/KafkaProduce.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/RcppExports.R +\name{KafkaProduce} +\alias{KafkaProduce} +\title{KafkaProduce} +\usage{ +KafkaProduce(producer_pointer, topic, partition, keys, values) +} +\arguments{ +\item{producer_pointer}{a Rcpp::XPtr} + +\item{topic}{a string indicating the topic to produce to} + +\item{partition}{an integer indicating the partition to produce to} + +\item{keys}{a character vector for all the keys for the messages} + +\item{values}{a character vector for all the values for the messages. Must be of same length as keys} +} +\value{ +returns the number of messages succesfully sent +} +\description{ +Produces key/values to a particular topic on a particular partition +} diff --git a/fRanz/man/RdSubscribe.Rd b/fRanz/man/RdSubscribe.Rd new file mode 100644 index 0000000..2f6b355 --- /dev/null +++ b/fRanz/man/RdSubscribe.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/RcppExports.R +\name{RdSubscribe} +\alias{RdSubscribe} +\title{RdSubscribe} +\usage{ +RdSubscribe(consumerPtr, Rtopics) +} +\arguments{ +\item{consumerPtr}{a reference to a Rcpp::XPtr} + +\item{Rtopics}{a character vector listing the topics to subscribe to} +} +\value{ +the int representation of the librdkafka error code of the response to subscribe. 0 is good +} +\description{ +A method to register a consumer with a set amount of topics as consumers. +This is important so the broker can track offsets and register it in a consumer group +} diff --git a/fRanz/src/consumer.cpp b/fRanz/src/consumer.cpp index 0b872cc..63576b1 100644 --- a/fRanz/src/consumer.cpp +++ b/fRanz/src/consumer.cpp @@ -62,11 +62,11 @@ Rcpp::List KafkaConsume(SEXP consumerPtr, int numResults, int timeout = 10000) { messages[i] = message; break; } case RdKafka::ERR__PARTITION_EOF: { - printf("No additional messages available\n"); + //Trim the messages to the appropriate size + messages = messages[Rcpp::Range(0,i-1)]; goto exit_loop; } default: { - /* Errors */ - printf("Consume failed: %s", msg->errstr().c_str()); + Rcpp::stop("Consume failed: %s", msg->errstr().c_str()); goto exit_loop; } } diff --git a/fRanz/src/producer.cpp b/fRanz/src/producer.cpp index 69c73a3..70295fc 100644 --- a/fRanz/src/producer.cpp +++ b/fRanz/src/producer.cpp @@ -39,8 +39,7 @@ int KafkaProduce(SEXP producer_pointer, std::string s_topic = Rcpp::as(topic); if (keys.size() != values.size()) { - std::cout << "keys and values must be same size" << std::endl; - return -1; + Rcpp::stop("keys and values must be same size"); } int numMsgs = keys.size(); int numSent = 0; From d7b40fbd857d41e27119c21ec53cf32380cdbe21 Mon Sep 17 00:00:00 2001 From: Paxanator Date: Mon, 27 May 2019 14:25:59 -0500 Subject: [PATCH 10/11] fix(check): Trying to ship librdkafka binaries with fRanz.so in install process --- .gitignore | 5 +++-- fRanz/configure | 4 ++++ fRanz/src/Makevars | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d1ca494..2f21926 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ *.o *.so *.DS_Store -franz/src/librdkafka -librdkafka* \ No newline at end of file +franz/src/librdkafa +franz/src/pkgconfig +librdkafka* diff --git a/fRanz/configure b/fRanz/configure index 0ddbdf8..c374d73 100755 --- a/fRanz/configure +++ b/fRanz/configure @@ -13,6 +13,10 @@ if [ ! -d "src/librdkafka" ]; then make && \ make install && \ cd .. && \ + # We must also copy librdkafka/libs into src so that they are installed by default, otherwise we need to write a custom install.package.R + # Which is a larger more daunting task that could be error prone https://github.com/wch/r-source/blob/245613c0833040e05cb5c6a07205ff0fd9248577/src/library/tools/R/install.R#L513 + # and also looks like it's deprecated https://github.com/r-lib/devtools/issues/1444 + cp -r src/librdkafka/lib/* src/ && \ rm -r librdkafka-1.0.0 && \ rm librdkafka-1.0.0.tar.gz; else diff --git a/fRanz/src/Makevars b/fRanz/src/Makevars index 534435d..6d873bd 100644 --- a/fRanz/src/Makevars +++ b/fRanz/src/Makevars @@ -1,5 +1,5 @@ LIBRDKAFKA_LOC = librdkafka -PKG_LIBS = -L$(LIBRDKAFKA_LOC)/lib -lrdkafka++ +PKG_LIBS = -L. -lrdkafka++ PKG_CXXFLAGS = -I$(LIBRDKAFKA_LOC)/include CXX_STD=CXX11 \ No newline at end of file From e060e7636f851ed3162f93e6947c8f4547c4bcc3 Mon Sep 17 00:00:00 2001 From: Paxanator Date: Sun, 9 Jun 2019 17:50:53 -0500 Subject: [PATCH 11/11] chore(install): Added devel/docker to help with installation checking --- .dockerignore | 3 +++ .gitkeep | 0 .travis.yml | 3 ++- CONTRIBUTING.md | 1 + Makefile | 14 ++++++-------- devel/Dockerfile | 12 ++++++++++++ fRanz/NAMESPACE | 3 --- fRanz/cleanup | 3 ++- fRanz/configure | 17 +++++++---------- fRanz/src/Makevars | 2 +- 10 files changed, 34 insertions(+), 24 deletions(-) create mode 100644 .dockerignore create mode 100644 .gitkeep create mode 100644 devel/Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fe63811 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.vscode/ +.git/ + diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.travis.yml b/.travis.yml index 7a990de..2b38878 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ services: - docker -before_install: +before_install: + - docker-compose up -d - sleep 20 - cd fRanz diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 387d75a..fcc807a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,6 +76,7 @@ For more details see [conventional commits](https://conventionalcommits.org) ### Development `NOTE TO DEVELOPERS: Please add tips as you find them` +- You need to have g++ to work with this package properly. The clang compiler is not supported because of differences in static linking in the namespace. See this issue for more details #### Speeding up development times diff --git a/Makefile b/Makefile index 32ae951..c99b842 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,8 @@ PACKAGE = fRanz .PHONY: install test docs clean distclean cleanRcpp unlock build check -install: unlock clean cleanRcpp - # Install fRanz R package - Rscript -e 'if (!"Rcpp" %in% rownames(installed.packages())) {install.packages("Rcpp", repos = "https://cran.rstudio.com")}' && \ - Rscript -e "Rcpp::compileAttributes(file.path(getwd(),'$(PACKAGE)'))" && \ - Rscript -e 'if (!"devtools" %in% rownames(installed.packages())) {install.packages("devtools", repos = "https://cran.rstudio.com")}' && \ - Rscript -e "devtools::install(file.path(getwd(),'$(PACKAGE)'), force = TRUE, upgrade = FALSE)" +install: docs + R CMD INSTALL $(PACKAGE) check: docs clean R CMD build fRanz @@ -19,7 +15,9 @@ clean: # Remove cpp object files find $(PACKAGE)/src -name '*.o' -delete find $(PACKAGE)/src -name '*.so' -delete - rm -r $(PACKAGE)/src/librdkafka + find $(PACKAGE)/src -name '*.a' -delete + find $(PACKAGE)/src -name '*.dylib' -delete + rm -r $(PACKAGE)/src/librdkafka distclean: clean cleanRcpp @@ -35,4 +33,4 @@ docs: Rscript -e "roxygen2::roxygenize('$(PACKAGE)')" test: # Run unit tests - Rscript -e "devtools::test('$(PACKAGE)')" \ No newline at end of file + Rscript -e "devtools::test('$(PACKAGE)')" diff --git a/devel/Dockerfile b/devel/Dockerfile new file mode 100644 index 0000000..d424426 --- /dev/null +++ b/devel/Dockerfile @@ -0,0 +1,12 @@ +FROM rocker/r-base +ARG test + +# For roxygen2 +RUN apt-get update && apt-get install -y libxml2-dev +RUN Rscript -e "install.packages(c('Rcpp','R6','testthat','uuid','roxygen2'))" + +RUN echo "$test" +COPY ./fRanz fRanz + +RUN R CMD build fRanz +RUN R CMD check --install-args='--no-clean-on-error' fRanz_0.1.0.tar.gz || cat fRanz.Rcheck/00install.out \ No newline at end of file diff --git a/fRanz/NAMESPACE b/fRanz/NAMESPACE index b040fef..a92293e 100644 --- a/fRanz/NAMESPACE +++ b/fRanz/NAMESPACE @@ -1,8 +1,5 @@ # Generated by roxygen2: do not edit by hand -export(KafkaBroker) -export(KafkaConsumer) -export(KafkaProducer) importFrom(R6,R6Class) importFrom(Rcpp,sourceCpp) useDynLib(fRanz) diff --git a/fRanz/cleanup b/fRanz/cleanup index c23a42f..71c25ab 100755 --- a/fRanz/cleanup +++ b/fRanz/cleanup @@ -1,4 +1,5 @@ #!/bin/bash set -eo -rm -r librdkafka-1.0.0 librdkafka-1.0.0.tar.gz \ No newline at end of file +rm -r librdkafka-1.0.0 librdkafka-1.0.0.tar.gz +rm -r src/librdkafka \ No newline at end of file diff --git a/fRanz/configure b/fRanz/configure index c374d73..1fb435d 100755 --- a/fRanz/configure +++ b/fRanz/configure @@ -8,16 +8,13 @@ if [ ! -d "src/librdkafka" ]; then wget https://github.com/edenhill/librdkafka/archive/v1.0.0.tar.gz -O librdkafka-1.0.0.tar.gz && \ tar xzf librdkafka-1.0.0.tar.gz && \ INSTALL_PATH="$PWD/src/librdkafka" && \ - cd librdkafka-1.0.0 && \ - ./configure $(LIBRDKAFKA_CONF) --prefix="$INSTALL_PATH" && \ - make && \ - make install && \ - cd .. && \ - # We must also copy librdkafka/libs into src so that they are installed by default, otherwise we need to write a custom install.package.R - # Which is a larger more daunting task that could be error prone https://github.com/wch/r-source/blob/245613c0833040e05cb5c6a07205ff0fd9248577/src/library/tools/R/install.R#L513 - # and also looks like it's deprecated https://github.com/r-lib/devtools/issues/1444 - cp -r src/librdkafka/lib/* src/ && \ - rm -r librdkafka-1.0.0 && \ + mkdir $INSTALL_PATH &&\ + mv librdkafka-1.0.0/* $INSTALL_PATH && \ + cd $INSTALL_PATH && \ + ./configure --prefix=. && make && make install && \ + cd ../.. && + cp -r src/librdkafka/lib inst/ && \ + cp -r inst/lib inst/libs && \ rm librdkafka-1.0.0.tar.gz; else echo "Detected librdkafka in src" diff --git a/fRanz/src/Makevars b/fRanz/src/Makevars index 6d873bd..ef0e28f 100644 --- a/fRanz/src/Makevars +++ b/fRanz/src/Makevars @@ -1,5 +1,5 @@ LIBRDKAFKA_LOC = librdkafka -PKG_LIBS = -L. -lrdkafka++ +PKG_LIBS = -L$(LIBRDKAFKA_LOC)/lib -lrdkafka++ PKG_CXXFLAGS = -I$(LIBRDKAFKA_LOC)/include CXX_STD=CXX11 \ No newline at end of file