Working Travis Build

.dockerignore

@@ -0,0 +1,3 @@
+.vscode/
+.git/
+

.gitignore

@@ -5,4 +5,6 @@
 *.o
 *.so
 *.DS_Store
-fRanz/inst/librdkafka*
+franz/src/librdkafa
+franz/src/pkgconfig
+librdkafka*

.travis.yml

@@ -2,14 +2,12 @@ services:
   - docker
 
 before_install:
+
   - docker-compose up -d 
   - sleep 20
+  - cd fRanz
 
 language: r
 r: 
   - release
-cache: packages
-
-os:
-  - linux
-  - osx
+cache: packages

CONTRIBUTING.md

@@ -56,29 +56,43 @@ To submit a PR, please follow these steps:
 1. Fork `fRanz` to your GitHub account
 2. Create a branch on your fork and add your changes
 3. If you are changing or adding to the R code in the package, add unit tests and integration tests confirming that your code works as expected
-3. When you are ready, click "Compare & Pull Request". Open A PR comparing your branch to the `master` branch in this repo
-4. In the description section on your PR, please indicate the following:
+4. Any commits added should ideally follow [conventional commits](https://conventionalcommits.org). See the Conventional Commits section below for more detail
+5. When you are ready, click "Compare & Pull Request". Open A PR comparing your branch to the `master` branch in this repo
+6. In the description section on your PR, please indicate the following:
     - description of what the PR is trying to do and how it improves `fRanz`
     - links to any open [issues](https://github.com/UptakeOpenSource/fRanz/issues) that your PR is addressing
 
 We will try to review PRs promptly and get back to you within a few days.
 
+### Conventional Commits
+
+We strive to follow conventional commits to make creating NEWS.md and other files easy to maintain. Additionally it provides ease to PR reviewers to semantically check which commit they want to review and sift through the changes. Though we are not terribly strict around this rule since it will cause lots of friction for first time contributors to have to re-author commits, and that is more important than havea   pure git history, we do ask frequent commiters to follow this convention. 
+
+In other words, this is a good useful convention but not at the expense of reducing interest or involvement in the project. This will come at the expense of automation, but that's ok.
+
+For more details see [conventional commits](https://conventionalcommits.org)
+
 ## Running Tests Locally <a name="testing"></a>
 
 ### Development
-> WIP: Document Local Development Setup
+`NOTE TO DEVELOPERS: Please add tips as you find them`
+- You need to have g++ to work with this package properly. The clang compiler is not supported because of differences in static linking in the namespace. See this issue for more details 
+
+#### Speeding up development times
 
-### Running Unit Tests
-> WIP: Document How to Run Unit Tests
+Currently `fRanz` installs the entire pulls and compiles the entire `librdkafka` source each time it is installed. This is good for portability but adds significant install time each time. In order to improve the experience developing a guard is put in ./configure which might cause caching issues. Use `make clean` to purge your install.
 
-### Running Integration Tests
-> WIP: Document How to Run Integration Tests in Docker
+### Running Unit and Integration Tests
+> You must have `docker` and `docker-compose` configured correctly
+> run `docker-compose up -d` from the root of the repository. This will start kafka running on your machine
+> run `make test` or other R testing tools
+> when done, make sure to call `docker-compose down` to shutdown any running docker instances
 
 ### Creating Releases
-> WIP: Document How to Create Releases and Versioning
+> Currently the best practice known is under `make check`
+> When we figure this out we will document it!
 
 ## Package Versioning <a name="version"></a>
-> WIP: Document Package Versioning format
 
 ### Version Format
 We follow semantic versioning for `fRanz` releases, `MAJOR`.`MINOR`.`PATCH`: 

Makefile

@@ -1,15 +1,12 @@
 PACKAGE = fRanz
-INSTALLDIR = $(HOME)/.$(PACKAGE)/local
-OS = $(shell uname -s)
+.PHONY: install test docs clean distclean cleanRcpp unlock build check
 
-.PHONY: install smoke test docs roxygen pdf version clean distclean cleanRcpp unlock
+install: docs
+	R CMD INSTALL $(PACKAGE)
 
-install: unlock clean cleanRcpp
-	# Install fRanz R package
-	Rscript -e 'if (!"Rcpp" %in% rownames(installed.packages())) {install.packages("Rcpp", repos = "https://cran.rstudio.com")}' && \
-	Rscript -e "Rcpp::compileAttributes(file.path(getwd(),'$(PACKAGE)'))" && \
-	Rscript -e 'if (!"devtools" %in% rownames(installed.packages())) {install.packages("devtools", repos = "https://cran.rstudio.com")}' && \
-	Rscript -e "devtools::install(file.path(getwd(),'$(PACKAGE)'), force = TRUE, upgrade = FALSE)"
+check: docs clean
+	R CMD build fRanz
+	R CMD check --as-cran `ls | grep  fRanz*.tar.gz`
 
 cleanRcpp:
 	rm -f fRanz/R/RcppExport.R fRanz/src/RcppExport.cpp
@@ -18,20 +15,22 @@ clean:
 	# Remove cpp object files
 	find $(PACKAGE)/src -name '*.o' -delete
 	find $(PACKAGE)/src -name '*.so' -delete
+	find $(PACKAGE)/src -name '*.a' -delete
+	find $(PACKAGE)/src -name '*.dylib' -delete
+	rm -r $(PACKAGE)/src/librdkafka 
 
 distclean: clean cleanRcpp
 
 unlock:
-	# Remove 00LOCK-cpproll directory
+	# Remove 00LOCK directory
 	for libpath in $$(Rscript -e "noquote(paste(.libPaths(), collapse = ' '))"); do \
 		echo "Unlocking $$libpath..." && \
 		rm -rf $$libpath/00LOCK-$(PACKAGE); \
 	done
 
-docs roxygen:
+docs:
 	# Regenerate documentation with roxygen
 	Rscript -e "roxygen2::roxygenize('$(PACKAGE)')"
 test:
 	# Run unit tests
 	Rscript -e "devtools::test('$(PACKAGE)')"
-

README.md

@@ -48,3 +48,5 @@ result
 # [[1]]$payload
 # [1] "My First Message" 
 ```
+
+

devel/Dockerfile

@@ -0,0 +1,12 @@
+FROM rocker/r-base
+ARG test
+
+# For roxygen2
+RUN apt-get update && apt-get install -y libxml2-dev 
+RUN Rscript -e "install.packages(c('Rcpp','R6','testthat','uuid','roxygen2'))"
+
+RUN echo "$test"
+COPY ./fRanz fRanz
+
+RUN R CMD build fRanz
+RUN R CMD check --install-args='--no-clean-on-error' fRanz_0.1.0.tar.gz || cat fRanz.Rcheck/00install.out 

fRanz/DESCRIPTION

@@ -1,17 +1,16 @@
 Package: fRanz
 Type: Package
 Title: An R Kafka Client
+Description: This is an R package for interacting with Kafka. It provides a wrapper around librdkafka with idiomatic R classes to consume and produce to Kafka topics.
 Version: 0.1.0
 Date: 2019-05-13
 Authors@R: c(
-    person("Patrick", "Boueri", email = "patrick.boueri@uptake.com", role = c("aut")),
-    person("Mike", "Jermann", email = "mike.jermann@uptake.com", role = c("cre"))
+    person("Patrick", "Boueri", email = "patrick.boueri@uptake.com", role = c("cre")),
+    person("Mike", "Jermann", email = "mike.jermann@uptake.com", role = c("aut"))
     )
 Maintainer: Patrick Boueri <patrick.boueri@uptake.com>
-Description: An R Kafka Client
 License: BSD_3_clause + file LICENSE
 Imports: 
-    jsonlite,
     Rcpp,
     R6,
     methods

fRanz/NAMESPACE

@@ -1,8 +1,5 @@
 # Generated by roxygen2: do not edit by hand
 
-export(KafkaBroker)
-export(KafkaConsumer)
-export(KafkaProducer)
 importFrom(R6,R6Class)
 importFrom(Rcpp,sourceCpp)
 useDynLib(fRanz)

fRanz/R/KafkaBroker.R

@@ -13,7 +13,6 @@
 #' BROKER_HOST <- 'localhost'
 #' BROKER_PORT <- 9092
 #' TOPIC_NAME <- 'myTestTopic'
-
 #' # KafkaBroker
 #' broker <- KafkaBroker$new(host=BROKER_HOST, port=BROKER_PORT)
 #' 
@@ -27,7 +26,9 @@
 #' 
 #' 
 #' # KafkaConsumer
-#' consumer <- KafkaConsumer$new(brokers = list(broker), groupId = "test", extraOptions=list(`auto.offset.reset`="earliest"))
+#' consumer <- KafkaConsumer$new(brokers = list(broker),
+#'                                groupId = "test",
+#'                                extraOptions=list(`auto.offset.reset`="earliest"))
 #' consumer$subscribe(topics = c(TOPIC_NAME))
 #' result <- consumer$consume(topic=TOPIC_NAME)
 #' 

fRanz/R/KafkaConsumer.R

@@ -35,7 +35,9 @@
 #' 
 #' 
 #' # KafkaConsumer
-#' consumer <- KafkaConsumer$new(brokers = list(broker), groupId = "test", extraOptions=list(`auto.offset.reset`="earliest"))
+#' consumer <- KafkaConsumer$new(brokers = list(broker), 
+#'                               groupId = "test",
+#'                               extraOptions=list(`auto.offset.reset`="earliest"))
 #' consumer$subscribe(topics = c(TOPIC_NAME))
 #' result <- consumer$consume(topic=TOPIC_NAME)
 #' 
@@ -71,7 +73,7 @@ KafkaConsumer <- R6::R6Class(
         }
 
         , consume = function(topic, numResults=100) {
-            Filter(function(msg) !is.null(msg), KafkaConsume(private$consumerPtr, numResults))
+            return(KafkaConsume(private$consumerPtr, numResults))
         }
 
         , getTopics = function() {

fRanz/R/RcppExports.R

@@ -3,7 +3,7 @@
 
 #' @title GetRdConsumer
 #' @name GetRdConsumer
-#' @description Creates an Rcpp::XPtr<RdKafka::Consumer>. For more details on options \link{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}
+#' @description Creates an Rcpp::XPtr<RdKafka::Consumer>. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration}
 #' @param keys a character vector indicating option keys to parameterize the RdKafka::Consumer
 #' @param values a character vector indicating option values to parameterize the RdKafka::Consumer. Must be of same length as keys.
 #' @return a Rcpp::XPtr<RdKafka::Consumer>
@@ -28,14 +28,15 @@ RdSubscribe <- function(consumerPtr, Rtopics) {
 #'              the provided consumer is subscribed to.
 #' @param consumerPtr a reference to a Rcpp::XPtr<RdKafka::KafkaConsumer>
 #' @param numResults how many results should be consumed before returning. Will return early if offset is at maximum
+#' @param timeout the timeout in milliseconds. Default is 10000
 #' @return a list of length numResults with values list(key=key,value=value)
-KafkaConsume <- function(consumerPtr, numResults) {
-    .Call('_fRanz_KafkaConsume', PACKAGE = 'fRanz', consumerPtr, numResults)
+KafkaConsume <- function(consumerPtr, numResults, timeout = 10000L) {
+    .Call('_fRanz_KafkaConsume', PACKAGE = 'fRanz', consumerPtr, numResults, timeout)
 }
 
 #' @title GetRdProducer
 #' @name GetRdProducer
-#' @description Creates an Rcpp::XPtr<RdKafka::Producer>. For more details on options \link{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}
+#' @description Creates an Rcpp::XPtr<RdKafka::Producer>. For more details on options \href{https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md}{Configuration}
 #' @param keys a character vector indicating option keys to parameterize the RdKafka::Producer
 #' @param values a character vector indicating option values to parameterize the RdKafka::Producer. Must be of same length as keys.
 #' @return a Rcpp::XPtr<RdKafka::Producer>

fRanz/cleanup

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -eo
+
+rm -r librdkafka-1.0.0 librdkafka-1.0.0.tar.gz
+rm -r src/librdkafka

fRanz/configure

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -eo
+
+
+### Install librdkafka. Check simply if the library exists if need be
+if [ ! -d "src/librdkafka" ]; then
+    wget https://github.com/edenhill/librdkafka/archive/v1.0.0.tar.gz -O librdkafka-1.0.0.tar.gz && \
+    tar xzf librdkafka-1.0.0.tar.gz && \
+    INSTALL_PATH="$PWD/src/librdkafka" && \
+    mkdir $INSTALL_PATH &&\
+    mv librdkafka-1.0.0/* $INSTALL_PATH && \
+    cd $INSTALL_PATH && \
+    ./configure --prefix=. && make && make install && \
+    cd ../.. &&
+    cp -r src/librdkafka/lib inst/ && \
+    cp -r inst/lib inst/libs && \
+    rm librdkafka-1.0.0.tar.gz;
+else
+    echo "Detected librdkafka in src"
+fi

fRanz/src/Makevars

@@ -1,23 +1,5 @@
-INSTALLDIR = $(HOME)/.fRanz/librdkafka
-LIBRDKAFKADIR = $(PWD)/../inst/librdkafka-0.11.6
-
-PKG_LIBS = -L$(INSTALLDIR)/src-cpp -lrdkafka++
-PKG_CXXFLAGS = -std=c++11 -I$(INSTALLDIR)/src-cpp
-
-.PHONY: all install_librdkadka
-
-all: install_librdkadka
-
-install_librdkadka:
-	if [ ! -s $(INSTALLDIR)/src-cpp/librdkafka++.a ] ; \
-	then \
-		mkdir -p $(INSTALLDIR) && \
-		cd $(shell dirname $(LIBRDKAFKADIR)) && \
-		tar xzf $(LIBRDKAFKADIR).tar.gz && \
-		cd $(LIBRDKAFKADIR) && \
-		./configure && \
-		$(MAKE) && \
-                $(MAKE) install && \
-		mv * $(INSTALLDIR) ; \
-	fi
+LIBRDKAFKA_LOC = librdkafka
 
+PKG_LIBS = -L$(LIBRDKAFKA_LOC)/lib -lrdkafka++ 
+PKG_CXXFLAGS = -I$(LIBRDKAFKA_LOC)/include
+CXX_STD=CXX11