diff --git a/README-gradle.md b/README-gradle.md deleted file mode 100644 index a5644279..00000000 --- a/README-gradle.md +++ /dev/null @@ -1,51 +0,0 @@ - -To use gradle to build/run the hello-samza project: - -1) the project is configured to download and use gradle version 2.3 - on first task execution, it will download the required gradle jars. - -2) download/install yarn/kafka/zookeeper: - - $ ./gradlew installGrid - -3) build hello-samza job package: - - $ ./gradlew distTar - -4) deploy hello-samza project to grid: - - $ ./gradlew deployHelloSamza - -5) start the grid (starts up yarn/kafka/zookeeper): - - $ ./gradlew startGrid - -6) run the various Samza tasks that are part of hello-samza project: - - $ ./gradlew runWikiFeed - $ ./gradlew runWikiParser - $ ./gradlew runWikiStats - -7) view all the current Kafka topics: - - $ ./gradlew listKafkaTopics - -8) view the Kafka topics output by the various Samza tasks: - - $ ./gradlew dumpWikiRaw - ( output of Kafka topic scrolls by) - CTRL-c - - $ ./gradlew dumpWikiEdits - ( output of Kafka topic scrolls by) - CTRL-c - - $ ./gradlew dumpWikiStats - ( output of Kafka topic scrolls by) - CTRL-c - -9) stop all the components: - - $ ./gradlew stopGrid - -Shortcut: using the 'runWiki*' tasks directly will do steps 3-6 automatically. - diff --git a/bin/grid b/bin/grid index 5c590e41..62ae719f 100755 --- a/bin/grid +++ b/bin/grid @@ -34,9 +34,10 @@ DEPLOY_ROOT_DIR=$BASE_DIR/deploy DOWNLOAD_CACHE_DIR=$HOME/.samza/download COMMAND=$1 SYSTEM=$2 +YARN_VERSION="3.3.4" DOWNLOAD_KAFKA=https://archive.apache.org/dist/kafka/2.1.1/kafka_2.11-2.1.1.tgz -DOWNLOAD_YARN=https://archive.apache.org/dist/hadoop/common/hadoop-2.9.2/hadoop-2.9.2.tar.gz +DOWNLOAD_YARN=https://archive.apache.org/dist/hadoop/common/hadoop-$YARN_VERSION/hadoop-$YARN_VERSION.tar.gz DOWNLOAD_ZOOKEEPER=https://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz SERVICE_WAIT_TIMEOUT_SEC=20 @@ -52,6 +53,7 @@ bootstrap() { mkdir "$DEPLOY_ROOT_DIR" install_all start_all + echo "Visit http://localhost:8088/ for Hadoop UI" exit 0 } @@ -81,17 +83,20 @@ install_all_without_yarn() { install_samza() { echo "Building samza from master..." mkdir -p "$DEPLOY_ROOT_DIR" + echo "DOWNLOAD_CACHE_DIR: $DOWNLOAD_CACHE_DIR" if [ -d "$DOWNLOAD_CACHE_DIR/samza/.git" ]; then + echo "Resetting local samza to master" pushd "$DOWNLOAD_CACHE_DIR/samza" git fetch origin git reset --hard origin/master else + echo "Cloning samza git repo to build locally" mkdir -p $DOWNLOAD_CACHE_DIR pushd $DOWNLOAD_CACHE_DIR git clone https://gitbox.apache.org/repos/asf/samza.git cd samza fi - ./gradlew -PscalaSuffix=2.11 clean publishToMavenLocal + ./gradlew -PscalaSuffix=2.12 clean publishToMavenLocal popd } @@ -103,7 +108,7 @@ install_zookeeper() { install_yarn() { mkdir -p "$DEPLOY_ROOT_DIR" - install yarn $DOWNLOAD_YARN hadoop-2.9.2 + install yarn $DOWNLOAD_YARN hadoop-$YARN_VERSION cp "$BASE_DIR/conf/yarn-site.xml" "$DEPLOY_ROOT_DIR/yarn/etc/hadoop/yarn-site.xml" if [ ! -f "$HOME/.samza/conf/yarn-site.xml" ]; then mkdir -p "$HOME/.samza/conf" diff --git a/build.gradle b/build.gradle deleted file mode 100644 index 07355765..00000000 --- a/build.gradle +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -apply plugin: 'eclipse' -apply plugin: 'idea' -apply plugin: 'java' - -defaultTasks 'distTar' - -task wrapper(type: Wrapper) { - description = "Updates gradlew and supporting files." - gradleVersion = '2.3' -} - -version = "$SAMZA_VERSION" - -repositories { - mavenLocal() - mavenCentral() - maven { url "https://repository.apache.org/content/groups/public" } -} - - -idea { - module { - sourceDirs += file('src/main/java') - testSourceDirs += file('src/test/java') - } -} - -// a configuration for dependencies that need exploding into package -configurations { - explode -} - -configurations.all { - // using log4j1 to log4j2 bridge so need to exclude log4j1 - exclude group: 'log4j', module: 'log4j' - // exclude all other slf4j bindings that are transitively pulled in - exclude group: 'org.slf4j', module: 'slf4j-log4j12' -} - -dependencies { - compile(group: 'org.codehaus.jackson', name: 'jackson-jaxrs', version: '1.9.13') - compile(group: 'org.slf4j', name: 'slf4j-api', version: "$SLF4J_VERSION") - compile(group: 'org.schwering', name: 'irclib', version: '1.10') - compile(group: 'org.apache.samza', name: 'samza-api', version: "$SAMZA_VERSION") - compile(group: 'org.apache.samza', name: 'samza-aws_2.11', version: "$SAMZA_VERSION") - compile(group: 'org.apache.samza', name: 'samza-azure_2.11', version: "$SAMZA_VERSION") - compile(group: 'org.apache.samza', name: 'samza-kafka_2.11', version: "$SAMZA_VERSION") - compile(group: 'org.apache.samza', name: 'samza-kv_2.11', version: "$SAMZA_VERSION") - compile(group: 'org.apache.samza', name: 'samza-kv-couchbase_2.11', version: "$SAMZA_VERSION") - compile(group: 'org.apache.samza', name: 'samza-kv-rocksdb_2.11', version: "$SAMZA_VERSION") - explode(group: 'org.apache.samza', name: 'samza-shell', ext: 'tgz', classifier: 'dist', version: "$SAMZA_VERSION") - runtime(group: 'org.apache.samza', name: 'samza-core_2.11', version: "$SAMZA_VERSION") - runtime(group: 'org.apache.samza', name: 'samza-log4j2_2.11', version: "$SAMZA_VERSION") - runtime(group: 'org.apache.samza', name: 'samza-shell', version: "$SAMZA_VERSION") - runtime(group: 'org.apache.samza', name: 'samza-yarn_2.11', version: "$SAMZA_VERSION") - runtime(group: 'org.apache.kafka', name: 'kafka_2.11', version: "$KAFKA_VERSION") - runtime(group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: "$HADOOP_VERSION") - testCompile(group: 'org.apache.samza', name: 'samza-test_2.11', version: "$SAMZA_VERSION") - testCompile(group: 'junit', name: 'junit', version: "4.12") -} - -// make the samza distribution .tgz file -task distTar(dependsOn: build, type: Tar) { - destinationDir(new File(project.buildDir, "/distributions")) - compression(Compression.GZIP) - classifier('dist') - extension('tar.gz') - into("config") { - from("src/main/config") { - include "wikipedia-feed.properties" - include "wikipedia-parser.properties" - include "wikipedia-stats.properties" - include "wikipedia-application.properties" - - // expand the Maven tokens with Gradle equivalents. Also change 'target' (Maven) to 'build/distributions' (Gradle) - filter { String line -> - line.replaceAll('[\$][{]basedir[}]', project.projectDir.toString()).replaceAll('[\$][{]project.artifactId[}]', project.name.toString()).replaceAll('/target/', '/build/distributions/').replaceAll('[\$][{]pom.version[}]', version) - } - } - } - - into("bin") { - from { - configurations.explode.collect { tarTree(it) } - } - } - - into("lib") { - from configurations.runtime - from configurations.runtime.artifacts.files - from("src/main/resources/") { - include "log4j2.xml" - } - } -} - -// install everything -task installGrid(type: Exec) { - workingDir(project.projectDir) - commandLine("bin/grid", "install", "all") - outputs.upToDateWhen { - ["kafka", "zookeeper", "yarn"].every { - (new File(project.projectDir, "deploy/" + it)).exists() - } - } -} - -// update the Samza job -task deployHelloSamza(dependsOn: [distTar, installGrid], type: Sync) { - into(new File(project.projectDir, "/deploy/samza")) - from(tarTree(distTar.archivePath)) -} - - -// run everything -task startGrid(type: Exec) { - workingDir(project.projectDir) - commandLine("bin/grid", "start", "all") - outputs.upToDateWhen { - // use running zookeeper as proxy - File zookeeperPidFile = new File("/tmp/zookeeper/zookeeper_server.pid") - zookeeperPidFile.exists() && - "kill -0 ${zookeeperPidFile.text}".execute().waitFor() == 0 - } -} - -// stop everything -task stopGrid(type: Exec) { - workingDir(project.projectDir) - commandLine("bin/grid", "stop", "all") -} - -// -// Samza helpers -// - -// helper task to run Samza jobs -class SamzaTask extends DefaultTask { - String configFile; - - @TaskAction - def startSamza() { - project.exec { - workingDir(project.projectDir) - commandLine("deploy/samza/bin/run-app.sh", - "--config-path=${project.projectDir}/deploy/samza/config/${configFile}") - - } - } -} - -// helper to run the Samza job to retrieve Wikipedia data -task runWikiFeed(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) { - configFile("wikipedia-feed.properties") -} - -// helper to run the Samza job process Wikipedia data -task runWikiParser(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) { - configFile("wikipedia-parser.properties") -} - -// helper to run the Samza job to summarize stats on Wikipedia edits -task runWikiStats(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) { - configFile("wikipedia-stats.properties") -} - - -// -// Kafka helpers -// - -// show all Kafka topics -task listKafkaTopics(type: Exec) { - workingDir(project.projectDir) - commandLine("deploy/kafka/bin/kafka-topics.sh", - "--zookeeper", "localhost:2181", - "--list") -} - -// helper task to monitor a Kafka topic -class KafkaDumpTask extends DefaultTask { - String topic; - - @TaskAction - def dumpTopic() { - project.exec { - workingDir(project.projectDir) - commandLine("deploy/kafka/bin/kafka-console-consumer.sh", - "--zookeeper", "localhost:2181", - "--topic", "${topic}") - } - } -} - -// helper to dump the wikipedia-raw topic -task dumpWikiRaw(dependsOn: startGrid, type: KafkaDumpTask) { - topic("wikipedia-raw") -} - -// helper to dump the wikipedia-edits topic -task dumpWikiEdits(dependsOn: startGrid, type: KafkaDumpTask) { - topic("wikipedia-edits") -} - -// helper to dump the wikipedia-stats topic -task dumpWikiStats(dependsOn: startGrid, type: KafkaDumpTask) { - topic("wikipedia-stats") -} diff --git a/gradle.properties b/gradle.properties deleted file mode 100644 index 6435c330..00000000 --- a/gradle.properties +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -SAMZA_VERSION=1.6.0 -KAFKA_VERSION=0.11.0.2 -HADOOP_VERSION=2.7.1 - -SLF4J_VERSION = 1.7.7 - diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index fd7e590e..00000000 Binary files a/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index c91c2cc7..00000000 --- a/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,6 +0,0 @@ -#Fri Mar 27 16:28:33 PDT 2020 -distributionUrl=https\://services.gradle.org/distributions/gradle-2.3-all.zip -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -zipStorePath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew deleted file mode 100755 index 91a7e269..00000000 --- a/gradlew +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env bash - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn ( ) { - echo "$*" -} - -die ( ) { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; -esac - -# For Cygwin, ensure paths are in UNIX format before anything is touched. -if $cygwin ; then - [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` -fi - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >&- -APP_HOME="`pwd -P`" -cd "$SAVED" >&- - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules -function splitJvmOpts() { - JVM_OPTS=("$@") -} -eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS -JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" - -exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/gradlew.bat b/gradlew.bat deleted file mode 100644 index aec99730..00000000 --- a/gradlew.bat +++ /dev/null @@ -1,90 +0,0 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windowz variants - -if not "%OS%" == "Windows_NT" goto win9xME_args -if "%@eval[2+2]" == "4" goto 4NT_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* -goto execute - -:4NT_args -@rem Get arguments from the 4NT Shell from JP Software -set CMD_LINE_ARGS=%$ - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/pom.xml b/pom.xml index d7d83cac..df8d58ee 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ under the License. org.apache.samza hello-samza - 1.6.0 + 1.8.0 jar Samza Example @@ -48,7 +48,7 @@ under the License. org.apache.samza - samza-azure_2.11 + samza-azure_${samza.scala.version} ${samza.version} @@ -59,7 +59,7 @@ under the License. org.apache.samza - samza-core_2.11 + samza-core_${samza.scala.version} ${samza.version} @@ -75,7 +75,7 @@ under the License. org.apache.samza - samza-test_2.11 + samza-test_${samza.scala.version} ${samza.version} @@ -106,7 +106,7 @@ under the License. org.apache.samza - samza-log4j2_2.11 + samza-log4j2_${samza.scala.version} ${samza.version} @@ -118,7 +118,7 @@ under the License. org.apache.samza - samza-yarn_2.11 + samza-yarn3_${samza.scala.version} ${samza.version} @@ -134,22 +134,22 @@ under the License. org.apache.samza - samza-kv_2.11 + samza-kv_${samza.scala.version} ${samza.version} org.apache.samza - samza-kv-rocksdb_2.11 + samza-kv-rocksdb_${samza.scala.version} ${samza.version} org.apache.samza - samza-kafka_2.11 + samza-kafka_${samza.scala.version} ${samza.version} org.apache.samza - samza-aws_2.11 + samza-aws_${samza.scala.version} ${samza.version} @@ -163,9 +163,9 @@ under the License. 1.6.2 - org.codehaus.jackson - jackson-jaxrs - 1.9.13 + com.fasterxml.jackson.core + jackson-core + 2.12.2 org.apache.httpcomponents @@ -294,8 +294,9 @@ under the License. UTF-8 - 1.6.0 - 2.6.1 + 1.8.0 + 2.12 + 3.3.4 diff --git a/src/main/java/samza/examples/cookbook/CouchbaseTableExample.java b/src/main/java/samza/examples/cookbook/CouchbaseTableExample.java index fdc5bc6c..33ccea69 100644 --- a/src/main/java/samza/examples/cookbook/CouchbaseTableExample.java +++ b/src/main/java/samza/examples/cookbook/CouchbaseTableExample.java @@ -41,6 +41,7 @@ import org.apache.samza.system.kafka.descriptors.KafkaInputDescriptor; import org.apache.samza.system.kafka.descriptors.KafkaOutputDescriptor; import org.apache.samza.system.kafka.descriptors.KafkaSystemDescriptor; +import org.apache.samza.table.ReadWriteTable; import org.apache.samza.table.descriptors.RemoteTableDescriptor; import org.apache.samza.table.remote.NoOpTableReadFunction; import org.apache.samza.table.remote.RemoteTable; @@ -194,7 +195,8 @@ static class MyCountFunction implements MapFunction { @Override public void init(Context context) { - RemoteTable table = (RemoteTable) context.getTaskContext().getTable("couchbase-table"); + context.getTaskContext().getTable("couchbase-table"); + RemoteTable table = (RemoteTable) context.getTaskContext().getUpdatableTable("couchbase-table"); writeFn = (MyCouchbaseTableWriteFunction) table.getWriteFunction(); } diff --git a/src/main/java/samza/examples/cookbook/RemoteTableJoinExample.java b/src/main/java/samza/examples/cookbook/RemoteTableJoinExample.java index 4f5c5f77..06805442 100644 --- a/src/main/java/samza/examples/cookbook/RemoteTableJoinExample.java +++ b/src/main/java/samza/examples/cookbook/RemoteTableJoinExample.java @@ -128,7 +128,7 @@ public void describe(StreamApplicationDescriptor appDescriptor) { MessageStream stockSymbolStream = appDescriptor.getInputStream(stockSymbolInputDescriptor); OutputStream stockPriceStream = appDescriptor.getOutputStream(stockPriceOutputDescriptor); - RemoteTableDescriptor remoteTableDescriptor = + RemoteTableDescriptor remoteTableDescriptor = new RemoteTableDescriptor("remote-table") .withReadRateLimit(10) .withReadFunction(new StockPriceReadFunction()); diff --git a/src/main/java/samza/examples/cookbook/data/AdClick.java b/src/main/java/samza/examples/cookbook/data/AdClick.java index 82925c71..7345651b 100644 --- a/src/main/java/samza/examples/cookbook/data/AdClick.java +++ b/src/main/java/samza/examples/cookbook/data/AdClick.java @@ -30,6 +30,9 @@ public class AdClick { private String adId; // an unique id for the ad private String userId; // the user that clicked the ad + public AdClick() { + } + public AdClick( @JsonProperty("pageId") String pageId, @JsonProperty("adId") String adId, diff --git a/src/main/java/samza/examples/cookbook/data/PageView.java b/src/main/java/samza/examples/cookbook/data/PageView.java index 96406944..c82bb5d2 100644 --- a/src/main/java/samza/examples/cookbook/data/PageView.java +++ b/src/main/java/samza/examples/cookbook/data/PageView.java @@ -24,9 +24,12 @@ * A page view event */ public class PageView { - public final String userId; - public final String country; - public final String pageId; + public String userId; + public String country; + public String pageId; + + public PageView() { + } /** * Constructs a page view event. diff --git a/src/main/java/samza/examples/wikipedia/system/WikipediaFeed.java b/src/main/java/samza/examples/wikipedia/system/WikipediaFeed.java index 16e302e3..3238cbf0 100644 --- a/src/main/java/samza/examples/wikipedia/system/WikipediaFeed.java +++ b/src/main/java/samza/examples/wikipedia/system/WikipediaFeed.java @@ -19,14 +19,9 @@ package samza.examples.wikipedia.system; -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Random; -import java.util.Set; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.paranamer.ParanamerModule; import org.apache.samza.SamzaException; -import org.codehaus.jackson.map.ObjectMapper; import org.schwering.irc.lib.IRCConnection; import org.schwering.irc.lib.IRCEventListener; import org.schwering.irc.lib.IRCModeParser; @@ -34,10 +29,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Random; +import java.util.Set; + public class WikipediaFeed { private static final Logger log = LoggerFactory.getLogger(WikipediaFeed.class); private static final Random random = new Random(); private static final ObjectMapper jsonMapper = new ObjectMapper(); + static { + jsonMapper.registerModule(new ParanamerModule()); + } private final Map> channelListeners; private final String host; diff --git a/src/test/java/samza/examples/test/utils/TestUtils.java b/src/test/java/samza/examples/test/utils/TestUtils.java index d5e957ee..777ad382 100644 --- a/src/test/java/samza/examples/test/utils/TestUtils.java +++ b/src/test/java/samza/examples/test/utils/TestUtils.java @@ -19,6 +19,8 @@ package samza.examples.test.utils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.paranamer.ParanamerModule; import com.google.common.io.Resources; import java.io.BufferedReader; import java.io.IOException; @@ -28,7 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.stream.Collectors; -import org.codehaus.jackson.map.ObjectMapper; + import samza.examples.cookbook.data.PageView; import samza.examples.wikipedia.application.WikipediaApplication; @@ -53,6 +55,7 @@ public static List genWikipediaFeedEvents(String channel) { break; } ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new ParanamerModule()); return wikiEvents.stream().map(event -> { try { return new WikipediaFeedEvent(mapper.readValue(event, HashMap.class)); diff --git a/src/test/java/samza/examples/wikipedia/task/test/TestWikipediaTask.java b/src/test/java/samza/examples/wikipedia/task/test/TestWikipediaTask.java index 0fc992ad..66cbd9a8 100644 --- a/src/test/java/samza/examples/wikipedia/task/test/TestWikipediaTask.java +++ b/src/test/java/samza/examples/wikipedia/task/test/TestWikipediaTask.java @@ -24,12 +24,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.paranamer.ParanamerModule; import org.apache.samza.serializers.NoOpSerde; import org.apache.samza.test.framework.TestRunner; import org.apache.samza.test.framework.system.descriptors.InMemoryInputDescriptor; import org.apache.samza.test.framework.system.descriptors.InMemoryOutputDescriptor; import org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor; -import org.codehaus.jackson.map.ObjectMapper; + import org.junit.Assert; import org.junit.Test; import samza.examples.wikipedia.system.WikipediaFeed.WikipediaFeedEvent; @@ -62,6 +65,7 @@ public void testWikipediaFeedTask() throws Exception { public static List> parseJSONToMap(String[] lines) throws Exception{ List> wikiRawEvents = new ArrayList<>(); ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new ParanamerModule()); for (String line : lines) { wikiRawEvents.add(mapper.readValue(line, HashMap.class)); }