From 7a1ace94292bd788bd2a0a4fd470c9584629a1d1 Mon Sep 17 00:00:00 2001 From: Hemant Bhanawat Date: Sat, 16 Apr 2016 23:43:32 -0700 Subject: [PATCH 001/142] [SPARK-13904][SCHEDULER] Add support for pluggable cluster manager This commit adds support for pluggable cluster manager. And also allows a cluster manager to clean up tasks without taking the parent process down. To plug a new external cluster manager, ExternalClusterManager trait should be implemented. It returns task scheduler and backend scheduler that will be used by SparkContext to schedule tasks. An external cluster manager is registered using the java.util.ServiceLoader mechanism (This mechanism is also being used to register data sources like parquet, json, jdbc etc.). This allows auto-loading implementations of ExternalClusterManager interface. Currently, when a driver fails, executors exit using system.exit. This does not bode well for cluster managers that would like to reuse the parent process of an executor. Hence, 1. Moving system.exit to a function that can be overriden in subclasses of CoarseGrainedExecutorBackend. 2. Added functionality of killing all the running tasks in an executor. ExternalClusterManagerSuite.scala was added to test this patch. Author: Hemant Bhanawat Closes #11723 from hbhanawat/pluggableScheduler. Conflicts: core/src/main/scala/org/apache/spark/SparkContext.scala core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala core/src/main/scala/org/apache/spark/executor/Executor.scala core/src/main/scala/org/apache/spark/scheduler/ExternalClusterManager.scala core/src/test/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala dev/.rat-excludes --- .../CoarseGrainedExecutorBackend.scala | 16 +++++--- .../org/apache/spark/executor/Executor.scala | 15 +++++++ .../scheduler/ExternalClusterManager.scala | 9 +++-- .../ExternalClusterManagerSuite.scala | 40 +++++-------------- dev/.rat-excludes | 1 + 5 files changed, 42 insertions(+), 39 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala index 9b62e4b1b7150..8cc65f6e4efa7 100644 --- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala +++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala @@ -66,7 +66,8 @@ private[spark] class CoarseGrainedExecutorBackend( case Success(msg) => // Always receive `true`. Just ignore it case Failure(e) => - exitExecutor(1, s"Cannot register with driver: $driverUrl", e, notifyDriver = false) + logError(s"Cannot register with driver: $driverUrl", e) + exitExecutor() }(ThreadUtils.sameThread) } @@ -87,11 +88,13 @@ private[spark] class CoarseGrainedExecutorBackend( } case RegisterExecutorFailed(message) => - exitExecutor(1, "Slave registration failed: " + message) + logError("Slave registration failed: " + message) + exitExecutor() case LaunchTask(data) => if (executor == null) { - exitExecutor(1, "Received LaunchTask command but executor was null") + logError("Received LaunchTask command but executor was null") + exitExecutor() } else { val taskDesc = TaskDescription.decode(data.value) logInfo("Got assigned task " + taskDesc.taskId) @@ -100,7 +103,8 @@ private[spark] class CoarseGrainedExecutorBackend( case KillTask(taskId, _, interruptThread, reason) => if (executor == null) { - exitExecutor(1, "Received KillTask command but executor was null") + logError("Received KillTask command but executor was null") + exitExecutor() } else { executor.killTask(taskId, interruptThread, reason) } @@ -133,8 +137,8 @@ private[spark] class CoarseGrainedExecutorBackend( if (stopping.get()) { logInfo(s"Driver from $remoteAddress disconnected during shutdown") } else if (driver.exists(_.address == remoteAddress)) { - exitExecutor(1, s"Driver $remoteAddress disassociated! Shutting down.", null, - notifyDriver = false) + logError(s"Driver $remoteAddress disassociated! Shutting down.") + exitExecutor() } else { logWarning(s"An unknown ($remoteAddress) driver disconnected.") } diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 2c3a8ef74800b..05ec6565dfe25 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -214,6 +214,21 @@ private[spark] class Executor( killTask(t, interruptThread = interruptThread, reason = reason)) } + /** + * Function to kill the running tasks in an executor. + * This can be called by executor back-ends to kill the + * tasks instead of taking the JVM down. + * @param interruptThread whether to interrupt the task thread + */ + def killAllTasks(interruptThread: Boolean) : Unit = { + // kill all the running tasks + for (taskRunner <- runningTasks.values().asScala) { + if (taskRunner != null) { + taskRunner.kill(interruptThread) + } + } + } + def stop(): Unit = { env.metricsSystem.report() heartbeater.shutdown() diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExternalClusterManager.scala b/core/src/main/scala/org/apache/spark/scheduler/ExternalClusterManager.scala index 47f3527a32c01..bd9a3bf67d190 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ExternalClusterManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ExternalClusterManager.scala @@ -18,11 +18,14 @@ package org.apache.spark.scheduler import org.apache.spark.SparkContext +import org.apache.spark.annotation.DeveloperApi /** + * :: DeveloperApi :: * A cluster manager interface to plugin external scheduler. */ -private[spark] trait ExternalClusterManager { +@DeveloperApi +trait ExternalClusterManager { /** * Check if this cluster manager instance can create scheduler components @@ -42,7 +45,7 @@ private[spark] trait ExternalClusterManager { /** * Create a scheduler backend for the given SparkContext and scheduler. This is - * called after task scheduler is created using `ExternalClusterManager.createTaskScheduler()`. + * called after task scheduler is created using [[ExternalClusterManager.createTaskScheduler()]]. * @param sc SparkContext * @param masterURL the master URL * @param scheduler TaskScheduler that will be used with the scheduler backend. @@ -59,4 +62,4 @@ private[spark] trait ExternalClusterManager { * @param backend SchedulerBackend that works with a TaskScheduler */ def initialize(scheduler: TaskScheduler, backend: SchedulerBackend): Unit -} +} \ No newline at end of file diff --git a/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala index a4e4ea7cd2894..c57a783f2f99f 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala @@ -20,31 +20,19 @@ package org.apache.spark.scheduler import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite} import org.apache.spark.scheduler.SchedulingMode.SchedulingMode import org.apache.spark.storage.BlockManagerId -import org.apache.spark.util.AccumulatorV2 -class ExternalClusterManagerSuite extends SparkFunSuite with LocalSparkContext { +class ExternalClusterManagerSuite extends SparkFunSuite with LocalSparkContext +{ test("launch of backend and scheduler") { val conf = new SparkConf().setMaster("myclusterManager"). setAppName("testcm").set("spark.driver.allowMultipleContexts", "true") sc = new SparkContext(conf) - // check if the scheduler components are created and initialized - sc.schedulerBackend match { - case dummy: DummySchedulerBackend => assert(dummy.initialized) - case other => fail(s"wrong scheduler backend: ${other}") - } - sc.taskScheduler match { - case dummy: DummyTaskScheduler => assert(dummy.initialized) - case other => fail(s"wrong task scheduler: ${other}") - } + // check if the scheduler components are created + assert(sc.schedulerBackend.isInstanceOf[DummySchedulerBackend]) + assert(sc.taskScheduler.isInstanceOf[DummyTaskScheduler]) } } -/** - * Super basic ExternalClusterManager, just to verify ExternalClusterManagers can be configured. - * - * Note that if you want a special ClusterManager for tests, you are probably much more interested - * in [[MockExternalClusterManager]] and the corresponding [[SchedulerIntegrationSuite]] - */ private class DummyExternalClusterManager extends ExternalClusterManager { def canCreate(masterURL: String): Boolean = masterURL == "myclusterManager" @@ -56,15 +44,11 @@ private class DummyExternalClusterManager extends ExternalClusterManager { masterURL: String, scheduler: TaskScheduler): SchedulerBackend = new DummySchedulerBackend() - def initialize(scheduler: TaskScheduler, backend: SchedulerBackend): Unit = { - scheduler.asInstanceOf[DummyTaskScheduler].initialized = true - backend.asInstanceOf[DummySchedulerBackend].initialized = true - } + def initialize(scheduler: TaskScheduler, backend: SchedulerBackend): Unit = {} } private class DummySchedulerBackend extends SchedulerBackend { - var initialized = false def start() {} def stop() {} def reviveOffers() {} @@ -72,22 +56,18 @@ private class DummySchedulerBackend extends SchedulerBackend { } private class DummyTaskScheduler extends TaskScheduler { - var initialized = false - override def schedulingMode: SchedulingMode = SchedulingMode.FIFO - override def rootPool: Pool = new Pool("", schedulingMode, 0, 0) + override def rootPool: Pool = null + override def schedulingMode: SchedulingMode = SchedulingMode.NONE override def start(): Unit = {} override def stop(): Unit = {} override def submitTasks(taskSet: TaskSet): Unit = {} override def cancelTasks(stageId: Int, interruptThread: Boolean): Unit = {} - override def killTaskAttempt( - taskId: Long, interruptThread: Boolean, reason: String): Boolean = false override def setDAGScheduler(dagScheduler: DAGScheduler): Unit = {} override def defaultParallelism(): Int = 2 override def executorLost(executorId: String, reason: ExecutorLossReason): Unit = {} - override def workerRemoved(workerId: String, host: String, message: String): Unit = {} override def applicationAttemptId(): Option[String] = None def executorHeartbeatReceived( execId: String, - accumUpdates: Array[(Long, Seq[AccumulatorV2[_, _]])], + accumUpdates: Array[(Long, Seq[AccumulableInfo])], blockManagerId: BlockManagerId): Boolean = true -} +} \ No newline at end of file diff --git a/dev/.rat-excludes b/dev/.rat-excludes index 607234b4068d0..1043aed01d9a4 100644 --- a/dev/.rat-excludes +++ b/dev/.rat-excludes @@ -103,3 +103,4 @@ META-INF/* spark-warehouse structured-streaming/* kafka-source-initial-offset-version-2.1.0.bin +org.apache.spark.scheduler.ExternalClusterManager From 06eca13c64d1ce5e9ad44d556bc7ee21ed40eef9 Mon Sep 17 00:00:00 2001 From: Hemant Bhanawat Date: Wed, 27 Apr 2016 10:59:23 -0700 Subject: [PATCH 002/142] [SPARK-14729][SCHEDULER] Refactored YARN scheduler creation code to use newly added ExternalClusterManager With the addition of ExternalClusterManager(ECM) interface in PR #11723, any cluster manager can now be integrated with Spark. It was suggested in ExternalClusterManager PR that one of the existing cluster managers should start using the new interface to ensure that the API is correct. Ideally, all the existing cluster managers should eventually use the ECM interface but as a first step yarn will now use the ECM interface. This PR refactors YARN code from SparkContext.createTaskScheduler function into YarnClusterManager that implements ECM interface. Since this is refactoring, no new tests has been added. Existing tests have been run. Basic manual testing with YARN was done too. Author: Hemant Bhanawat Closes #12641 from hbhanawat/yarnClusterMgr. Conflicts: core/src/main/scala/org/apache/spark/SparkContext.scala core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala --- .../scala/org/apache/spark/SparkContext.scala | 64 +++++++++++++++++++ .../SparkContextSchedulerCreationSuite.scala | 30 +++++++++ ...che.spark.scheduler.ExternalClusterManager | 1 + .../cluster/YarnClusterManager.scala | 56 ++++++++++++++++ 4 files changed, 151 insertions(+) create mode 100644 yarn/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager create mode 100644 yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 3828d4f703247..c44faa5a519ab 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -2731,6 +2731,70 @@ object SparkContext extends Logging { } (backend, scheduler) + case "yarn" if deployMode == "cluster" => + val scheduler = try { + val clazz = Utils.classForName("org.apache.spark.scheduler.cluster.YarnClusterScheduler") + val cons = clazz.getConstructor(classOf[SparkContext]) + cons.newInstance(sc).asInstanceOf[TaskSchedulerImpl] + } catch { + // TODO: Enumerate the exact reasons why it can fail + // But irrespective of it, it means we cannot proceed ! + case e: Exception => + throw new SparkException("YARN mode not available ?", e) + } + val backend = try { + val clazz = + Utils.classForName("org.apache.spark.scheduler.cluster.YarnClusterSchedulerBackend") + val cons = clazz.getConstructor(classOf[TaskSchedulerImpl], classOf[SparkContext]) + cons.newInstance(scheduler, sc).asInstanceOf[CoarseGrainedSchedulerBackend] + } catch { + case e: Exception => + throw new SparkException("YARN mode not available ?", e) + } + scheduler.initialize(backend) + (backend, scheduler) + + case "yarn" if deployMode == "client" => + val scheduler = try { + val clazz = Utils.classForName("org.apache.spark.scheduler.cluster.YarnScheduler") + val cons = clazz.getConstructor(classOf[SparkContext]) + cons.newInstance(sc).asInstanceOf[TaskSchedulerImpl] + + } catch { + case e: Exception => + throw new SparkException("YARN mode not available ?", e) + } + + val backend = try { + val clazz = + Utils.classForName("org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend") + val cons = clazz.getConstructor(classOf[TaskSchedulerImpl], classOf[SparkContext]) + cons.newInstance(scheduler, sc).asInstanceOf[CoarseGrainedSchedulerBackend] + } catch { + case e: Exception => + throw new SparkException("YARN mode not available ?", e) + } + + scheduler.initialize(backend) + (backend, scheduler) + + case MESOS_REGEX(mesosUrl) => + MesosNativeLibrary.load() + val scheduler = new TaskSchedulerImpl(sc) + val coarseGrained = sc.conf.getBoolean("spark.mesos.coarse", defaultValue = true) + val backend = if (coarseGrained) { + new CoarseMesosSchedulerBackend(scheduler, sc, mesosUrl, sc.env.securityManager) + } else { + new MesosSchedulerBackend(scheduler, sc, mesosUrl) + } + scheduler.initialize(backend) + (backend, scheduler) + + case zkUrl if zkUrl.startsWith("zk://") => + logWarning("Master URL for a multi-master Mesos cluster managed by ZooKeeper should be " + + "in the form mesos://zk://host:port. Current Master URL will stop working in Spark 2.0.") + createTaskScheduler(sc, "mesos://" + zkUrl, deployMode) + case masterUrl => val cm = getClusterManager(masterUrl) match { case Some(clusterMgr) => clusterMgr diff --git a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala index f8938dfedee5b..43be36966d922 100644 --- a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala @@ -129,4 +129,34 @@ class SparkContextSchedulerCreationSuite case _ => fail() } } + + def testMesos(master: String, expectedClass: Class[_], coarse: Boolean) { + val conf = new SparkConf().set("spark.mesos.coarse", coarse.toString) + try { + val sched = createTaskScheduler(master, "client", conf) + assert(sched.backend.getClass === expectedClass) + } catch { + case e: UnsatisfiedLinkError => + assert(e.getMessage.contains("mesos")) + logWarning("Mesos not available, could not test actual Mesos scheduler creation") + case e: Throwable => fail(e) + } + } + + test("mesos fine-grained") { + testMesos("mesos://localhost:1234", classOf[MesosSchedulerBackend], coarse = false) + } + + test("mesos coarse-grained") { + testMesos("mesos://localhost:1234", classOf[CoarseMesosSchedulerBackend], coarse = true) + } + + test("mesos with zookeeper") { + testMesos("mesos://zk://localhost:1234,localhost:2345", + classOf[MesosSchedulerBackend], coarse = false) + } + + test("mesos with zookeeper and Master URL starting with zk://") { + testMesos("zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false) + } } diff --git a/yarn/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager b/yarn/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager new file mode 100644 index 0000000000000..6e8a1ebfc61da --- /dev/null +++ b/yarn/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager @@ -0,0 +1 @@ +org.apache.spark.scheduler.cluster.YarnClusterManager diff --git a/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala new file mode 100644 index 0000000000000..64cd1bd088001 --- /dev/null +++ b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.scheduler.cluster + +import org.apache.spark.{SparkContext, SparkException} +import org.apache.spark.scheduler.{ExternalClusterManager, SchedulerBackend, TaskScheduler, TaskSchedulerImpl} + +/** + * Cluster Manager for creation of Yarn scheduler and backend + */ +private[spark] class YarnClusterManager extends ExternalClusterManager { + + override def canCreate(masterURL: String): Boolean = { + masterURL == "yarn" + } + + override def createTaskScheduler(sc: SparkContext, masterURL: String): TaskScheduler = { + sc.deployMode match { + case "cluster" => new YarnClusterScheduler(sc) + case "client" => new YarnScheduler(sc) + case _ => throw new SparkException(s"Unknown deploy mode '${sc.deployMode}' for Yarn") + } + } + + override def createSchedulerBackend(sc: SparkContext, + masterURL: String, + scheduler: TaskScheduler): SchedulerBackend = { + sc.deployMode match { + case "cluster" => + new YarnClusterSchedulerBackend(scheduler.asInstanceOf[TaskSchedulerImpl], sc) + case "client" => + new YarnClientSchedulerBackend(scheduler.asInstanceOf[TaskSchedulerImpl], sc) + case _ => + throw new SparkException(s"Unknown deploy mode '${sc.deployMode}' for Yarn") + } + } + + override def initialize(scheduler: TaskScheduler, backend: SchedulerBackend): Unit = { + scheduler.asInstanceOf[TaskSchedulerImpl].initialize(backend) + } +} From e9f80e697711fb78de85b7ddf03c2cf17f4bd515 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 21 Nov 2015 22:18:18 +0530 Subject: [PATCH 003/142] [SNAPPYDATA] increasing visibility of SparkContext.activeContext --- core/src/main/scala/org/apache/spark/SparkContext.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index c44faa5a519ab..1dcbb87405f3f 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -2419,7 +2419,7 @@ object SparkContext extends Logging { * * Access to this field is guarded by SPARK_CONTEXT_CONSTRUCTOR_LOCK. */ - private val activeContext: AtomicReference[SparkContext] = + private[spark] val activeContext: AtomicReference[SparkContext] = new AtomicReference[SparkContext](null) /** From 009ab91bcfd98d6f94f182d9cff5e0750e7db7b2 Mon Sep 17 00:00:00 2001 From: Neeraj Kumar Date: Wed, 30 Dec 2015 19:17:11 +0530 Subject: [PATCH 004/142] [SNAPPYDATA] add SnappyData's modification headers in updated files --- .../scala/org/apache/spark/SparkContext.scala | 18 ++++++++++++++++++ .../spark/scheduler/TaskSetManager.scala | 18 ++++++++++++++++++ .../apache/spark/sql/AnalysisException.scala | 18 ++++++++++++++++++ .../catalyst/plans/physical/partitioning.scala | 18 ++++++++++++++++++ .../spark/sql/hive/thriftserver/CliSuite.scala | 18 ++++++++++++++++++ .../thriftserver/HiveThriftServer2Suites.scala | 18 ++++++++++++++++++ .../spark/sql/hive/HiveSparkSubmitSuite.scala | 18 ++++++++++++++++++ .../spark/sql/hive/StatisticsSuite.scala | 18 ++++++++++++++++++ 8 files changed, 144 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 1dcbb87405f3f..fcc5fce4bb798 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index c3ed11bfe352a..5149cbcdfb3c0 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.scheduler diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 50ee6cd4085ea..741d2aaa3061d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index 4d9a9925fe3ff..77b40e6452880 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.catalyst.plans.physical diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 933fd7369380a..521facc027226 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.hive.thriftserver diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 496f8c82a6c61..b445cc359049f 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.hive.thriftserver diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index 21b3e281490cf..fd7f1a535b0a9 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.hive diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala index 3af8af0814bb4..35f13a0626666 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.hive From 8998a02fd97435d3187a7eab7938db16a0052987 Mon Sep 17 00:00:00 2001 From: Soubhik Chakraborty Date: Wed, 13 Jan 2016 03:09:56 +0530 Subject: [PATCH 005/142] [SNAP-404] Address #comment about increasing decimal precision Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala --- .../main/scala/org/apache/spark/sql/types/DecimalType.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala index ef3b67c0d48d0..fbe24995dc51b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala @@ -118,6 +118,10 @@ object DecimalType extends AbstractDataType { val SYSTEM_DEFAULT: DecimalType = DecimalType(MAX_PRECISION, 18) val USER_DEFAULT: DecimalType = DecimalType(10, 0) val MINIMUM_ADJUSTED_SCALE = 6 +// val MAX_PRECISION = 127 +// val MAX_SCALE = 63 +// val SYSTEM_DEFAULT: DecimalType = DecimalType(38, 18) +// val USER_DEFAULT: DecimalType = DecimalType(38, 18) // The decimal types compatible with other numeric types private[sql] val ByteDecimal = DecimalType(3, 0) From cc40bc967be18eab378d1ffddbc0135743631d3c Mon Sep 17 00:00:00 2001 From: SachinJanani Date: Thu, 14 Jul 2016 16:55:06 +0530 Subject: [PATCH 006/142] [SNAP-860] Removed hardcoding of size of Array used for storing DecimalType (#38) Created a jira for -Pspark precheckin SNAP-914 (https://jira.snappydata.io/browse/SNAP-914) --- .../main/scala/org/apache/spark/sql/types/DecimalType.scala | 4 ++-- .../sql/execution/datasources/json/JsonInferSchema.scala | 4 ++-- .../datasources/parquet/ParquetSchemaConverter.scala | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala index fbe24995dc51b..5ad10d9cbf9b9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} * A Decimal that must have fixed precision (the maximum number of digits) and scale (the number * of digits on right side of dot). * - * The precision can be up to 38, scale can also be up to 38 (less or equal to precision). + * The precision can be up to 127, scale can also be up to 127 (less or equal to precision). * * The default precision and scale is (10, 0). * @@ -48,7 +48,7 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType { } if (precision > DecimalType.MAX_PRECISION) { - throw new AnalysisException(s"DecimalType can only support precision up to 38") + throw new AnalysisException(s"DecimalType can only support precision up to ${DecimalType.MAX_PRECISION}") } // default constructor for Java diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala index a270a6451d5dd..1c1b9f0520179 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala @@ -279,8 +279,8 @@ private[sql] object JsonInferSchema { case (t1: DecimalType, t2: DecimalType) => val scale = math.max(t1.scale, t2.scale) val range = math.max(t1.precision - t1.scale, t2.precision - t2.scale) - if (range + scale > 38) { - // DecimalType can't support precision > 38 + if (range + scale > DecimalType.MAX_PRECISION) { + // DecimalType can't support precision > DecimalType.MAX_PRECISION DoubleType } else { DecimalType(range + scale, scale) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala index c61be077d309f..4f64005852110 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala @@ -594,7 +594,7 @@ private[sql] object ParquetSchemaConverter { } // Returns the minimum number of bytes needed to store a decimal with a given `precision`. - val minBytesForPrecision = Array.tabulate[Int](39)(computeMinBytesForPrecision) + val minBytesForPrecision = Array.tabulate[Int](DecimalType.MAX_PRECISION + 1)(computeMinBytesForPrecision) // Max precision of a decimal value stored in `numBytes` bytes def maxPrecisionForBytes(numBytes: Int): Int = { From 79130cb2762d260f80bfbdcf54fba92f5e80000c Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Fri, 29 Jul 2016 10:30:42 +0530 Subject: [PATCH 007/142] [SNAPPYDATA] Try hard to not schedule on others if ExecutorCacheTaskLocation is alive Conflicts: core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala --- .../spark/scheduler/TaskSetManager.scala | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 5149cbcdfb3c0..3da2116f10d97 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -351,7 +351,9 @@ private[spark] class TaskSetManager( // Check for node-local tasks if (TaskLocality.isAllowed(locality, TaskLocality.NODE_LOCAL)) { - for (index <- speculatableTasks if canRunOnHost(index)) { + for (index <- speculatableTasks if canRunOnHost(index) && + // don't return executor-local tasks that are still alive + canRunOnExecutor(execId, index)) { val locations = tasks(index).preferredLocations.map(_.host) if (locations.contains(host)) { speculatableTasks -= index @@ -374,7 +376,9 @@ private[spark] class TaskSetManager( // Check for rack-local tasks if (TaskLocality.isAllowed(locality, TaskLocality.RACK_LOCAL)) { for (rack <- sched.getRackForHost(host)) { - for (index <- speculatableTasks if canRunOnHost(index)) { + for (index <- speculatableTasks if canRunOnHost(index) + // don't return executor-local tasks that are still alive + if canRunOnExecutor(execId, index)) { val racks = tasks(index).preferredLocations.map(_.host).flatMap(sched.getRackForHost) if (racks.contains(rack)) { speculatableTasks -= index @@ -386,7 +390,9 @@ private[spark] class TaskSetManager( // Check for non-local tasks if (TaskLocality.isAllowed(locality, TaskLocality.ANY)) { - for (index <- speculatableTasks if canRunOnHost(index)) { + for (index <- speculatableTasks if canRunOnHost(index) && + // don't return executor-local tasks that are still alive + canRunOnExecutor(execId, index)) { speculatableTasks -= index return Some((index, TaskLocality.ANY)) } @@ -396,6 +402,17 @@ private[spark] class TaskSetManager( None } + private def canRunOnExecutor(execId: String, taskId: Int): Boolean = { + val locations = tasks(taskId).preferredLocations + locations.isEmpty || locations.exists { + case e: ExecutorCacheTaskLocation => execId == e.executorId + case _ => false + } || locations.collectFirst { + case e: ExecutorCacheTaskLocation if sched.isExecutorAlive(e.executorId) + && !executorIsBlacklisted(e.executorId, taskId) => false + }.getOrElse(true) + } + /** * Dequeue a pending task for a given node and return its index and locality level. * Only search for tasks matching the given locality constraint. @@ -410,7 +427,9 @@ private[spark] class TaskSetManager( } if (TaskLocality.isAllowed(maxLocality, TaskLocality.NODE_LOCAL)) { - for (index <- dequeueTaskFromList(execId, host, getPendingTasksForHost(host))) { + for (index <- dequeueTaskFromList(execId, getPendingTasksForHost(host)) + // don't return executor-local tasks that are still alive + if canRunOnExecutor(execId, index)) { return Some((index, TaskLocality.NODE_LOCAL, false)) } } @@ -425,14 +444,18 @@ private[spark] class TaskSetManager( if (TaskLocality.isAllowed(maxLocality, TaskLocality.RACK_LOCAL)) { for { rack <- sched.getRackForHost(host) - index <- dequeueTaskFromList(execId, host, getPendingTasksForRack(rack)) + index <- dequeueTaskFromList(execId, getPendingTasksForRack(rack)) + // don't return executor-local tasks that are still alive + if canRunOnExecutor(execId, index) } { - return Some((index, TaskLocality.RACK_LOCAL, false)) + return Some((index, TaskLocality.RACK_LOCAL, false)) } } if (TaskLocality.isAllowed(maxLocality, TaskLocality.ANY)) { - for (index <- dequeueTaskFromList(execId, host, allPendingTasks)) { + for (index <- dequeueTaskFromList(execId, allPendingTasks) + // don't return executor-local tasks that are still alive + if canRunOnExecutor(execId, index)) { return Some((index, TaskLocality.ANY, false)) } } From 596ac8930efcfcffd0857d3e88b2e3a9e9644980 Mon Sep 17 00:00:00 2001 From: nthanvi Date: Tue, 17 May 2016 14:43:35 +0530 Subject: [PATCH 008/142] [SNAPPYDATA] Use SnappyContext as default SQLContext on shell (#35) Conflicts: python/pyspark/shell.py --- python/pyspark/shell.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py index b5fcf7092d93a..516185f10cdce 100644 --- a/python/pyspark/shell.py +++ b/python/pyspark/shell.py @@ -31,6 +31,8 @@ from pyspark import SparkConf from pyspark.context import SparkContext from pyspark.sql import SparkSession, SQLContext +from pyspark.sql.snappy import SnappyContext +from pyspark.storagelevel import StorageLevel if os.environ.get("SPARK_EXECUTOR_URI"): SparkContext.setSystemProperty("spark.executor.uri", os.environ["SPARK_EXECUTOR_URI"]) @@ -38,6 +40,8 @@ SparkContext._ensure_initialized() try: + sqlContext = SnappyContext(sc) +except py4j.protocol.Py4JError: # Try to access HiveConf, it will raise exception if Hive is not added conf = SparkConf() if conf.get('spark.sql.catalogImplementation', 'hive').lower() == 'hive': From f5bc2d6b4c7e48a3ed001d96cc7b7f25f85914b2 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 29 Mar 2016 00:06:36 +0530 Subject: [PATCH 009/142] [SNAP-643] Increase visibility of some methods in GenerateUnsafeProjection * increase visibility of complex type write methods (that perform code generation) in GenerateUnsafeProjection to allow using from outside * allow for internal types (ArrayData, MapData, InternalRow) directly in Row->InternalRow conversions in CatalystTypeConverters for complex types Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala --- .../spark/sql/catalyst/CatalystTypeConverters.scala | 4 ++++ .../expressions/codegen/GenerateUnsafeProjection.scala | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala index 474ec592201d9..6a96aab7ad9dd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala @@ -170,6 +170,7 @@ object CatalystTypeConverters { convertedIterable += elementConverter.toCatalyst(item) } new GenericArrayData(convertedIterable.toArray) + case a: ArrayData => a } } @@ -206,6 +207,7 @@ object CatalystTypeConverters { scalaValue match { case map: Map[_, _] => ArrayBasedMapData(map, keyFunction, valueFunction) case javaMap: JavaMap[_, _] => ArrayBasedMapData(javaMap, keyFunction, valueFunction) + case m: MapData => m } } @@ -252,6 +254,8 @@ object CatalystTypeConverters { idx += 1 } new GenericInternalRow(ar) + + case row: InternalRow => row } override def toScala(row: InternalRow): Row = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala index 36ffa8dcdd2b6..396d66ba92542 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala @@ -44,7 +44,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro } // TODO: if the nullability of field is correct, we can use it to save null check. - private def writeStructToBuffer( + private[sql] def writeStructToBuffer( ctx: CodegenContext, input: String, fieldTypes: Seq[DataType], @@ -177,7 +177,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro } // TODO: if the nullability of array element is correct, we can use it to save null check. - private def writeArrayToBuffer( + private[sql] def writeArrayToBuffer( ctx: CodegenContext, input: String, elementType: DataType, @@ -213,11 +213,13 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro $arrayWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); """ - case a @ ArrayType(et, _) => + case a @ ArrayType(at, _) => s""" final int $tmpCursor = $bufferHolder.cursor; ${writeArrayToBuffer(ctx, element, et, bufferHolder)} $arrayWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); + $arrayWriter.setOffset($index); + ${writeArrayToBuffer(ctx, element, at, bufferHolder)} """ case m @ MapType(kt, vt, _) => @@ -256,7 +258,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro } // TODO: if the nullability of value element is correct, we can use it to save null check. - private def writeMapToBuffer( + private[sql] def writeMapToBuffer( ctx: CodegenContext, input: String, keyType: DataType, From 3104dd1d7ffbec98c6ede50d11fabc7a5c6e45f3 Mon Sep 17 00:00:00 2001 From: Asif Shahid Date: Wed, 27 Apr 2016 21:54:33 -0700 Subject: [PATCH 010/142] [SNAPPYDATA] Fixing sequence of expression in an option Fixing TakeOrderedAndProject that contains Sequence of Expression in an option, but spark plan expression transformation used to skip it as it does not handle sequence in an Option. Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala --- .../spark/sql/catalyst/plans/QueryPlan.scala | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala index ddf2cbf2ab911..d3dbb7851b828 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.catalyst.plans @@ -113,8 +131,9 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT } def recursiveTransform(arg: Any): AnyRef = arg match { - case e: Expression => transformExpression(e) - case Some(value) => Some(recursiveTransform(value)) + case e: Expression => transformExpressionUp(e) + case Some(e: Expression) => Some(transformExpressionUp(e)) + case Some(seq: Traversable[_]) => Some(seq.map(recursiveTransform)) case m: Map[_, _] => m case d: DataType => d // Avoid unpacking Structs case seq: Traversable[_] => seq.map(recursiveTransform) @@ -148,7 +167,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT productIterator.flatMap { case e: Expression => e :: Nil - case s: Some[_] => seqToExpressions(s.toSeq) + case Some(seq: Traversable[_] ) => seqToExpressions(seq) case seq: Traversable[_] => seqToExpressions(seq) case other => Nil }.toSeq From 4cf0d5d721797f7101399f6b2b7193ad01713983 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 28 Jul 2016 21:51:57 +0530 Subject: [PATCH 011/142] [SNAP-931] Use non-secure randomUUID where appropriate (#40) - adding a non-secure version for random UUID adapted from Android UUID.java - use the same for file name in DiskBlockManager, Utils methods, WriteAheadLogBackedBlockRDD Conflicts: core/src/main/scala/org/apache/spark/util/Utils.scala Conflicts: core/src/main/scala/org/apache/spark/util/Utils.scala streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala --- .../spark/storage/DiskBlockManager.scala | 9 ++-- .../apache/spark/storage/StorageUtils.scala | 49 +++++++++++++++++++ .../scala/org/apache/spark/util/Utils.scala | 26 +++++++++- .../rdd/WriteAheadLogBackedBlockRDD.scala | 6 +-- 4 files changed, 79 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index a69bcc9259995..84ee3749cc1f4 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -18,7 +18,6 @@ package org.apache.spark.storage import java.io.{File, IOException} -import java.util.UUID import org.apache.spark.SparkConf import org.apache.spark.executor.ExecutorExitCode @@ -114,18 +113,18 @@ private[spark] class DiskBlockManager(conf: SparkConf, deleteFilesOnStop: Boolea /** Produces a unique block id and File suitable for storing local intermediate results. */ def createTempLocalBlock(): (TempLocalBlockId, File) = { - var blockId = new TempLocalBlockId(UUID.randomUUID()) + var blockId = new TempLocalBlockId(StorageUtils.newNonSecureRandomUUID()) while (getFile(blockId).exists()) { - blockId = new TempLocalBlockId(UUID.randomUUID()) + blockId = new TempLocalBlockId(StorageUtils.newNonSecureRandomUUID()) } (blockId, getFile(blockId)) } /** Produces a unique block id and File suitable for storing shuffled intermediate results. */ def createTempShuffleBlock(): (TempShuffleBlockId, File) = { - var blockId = new TempShuffleBlockId(UUID.randomUUID()) + var blockId = new TempShuffleBlockId(StorageUtils.newNonSecureRandomUUID()) while (getFile(blockId).exists()) { - blockId = new TempShuffleBlockId(UUID.randomUUID()) + blockId = new TempShuffleBlockId(StorageUtils.newNonSecureRandomUUID()) } (blockId, getFile(blockId)) } diff --git a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala index e9694fdbca2de..4b0da68f59b4d 100644 --- a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala +++ b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala @@ -18,6 +18,7 @@ package org.apache.spark.storage import java.nio.{ByteBuffer, MappedByteBuffer} +import java.util.UUID import scala.collection.Map import scala.collection.mutable @@ -331,4 +332,52 @@ private[spark] object StorageUtils extends Logging { blockLocations } + /** static random number generator for UUIDs */ + private val uuidRnd = new java.util.Random + + /** + * Generate a random UUID for file names etc. Uses non-secure version + * of random number generator to be more efficient given that its not + * critical to have this unique. + * + * Adapted from Android's java.util.UUID source. + */ + final def newNonSecureRandomUUID(): UUID = { + val randomBytes: Array[Byte] = new Array[Byte](16) + uuidRnd.nextBytes(randomBytes) + + var msb = getLong(randomBytes, 0) + var lsb = getLong(randomBytes, 8) + // Set the version field to 4. + msb &= ~(0xfL << 12) + msb |= (4L << 12) + // Set the variant field to 2. Note that the variant field is + // variable-width, so supporting other variants is not just a matter + // of changing the constant 2 below! + lsb &= ~(0x3L << 62) + lsb |= 2L << 62 + new UUID(msb, lsb) + } + + final def getLong(src: Array[Byte], offset: Int): Long = { + var index = offset + var h: Int = (src(index) & 0xff) << 24 + index += 1 + h |= (src(index) & 0xff) << 16 + index += 1 + h |= (src(index) & 0xff) << 8 + index += 1 + h |= (src(index) & 0xff) + index += 1 + + var l = (src(index) & 0xff) << 24 + index += 1 + l |= (src(index) & 0xff) << 16 + index += 1 + l |= (src(index) & 0xff) << 8 + index += 1 + l |= (src(index) & 0xff) + + (h.toLong << 32L) | (l.toLong & 0xffffffffL) + } } diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 5853302973140..20622010ea60a 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -63,6 +63,8 @@ import org.apache.spark.internal.config._ import org.apache.spark.launcher.SparkLauncher import org.apache.spark.network.util.JavaUtils import org.apache.spark.serializer.{DeserializationStream, SerializationStream, SerializerInstance} +import org.apache.spark.util.logging.RollingFileAppender +import org.apache.spark.storage.StorageUtils /** CallSite represents a place in user code. It can have a short and a long form. */ private[spark] case class CallSite(shortForm: String, longForm: String) @@ -296,7 +298,8 @@ private[spark] object Utils extends Logging { maxAttempts + " attempts!") } try { - dir = new File(root, namePrefix + "-" + UUID.randomUUID.toString) + dir = new File(root, namePrefix + "-" + + StorageUtils.newNonSecureRandomUUID().toString) if (dir.exists() || !dir.mkdirs()) { dir = null } @@ -2589,7 +2592,26 @@ private[spark] object Utils extends Logging { * Returns a path of temporary file which is in the same directory with `path`. */ def tempFileWith(path: File): File = { - new File(path.getAbsolutePath + "." + UUID.randomUUID()) + var temp: File = null + do { + temp = new File(path.getAbsolutePath + "." + + StorageUtils.newNonSecureRandomUUID()) + } while (temp.exists()) + } + + /** + * Returns a path of temporary file which is in the same directory with `path`. + */ + def tempFileWith(parent: String, prefix: String): File = { + var temp: File = null + do { + val name = if (prefix == null) { + StorageUtils.newNonSecureRandomUUID().toString + } else { + prefix + '.' + StorageUtils.newNonSecureRandomUUID().toString + } + temp = new File(parent, name) + } while (temp.exists()) } /** diff --git a/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala b/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala index 844760ab61d2e..41f7c60e1cc35 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala @@ -16,9 +16,7 @@ */ package org.apache.spark.streaming.rdd -import java.io.File import java.nio.ByteBuffer -import java.util.UUID import scala.reflect.ClassTag import scala.util.control.NonFatal @@ -135,8 +133,8 @@ class WriteAheadLogBackedBlockRDD[T: ClassTag]( // FileBasedWriteAheadLog will not create any file or directory at that path. Also, // this dummy directory should not already exist otherwise the WAL will try to recover // past events from the directory and throw errors. - val nonExistentDirectory = new File( - System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString).getAbsolutePath + val nonExistentDirectory = Utils.tempFileWith( + System.getProperty("java.io.tmpdir"), prefix = null).getAbsolutePath writeAheadLog = WriteAheadLogUtils.createLogForReceiver( SparkEnv.get.conf, nonExistentDirectory, hadoopConf) dataRead = writeAheadLog.read(partition.walRecordHandle) From a6d7679a54ad4d741fa7bb320681ebb55a056c81 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Fri, 29 Jul 2016 12:10:09 +0530 Subject: [PATCH 012/142] [SNAPPYDATA] Adding SnappyData modification headers for missing files --- python/pyspark/shell.py | 19 +++++++++++++++++++ .../sql/catalyst/CatalystTypeConverters.scala | 18 ++++++++++++++++++ .../codegen/GenerateUnsafeProjection.scala | 18 ++++++++++++++++++ .../apache/spark/sql/types/DecimalType.scala | 18 ++++++++++++++++++ .../datasources/json/JsonInferSchema.scala | 18 ++++++++++++++++++ .../parquet/ParquetSchemaConverter.scala | 18 ++++++++++++++++++ 6 files changed, 109 insertions(+) diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py index 516185f10cdce..8122533f66cb1 100644 --- a/python/pyspark/shell.py +++ b/python/pyspark/shell.py @@ -15,6 +15,25 @@ # limitations under the License. # +# +# Changes for SnappyData data platform. +# +# Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. +# + """ An interactive shell. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala index 6a96aab7ad9dd..86a625a6c8272 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.catalyst diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala index 396d66ba92542..d9e9c866aeb5b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.catalyst.expressions.codegen diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala index 5ad10d9cbf9b9..c19067c73955f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.types diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala index 1c1b9f0520179..f3f0d7046de06 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.execution.datasources.json diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala index 4f64005852110..6eeaa02388379 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.execution.datasources.parquet From b1dafd55c8eac6fd022c047942d35a487cf92185 Mon Sep 17 00:00:00 2001 From: Hemant Bhanawat Date: Mon, 1 Feb 2016 19:03:27 +0530 Subject: [PATCH 013/142] [SNAPPYDATA] Updated README.md with information on SnappyData's changes --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 1e521a7e7b178..ba75637ba4a8e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ +## SnappyData's extensions to Spark + +- SnappyData collocates Spark executors with its in-memory data store in the same JVM. To achieve this, support for external cluster manager in Spark 2.0 is used to add a SnappyData cluster manager. +- SnappyData's MemoryManager was needed to generate and handle memory events. A property spark.memory.manager is now used to specify a memory manager other than Spark's own. +- To display the consumption of memory in an external embedded store, Spark's storage UI was updated. +- Support for getting length of type (for VARCHAR) was added in the JDBCDialect class. +- For SnappyData, dynamic continous queries on streams would be enabled in future. For that, support for registering DStreams after streaming context has started is added. +- For partitioning, sequence of expressions can be provided. SnappyData adds OrderlessHashPartitioning that does not take into account order of expressions while partitioning. +- Hive client thread-local configuration changed to be instance specific. +- Hive client added support for dropTable and listing tables for all databases. +- RDD partitions with executor specific preferred locations will be forced to be routed to one of those executors if alive. +- An "unsecure" version of random UUID added in DiskBlockManager for temporary file names. +- Added a fix for SPARK-13116. +- Increased visibility of some classes/methods. + + # Apache Spark Spark is a fast and general cluster computing system for Big Data. It provides From 71b22d0f646036efe8cbfb927f15769fa1a71054 Mon Sep 17 00:00:00 2001 From: ahshahid Date: Tue, 7 Jun 2016 14:15:36 -0700 Subject: [PATCH 014/142] [SNAPPYDATA] Optimizations for bootstrap * Added a method to bump up the expr id counter by a given number, so as to reserve the ExprID * Optimizing the Declarative aggregate function to have predictable input buffer aggregte attribute reference by using reservation in the ExprID being generated * Changes to minimize the query plan size for bootstrap and some more optimizations which aids in perf improvement * fixed scala style failures Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala --- .../expressions/aggregate/interfaces.scala | 30 +++++++++++++- .../expressions/namedExpressions.scala | 21 +++++++++- .../sql/execution/aggregate/AggUtils.scala | 40 ++++++++++++++++++- .../aggregate/HashAggregateExec.scala | 24 ++++++++++- .../aggregate/SortAggregateExec.scala | 24 ++++++++++- 5 files changed, 131 insertions(+), 8 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala index e1d16a2cd38b0..d6d394f5b8ae9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.catalyst.expressions.aggregate @@ -379,8 +397,16 @@ abstract class DeclarativeAggregate /** An expression-based aggregate's bufferSchema is derived from bufferAttributes. */ final override def aggBufferSchema: StructType = StructType.fromAttributes(aggBufferAttributes) - final lazy val inputAggBufferAttributes: Seq[AttributeReference] = - aggBufferAttributes.map(_.newInstance()) + lazy val inputAggBufferbaseExprID = NamedExpression.allocateExprID(aggBufferAttributes.length) + + /* final lazy val inputAggBufferAttributes: Seq[AttributeReference] = + aggBufferAttributes.map(_.newInstance()) */ + + @transient final lazy val inputAggBufferAttributes: Seq[AttributeReference] = + aggBufferAttributes.zipWithIndex.map { + case ( attr, i) => attr.withExprId( ExprId( inputAggBufferbaseExprID.id + i, + inputAggBufferbaseExprID.jvmId)) + } /** * A helper class for representing an attribute used in merging two diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala index 8df870468c2ad..8f66478011a84 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.catalyst.expressions @@ -30,7 +48,8 @@ object NamedExpression { private val curId = new java.util.concurrent.atomic.AtomicLong() private[expressions] val jvmId = UUID.randomUUID() def newExprId: ExprId = ExprId(curId.getAndIncrement(), jvmId) - def unapply(expr: NamedExpression): Option[(String, DataType)] = Some((expr.name, expr.dataType)) + def unapply(expr: NamedExpression): Option[(String, DataType)] = Some(expr.name, expr.dataType) + def allocateExprID(quota: Int): ExprId = ExprId(curId.getAndAdd(quota), jvmId) } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala index ebbdf1aaa024d..0c3161662e717 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.execution.aggregate @@ -27,6 +45,26 @@ import org.apache.spark.sql.internal.SQLConf * Utility functions used by the query planner to convert our plan to new aggregation code path. */ object AggUtils { + + def planAggregateWithoutPartial( + groupingExpressions: Seq[NamedExpression], + aggregateExpressions: Seq[AggregateExpression], + resultExpressions: Seq[NamedExpression], + child: SparkPlan): Seq[SparkPlan] = { + + val completeAggregateExpressions = aggregateExpressions.map(_.copy(mode = Complete)) + val completeAggregateAttributes = completeAggregateExpressions.map(_.resultAttribute) + SortAggregateExec( + requiredChildDistributionExpressions = Some(groupingExpressions), + groupingExpressions = groupingExpressions, + aggregateExpressions = completeAggregateExpressions, + aggregateAttributes = completeAggregateAttributes, + initialInputBufferOffset = 0, + resultExpressions = resultExpressions, + child = child + ) :: Nil + } + private def createAggregate( requiredChildDistributionExpressions: Option[Seq[Expression]] = None, groupingExpressions: Seq[NamedExpression] = Nil, @@ -44,7 +82,7 @@ object AggUtils { aggregateExpressions = aggregateExpressions, aggregateAttributes = aggregateAttributes, initialInputBufferOffset = initialInputBufferOffset, - resultExpressions = resultExpressions, + __resultExpressions = resultExpressions, child = child) } else { val objectHashEnabled = child.sqlContext.conf.useObjectHashAggregation diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala index ce3c68810f3b6..cfdda6a046f80 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.execution.aggregate @@ -42,11 +60,13 @@ case class HashAggregateExec( aggregateExpressions: Seq[AggregateExpression], aggregateAttributes: Seq[Attribute], initialInputBufferOffset: Int, - resultExpressions: Seq[NamedExpression], + __resultExpressions: Seq[NamedExpression], child: SparkPlan) extends UnaryExecNode with CodegenSupport { - private[this] val aggregateBufferAttributes = { + @transient lazy val resultExpressions = __resultExpressions + + @transient lazy private[this] val aggregateBufferAttributes = { aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala index fc87de2c52e41..00de22e54ab07 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.execution.aggregate @@ -36,11 +54,13 @@ case class SortAggregateExec( aggregateExpressions: Seq[AggregateExpression], aggregateAttributes: Seq[Attribute], initialInputBufferOffset: Int, - resultExpressions: Seq[NamedExpression], + __resultExpressions: Seq[NamedExpression], child: SparkPlan) extends UnaryExecNode { - private[this] val aggregateBufferAttributes = { + @transient lazy val resultExpressions = __resultExpressions + + @transient lazy private[this] val aggregateBufferAttributes = { aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes) } From 490663f76ee47b3c181bd8e80f827e74e54102f2 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sun, 31 Jul 2016 21:54:37 +0530 Subject: [PATCH 015/142] [SNAPPYDATA] Gradle build scripts and build fixes - Adding gradle build scripts with gradle wrapper invocation for all projects/subprojects - Added product target to pack snappy-spark distribution (like dev/make-distribution.sh) - Changes to make it compatible with top-level SnappyData build - Add SnappyData modification headers to remaining modified files - Fixed compilation and few test issues --- .gitignore | 7 +- assembly/build.gradle | 132 ++++++ build.gradle | 376 ++++++++++++++++++ common/network-common/build.gradle | 26 ++ common/network-shuffle/build.gradle | 30 ++ common/network-yarn/build.gradle | 100 +++++ common/sketch/build.gradle | 28 ++ common/tags/build.gradle | 22 + common/unsafe/build.gradle | 38 ++ core/build.gradle | 171 ++++++++ .../spark/storage/DiskBlockManager.scala | 18 + .../apache/spark/storage/StorageUtils.scala | 18 + .../scala/org/apache/spark/util/Utils.scala | 20 + examples/build.gradle | 43 ++ .../docker-integration-tests/build.gradle | 52 +++ external/flume-sink/build.gradle | 50 +++ external/flume/build.gradle | 40 ++ external/kafka-0-10/build.gradle | 34 ++ external/kafka-0-8/build.gradle | 33 ++ external/spark-ganglia-lgpl/build.gradle | 24 ++ gradle.properties | 5 + gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 53638 bytes gradle/wrapper/gradle-wrapper.properties | 6 + gradlew | 160 ++++++++ gradlew.bat | 90 +++++ graphx/build.gradle | 30 ++ launcher/build.gradle | 39 ++ mllib-local/build.gradle | 29 ++ mllib/build.gradle | 46 +++ repl/build.gradle | 43 ++ settings.gradle | 82 ++++ sql/catalyst/build.gradle | 47 +++ .../apache/spark/sql/AnalysisException.scala | 5 + .../sql/catalyst/expressions/Projection.scala | 2 +- sql/core/build.gradle | 43 ++ sql/hive-thriftserver/build.gradle | 85 ++++ sql/hive/build.gradle | 117 ++++++ .../spark/sql/hive/StatisticsSuite.scala | 18 - streaming/build.gradle | 44 ++ .../rdd/WriteAheadLogBackedBlockRDD.scala | 19 + tools/build.gradle | 25 ++ yarn/build.gradle | 137 +++++++ 42 files changed, 2312 insertions(+), 22 deletions(-) create mode 100644 assembly/build.gradle create mode 100644 build.gradle create mode 100644 common/network-common/build.gradle create mode 100644 common/network-shuffle/build.gradle create mode 100644 common/network-yarn/build.gradle create mode 100644 common/sketch/build.gradle create mode 100644 common/tags/build.gradle create mode 100644 common/unsafe/build.gradle create mode 100644 core/build.gradle create mode 100644 examples/build.gradle create mode 100644 external/docker-integration-tests/build.gradle create mode 100644 external/flume-sink/build.gradle create mode 100644 external/flume/build.gradle create mode 100644 external/kafka-0-10/build.gradle create mode 100644 external/kafka-0-8/build.gradle create mode 100644 external/spark-ganglia-lgpl/build.gradle create mode 100644 gradle.properties create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100755 gradlew create mode 100644 gradlew.bat create mode 100644 graphx/build.gradle create mode 100644 launcher/build.gradle create mode 100644 mllib-local/build.gradle create mode 100644 mllib/build.gradle create mode 100644 repl/build.gradle create mode 100644 settings.gradle create mode 100644 sql/catalyst/build.gradle create mode 100644 sql/core/build.gradle create mode 100644 sql/hive-thriftserver/build.gradle create mode 100644 sql/hive/build.gradle create mode 100644 streaming/build.gradle create mode 100644 tools/build.gradle create mode 100644 yarn/build.gradle diff --git a/.gitignore b/.gitignore index 903297db96901..6e8668c46a350 100644 --- a/.gitignore +++ b/.gitignore @@ -71,7 +71,7 @@ spark-*-bin-*.tgz spark-tests.log src_managed/ streaming-tests.log -target/ +build-artifacts/ unit-tests.log work/ @@ -86,8 +86,9 @@ spark-warehouse/ # For R session data .RData .RHistory -.Rhistory *.Rproj *.Rproj.* - .Rproj.user + +# gradle specific +.gradle/ diff --git a/assembly/build.gradle b/assembly/build.gradle new file mode 100644 index 0000000000000..5a0f18c45e3f6 --- /dev/null +++ b/assembly/build.gradle @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Assembly' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-hive_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-hive-thriftserver_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-repl_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-yarn_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-mllib_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) + if (rootProject.hasProperty('ganglia')) { + compile project(subprojectBase + 'snappy-spark-ganglia-lgpl_' + scalaBinaryVersion) + } +} + +def cleanProduct() { + delete "${sparkProjectRootDir}/python/lib/pyspark.zip" + delete snappyProductDir +} +clean.doLast { + cleanProduct() +} + +task product(type: Zip) { + def examplesProject = project(subprojectBase + 'snappy-spark-examples_' + scalaBinaryVersion) + String yarnShuffleProject = subprojectBase + 'snappy-spark-network-yarn_' + scalaBinaryVersion + dependsOn jar, examplesProject.jar, "${yarnShuffleProject}:shadowJar" + // create python zip + destinationDir = file("${snappyProductDir}/python/lib") + archiveName = 'pyspark.zip' + from("${sparkProjectRootDir}/python") { + include 'pyspark/**/*' + } + + doFirst { + cleanProduct() + } + doLast { + // copy all runtime dependencies (skip for top-level snappydata builds) + if (rootProject.name == 'snappy-spark') { + copy { + from(configurations.runtime) { + // exclude antlr4 explicitly (runtime is still included) + // that gets pulled by antlr gradle plugin + exclude '**antlr4-4*.jar' + // exclude scalatest included by spark-tags + exclude '**scalatest*.jar' + } + into "${snappyProductDir}/jars" + } + } + // copy scripts, data and other files that are part of distribution + copy { + from(sparkProjectRootDir) { + include 'bin/**' + include 'sbin/**' + include 'conf/**' + include 'data/**' + include 'licenses/**' + include 'python/**' + include 'examples/src/**' + } + into snappyProductDir + } + def sparkR = 'sparkProjectRootDir/R/lib/SparkR' + if (file(sparkR).exists()) { + copy { + from sparkR + into "${snappyProductDir}/R/lib" + } + } + + // copy yarn shuffle shadow jar + copy { + from "${project(yarnShuffleProject).buildDir}/jars" + into "${snappyProductDir}/yarn" + } + // copy examples jars + copy { + from "${examplesProject.buildDir}/jars" + into "${snappyProductDir}/examples/jars" + } + // create RELEASE file, copy README etc for top-level snappy-spark project + if (rootProject.name == 'snappy-spark') { + copy { + from(sparkProjectRootDir) { + include 'LICENSE' + include 'NOTICE' + include 'README.md' + } + into snappyProductDir + } + def releaseFile = file("${snappyProductDir}/RELEASE") + String buildFlags = "" + if (rootProject.hasProperty('docker')) { + buildFlags += ' -Pdocker' + } + if (rootProject.hasProperty('ganglia')) { + buildFlags += ' -Pganglia' + } + String gitRevision = "${gitCmd} rev-parse --short HEAD".execute().text.trim() + if (gitRevision.length() > 0) { + gitRevision = " (git revision ${gitRevision})" + } + + releaseFile.append("Spark ${version}${gitRevision} built for Hadoop ${hadoopVersion}\n") + releaseFile.append("Build flags:${buildFlags}\n") + } + } +} diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000000000..c235dcf712cc4 --- /dev/null +++ b/build.gradle @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +apply plugin: 'wrapper' + +// TODO: profiles and allow changing hadoopVersion + +buildscript { + repositories { + maven { url 'https://plugins.gradle.org/m2' } + mavenCentral() + } + dependencies { + classpath 'io.snappydata:gradle-scalatest:0.13-1' + classpath 'org.github.ngbinh.scalastyle:gradle-scalastyle-plugin_2.11:0.8.2' + } +} + +description = 'Spark Project' + +allprojects { + // We want to see all test results. This is equivalent to setting --continue + // on the command line. + gradle.startParameter.continueOnFailure = true + + repositories { + mavenCentral() + maven { url 'http://repository.apache.org/snapshots' } + } + + apply plugin: 'idea' + + group = 'io.snappydata' + version = '2.0.1-1' + + ext { + scalaBinaryVersion = '2.11' + scalaVersion = scalaBinaryVersion + '.8' + hadoopVersion = '2.7.2' + protobufVersion = '2.6.1' + jerseyVersion = '2.22.2' + sunJerseyVersion = '1.19.1' + jettyVersion = '9.2.16.v20160414' + log4jVersion = '1.2.17' + slf4jVersion = '1.7.21' + junitVersion = '4.12' + javaxServletVersion = '3.1.0' + guavaVersion = '14.0.1' + hiveVersion = '1.2.1.spark2' + chillVersion = '0.8.0' + nettyVersion = '3.8.0.Final' + nettyAllVersion = '4.0.29.Final' + derbyVersion = '10.12.1.1' + httpClientVersion = '4.5.2' + httpCoreVersion = '4.4.4' + fasterXmlVersion = '2.6.5' + snappyJavaVersion = '1.1.2.6' + parquetVersion = '1.7.0' + hiveParquetVersion = '1.6.0' + metricsVersion = '3.1.2' + thriftVersion = '0.9.3' + antlrVersion = '4.5.3' + jpamVersion = '1.1' + seleniumVersion = '2.52.0' + curatorVersion = '2.7.1' + commonsCodecVersion = '1.10' + avroVersion = '1.7.7' + jsr305Version = '3.0.1' + scalatestVersion = '2.2.6' + pegdownVersion = '1.6.0' + + shadePackageName = 'org.spark_project' + } + + // default output directory like in sbt/maven + buildDir = 'build-artifacts/scala-' + scalaBinaryVersion + + ext { + if (rootProject.name == 'snappy-spark') { + subprojectBase = ':' + sparkProjectRoot = ':' + sparkProjectRootDir = project(':').projectDir + testResultsBase = "${rootProject.buildDir}/tests" + gitCmd = "git --git-dir=${rootDir}/.git --work-tree=${rootDir}" + } else { + subprojectBase = ':snappy-spark:' + sparkProjectRoot = ':snappy-spark' + sparkProjectRootDir = project(':snappy-spark').projectDir + testResultsBase = "${rootProject.buildDir}/tests/spark" + gitCmd = "git --git-dir=${project(sparkProjectRoot).projectDir}/.git --work-tree=${project(sparkProjectRoot).projectDir}" + } + snappyProductDir = "${rootProject.buildDir}/snappy" + } +} + +def getStackTrace(def t) { + java.io.StringWriter sw = new java.io.StringWriter() + java.io.PrintWriter pw = new java.io.PrintWriter(sw) + org.codehaus.groovy.runtime.StackTraceUtils.sanitize(t).printStackTrace(pw) + return sw.toString() +} + +task cleanSparkScalaTest << { + def workingDir = "${testResultsBase}/scalatest" + delete workingDir + file(workingDir).mkdirs() +} +task cleanSparkJUnit << { + def workingDir = "${testResultsBase}/junit" + delete workingDir + file(workingDir).mkdirs() +} + +subprojects { + apply plugin: 'scala' + apply plugin: 'maven' + apply plugin: 'scalaStyle' + + // apply compiler options + compileJava.options.encoding = 'UTF-8' + compileJava.options.compilerArgs << '-Xlint:all,-serial,-path,-deprecation' + // compileScala.scalaCompileOptions.optimize = true + compileScala.options.encoding = 'UTF-8' + + javadoc.options.charSet = 'UTF-8' + + scalaStyle { + configLocation = "${sparkProjectRootDir}/scalastyle-config.xml" + inputEncoding = 'UTF-8' + outputEncoding = 'UTF-8' + outputFile = "${buildDir}/scalastyle-output.xml" + includeTestSourceDirectory = false + source = 'src/main/scala' + testSource = 'src/test/scala' + failOnViolation = true + failOnWarning = false + } + + configurations { + runtimeJar { + description 'a dependency to include additional jars at runtime' + visible true + } + } + + // when invoking from snappydata, below are already defined at top-level + if (rootProject.name == 'snappy-spark') { + task packageSources(type: Jar, dependsOn: classes) { + classifier = 'sources' + from sourceSets.main.allSource + } + + configurations { + provided { + description 'a dependency that is provided externally at runtime' + visible true + } + testOutput { + extendsFrom testCompile + description 'a dependency that exposes test artifacts' + } + } + + task packageTests(type: Jar, dependsOn: testClasses) { + description 'Assembles a jar archive of test classes.' + from sourceSets.test.output.classesDir + classifier = 'tests' + } + artifacts { + testOutput packageTests + } + + idea { + module { + scopes.PROVIDED.plus += [ configurations.provided ] + } + } + + sourceSets { + main.compileClasspath += configurations.provided + main.runtimeClasspath -= configurations.provided + test.compileClasspath += configurations.provided + test.runtimeClasspath += configurations.provided + } + + javadoc.classpath += configurations.provided + } + task packageScalaDocs(type: Jar, dependsOn: scaladoc) { + classifier = 'javadoc' + from scaladoc + } + if (rootProject.hasProperty('enablePublish')) { + artifacts { + archives packageScalaDocs, packageSources + } + } + + // fix scala+java mix to all use compileScala which use correct dependency order + sourceSets.main.scala.srcDir 'src/main/java' + sourceSets.main.java.srcDirs = [] + + dependencies { + // This is a dummy dependency that is used along with the shading plug-in + // to create effective poms on publishing (see SPARK-3812). + //compile group: 'org.spark-project.spark', name: 'unused', version: '1.0.0' + compile 'org.scala-lang:scala-library:' + scalaVersion + compile 'org.scala-lang:scala-reflect:' + scalaVersion + + compile group: 'log4j', name:'log4j', version: log4jVersion + compile 'org.slf4j:slf4j-api:' + slf4jVersion + compile 'org.slf4j:slf4j-log4j12:' + slf4jVersion + + testCompile "junit:junit:${junitVersion}" + testCompile "org.scalatest:scalatest_${scalaBinaryVersion}:${scalatestVersion}" + testCompile 'org.mockito:mockito-core:1.10.19' + testCompile 'org.scalacheck:scalacheck_' + scalaBinaryVersion + ':1.12.5' + testCompile 'com.novocode:junit-interface:0.11' + + testRuntime "org.pegdown:pegdown:${pegdownVersion}" + } + + if (rootProject.name == 'snappy-spark') { + task scalaTest(type: Test) { + actions = [ new com.github.maiflai.ScalaTestAction() ] + + List suites = [] + extensions.add(com.github.maiflai.ScalaTestAction.SUITES, suites) + extensions.add('suite', { String name -> suites.add(name) } ) + extensions.add('suites', { String... name -> suites.addAll(name) } ) + + def result = new StringBuilder() + extensions.add(com.github.maiflai.ScalaTestAction.TESTRESULT, result) + extensions.add('testResult', { String name -> result.append(name) } ) + + def output = new StringBuilder() + extensions.add(com.github.maiflai.ScalaTestAction.TESTOUTPUT, output) + extensions.add('testOutput', { String name -> output.append(name) } ) + + def errorOutput = new StringBuilder() + extensions.add(com.github.maiflai.ScalaTestAction.TESTERROR, errorOutput) + extensions.add('testError', { String name -> errorOutput.append(name) } ) + + // running a single scala suite + if (rootProject.hasProperty('singleSuite')) { + suite singleSuite + } + } + } + scalaTest { + // top-level default is single process run since scalatest does not + // spawn separate JVMs + maxParallelForks = 1 + systemProperties 'test.src.tables': '__not_used__' + + workingDir = "${testResultsBase}/scalatest" + + testResult '/dev/tty' + testOutput "${workingDir}/output.txt" + testError "${workingDir}/error.txt" + binResultsDir = file("${workingDir}/binary/${project.name}") + reports.html.destination = file("${workingDir}/html/${project.name}") + reports.junitXml.destination = file(workingDir) + } + test { + jvmArgs '-Xss4096k' + maxParallelForks = (2 * Runtime.getRuntime().availableProcessors()) + systemProperties 'spark.master.rest.enabled': 'false', + 'test.src.tables': 'src' + + workingDir = "${testResultsBase}/junit" + + binResultsDir = file("${workingDir}/binary/${project.name}") + reports.html.destination = file("${workingDir}/html/${project.name}") + reports.junitXml.destination = file(workingDir) + } + // need to do below after graph is ready else it will give an error about + // runtimeClaspath being set after being finalized + gradle.taskGraph.whenReady({ graph -> + tasks.withType(Test).each { test -> + test.configure { + onlyIf { ! Boolean.getBoolean('skip.tests') } + + jvmArgs '-XX:+HeapDumpOnOutOfMemoryError', '-XX:MaxPermSize=512m', '-XX:ReservedCodeCacheSize=512m' + maxHeapSize '3g' + // disable assertions for hive tests as in Spark's pom.xml because HiveCompatibilitySuite currently fails (SPARK-4814) + if (test.project.name.contains('snappy-spark-hive_')) { + jvmArgs '-da' + maxParallelForks = 1 + } else { + jvmArgs '-ea' + } + environment 'SPARK_DIST_CLASSPATH': "${sourceSets.test.runtimeClasspath.asPath}", + 'SPARK_PREPEND_CLASSES': '1', + 'SPARK_SCALA_VERSION': scalaBinaryVersion, + 'SPARK_TESTING': '1', + 'JAVA_HOME': System.getProperty('java.home') + systemProperties 'log4j.configuration': "file:${projectDir}/src/test/resources/log4j.properties", + 'derby.system.durability': 'test', + 'java.awt.headless': 'true', + 'java.io.tmpdir': "${rootProject.buildDir}/tmp", + 'spark.test.home': snappyProductDir, + 'spark.project.home': "${project(sparkProjectRoot).projectDir}", + 'spark.testing': '1', + 'spark.ui.enabled': 'false', + 'spark.ui.showConsoleProgress': 'false', + 'spark.driver.allowMultipleContexts': 'true', + 'spark.unsafe.exceptionOnMemoryLeak': 'true' + + testLogging.exceptionFormat = 'full' + + if (rootProject.name == 'snappy-spark') { + def eol = System.getProperty('line.separator') + beforeTest { desc -> + def now = new Date().format('yyyy-MM-dd HH:mm:ss.SSS Z') + def progress = new File(workingDir, "progress.txt") + def output = new File(workingDir, "output.txt") + progress << "$now Starting test $desc.className $desc.name$eol" + output << "${now} STARTING TEST ${desc.className} ${desc.name}${eol}${eol}" + } + onOutput { desc, event -> + def output = new File(workingDir, "output.txt") + output << event.message + } + afterTest { desc, result -> + def now = new Date().format('yyyy-MM-dd HH:mm:ss.SSS Z') + def progress = new File(workingDir, "progress.txt") + def output = new File(workingDir, "output.txt") + progress << "${now} Completed test ${desc.className} ${desc.name} with result: ${result.resultType}${eol}" + output << "${eol}${now} COMPLETED TEST ${desc.className} ${desc.name} with result: ${result.resultType}${eol}${eol}" + result.exceptions.each { t -> + progress << " EXCEPTION: ${getStackTrace(t)}${eol}" + output << "${getStackTrace(t)}${eol}" + } + } + } + } + } + }) + test.dependsOn subprojectBase + 'cleanSparkJUnit' + scalaTest.dependsOn subprojectBase + 'cleanSparkScalaTest' + check.dependsOn scalaTest + if (rootProject.name == 'snappy-spark') { + check.dependsOn "${subprojectBase}snappy-spark-assembly_${scalaBinaryVersion}:product" + } +} + +task generateSources { + dependsOn subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion + ':generateGrammarSource' + dependsOn subprojectBase + 'snappy-spark-streaming-flume-sink_' + scalaBinaryVersion + ':generateAvroJava' +} + +if (rootProject.name == 'snappy-spark') { + task scalaStyle { + dependsOn subprojects.scalaStyle + } + task check { + dependsOn subprojects.check + } +} else { + scalaStyle.dependsOn subprojects.scalaStyle + check.dependsOn subprojects.check +} diff --git a/common/network-common/build.gradle b/common/network-common/build.gradle new file mode 100644 index 0000000000000..63c23210f801e --- /dev/null +++ b/common/network-common/build.gradle @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Networking' + +dependencies { + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion + compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version + compile group: 'com.google.guava', name: 'guava', version: guavaVersion +} diff --git a/common/network-shuffle/build.gradle b/common/network-shuffle/build.gradle new file mode 100644 index 0000000000000..0ffbc3414ad14 --- /dev/null +++ b/common/network-shuffle/build.gradle @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Shuffle Streaming Service' + +dependencies { + compile project(subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: '1.8' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: fasterXmlVersion + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + + testCompile project(path: subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion, configuration: 'testOutput') +} diff --git a/common/network-yarn/build.gradle b/common/network-yarn/build.gradle new file mode 100644 index 0000000000000..323463acad43c --- /dev/null +++ b/common/network-yarn/build.gradle @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +plugins { + id 'com.github.johnrengelman.shadow' version '1.2.3' +} + +description = 'Spark Project YARN Shuffle Service' + +dependencies { + compile project(subprojectBase + 'snappy-spark-network-shuffle_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion + provided (group: 'org.apache.hadoop', name: 'hadoop-client', version: hadoopVersion) { + exclude(group: 'asm', module: 'asm') + exclude(group: 'org.codehaus.jackson', module: 'jackson-core-asl') + exclude(group: 'org.codehaus.jackson', module: 'jackson-mapper-asl') + exclude(group: 'org.ow2.asm', module: 'asm') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'org.mockito', module: 'mockito-all') + exclude(group: 'org.mortbay.jetty', module: 'servlet-api-2.5') + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'junit', module: 'junit') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'com.sun.jersey') + exclude(group: 'com.sun.jersey.jersey-test-framework') + exclude(group: 'com.sun.jersey.contribs') + } + + /* + runtimeJar project(subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion) + runtimeJar project(subprojectBase + 'snappy-spark-network-shuffle_' + scalaBinaryVersion) + runtimeJar group: 'io.netty', name: 'netty-all', version: nettyAllVersion + runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion + runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: fasterXmlVersion + */ +} + +shadowJar { + baseName 'snappy-spark' + classifier 'yarn-shuffle' + + mergeServiceFiles { + exclude 'META-INF/*.SF' + exclude 'META-INF/*.DSA' + exclude 'META-INF/*.RSA' + } + + dependencies { + exclude(dependency('org.scala-lang:.*')) + exclude(dependency('org.scala-lang.modules:.*')) + exclude(dependency('org.slf4j:.*')) + exclude(dependency('log4j:.*')) + exclude(dependency('org.scalatest:.*')) + } + //configurations = [ project.configurations.runtimeJar ] + + relocate 'io.netty', "${shadePackageName}.io.netty" + relocate 'com.fasterxml.jackson', "${shadePackageName}.com.fasterxml.jackson" + relocate 'com.google.common', "${shadePackageName}.guava" + + String createdBy = "" + if (rootProject.hasProperty('enablePublish')) { + createdBy = "SnappyData Build Team" + } else { + createdBy = System.getProperty("user.name") + } + manifest { + attributes( + "Manifest-Version" : "1.0", + "Created-By" : createdBy, + "Title" : project.name, + "Version" : version, + "Vendor" : "SnappyData, Inc." + ) + } + + doLast { + copy { + from outputs + into "${buildDir}/jars" + } + } +} diff --git a/common/sketch/build.gradle b/common/sketch/build.gradle new file mode 100644 index 0000000000000..a5e5efff08b5a --- /dev/null +++ b/common/sketch/build.gradle @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Sketch' + +dependencies { + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) +} + +tasks.withType(JavaCompile) { + options.compilerArgs << '-XDignore.symbol.file' + options.fork = true + options.forkOptions.executable = 'javac' +} diff --git a/common/tags/build.gradle b/common/tags/build.gradle new file mode 100644 index 0000000000000..e272cfbaa6382 --- /dev/null +++ b/common/tags/build.gradle @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Tags' + +dependencies { + compile "org.scalatest:scalatest_${scalaBinaryVersion}:${scalatestVersion}" +} diff --git a/common/unsafe/build.gradle b/common/unsafe/build.gradle new file mode 100644 index 0000000000000..69d29942f5f1c --- /dev/null +++ b/common/unsafe/build.gradle @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Unsafe' + +dependencies { + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'com.twitter', name: 'chill_' + scalaBinaryVersion, version: chillVersion + compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + + testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.2' +} + +// reset the srcDirs to allow javac compilation with specific args below +sourceSets.main.scala.srcDirs = [ 'src/main/scala' ] +sourceSets.main.java.srcDirs = [ 'src/main/java' ] + +tasks.withType(JavaCompile) { + options.compilerArgs << '-XDignore.symbol.file' + options.fork = true + options.forkOptions.executable = 'javac' +} diff --git a/core/build.gradle b/core/build.gradle new file mode 100644 index 0000000000000..e1b52bfec0799 --- /dev/null +++ b/core/build.gradle @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Core' + +dependencies { + compile project(subprojectBase + 'snappy-spark-launcher_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-network-shuffle_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-unsafe_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile(group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion) { + exclude(group: 'io.netty', module: 'netty') + exclude(group: 'org.mortbay.jetty', module: 'jetty') + exclude(group: 'org.mortbay.jetty', module: 'jetty-util') + exclude(group: 'org.mortbay.jetty', module: 'servlet-api') + exclude(group: 'org.apache.velocity', module: 'velocity') + } + compile(group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion, classifier: 'hadoop2') { + exclude(group: 'io.netty', module: 'netty') + exclude(group: 'org.mortbay.jetty', module: 'jetty') + exclude(group: 'org.mortbay.jetty', module: 'jetty-util') + exclude(group: 'org.mortbay.jetty', module: 'servlet-api') + exclude(group: 'org.apache.velocity', module: 'velocity') + exclude(group: 'org.apache.avro', module: 'avro-ipc') + } + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile group: 'com.twitter', name: 'chill_' + scalaBinaryVersion, version: chillVersion + compile group: 'com.twitter', name: 'chill-java', version: chillVersion + compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' + // explicitly include netty from akka-remote to not let zookeeper override it + compile group: 'io.netty', name: 'netty', version: nettyVersion + // explicitly exclude old netty from zookeeper + compile(group: 'org.apache.zookeeper', name: 'zookeeper', version: '3.4.8') { + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'jline', module: 'jline') + } + compile group: 'com.google.protobuf', name: 'protobuf-java', version: protobufVersion + compile(group: 'org.apache.hadoop', name: 'hadoop-client', version: hadoopVersion) { + exclude(group: 'asm', module: 'asm') + exclude(group: 'org.codehaus.jackson', module: 'jackson-mapper-asl') + exclude(group: 'org.ow2.asm', module: 'asm') + exclude(group: 'org.apache.zookeeper', module: 'zookeeper') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'org.mockito', module: 'mockito-all') + exclude(group: 'org.mortbay.jetty', module: 'servlet-api-2.5') + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'junit', module: 'junit') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'com.sun.jersey') + exclude(group: 'com.sun.jersey.jersey-test-framework') + exclude(group: 'com.sun.jersey.contribs') + exclude(group: 'com.google.protobuf', module: 'protobuf-java') + } + compile(group: 'net.java.dev.jets3t', name: 'jets3t', version: '0.9.3') { + exclude(group: 'commons-logging', module: 'commons-logging') + } + compile(group: 'org.apache.curator', name: 'curator-recipes', version: curatorVersion) { + exclude(group: 'org.apache.zookeeper', module: 'zookeeper') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'jline', module: 'jline') + exclude(group: 'com.google.guava', module: 'guava') + } + + compile 'org.scala-lang:scalap:' + scalaVersion + compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded' , version: '4.4' + compile group: 'org.roaringbitmap', name: 'RoaringBitmap' , version: '0.5.11' + + compile group: 'org.eclipse.jetty', name: 'jetty-server', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-plus', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-util', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-http', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-servlets', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-security', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-continuation', version: jettyVersion + compile group: 'javax.servlet', name: 'javax.servlet-api', version: javaxServletVersion + compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.2' + compile group: 'org.apache.commons', name: 'commons-math3', version: '3.4.1' + compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version + compile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion + compile group: 'org.slf4j', name: 'jcl-over-slf4j', version: slf4jVersion + compile group: 'com.ning', name: 'compress-lzf', version: '1.0.3' + compile group: 'org.xerial.snappy', name: 'snappy-java', version: snappyJavaVersion + compile group: 'net.jpountz.lz4', name: 'lz4', version: '1.3.0' + compile group: 'commons-net', name: 'commons-net', version: '2.2' + compile group: 'org.json4s', name: 'json4s-jackson_' + scalaBinaryVersion, version: '3.2.11' + compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: jerseyVersion + compile group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion + compile group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion + compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet', version: jerseyVersion + compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet-core', version: jerseyVersion + compile(group: 'org.apache.mesos', name: 'mesos', version: '0.21.1', classifier: 'shaded-protobuf') { + exclude(group: 'com.google.protobuf', module: 'protobuf-java') + } + compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion + compile(group: 'com.clearspring.analytics', name: 'stream', version: '2.7.0') { + exclude(group: 'it.unimi.dsi', module: 'fastutil') + } + compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion + compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: metricsVersion + compile group: 'io.dropwizard.metrics', name: 'metrics-json', version: metricsVersion + compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: metricsVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion + compile(group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_' + scalaBinaryVersion, version: fasterXmlVersion) { + exclude(group: 'com.google.guava', module: 'guava') + } + compile group: 'org.apache.ivy', name: 'ivy', version: '2.4.0' + compile group: 'oro', name: 'oro', version: '2.0.8' + compile(group: 'net.razorvine', name: 'pyrolite', version: '4.9') { + exclude(group: 'net.razorvine', module: 'serpent') + } + compile group: 'net.sf.py4j', name: 'py4j', version: '0.10.1' + + testCompile group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion, classifier: 'tests' + testCompile "org.apache.derby:derby:${derbyVersion}" + testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-java', version: seleniumVersion) { + exclude(group: 'com.google.guava', module: 'guava') + } + testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-htmlunit-driver', version: seleniumVersion) { + exclude(group: 'com.google.guava', module: 'guava') + } + testCompile group: 'xml-apis', name: 'xml-apis', version: '1.4.01' + testCompile group: 'org.hamcrest', name: 'hamcrest-core', version: '1.3' + testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '1.3' + testCompile(group: 'org.apache.curator', name: 'curator-test', version: curatorVersion) { + exclude(group: 'org.apache.zookeeper', module: 'zookeeper') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'com.google.guava', module: 'guava') + } +} + +// TODO: sparkr profile, copy-dependencies target? + +// fix scala+java test ordering +sourceSets.test.scala.srcDir 'src/test/java' +sourceSets.test.java.srcDirs = [] + +// generate properties using spark-build-info and add to project resources +String extraResourceDir = "${buildDir}/extra-resources" + +task generateBuildInfo << { + file(extraResourceDir).mkdirs() + exec { + executable 'bash' + workingDir = buildDir + args "${projectDir}/../build/spark-build-info", extraResourceDir, version + } +} +sourceSets { + main { + // register generated resources on the main SourceSet + output.dir(extraResourceDir, builtBy: 'generateBuildInfo') + } +} diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index 84ee3749cc1f4..25ba6c2353f54 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.storage diff --git a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala index 4b0da68f59b4d..dab5bbcfece70 100644 --- a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala +++ b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.storage diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 20622010ea60a..be5becf145d0f 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.util @@ -2597,6 +2615,7 @@ private[spark] object Utils extends Logging { temp = new File(path.getAbsolutePath + "." + StorageUtils.newNonSecureRandomUUID()) } while (temp.exists()) + temp } /** @@ -2612,6 +2631,7 @@ private[spark] object Utils extends Logging { } temp = new File(parent, name) } while (temp.exists()) + temp } /** diff --git a/examples/build.gradle b/examples/build.gradle new file mode 100644 index 0000000000000..eeeee87812fed --- /dev/null +++ b/examples/build.gradle @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Examples' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-mllib_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-hive_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming-flume_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion) + + compile group: 'org.apache.commons', name: 'commons-math3', version: '3.4.1' + compile group: 'com.github.scopt', name: 'scopt_' + scalaBinaryVersion, version: '3.3.0' + compile group: 'com.twitter', name: 'parquet-hadoop-bundle', version: hiveParquetVersion + + runtimeJar group: 'com.github.scopt', name: 'scopt_' + scalaBinaryVersion, version: '3.3.0' +} + +jar.doLast { + copy { + from configurations.runtimeJar + from outputs + exclude 'scala-*' + into "${buildDir}/jars" + } +} diff --git a/external/docker-integration-tests/build.gradle b/external/docker-integration-tests/build.gradle new file mode 100644 index 0000000000000..93ae7e08befab --- /dev/null +++ b/external/docker-integration-tests/build.gradle @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Docker Integration Tests' + +dependencies { + compile group: 'com.ibm.db2.jcc', name: 'db2jcc4', version: '10.5.0.5' + + testCompile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + testCompile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) + testCompile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + testCompile(group: 'com.spotify', name: 'docker-client', version: '3.6.6', classifier: 'shaded') { + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'com.fasterxml.jackson.jaxrs', module: 'jackson-jaxrs-json-provider') + exclude(group: 'com.fasterxml.jackson.datatype', module: 'jackson-datatype-guava') + exclude(group: 'com.fasterxml.jackson.core', module: 'jackson-databind') + exclude(group: 'org.glassfish.jersey.core', module: 'jersey-client') + exclude(group: 'org.glassfish.jersey.connectors', module: 'jersey-apache-connector') + exclude(group: 'org.glassfish.jersey.media', module: 'jersey-media-json-jackson') + } + testCompile group: 'org.apache.httpcomponents', name: 'httpclient', version: httpClientVersion + testCompile group: 'org.apache.httpcomponents', name: 'httpcore', version: httpCoreVersion + testCompile group: 'mysql', name: 'mysql-connector-java', version: '5.1.38' + testCompile group: 'org.postgresql', name: 'postgresql', version: '9.4.1207.jre7' + testCompile group: 'com.oracle', name: 'ojdbc6', version: '11.2.0.1.0' + testCompile group: 'com.sun.jersey', name: 'jersey-server', version: sunJerseyVersion + testCompile group: 'com.sun.jersey', name: 'jersey-core', version: sunJerseyVersion + testCompile group: 'com.sun.jersey', name: 'jersey-servlet', version: sunJerseyVersion + testCompile(group: 'com.sun.jersey', name: 'jersey-json', version: sunJerseyVersion) { + exclude(group: 'stax', module: 'stax-api') + } + testCompile group: 'com.google.guava', name: 'guava', version: '18.0' + + testCompile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion, configuration: 'testOutput') +} diff --git a/external/flume-sink/build.gradle b/external/flume-sink/build.gradle new file mode 100644 index 0000000000000..99a0357048896 --- /dev/null +++ b/external/flume-sink/build.gradle @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +plugins { + id 'com.commercehub.gradle.plugin.avro' version '0.8.0' +} + +description = 'Spark Project External Flume Sink' + +dependencies { + compile(group: 'org.apache.flume', name: 'flume-ng-sdk', version: '1.6.0') { + exclude(group: 'io.netty', module: 'netty') + exclude(group: 'org.apache.flume', module: 'flume-ng-auth') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'javax.servlet', module: 'servlet-api') + } + compile(group: 'org.apache.flume', name: 'flume-ng-core', version: '1.6.0') { + exclude(group: 'io.netty', module: 'netty') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'javax.servlet', module: 'servlet-api') + } + + compile(group: 'org.apache.avro', name: 'avro', version: avroVersion) + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile group: 'io.netty', name: 'netty', version: nettyVersion +} + +// for compatibility with maven generated code, though default "string" seems +// more efficient requiring no conversions +avro.stringType = "charSequence" + +tasks.withType(JavaCompile) { + options.compilerArgs << '-Xlint:all,-serial,-path,-deprecation,-unchecked' +} diff --git a/external/flume/build.gradle b/external/flume/build.gradle new file mode 100644 index 0000000000000..6e2dc5a974d82 --- /dev/null +++ b/external/flume/build.gradle @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project External Flume' + +dependencies { + compile project(subprojectBase + 'snappy-spark-streaming-flume-sink_' + scalaBinaryVersion) + compile group: 'io.netty', name: 'netty', version: nettyVersion + compile(group: 'org.apache.flume', name: 'flume-ng-core', version: '1.6.0') { + exclude(group: 'io.netty', module: 'netty') + exclude(group: 'org.apache.flume', module: 'flume-ng-auth') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'javax.servlet', module: 'servlet-api') + } + compile(group: 'org.apache.flume', name: 'flume-ng-sdk', version: '1.6.0') { + exclude(group: 'io.netty', module: 'netty') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'javax.servlet', module: 'servlet-api') + } + + compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') +} diff --git a/external/kafka-0-10/build.gradle b/external/kafka-0-10/build.gradle new file mode 100644 index 0000000000000..c898dfce495c5 --- /dev/null +++ b/external/kafka-0-10/build.gradle @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Integration for Kafka 0.10' + +dependencies { + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) + + compile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.0') { + exclude(group: 'com.sun.jmx', module: 'jmxri') + exclude(group: 'com.sun.jdmk ', module: 'jmxtools') + exclude(group: 'net.sf.jopt-simple', module: 'jopt-simple') + exclude(group: 'org.slf4j', module: 'slf4j-simple') + exclude(group: 'org.apache.zookeeper', module: 'zookeeper') + } + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile group: 'net.sf.jopt-simple', name: 'jopt-simple', version: '3.2' +} diff --git a/external/kafka-0-8/build.gradle b/external/kafka-0-8/build.gradle new file mode 100644 index 0000000000000..f756511358b54 --- /dev/null +++ b/external/kafka-0-8/build.gradle @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Integration for Kafka 0.8' + +dependencies { + compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) + + compile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.8.2.1') { + exclude(group: 'com.sun.jmx', module: 'jmxri') + exclude(group: 'com.sun.jdmk ', module: 'jmxtools') + exclude(group: 'net.sf.jopt-simple', module: 'jopt-simple') + exclude(group: 'org.slf4j', module: 'slf4j-simple') + exclude(group: 'org.apache.zookeeper', module: 'zookeeper') + } + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile group: 'net.sf.jopt-simple', name: 'jopt-simple', version: '3.2' +} diff --git a/external/spark-ganglia-lgpl/build.gradle b/external/spark-ganglia-lgpl/build.gradle new file mode 100644 index 0000000000000..39e0a747ce432 --- /dev/null +++ b/external/spark-ganglia-lgpl/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Ganglia Integration' + +dependencies { + compile project(subprojectBase + 'spark-core_' + scalaBinaryVersion) + + compile group: 'io.dropwizard.metrics', name: 'metrics-ganglia', version: metricsVersion +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000000000..53c56bd3da6f3 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,5 @@ +org.gradle.daemon = false +#org.gradle.parallel=true + +# added below options to gradlew* scripts +# org.gradle.jvmargs = -Xmx2g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..5ccda13e9cb94678ba179b32452cf3d60dc36353 GIT binary patch literal 53638 zcmafaW0a=B^559DjdyI@wr$%scWm3Xy<^+Pj_sKpY&N+!|K#4>Bz;ajPk*RBjZ;RV75EK*;j-;d{(BB5~-#>pF^k0$_Qx&35mhPeng zP5V`%P1S)(UiPcRczm!G=Nud7!TH`9_!bdReTmO0lO(Zfn zfMqP~+s2VTE#?sl=$9e(CuBOAH2n}=c4idsipAKKQIO=pSsMiScd0TvKs^d1xMCym`IZxg&Xd3ii(^3$M#K)myV2qM z{o5&z?1rtP{gnX6zegV2 z$03xe*g2pGA^BqfBE}XDD-CN?H&?w?kE0wG3~``ie?T+IRmsT;*cpLZ)OnaXBtFzX zMcc97L%tQqGz+E@0i)gy&0g_7PV@3~zaE{g-2zQ|T9d>EL&JiD+E8t?H^#-Hv8!mV z-r%F^bl8v}j19B}`M^zJ>hD+Tg?Fg%no2GYmKkbR`2=|`@o`~1&VbKUJ@ZSojd|ihP|{9&8F<6 zcAvtwl6yo{Js7UQ{l~MjA7Rz8(sYEcRXX(n*(Mp75OBEijo(25zwC)pc=#%f_xV93 z`&d+DI*TAtd3KBd(^964h=X;uks5gf3MU}cR#X_wnk6_crcAVLsTV2SzsGM$h~aq~ z6oX|v?jSq%txd-SHwHy`7x4*jJqH^;0*1`Sztp*aYi4tRMBD|Ryyl%F{IC{(=Y{Y5 zhSQPvqZN~4uvCi*h``E|llqjfc4rnRQPs3@(MnB9jto`dtz!CB-ojaReyN}7BwMp! zU@jEkH2hS%NUB|wE0d;=hS4^M^dTz`r=^`7LNsQkU26nlt4o?Ki13cwDXkQH+)w#uNVQo2o@pEJOAZV3Uf z8WWqpN|lDuGdkokHkKLwmo@qCdV6}M=~DGq+P3}@$$yqQssE{3|BxxM*q?tD3oiW6 z^!W)Iau1CDv+;dTH4Lbb;*)+mGrKg;g)4tHB;h~=3QsCF)I|E{`=jp;ArQuy&zUzA zlz$NoIhz7h@;Sw+#%u~;!w56XV3JkGLOHaVlvs1eSSck_-2#zs%EynXvEnsUsO3{@ z=2B!(Gdra;oKm@A@~#LeoDFC2&V->;dgCP}x`Qm{yZA&ULeNnWvNIGzcgjx2?Rx#m z_I4lu^j~)hR_VQ?`&Yk|{^}Rqf8MFY|1el;E@sY>4t8d;4h}YMj{n$ntcs2Tju6_n zc%t6wvvLifwar=wOlL#;T5V}~s_KU-6cMz7X&7`JeYdHW?WaaBnYH!e82^(58{d#J z&3H)nMCXi0pUcVg^sRt^KZxdFRj|_ZglEw{Ri0EN6_laAxbE8zB=H8KgU;Xtpk5?z zC2?g-xj`9d8MtJf-!H#~s0}tJ>Ksa+7KP;J(%hHwUBewO);ZZ&ry8oYXI%5+YQgNQeyS*ViKL>Yy2`MsK? zB7Y$Zk@YAy#-Kwyo5KSK$lcvER(OV>qrW1VXPo7Ih%dEJZ<|5sEmeC)do0(dJ;7Fc z#v{T#df-92-StcUzRO7OjZ?g-Ik?9eGEDWsUL(f2jUmS9_ajH?wV&{0Aj)-0IP36} z4!4}CW2D{v(ZmPjB$#&;fps(Hvph<>^IORq|0^=eDhYiA%W5!rM_K_y(bsu@*)m3P++I=?)h!HA@uUc{zxJ0ibvxU%Ke8OQ+KDRndS#XDA4T zto-I$zC-%q0v8ZL`!Z;MMK0`Irsn?gZwiTbsJAr&4g~c3FEn8J&tfX(X=3ZOyEhpng#DDSOc6XLr%uGtB2|0=_Au$%heH3&*ID*ZPhs8iJyw{$c z)`ySqPVndS_Rnv2f$xtMcp${1WTLjhW)S3(;l*PK4#cXz09;@vNj*6?$Q>%5jIboV2fgAyb?c(W#K{@rj#6OKF&J#QQ9 zboB4HeJ?hXrHp)H9rx@Rta#*PdvkHJ<<2Asc#ClKA;st5qadT0NZHEA111(&qsaLb zkeTH_h(yr92XkyqqslQgTo(|RN$hhQ*IL7<12W?$+q6R2jtnWadKrIyeA>bj=;9mM zaPK&0{1#djnc2d@4fR{7K872i*IvH0mK#eqf4=iu8F5=2P#fG-GBZ|`J2MyJ(^^*5 z$tQaAS;Y(2k!j10=adaH9^!>^k+iBMVWD9#+F=&Q(yZ5NVJ>A}t>1R@32TZ0JTMas z%42sos08y0NMkb$BsDnQo8nVhd#ksaq8UyBjAO1FHRfW*u`ojc^y3)=(f&PTM`k@F zMoZFS>HCeNX1c@G{<=x`IQr>{11kPK#7AZWy_q&GQYwk*t|uTE9H*TVi|@g8P95wBlwf};`RANPqd z@rf3B=~Q8%Wgl5i2t$W?Ns1WgZ1t0sFVJF73Rc!d@X---3W@e+Dbvpj;l~8r`F9Sc zKd8G>dR2>61(|Bw&XdTlG}}fnu~6{2xsz6Efmc;nRDupK!KI-q=^*h{`b$W=VCBWe|mhK3YN$PO28ZaH@2V;Nbgpwl8Fig6xxkWN7UhWjM%G z<^O`4VX45jgsIIO-R$7F$`uCa6O>(WOZ>i>Gor3X^yySAwSB;0*X`pWy<(jya4!HO zYDvrso1n6V3G<>TnrSv{+unA;cSAWuH!9k`z#^j306@cy?0{jP-p4NUCSRP<_wNHG6^axCR zMECIg`Vz^ja7F}a@eRp)X%2>D5;2HR8kL?^&npLgqU9<#pY)7;V<(`jjbZL5j>8!3 zLF`9aB2GDCO#Q|6P6_x!My0QO@k9M-2f2-|AF33CZ))eL<;OOi;76DiE|3}S)dWs znAG6VC4EKe9MedTp6d0%J409Iu^T12e)n(N?^s^DlM+cUS5h4SEqq4NjK+%c6KPr%EUeiN1v_&WkfIr9Do+Q{2=Ap?{FS|6D&*Au8=PW}~Rt~+3EJpNK(;7R!k z%&9kpQ;0c6#VF?+W0D~mUp&bc96HK5g~~9ch$%)z5Z@Wq`!V zLK|>+`YZuZ?H_46y1zmqL~@TLC0lsiaSZzy-7!W4m2VsY*S@u zW}zeWzr7N1oL)qzKa8J;I?@RxaLmPEowX2BlSD2hnYW}WyvQ$FK%;PTPl4|ftNkVB-&5MvFWP;Yx2`zBS8o_7QfB#MLS!;fRlOU$t z%iCYD|0>%`e)v|$e?d9+U&O=spApZ$`@51x_J1P_|AafS>X$C4YG{A#vQ3BFr-~J& zrNf{=tbRuZqT9ky&r6pgk}1*#cgZ;@Dve26=%&Eu887bdn12Lfj~ET zV5IjzrKV8JQNK?MSC0X660d`nnw55z4~eTJnzix7U0S>nhTZ)+f}uOuozz3gz(ZMj zYs1J$Dy2L`8>*UTBLriGUKTY410qsp)+K??D$!Mr_7=C*Ec`Q^X$b{2>?+1_7Ka7f zd2{rtBr%g2PbF_kHb0yF5ypGWmJ(ftYt&YJ_ynIc9aWyTF~dXbrN!iNKZ_(_u>s(I z2*X|;szamJz{2dl(P2I;%foJLF51)WO50&h!EXP{5|gzy^b(4+nn<;j$NCt?UP_cL zEU1-XN`f*1Gl}Fht4&Pn5xo)Ma$kZyBt2qju1yuSTysFNG^pi%vvU=hqPhQ-Cpl`! zSt+cpY2TVYkvTflcqwieu*7MHVIVSQSivl850|@O!h+yWxUM;@nYhP*y8k$MMX>^Vzs= zqdctvR$e(}|H4~b=2=Dszgy^xqjuY6alG6sJ&J~#*>K6$Dl|Pf&Y~sh$@07*amh*T z_wIHl$BbJK)7B2V0P=;_iRjD@X9TGO4h+mK7laU=qy zmy|9(xNb7=49$9mLHL1jK#9O+t+CJ;3IDtkK4I0H_{C zMZMuxRK)|E4155y$!RGIcAVHwpQK;2>ssTa8^Y0_8A*sk}a3gZD5Il8pD}4GR1;_@6*%Y^za=h*V zu5|)zJI%1zR^G!TWc5ioA9xPwtfcqbul(N(r%J#%9D&&zkuV4xj0VXfymNc> zXa2;S1FBKAgNu0n(1&iyT^XFajIG9aRR$`eVEejDBr(KR%NpK5M4^XoP2r^{5!QII z=({hFfVnU7w+W~i2BF-t(|u6~1LVuM`kfH|v*hfA+X=o<%CrInhQid%%Pn@a#zcB{ z^o7+|r0o0_JFn1}AG;)N{NfS8LI&fnSX*e@KcPEe6OV6U-=oiHjXw;D&;Ui32rr=^ zeI$9fsv_3NJU2zXlqE0HwYjRkF^(*=dmpb#bAGI-iGF7mAYYyfIwG8Z;|+==LK(t5 z=LlsTsJKX@6c-VZ+w{_}$Z-T9i*wIN-d5;hIXRENbV;h#&hIG%+U%XScSeP_M`St_ zW*|AQg18~yjh5x7_;)cRFt3fbcf={_ULTh(DfkEq%mVu$EcpgdN}OOAmZBAc2`mzY-_S)k2M=`*UssiSqyy?xVKo=MbGuFS6XRVx2Djkn-AEcIgSQvVAF zfc`p)g#X!O$~sv5mqQqq{Nb>uh&I-rK1N;7H0mguftm{=rV;MIL=kQZjZ6q_PVrHj zl3gcbRfrb*Cn_KeXJnZ06ZEq<=ClnSMYA~}FVE$cEB}!?;QRYG{63OGvgE!wNV+3U z&{hS7QV6Z6UL=q3cB&(hP~yB{WPCY<2lhJj&?d_e^Y$rRxGZ6c1-iXZdJo;c_d!-U7axq4RNZah&+JQde7@OZf;An_X_9 z;&omnq3Vb#o!)t8oY6U5q9V@2gy>2y91VhMexMd;_^7e(hpL;mFYj^Ms7`BGG-h5q zK%Q)s!X(^S6HkOCW#c?l`3P0RZTN{hKdkGb8X4YR z`Q>!^OJIGZmB9N^htzi$VRW^@G$NKIzjI7BcO^B;MhbLW^3e|uMwSeIP|Zs8x`x5{ zeL>F`oNu5Vl>~_AJi)C>nNM&Fhx!XDe?wEG`x8B+)*W-m%b^U0@g03H+h=q%dPTeRj5TVw;2U@yvqaijNc-yOdE0sdE!oOQm8LWcwb zvPS>{qWI4usc7VCDdgf}W@r3gcXz7#y9ura^7ra0x>qu1l*@W+h%sd>?FNRF3P1|= zZbh`V{x`M!W`~UpsRQ+GS9kSrxHBr?)ej6Lo3uG_H zt<~-{2g_b|`=9T*FAm_G(f&ius6m395oJ6G`(dhHA`zwVV}R^Nn`tA;NVLNfW^Hs9AWg~k&`b-e$0W2lP^ww;)sP<7yihp z>9;T5*j*ExLF8eyk}p*_!`MPt{qUWd(sve|0cQ3d(s?$MugrLsb=F0U=Z>3QNKb~xQiuR|wEjrfU z+ zcuJ(5)AJajIhR#qf<4<5IRe+i9ySGsxMdU+)OPV`G3%LV?Y&rh6sS?3B$pjiq)Tp> z7}ce0mnh`&H|iOQ1Gb0uRJX^%qxxmUZ=$BivYy9aXQU8){*Y>2BZ9l-I9bd_Cb^=f%{ zmiWjT=K4anJYD<1{Uv&xtE!H~-PThqR_6Xk6JGHrBqqGQKH?tN;Y{bu{vvRtU8UsW z(llN(rJar!7lEQU6Og+kl{x(`fYfSOcD-A(iW^ymja51?n_e*XdzPU6$KMxkZcJK` zJda&tRCKm42Sb&*e*F*DU{{L^KV#|D}q2fz@SkOgh!`P8>llzU)uQmjI>w+ojmYH8DUYso9od6~Q z;4`O#*N}<@^x4|bmeg+8V z#FzePXp9j-$0LxzgYFM1c8Sfb>{Ps6=4+{2OSxf{aUT^)eNk{Itv=A6UH-NWAk zbZSac^MG}0zqXgc*4EY)bsLAv>)zhim))%o>em2scaCgRq(g@OFY^hHi1@3ZH?0RyOJwn?nXpt|EK zC7C$;BzYJGJRO=@=P;0~`pP4Xo506(sb4GFpOQ5bebMSZAyb9jZC0r^8@4#6@ zN1(csX24ZP&hIq|Wy)DqPP#K}FbsE(!eN*bF(bfPb223XmHFutE}fA+X)(F@1;%yH zk1$?!wEHfeYJqddk074aDxPkEX(5=S7o}DSWlDe=WSG+rui7V8D&L^N{0Eoj{52^F znZVhj=S5qnyRaUmgH@o~<9q*IvbmW>Rd7#O2emhnur1TYHlW(mYhdexX!Kp=0`-Hf z!SuIx<@XNREb6dlpjM~u!9pOrQ+I)gFWa|Hkzd!e`}FK$)l*+RDtXAr6jhG2g7^(& zS#O*-Cp0BefE>7adHa;nq#sma9#q??1@@V^r}|p+lu`QF`_*sO?>*YP{B-=C+6Oin z!SJy*7wjAkT`h%`ZcC$OE7=@uwGN%GB0FWRvSBmnm)%cUkp!1DR;?)JH7?*wZ@)ch ztfdr-ApH==lQ4l~Yl$%t!PHH1zsCEliydl^cX*{BK4Bwl&; zJx=3~4w4j}SylJRk^&{HD^znI={^oh5?NHd2d1H_jh;qml_=-(7Web$GL)idHHN(A!9a;z33kdFv;y;z&UxF)%AeR4wcX*@Dd`TbAx)+%j<6 z7<;WW)i;(q6bO+@sG3~TKii83s*~Hn6H8`K%#A+`Q1P~mlINmHgw45@lEne3jJ17P zZh;hz8;0&UQZnl9P!Lf~yjjLORIsk_d{Ii_YQEQ%aR@^GyDuwHeJ}ihLhY_)AdGHg zhFSd5RVy#l^BEgfnr@AE7^Ft-Psf|qCcFrebXbLZme1nEm7HrU@Z!uVjNSN%-b3af z7qZUkhP=v4wwvDaxCFp{JDGra@o7D-oGogt?x{pR{FrAwaZ3wj-{uzAmKMqt2&XDh ze1?H=?{VpKPF0hrr{{u7|~@}3h?pBn#AQcs3}tn(ld_+ zqfibU?{%p}DH=QlEiIzWnzdn6rJbfL=v^pfoHN&UQod6xHxUy033s>27|xW2!iAIt zRMTOs@W ztWJX=uuAhKLfXd7Xin6FH>PB>VwRXl4F(L&>kRYIq}#{TDkrvZbm;tKBkE*r5OWJg ze$J|ti%j?`N}(DXhW1uC21DuyX5QdoV|XVRKtk$BO(Z z!r{tOk9B6*;^R}Wbbq0T8n^5h^;eF6;UW=V@uJ%kc4}Rrjhc5Tf!euvA3@BRs^etQ zTvbc9z2dp|@0!7GwZ0$+)pv^B;=9vdN9L&x2Rdrsyn9jC@59nFpfyrwenQ6#59tMB zbxBZXtF3No6)I@oO_M(F>|pLUp*fD=$lgK+TWWnF{Y4KN@A9dy_j=>>H)bJ&9@U?0 z-(8chcmAoj#}1M%`IE2FD2NrGL|<^YKj#l)D>KZ*V{He`yRLk7DCyeO=TFkfPHhf4 z5|@pgWJH9l&&_3*O#Cv^2K;f;@ziVC$~X*WqHORbe`PI1`*AOp(@oD_R*Ppc4F~vr z9G|JbXZs4my(CQT9e&C3$OA;PJWyy4wV#ru3Lb_&?NV=uGtP(tB#tEv(Qle)=vYk4 z_2_mBJ16D~hlCE6WQ@rh*$0;0&BE>^ZBhShUv^)=m zh*`f{oNA2|<*ho;N|huZ(U*r;ql>cY94QNLs#2$sN*iX*R7#Fg49~f#84)>us9SYM z8@F_Gk7c(OJeB~Djj!~LB-D6=PR&OfBRuiHS{2D`rca#zAoHu@6%XY^snOZ`Pe{_!OQ!llj z(vX$_-^MCRyV?lYDAQT~lA!9eCC;-*J^9Q>{{U0rLXqySl({*)}PenaIaXeRr|kqvv>R|%fO}InxZ~VOfnRIr--(j~C;1hh zD*m7*eDdTK57}_}{UZVr*caO06;H%}gb&))zDo;^G(P1=7mFuZQv=`$A05DWSR}Yj zz8ZepdD~0R;-d)k(Aaw^lnN|J!2D4^-n)OM?w|7@Y0vZ25q!@+PfU=QX}7$BJdUxj zdp&-GJ8nv;ER~ac$O(wj00ZDTP2$N1C|hle^8}xhQXJzom#1|R)U+mBjk=y|-iY!q z#0nokHC_*isP`5cvj%!?Ooi>_cRXO*x2mYm_L4oGpC8HQUTzGFv%d@~!e9%SuPlpj z{+Ksf!<#*U0k`m-)`OUxR@_@EbDXB26M2=)J>8l$jiL)8FEthy`J}zQnE^>+4bdYbk){#$Ae;MY{p}>NLh`~ z%kMR=#_oMtE+mw(R^lc%8j^q`_5VQoF zdsg1*Je;fRn1)0{M!eq2XZ@bD?`m0tNP`8$^5dk~b~$&7lVu};YMm&UAe?26^F0+) zQ#M%s*_)O?gbLFsB}@In8QSnvrhwl_hj}eQ#Dg$d!fkT2Bpahm|HITp`fg&o5lq3; zIhE8y*U&7&SRM}j)U^6QBW7u5?zLzorLgcxF7fF%IbQblWkITm@JWRX`9%5#5AFm3 z?qVtRQf9en5;}Gc7cg0+4t*tiyZN8fM3#oZS>Ty_0y23t91&jU(~bv4MIo%-OCEV7 zdAXNo%|)mFOFE?n)-|vp2eT+$NF-%FZ)ZJTBUbSk05b~^3%&W1_W4{Qt~t9odd5QM z`Gc4TcYEVcVve}3n#zgRpAFF!s0U_MxOA0)q)trM{)o)oLggxismMzF;OEj^Q5kik z8U%h>Fvp&;ufkk5TM9Wz*OYP7p7KApanC7QeD2)v0iO3c8vKIb$02BzG$mvFyE6cnmvjQ_D+w=-K2%xW?Oqod6 zB7Pl(^a*8csuMhcKhn@tK~z-am+?uA#K$Hy!*QpDIFp;!7GbNxPhhS)*SU3=9qWo2 zlrRVt`EqCX$@&P93(B0audju)9=r1FK+l*9&iT1wn$MJGx^nFLKC#uj7$2`SLY%5e z5ZVhH0FiQ_$qNYobPI9S))@hsvAm~!l+P2Ld$W0r?(zDtGFM~l&o`%k#PGW1`c9uT z>??a$mVcqFF?ro?X7rTop_TB;- z-W57af`0;N3i0G!+UI{3p4KevSam`(@cxay(;!sWX!1I>p<3VQ>!lK81nJ%b+ zaC{l?Bb+#xHQjN8jlax_iYvts#}cHKzm8Gms}{A*qiP!gaY6SXmN-qK^hnk2{Y7k zpFB~8I{Ao7f!d|!hCoNV8t=r5DH_O%vQ-`O3Ij(>ItU^Rdpg~c^&*XyHm`-oj#j$< zs@56Irzq;y_xL50pG(TO$lB~^%`SH3y71v^^PHT<)Fe!in!K<@`4L#QKo1UWCIb=t zsSrf|{N{PF6+M&c%}mwiq0W$O579{QuZikvDg70zm`~43QGO-r5Pk{^lf0W@cj{ei)J8*ALTs_Cpj#}WC zi~dg`S)L3Gmwe@?Qcu2X?ANb%K6(0%i&Xuf^x$WIkRrJ)2)z*Dm7R3N4yu5;J@DJ% z7QO8HEGp|}R620#Xlf#k_WQ$EfvsI1ws3xNbliM)pTUuFnkB3bYRZLSJ}s3GFL*ww zxCqqp2--kYmf9t(woiP*gSs97*`>%XM1f~pqciQDv}yl|R>f0=Y3Ec#{H^n0b*cUf zT?z`b6|cYwVav`F|FlPc8)QB8n0b#WsS^%EKR8`vz8C8L6UjM}H(wCJX}}#Cy?6)l zat}6oCqnMj-)GcV)Q|4apKY~;bkGOsbd9*rK}hyiVwg7);0g8OkJUmj^nu#&Wb_FS z5;q1k>;crXcO>Jx1@HJmgwwCUFnFk!9`{KFcT?=lYbe}3PuIMZW7h5sWj$Zr^BlSB z%4zYVvbVE{aarM@2NJO7Gw;HP>m9}(Upz3!lm*cu2Wc%DVucXK>X1q?uj8(HJQ3!V zH5Zbg+|b!G7#Cs98{?Lz)=b?9ouh9*RPDS7)fF4rXSx;gGFo9;pdPe&uUvoVIw}%D(mbE03&j@<0#ahS#Y}6C zzg;$sg8j!*_paDJnNWjA0=H@ZrE9_p`T0i5lkMTLb2za(w`2TOvGGEQGJny0X}+=G zIc^9V2J;y%k<5M`x8M=}etGa@Bj}g|W?6U1l0!fE*BF9O zF}^F8q0pPw3rnD`Cu$zYPicTy1ByF1g1U6(!G#XLV)}q zAn5=>`!7;UzfkT=LZNd~RiuDfibAh->%HQ=!8V?TnQw$@@{!w_O(zDwgk8>;;Hm+8 zl#60Sp%qZH%iosK?uTL>B>OK|n=X$T9`1z&gVyfS11B_L)WxTSP-; z**~VIOErBGO|wcRK?Z##|U}R@;6~V7UR}1!CHv0ERDFhD zS$yjwLc7+Zqang$sdZ(wtE9>m%h$g6=5H?YkXxGFY|&C@Bekzio2iik-la8wIYYD? zd07T!tEv{~#%ZUUkyPkHnTeUT;?8!xt5DA(X<25(AnGPQoq;pz{QFozFr->fl}l7KL=wwU0c$$0u6|Tn_0ut2k%t6$hP)9U8FA)$YI>KADk7I00%L1I zYcZ~iyUCV8VSB-!$*VTizC66F^K@f~Y^$AZX_?R=SDnRP}HCj$}I|n5d4*6S5u5RX2efSh`aD zC&D6%JG$k1@||c5Ek6vkOqA2j_uxgC>~=dUOb!t z)k|iO1Ez1pL-pzrsG4hNmHDd15X9VKIwubXrwo6&@`@dHvzzuz84AM>0`s;8f~^=J zZ-p?3-Hb z(WDr+^=6inA%7u{Z#8R-&)(D^n&^RbVqjksyK+cLGE`Td?i{WWKzBukjHI{d0YW$L zyP0A~m8zHpd!;NwkYLNY^aUAtn~9FOib#_*o((EbtnS?Eao=%w;f3qjtYqG!g$ZI3`1M80Xw1433 zYDuT%h~5S7xgT!7BmQdwUvjmbZ0T*t0K%+9$P9QPGmyEzXEd?WB=Pso;#S*e+%jtxnvqYOP;|Zfp=h!2kf%MR+7= zqGGk}Lgx^XfkhZ2xVInt_k*Agcds+D?9E$t@BvrZuf4Cmw{L!9J|Dka5Cvcidr=;a z==`^l2XsOJKXd)J(M7QlAPV>GwK~V*+rb^{2^|m*@jWe&&^s6+rSYDQ^n_H848ghf z=!SeuImw26j-NhEJ^LGci2@NPUqzZ8j57Vm05goac}pbcX)}pAEqx_0{oc6 z8=P1J$q!$?Kn76z(ZIe`fees$sX%?yQws0*twdd+*Ow1p;cyCShuLpwlJ9MVd(cIg zd+6kQu!kRSK^ZSYME;?KkoTyctbzGd6?==g_}DksIQZxIc{ObIb|s5xX-)2y6ESs$W_f{ivzGA^!rv!~r{nB%Cs92! zXKk}PxU;-A98m+b+z6F5D3dnrRIuE_ACKCfjd-^`g|YBr=)6-s`VJpTJ3e+`EkKEw^^iEn=}HPap7lN6VM~4Fn$X zutC%X2g+Hcm1c95+mf}-GN=fN`3h;OEzTc`$%qs~2*Wfado zPzd<{tWY>h!$b`Jnc73EI=}cSth`#km4MnUlg+@_MUIxdOhZSKN!zt%hJ|sxRQADk zFUbNBn_vbW%%oMMYqxL88&xk7gJTD!|rdWQ_?n zcG*h4WcftfU7B*^ zyY;4$6>n_*L)?0fo}9@|C@S7{4d^Xh>|vOp5wG(Zj!RjtDxCHSve@dHdvl9c7BjTt zIC$wdXJZa(ys>*f2KSn+F}3)p-H*(uM;l~DN0mvWTB$P3 z9XjJ@R28*M3U$A7Ej1-T2Hh0_%c#3^9?yo^x6^+>DLZe~#MnLu0&OP8>q;|((mR>R zJ5cfFwZus71_JM?CeQoIW7#+QSWIP;)}&MK$|`$1RpSAHhAU^P*K_W9S6MzDu-kVT zwS%Q3g^RFVH76n$0&tc#C0D&$Z+A zV)?!i&p;#EX1K%24(pd_RiUjSdzbcw@h>av$Iixg@X}!=hrvzbITP@5uTpmfYx5M$ zw!ElwFZ1IsSS{^@hZh-XZWfVM-KSj3Q?*2R5HuMal(y6{efA5~ z1T&LnfJ0_jz-Suospc$;Ad73` zU*~dB@=2(oS@R*|EEY{oBTPB27Hv@+i<{SaOLxumq#~@R6R&mBJr8g+X={q1hNdUC zK#4cHD0yiVNLDS5+pk?E@ zs=i@E$z?;yV`F6OE{##0JED?1BA)FO+uXLQWz)2lg;;1*JQq%SIIuwCxcp(h@UZ^n zY65>V$ftP<()hOq-99k>a)lGWl`}+10G0B9wk{_GYL>hexp!TZlYMB|s{{B{MVzu! zH_*q4xaq77m^Wm8JNmc-E?tE{NEk(hJ*AnwB8o2=j=IqT#DVJ67uz|dV$@6i1qWKw zhEU1Y2jtoPgf}pYcC&NwC*0&=OJ-V_VYscGPURO#MT;vN{XJOALTgdXJ6|k~Wti0D zm_ycwy8NrV4?NPkNa3@VEoX443umrYw?Br(^x-tNwwt;U0=&|(d)ZlJt+JbF2X${0 zGnd#L^TzQnc*+lx=$3y}1#1QCo}v;{Hbov+$L zIy3nf)T*-#h8F%Tu{n9=)>ZZ12n>H{ z(3K&i=jM(hT8Ctl@Z;WeeO)<+kv*udON*kb5&nl;GdIaQ@E2TNIO#j zqsR@_F$d5C4A3F=bZla?E@R9+-RvJahY`JR_;wm_TQV@7nv_1ub8HLY{L)Q zX?Jd&ASuon4BbfjVo&Hk?tG-m#&h~1YY(;w=6jNgzFa_L|BbP?ii^9;wzUba!QCOa zyF+ky*Wm8%5TtN-cXxMpcc*Z73l<0;Z+Gu=y5GIe_uChBR~IE~t^b%~jrq(!ZybBv zwQF=k-wQqN&xDpW!dg-ze)0}Ndum9c=kfnMg6BR7Wx#%)c>UctYj zI2usyXd~!5>?j;+R ztcEQ})Fp88$?c(Ka#E~?%U+;S$hmo5Pn~x0yqLMUa0BUtesm*FU3PlBcX?bR$n)Rd z6o4#8L{T^I(xcH;(AOy3NTEZZ==iHsaqf>)yt_wjcBu`W+qV!tJ>Pml*eNKI z*NxwZdAQevbL>vRZsq6Cf9X*j`r0Xb=iQ;RMV#VR-immwbYlx~eY@`q<5?4a=$;es zRG|@!SgHYNfF)2+ByHOaL8N|;B}6PQ9STt`WFqb%KfH_8A$T+zkBKS0*+*{kQ|aSE zCmb@OCKwq}@y5$hxRvD1+fjz)uQFuGR=LIYUXGcOOrQCE3jy&XG3Q5oi2T?9gS6PE zSVkW^sqpod?OQ3La~nIv_1&cR>p2~1QSrvzR=m*_=%xtkso8^i&eQQN+#7ig3(wgz zgY~V>N9;i=UH8as>Z;hc_p=-MInd$R&hz!@;{5z#jRxt2yEtcdfQgSE<*Er~?s*jB zXFNMgcH=`UTkePw;5%hZXIDW@Q$s}o-#|&f=-T%7+FZ&{`V5FEQtC~dDDf&AP`L9m zC~$9^BgUd67t#IUt;JPj(zz1CFl;4Bmi!UO&x_(K-P{?RC~}Bp*R-&>8wnx8Xzbkt z7$5IVi6FQ24MDS&D)D{RagO01KgnD|?qr*b&cl%JDsDJ26m+LMeg%Cm3q)oc;K^qS zEGgQDNeEF}l#mMbk7tyv;FH?yci@X-eXiryo{^x%v1J(biDGOz?kqhSM%{&MD zndH(?z~n^%R>5tE7o~oXyK+427*`?N9C_Ey%POchfv>+=+y4&5?Ik}#_pTylu=i{v zqy3U4iH)h+K9A0#|1F+f?65I1v%2HdqAsWG-Q+euLw=j1&}Gw<{nbQ#C)@@tgk>w_iX~Pb z9ggj=EKc+;xmp37?l**~E?XzxBBq@n zX%vahMtu8cBAEVr71W(-rG;b!E_RNdbW0Sv5UqZ>NeR@H3uT7cc)oFm0y4iWTBwjL zWn}4%GMi%1hK5+RK`Sa7=+N3XA@remIit2z`6zXfFmfr;G?H<)Itg$z#k?@7It^rL zRzeg1}G&58g5hLuSGb5qO?ncqDnuu71PVYObpw4o7wpYII3oA1>GW< zkcHw>vt^n91MdgI}HISUKD#lo8X7i21k*=5)OK@xmFYr4YvPm1^qRo|t zx#WkfD8|}opr-7wi&myQ?PGB7b`FUs*~MNGW3b3Sj@X6X<2=Z4M5VXCzOZfyEi64b z&fMyM#o$|TagMHE+-GJWSW2i0i(5=ZZlN#K0@P%}M2m?l!RS@8d6;l)W3Ao6UX&Si zFS|!WXd1iD8PVV{RbEtU@B!=Ypdf~_jKHhY#S2FF`o~*~_{|tXN)wl;EA(>dF;j-C zyKg83O*}B6oOn8vt6aV0keW}c24fH>JQg}K4&`YZE~idCU+QiML@^J)QU&R9P#D#y zF#*5w%A2YBYWo|ms?8FIbk+|m2Q5VL&UCg1(;e^KckoO=AP~?xbzIYvE698mMW396 zlLx$bn3y$Y@7Opq=TSDK&@I+Jo;eyXq71vsq!dVzj0H})+`|^EFskziD5d{k2eV1I z9IT65*0e^dgu5)hPuJx0fl7$fC9%xA2*QeXOK)D{_9F{9SqA4o`w9F|sZ!aA8%&;S;<%gqRHjwi^5O`VaVmhQ${Rk#EYrMQY_K^ z$?dT~5p2_3S8Al!U=54VTFhN%3^oz&%cr6}#?fF3OG1T~HCR%Z75KS?+UgF=cF;r~ z)x?r~E#`lY!|h3DQ&lu@rWg!KD^5y3#C;*e^0;85Zp7EsfDD;eQVY{!$YLuFrx#Gg z8Ax-BrDby+#X~@{rO4wQbH~Lc6@#%$Eb>Aod`Co>T%aMr^|TK;7}`WQ!UW*#^5yHY zjh80o(vda?g*9n%pu`W(nPeLv7l+_JiN|kNl*UtQjg#vA!)=ONG>%zMbHmKh5I$Bt z_u|G*(Zk(vSF!pCH@bHOAEAa>Gp@!N(Rp~L+1SKCEX#u>g}(q`i_+9 z1kda0UH~h5v4aIpN>QD~j1T2$djC=0FpQ2vG_w|{_WY_RVmTS3ebF_tHLlj`EKk@a zs{1mRdfGCKj>~egs@%5Dm1TVZv%g+HB=n^)*=rhipiF*+haaM8r}|hdF#I^SJ5%yr zhd0K)$D6LBoqq}mgP18t6NRR@G2?O z^5vnT`obE9M@V}>x0BaJLs#bs-|?sqS@Ue){Fu!nLv7aB_S7}-vBqcy@)h-sI{%1_ z_KT`aVUo-cPO%fI4{mWvjs9W_qm3Jj!4soqQ4~Vrtc0{($z{D%(;kF7@h%OF4|0MA zQbQ_%dmVnkw=4V0Ewd8(_f;fHDWh#*sdIc%#=8YxEPsy3wah`*MI$9~aFNVM8oY+V z58N?ee-{1I6#dgY)kB;G44F)>07s1NZ^+iBk==7Yi35$FJzf7V zIMhF*=l?}r%Tl*;!(BoLuHa~f^Gf1<`JxyC&PYVFDg_VoO-=}v$6t_%SZ;4vR(p7! zD263lQJj<_4>7m1pL7bH>~;r@K{pX^^}#0*UZVccJSZb%W1~@CEky_9hCkeNP29gY8&>G4bZb>-K0PgFCJ8gq8z{Hv{Z% z&;l5@_GmGouX9+w0V9U)I(w0OVU*dCdsL^{NVkoXe&k|}Gh#P7;1s0pdU-zMwLCT) zR;vYSyK)ok=xWB~kyNBMZ5G&1RE=S>)6}nE$tCuhHV2wUWZRBeVA*^eV@&T4KYao- z8>J@etM7A@7(4*0RL&+k3^~(YV$bu9)@#Rx?= ze3jW(AYMZ^vk}*kf`#Mc!y~SdQi!|QxZ~^%Ew*}6dZp4A*w@^B#sC|%2y(WS&wldh zD4iBDGg=j|`ii=4Cck<{^>+oi@J&M(Y8?VB=HITKvm9-(E2#I913ZSFKVKY2Se=Hx zJe3-Ch{*{?p`=}7FdrpaBru39>m8F11i)gUN8&E9PYLndHdk*H#r-vY296s+4#_nZCzlQW4W3f}v`#qflyLh3FkP(s)bMd@6~${pwe2gSWmhZr**B{T z5En}1-PoKH;=5FCS$31_wiaX0pv?A7=y5rlOxGN88=X>E^ zQwDj#0*=q0i%R{1f3Rp(89KMi@xGG>xVdzOD06R(Kq0@Wja=V&;S=s5a{mckK=v-& zmw$EwCp=F@7BIOuHel3WFJ_RFz$IV&6}ID?EB&ThJKvZ1~x6MmPge5 zt(SYdXt(59X5{!;Xk>fv(4V&5h>9RVHi}I}qrn&%#?f~i&jPKw+a51q%CmSk1MDEX zKe-r6bb+qh+%7o2ekA2s)~|-R7IaU9lYp6gN^E_a&~$HO*Y`}0or?{lth{q)u=E;| zI`}+cO0H^nzhsK~>!OtU&k2N0OLVYInlPthjkzZ2?zWYiku)sQ47LbfGme|WDV;l* zb>mOd&2kDRZSsW(mrUaRVkLmJONrT*|Hs&Hz=dkdo6?%J&MdaFQHRVK`qY_&J%MA* z_bu1cSSTyp`fE@g{Xd~mM@Q{e2?*+Q0g zVIH8a>a3_Qfq!l3jX8{A+0iQ!cZm2NOV?bksI0G3Rp0dotW5Q7al~fl7a-JGHl9n% z;oE>%4dION#<$t49%%DllWnO8T5z?RpbziccFCUVExL|4hwyOGgT(ixrM1zBV+8!d z^X4{oDIEy<8`?G=1FBkjj!DT+Iz@+~qsPvnA3m$YYa44D5f~z?Td7(CU`g#BkaT5N z*g;A#HjrA>$k~FNN#(p?3L%cbt2&LNuOsx5&k0 zi`vIXe2K4EbSAVsA**>T=5S)3*c1CSl;av6fEaX3VUx=A;q+f7gC%w8uw>&q111fv zZ_bP?e{e|Fsm${(@fT7x=0DJ#{)DtoXz-Enn>m;)G5VqlC>JHg<*H(Yt{#e#jGs(-y04=5aS>Vo$1;BY%!$&ez*ZtGli<{b0rA5}r9)af6 z>H94&tGZUGoI89eu}dOeSbH=dzRfAM3u-hI)p$zs?nqeek+n_bh1t+8m z)Hb4(LrOb-pe5*2>HcVG!fJtDU#w>qFKY3{eoZdw1yHgW!0E(yWylc8FqYQA1TEXD zbf~B#yG!7!%30~Ut7m9o(CKG@2tdj|R&9;>!RGlD3(LDB$M;FGK#^s$JHht}O6NdN z5j`8AY_jR?2}2T60dlmdKv23$I4Du{j`3BsJ8Ex32ISc(jt3{Hr{-@x%a%l+Y?3b^ zAZlPBAe{fRv-2N4%im<|j0~(rEIuP47KSeW7qFA0q%Dp6Ne7%Z1ui*PI5ah{Yd3m? zTAi=&X-f;DfD<Q_M{%F8}X;?4MUJ}8nu+pvwssN^tipM>X-sA+cApuG5Qmea`I+!?Wz zULdjKq+Jc-)2)X^`dQ3fOXK??zua!=6M0RESPpv-s+8Zw1a+M~+)B=kS>MO*EY~-G zqh(d07C`WlE$e}SMClh>Mj6YS9Y5I5w~9o;u&J4TY_k07lvEV7LnK>NO%_n(VeD4k zG){-&tGycc@kilbQR7!qRoa)@uMNncI^CHCej*vD~KpBG?G4FFP`b=h3c<5 zL3cO5n2TX8Ns;w10gjIPBve?{hVeUA_R9O5-CF_W@8-PrpMl)(WyZAYU+qv3Uowv` zc<@@i8N{_o6&`Tq9z27D9!$ZXBG12K(1xTbPA-}(BGM8IhS8Xq8qs3B1V%+B8<7^+ zHWC<^RHEO@de4X)Q)d1leX>^6?%(m$myvMyrdaKQ;co-G*a_oPU!!}z9L&OeLum*7 zPmeF&ND3Dl8U#cd5d?(ze_jdyNx=+O5A#Da!}c|8=wRW2=fXhIpo}d>u)ZRJjc;81 z7VEmf+D8kctKd-2vqlS?v^Fz^QL*VN|7F`mpjxPht{s!yY)v`4r>erMRi&o=hWUoM zc$@*eKvtV%c!_@cy|wR^bL@3Ik>r=-2hr>LrfgyqTnolEx5{gqRP6DN}YZqiH^9hZ1}POOQ87awyCk^DBpT` z3l3r4lFjkr7P(P=KPd$5rY|V&t1lX0jgKn$b(q*dDD$8`!MP{PJ zE5l~1+~7lOtKRTIY^&dZLCB9i=Y=mk$3>Qy@RIJ2pb!a0q!k;)n7WKG;lbeFqDEWM zUbIwY8gyU}?4CS_?RN6mrP`90btKc}IeaW0;4A;$yCp4flat@16AYKs;ImANuE@Ch zgjeDskVwH)h!u|vKg!~VjeNZ*;rLlKir#Nb$sHWzp0HE zKRf85#yP2R6x6LR0UblQ6mtru36wz`)a5ZMZllv_vKg+$cXf8K8lG(R`W7itv%jz9 zv?A4R`t4b~3Ju5+WhP@p(!*%A3iYs7i5Z);*uc`Bg?t}Bdp&KIIwJcE@45+Nsj^<}B6FhfzPXOMZwm$|zrP`#no437g$~jT=c@NXr4r3@7zb;@7=b}XQ@+T0An$s3 zy-I}GgwuAIcFneODJl1gC!5vLa4zl^6=rI})~MKOwK&^^*Mp~{|Avzd_dePHu_OxR zWb)10*H+H|oQJnRcJi%1k+|P(d`(SDgv0VMSW)SSUaRS@I;h!^H^GA=#VyTK_(!gb zfYYG+%8EPLgbXQFz~R>)fb%Mo_f*X)0&>nVZ#B25(?d$K$|%>Rf;cJv@dw2r0QP_k zOmz*loRlwq%RV0tfGOWV zh1Ja{Gc3yusN*N6APmm1UxMQcFeAHY`&woD=fh3C)mv^D8)0T20-M2-Ga>*$mbt(f z1%u6C!5IU0aO@~2&ig$|g^hi%FkNU#y1!3!u9J2j9UkFJap6$er-4nKm)0;T+aZzzSI<)c~I+d3DgJ zBgQ(WOG%eh-479Awz40?Ic#qbcB9V~pVpl$g5StF&L3iwtG-RYO7M2I^rT*EYB@}5 zlr_g!Sw!cg0-U<5zV5mO>=Wnpf~@J-rwvuM1B40%d&fu0Q`&tx1iO7|^cpZmgR%yk z3R=5%0-k1ulIeTxr1o<2aCk%Rd4Trb91M(hfeW=!m(`+T%a$6+4Xmp63wOl+%y~JpiZ=VoB60)9h8{km2x5%$kR_#)es^mlcUlVs=*0cHly>6$+gT)P zh0gTZSy#^7GE9M#%m)`!ff}C}tp13BHR<3_=&N zhGv3Ic(b|b)`N`6#z8|8f5>iIN_kVdk@>3U3O0t2@!QapkmnpkH*md(+p zeb;W_Ru~AoJ_jTh!CuE~h&ESGm1Q@S!L83OCoMmdp1|3!FIQ_3KGq|gPHihQzCKSV zg;&`PH!e@vqQY+5n$J|QnYWd|K3A{+YYvAe)1X?2DDx2HFN^FgdwCser(jP74#!{E&2X00Yr`cdQbzgR(vimiMd zNA_!FBf*@-xW%lsWt^?AddX)l4RS4u`z}JI?_+2-`t*%_hFGZ=Kr#Eu%MP%k)L($G zmJ4d3&3jkMSjX|y*e+@=vvSxMI=NLgzw|}!zLAC4^qxh<=~(Ozou9>_&b@!|%YIaA z!KkG}X@w_4p(I*&!v^|5IA;yZ25P;#XjY0hN}IVag2>$kgZ$Bbcr_#;B*Go^li_D( z@pm&ocQqt+jYGXAm+EL^9-TvKv&W6w9thfs2KUHIKz)is_}cbNwDt_G{6cBKDQU_p zYRfCknnMP{ps>$tQlsP=T)#r;6N26xZw zKx7F=It;ntA6NaNhJ93Fl3J6j|F}!XFB~2F72h3((74h!`%-q2WUMX!@V^_rJG`Nz?Eck!gZGL3RRbAlc8PT{kyZ{bF%57iM2edLIrX5&C?+2PYI%I zXt99XAiTJJm?DmB^RJI}#5Q}!H_^xAW!E=&)3di-GJ>FVyjz|GE^vr#Nh25g#3JFv z;XjkU;_aN_Dfh|qxh!lASsQ>JbwyBB+W|-gsv>tNKR_GVL`ua{xA-x#D^v6cF*-SA zgy^IqVrH*F($Qbn1$&o1{-!ImiF^qZKJ|p$=j$(U{XdcTbZ-A-HPZcqRB!)@#i6(V zKP-3U=_z@tDM?icN!f|X3EHvAnSG_lsTpN^X;~%4S_Q`1MrOuZW_AU8R|G{yW=7V1 zYRd5$DOq}|W?2R)nK8*F+3B$ZYU(MPu}RscL(qSt34nd_zT?-~nPxse2;Qgc{k+cv zF?6x8Hl}klF*32RcmB6e!51TFH$Vg*eEpVG2E70o50kTr=lIgy6AqZLFN$?e<0)rt9fNOYGuj}+R!Ob5B z>a#gOe&>t)>d!@gDVMA8>96f(EjrrupF#rtS)(#1^q9F`VLKt~oQ}h(Ntw*GrQ*F% zNmvfD^=MC8BKh2eqbIo0+itJM@4)_GFL%kHP=t&qL%JK4{a!*JA}6P=Na=+yHxCHF zt^HMZU`>NPwoAy1&-40ky8o{q;q!a{pIHl^g6|gNK(z{uL-Kc)}-_4e4&hh*MQ z+LrPsZHwu@zg^JquZXyjv#_0w%_o;o#=_R*6T<2GN#_5ruy~S+?k9#HjSmbif~Gmt zrfPc@@vy*ogFj730(#C|s2q8IaKr?A#YR5`Ubw)oe>1R7IIWHU2UEG)%1~beL^mhq*kFr#ttMPg&%1fPIIwM)17ddY(dM{ zm|$7$c*Jfo;UWY`M3CXl*Efd-R7=iV(J%4Qjz&GvG3^8xSn2gn_H9Ekp>>@kCSg}n z1)p>MY^Jm@X4Lbk^Z3!;%`sayVqbB}olO=JP<^3q!Ja=3Qe-KJ z<-$Q;E<*?*5BcqYTTs~5X=TR$b>6UVlCe{Vv+>0vo~VIESk&zgxe7I2)8d1b<}1y& z??NZr$i&i9d2Es$IGSSYYONr@1?AA(%%h<6acCOJN4{u?hF<`sCDLgNrmGaQ^M8w} zk#@Ym#l>KTW}``wng=U61g&mC-(D}k(ijVRz*H{yl9sAq9m`Qu*=23j_IAk22jwff z`-db}Ovk}+vQr#mjR>R<3lu+799RYM4mEuD1Z3Bx45{h2{5xgqucgTR!z8Boa|xb& zzWx@{{cGj^+Y(eTaQx)h|CcWt94YwMGz`%X^Cm4g*TeQuE8!UhG;q;A)Irx$Itx(A zU-tC8{vjyJI_>uS_X32bTj%fYprc43(83}?C)T#jJ2q9RwWG2dQo2`GQxq2X8!n{R z@t^gm5VrdHSub!Yk=Qh-s4l6rk`_HeXld`*BMJF?sGC;9RFZQvKJjUskfd(de$j3| zY2k;2JDsw?cv(JVNXOn*=F_LuU7xd9aK4#e4tImyciMp9F6T66vLjC>80~na( zSo`z?)DbJw*nrCO7~2e;x5K#aM0yhWq__QuJ0Z;3C3H@>(!;e}M3E zgV52L4pavfLlaoZmMW-GQj^U@sis7jaEI6+ht^#Xq(zuU7#~&>a_l$eE)h~XxC0-* zLj(0#+V)Sr4(P1aR}7U;(G=@#GDYpBt5!HDQsD|cy^}|OPo^3VCx(B*7!YbE{BD~- zTyLsMnImY?+O58^d|BeXQJAn>-!w2KYm2Ld_?7euE^r>-!mplJ zT%MXtb>FVVW!`$lPh7G7Fy#%MAzeo$0=HV$cA||Az}Qm3+(XKM9iMD2XzlXd+5d6n zJ&0_X&H;;hj(K}YuFVZ`0f)#Wj69(uU#Q8@Oq5)or#TL_8c_TdK4ZYQo3WW5gdX+fWum-4aLQo-t7dCMT!enzWjL%H7HNP zV3~Z}`VNqJa%GeuFyizrFakKW3YT--=qjfenAY<#M4mMn))d2~5Cf;MEnfYHh}A7L z8h zh^<&eLd+I{lpnykh;*~L!yYL4S?^IILpVG5|M0g3-6@kH%*Em-Y)v#}nB>gs0ggx-FUGem-(sOAidI2#zY zh&d;jIr0lCjRE)_rdq$~Jj;U!_2a)q^8ecHot3i8e^H+UKOa1N|C=4~&!w;YX;0aj zeeMr?gHH_ff6ao)QknUrK%@OJZ8tHn$-67iZwk=LQ!u6?`iz(qhQZV!qr!)66<-#^ zYr2kIB|q2nj%7f3-GYXZ4>$NF-cGqz30YZkI&Wn>b$eWI*lj(G%rI<&pxsXPA{`-M zS!^k>IyfreOR5+j~VywzG;xQb0w z%vugs{rpJTD!(a9AwuTOB(Af-=UpDbhvy*Y&(y(tt2UUPr6OTg*NX)iA>l947mYoQfXk^i!S#*U@1J@ z2QxQ8j~Mw)cm)o6Wb1vaQpc)Mov+?fu#6`k=k{qr-AjZkiMVJ53n}^h_sdXxd(h?w z2sYr+WB^%EIjsH{L;2MA^&>*EmiFM)Vx{5kjR(tRxH4Nd;F7H*z7)|SOkwfFzHwu zjyQb=&j52A)7Pv%WBN0lYwa?HW( zarYIRMDIjG1c=2P^h@kN8wb9$KO$>cd);0G+mrbMnewtor?uCd4zr1?o4SR(Cg_nF zkUVpjEWC662=|IJO^(DK?x!B2j^0b6y`ZUikbi4jw#7kK+WJ;36wlKtOhDmSF}g!P zH$f8Obzo73Pm&!K+EXYKY3-PIga1nMPS3b{_VSCxM!!Teo>DGkDn2nojggH_aaXpU zH$LH4G7AvW8Fsi#Kl+Y|_v>v#Dx3|0kueJzq0pCt13sdNMIxa77x~y1i2v)r{k7Il zND=PDKSinZ=X(DiMApBZ_Ma!ai_Pbtqt7uICjU6<|9QkG#Z5_pF`)&^zp@lGHEY?> zob_KdszR+K1%w7Lw$>K?cE%}=OA}#cIkRe`ZoT>9P01uFjZPP!xp|Pi`5TCO_viK% zatD421$LF$U_%rr8raKq98kg+@S5i*PgsVji0t{U;(+WL0{{<}<}w9W&4F{x7$Pl( zbVjo%I-2ko6E1HZJ2oWFx(V7DOrd@d5*` z$^w|?A0wYs&yY_5+5hSUlD7Sv!}ZT&=${JY-yXt14J&uFLv-IB;{;7|FhDTKPqmWt zSYs!|FA=ki#QBAQ@3=CNjq3((GLRfB8)<9c7ei9omq5K)b~Ud$=ylJZQYb>5i+Z*B zzn1gmKOQ|^d3|2DbE^2vOkEo?fVE))zmGHSv#vM0oqkVW4Si($efiF_l~f1eR-H>m zT!3nXB7;q=5h;0h=b>ki`3J>AN^bpM0t?`p#cS6RwMP{$3 zf5YpWx9YH`H(#WeiS>XF^CvIoZmh-lGlgi-e+{_h8{Yi!^Lihu^G%^Y=J}bEP=IJ3 ziy-mF2-~H${~5-Q!o!0yGUx8e06S*KOy>7J>>KEl^fOt7hf06Vr(Ld4S!ktk$I!o38En7`E1Gm*Q%|uEZ}mZi#3G(m+FYjfWOvb#E__Z- z@(F2jsA4hk!V}px!d;J|`8r=x_bYB$J1ll8Ve2jmxjaLGP=xs4h~P@22`UkOuxu$Wok0kqIKZY z7v{k&6b-io3)$3e)lbt}Z_~fO@<4lyx#RLPgJs(e+HrP2uvELL0%L{k5$EjyTCw_+ ze))JVuA@XZtJY(UUm@n~7IQSIt*5nZU*U$mCqwT1D3z8{4@xE$)w^EVdEw%s?L;?S z2|Izmbm=3AO)bgLcxt0pe<;d7iUbn__kCyk1MyTdvmGeC8&7J}*?2$SH-VwY?XfoQ zqYq<9sSl<$9(~Gzm3ef2XRgI*4*Qs^-iOc{mq!7#_VR<*pu?P9q(kH6HYv$S-z zs5+&xfW0mB?!dqQgwI_$GFrdABguYE1tGtW{WA+#TpmLPEWl=W5pr5Aw28uX;H%y; zShKW7UZO*hqu~tPrf!AcpmE0#hMhK)=uhlz&ZpnD;{0lol*8qGnCqv4KG-m|@U3b4 zl?Xtc+Q$Xc06T3Y*XQIe)_)JH*Bb%rJJNTuo*~r=xnwEUULbeU+->~Au=|AaCwC|2 zB2ZvgDEj*SPVt7h<9)_mv`@5?y4II?s5&t&^ql?IDp)~#`3~D#aLNhVyyKm%5s< zL&hyL{c)Pza*a=VISCsC*4=Tp=uQtG2X4&bwB(@~G@9X0<6#s;1YfU8ceDfM>1HP@ zBy+SOJ62?4+$-QM&((;$^prmdeX|xh66wMGkkKp(BhH)FWT^D$W>Xk4W6R9ON7z@% zVRY~IWw$uox{cpl(D*&pG(%7Kh}q0AA?!&l`stAjp6n;qfF)3#d6PKfdL*zVS>}wq z$k(Z_&pyty-RR4EK022?p=*_MgQjvXnxjs<8w|nxC^pBs8a0=({ZRGv?T7#LmUdbo z5r#Hl!!S!-O;@J)d>IFHl5f20y!@7DrF^ z_z?7%h9EOc?w4A3|Xj1uxkVSJ`K>#xa_TW471++r*d$j(!vspAXkqQ`^ zoZD|(BIil5=7YRnGxHTWM533mjo-NK;i4vPo| zn?ra*s~AYnANQ59amix31-PTaLuxI)j@uiXoOo2?&_`tlv)X@EpUk}*8n&pEBNV@j zfuXm{97;!lkNX6+v9zp?5+KP7qR1!wDyh@V6iji)=-R7v3v}szoJ4rNwSbSc0fn-6 z(%(nYl;3B!l$3VIQB5c{^=Z(cU<^xg(h!)O2HJD0|Dx{h)-S(T7;?(y9>PQ`{RE6a zN^$cR<(fZBWlxS$nRU8TMF?c>_K?{Wy8Giv%3+vXj7lz=Xr6dgwdby=-Tc90f7Rmn z^RwmZ}Z^0gA} zpDV7;h)ustN7vN)e6%NhRNEoJ_;&(`T^2pSMT}J*hGEJ`LU99Th$e4vERN_Cl7g#1fFt#NO)B_sw+ciHwEO-DTLU;d;C;dye z;aOqSO?_&%jn68b<9}bH{JlC!Qnis^P(bswU71tl#@~x8%r(>Y)Y<4)?fVX?&>J1e zdSv`?r;hj;u^UCIqcnnP3%jix85v47(NBMO6yqQ~NFgO$8%!~D(h#Rsy^ znD7kV2Yq1PEh7pH_)>|VT8Le`$~LvRf){u#Ty)~YrURJVsxwRhFE305zP=MK(trR9WN;qn5XYoiV7$sVI!-sf{?Fc5ip&cf@ zGudnOY3pS~zMc(WnAgRHe8MTkv!F<~v{cir@{BfEiuf5tq};~7zttC~cORaG;|u%A z8FeY29j1eTQTh7pT&GEc=CMS64QhN+jpHJHpUzv5^nu@Wie3C#1CM*Fx&F-9i<)v5-{o*hh{<{gKx zd=UIIR`8%zBdo!+hVyj&mt4`*Z~b^{sH4B1EheAOX$fYz1wCYxsp=EgFdIn{NTQ?! z>J?uF%QS{ge*9-;b6~hcvHQ70Y(I6h!2f24sM=cn|ETCL)d{7~I-ZYj?3)4ecW9*B z;84mjBN46j zdtzrh_wdA5IPhahW^WKPnw#v7R@qkTkoZGzAsT?)f=Pb)y67CDIEV%IE>TE6pUgTg zL9ZdRW$gY5xw!Ci462Hx`Qj`7ql{#Fy#ut?rfFT}i$MKVFUxodF!_t)=KFcy*2A>u z_moV!M0=Ff>}`{iEgwBS$0#Yu7Ct48thL-)BjwNg@UxAr_*gIIEQ=QTa3jL`EsI>+ zjYuZFeNxBqt$JrBLk_Ion==8{S`XJPyZS~xDjB_0DuZHdF|98P6U&+5+OyYb`R`>$ zT&R0d)Shz`lbzwV-&7um%7g?{!VTcX>T>tUF$!9T7@uYq`WTncHb52^h>~r@jbC@! zF~bd5ftDhUhb}_yZU8OJ2*Xn2&1ia$Rn;Sm7sb%p)Pb+^?Qaz*iRYMB z{HOnwh5+*Sxe|YKy#HBs{>$%G({@@=L;915HBl#u0zbG*K;&xLx*mU+% zHqg$khwqNi@8cEMKjqd2kB5=Wh&!5H#zmiJ>!cUnH29&|eUi)`>>g%wknYx9dUyk7 z6F_YwD8?T>eO49_%SzluTA1%nJDfXWFHX4~)rVA0=CF*0C@m&A?1f@nad48XnLNwp zE8bOcL>+B=P|RwkAvBVx$EsRlXod2)+=E|ci?Bp{RE($15ivmFA*6U|Xd=mybqYmB zu#QxV!?x&{MxKG=LW=FctT;RFq3zZuAprm?YtBkFC#wgKRa#~&N`;=AcGrK;T9x2{ zZa3FXnyoEKCn}04|aPM{QB&kb#R?2`EaZAdyVs z4NDNhxCE(jNZ3uV>6F;=aw^FftX^83sJ&I4mXoE;^pDxhmL3*Or|-UBbQG$UN9wWJ z_Gr&8`m-^ODr@p$q=n>t?3;VRW@%$E@F%O+)E0^L7Bk0GL2V=ugD4wo6jw;B4HfBf z9$QHbDDg?Ad$rCSd7b5Atz^Z2FIX`vgL0-hQrrvRe!D7O@;)!aZEA8H(l^x>vYSzH zF35#}be?z1YA86ZtzpZPL0PsKsJ()yYc7G}B+!0WSBI8hz}D*h*3E68CD?Du(Q-3L zwfLiSYE_|iz$(4oxcw(>fV}v^$3TkoAXSCoanpT!u9y7iS?o>u4w3bIBgSsNmYy*h z1W%o`nC8#GF4)IVaWQjh*sHzl$ltI8`Z9gLpT7&a8Oa-am+p{yPkvq2i8(iE=2^<$ zfPFOR36Q@j3jh;(h0lq&#C%}-VI{P!Psq@u4Lgsezk6)uhVSyjP^C}(S*wqO#C1D@ zS~bMmaW^{IO-E_6a?kb0qz1PaVOn;FXEYz|4+s>)b~lTLRcW%R!an^43kyD5MPZhRfWFGU)S%1LxsUp2 zE;Us$5)4>8;;&AzRRzfgNC~-xa1+bPl^r;)_xBxv+pm=I0VetdtyTryvPe_gPxm>Q z_J?3wcRi!xZ1W~0RUF-rRWG7hZKtOXPpeMOs9HYA}f$?TK=>Aff z@~*63NyAagJAJClb{y!OE`fH6#4%WiTMHeu%x(UjE6ke>r+;+=D#;049D6mD``YoG z&czj|I5U=`*T2rGnx~>8-_;gEt>-kGvt!5mgZR9IqqNN_j4}M1cGl@?_vG#0PW6A?WBxxZ#Uv%&zmm6n4Vn$Eva*fodv1|bVpO(hKfaNee@6>|jpKc0 zmjqmnnyQ&nkbM;N6v_wlKZCs%guAhSL58z%C790YIL@BnbNlo5{tmx`^uyb@_HUue}m7G$vd zI5w&QbjX4RGH=e-Y&Izgagbzx3!ogC(g+bO^w;IE3jJuu$#Z`eX#9;(yw!S zpqe!4ErF|esQfMRrgi`X06a6k9B2myu~agt{0Whb(z^ZbHdNFoP1U`Rw+iEw`Z*Lh zc-Q<@l@U!(xBZegoF3uD1t$e-JVD%mQkJBR%(I{VA|=?!j6|KU9U{n>$@I~$??v)w z;98=w&&N+hv|};w=V$2FMt0i5)=+sWdIPSZ{3%RYClgv0faav`D_>A_k9tv--8&ZBr9J#qbF7|M*D%$3JfTZ-)QB{C)K` zKU{V6KVloB^CESsBhf%|)dkLE6E48FS@*JteR5g64Jrpodo5U0D-!uwv&6a4Fl{O6 zyxHKQB8c>4AC`!pg{&~-0?8cDx6`0ER% z(2ZLil$_5P#|*_SXgAVG1ao)lP8|c;Y;=mTH`h;FEC+AV&L7eH=(oPA=AGnnzHDJN zyY(8rx**=d!{-j2ao-WU(*`r#rBA%2dAsQd?8cs>gDy?imSqbZjXjo|oeM^@$|BC< zxiy-8&F?g75yZ_hS70J6RcoaOB}DxY2bxH-g$L62jwV{5Lq#NOQCbAvllj~@ER~xF z!#Fu8vcW31=Vdw4Mn@uZsWvx;o337|70o?Ynkpam4QGJ_evK`nOPAK_gOajPx0c3l z311T3b1G>1Y9>_!ebR;7%-$Crbq27Fkvy4Prpl~klji#|FZMbu5h{}kA?rSKV_F_@ z%ytfQD=XwVo~bTvrXr1LW!Ertm`NTalpqU3>{Ao;$bF4x)NFB^q2&7P43-oNqy|O~ zJWOA#xsyxq7&4qBfdI+9&&Exo9v_JS-<(9XP-HJzQOdHAaG)7)hIn4(~N< z(mX#}|4=|llvZ6>9**iSEQ8ArOdxb-R=bTgQ7?tQs&H+yls>WKwl6GCflZgFs#K3Y zRSVXd=Zag)y#b|hcAjYe`KOt-G)ar^a+KZbVJ>4CVyU8S!L}rhhya%(*@~j0-UWik zAX#d?bD)K@tIHUpq>-%Z;K@w9wmwK_Cqx#M5V{!~@3jp<mvau-CC&*inT)s~J(4bQwef5NKwgT8Q!5-TWq+{Q14#0wUy=9Y&Ypv{g%UCpcdp)X5E#mp<_EW} z=2<&j9^Fa z3EXedVfxY5Pe70wdS^%p+8aT|)B!sa1#u?2OPp=CV&*`*TT`71PI~dxxHQMLqlV+_ zWUMw+Ruw)|rszJSTi7>|fUNVG>}E*etC>g}1Y86wHL@tqh2Qqg*qt5}vw^D+3tB2+ zOlY&s^1h6tuT%NS6X}W?gx*ns*7yHkXhnJ+UWZ{Ae>r|Qbk_)fB06SPL%>VMWpwm+x zP?NliTb!q@+a~(@Q&ktpE>{)L@>Q;Wgg8=R663fb>J2Q!78L#C@8y9LY2?-4vk z_VD8&5wvW2OdCi?KW4s%6jHk~mEnZx$`Hrv=UxVFN3|Te!P+28e*l_yVLT?&m0tzF zmVCLhb12&m4(bWEq&P>sQiougHCV+$&>Ci9;0S1h?h-E;Zi(Cvg;uYtg2|5aE7~sA zlE;v1(-5Hja}8O%R<7?(FE7NU;?vf~)EeP)K62hF4l^F`dlP53jXB(ANaPMf(LA zfmVcBTft7P)(yoHv{zG1JXx74PpyZbHDuCh3Mb>T=t*v!oOq@r?DC-d_s}%(rFs^^ zZ%d6I;6Giz^0sukzltv_A#7%@4AKf&qE=|ox(x0C`?lw{Z^gvhO?*eU;u_V^d6E9W zaC>^PN#!|DT>$3Zp&}#PJ}eydoid0XvIwPB3pE)2kBgEmre!NXABiU*HxMV7`Uz$yH)SRse@1O zoS+s^kzqW=Bmk$B17sAN(<_%zh&u=%w9Bp3=QzeRq`5~r^rb{D=12B3v5=xPUc1&8 zCLh@^od>R{eDUR3l*po`zs;uh_X)JBHCGHG)wgq$QMZp10IY zY2=XbVB_DLnONUkW5OQ*=0V-wY%*MU)o$!<{o_(bY2-BxVcMdMSYM5)5F15&N}G z|I&m%7V5&f_J#mGGD9nSQwLKg!{7TN=Bk*{ZvfixElr(VcAT1vR{dY%M0Tljq9WV}=Z$6IiTT;#oFAJFftdiMJ#wEMeD@ z51w^Pmnw7ll|jVNT-J<1oIp)}pY^abX z$(&%+DJ@cldZ?NZnM|!;*O=buWpVUW))kWmtz4NgS%^jN&Ywi!&&4)DLbp!JEbFytdRjmC-honQklfu%p{-bM^z#od$M*bQzSw5j zX3^Lp$w2swMHoF}`Z9WIzG8}u;B1R!=LYOf1}#jITe~~nQ;Uw&c+fa3Lgwn6)nstW z)1$tR#Q}yfd>bWF+Ob&g+BT(j=L_^e7|fj#j6cl=k0M`f(ftD-n>(#4Nl@}rN~tYS`Skee3!%|*_nJ(j6D=FB{b~2vk%t2&mxW2z+zjLWk+q27P_>9h-);iNanhX7PBDbcujL#~5 z&?(YirpKTZD!9x6l*G3Iph2-*)mo%M7O15Y?K^}Ss$rFWCN3IIg9>mn9T@G zzu)<~9Bo<%^_osbyA{t=qjdLDcdWLqyImc2r20f(BX$ETg5YLvVam~48XGJG7W5fN zeWs9lWApcxW-#tWmXVqY?yLp20j9M$T>My(y|^v)LKO%s%ODWLrLz*#LpXDdEQUUd z+D6kuG2t%RG2}8P=+solX{al2{qh6tDmV_>7 zZ>CY~Nt`JNL&QdxG8121=3=w47?gNkfrjE=GvVnB4cFHzeYMiDr zF5_|pv(dl>#!gc}Y4?VQ?1O6TUQNUqR0*hUlY5Q4E~dlc>lW+{v_{@P5fZYFa1c#X z=!TAp#_q{|jfLfaxnemk*gTcZFm4I4BIwKCaX7oX z6_wEK72gdI5+u7z4z_teh3zKKJ{7F9Y3tIuRr_Ri@Bvf7W=@p9;Bjmp&Ktwe1go@g z{G?eWDaFAruknsF)IB5n9^l#&E2dmJK4$G4!9U`y0i_Q|(feC(gNFEr!T9rk;$TvS zqh@V8#8Bth!us%G-sNs{bO&Os3lgsVUB}zcmS08vt@n65^4Xp|reP<{!%*rXNk~aK)LSlNoJNI(_qdG2Jodh6AlOv+{ z;JW)ApZ^)A@XJnTtgl4;2A{h<2G$v-cp><^qRl*d)+?YIEFckv*!>B&DkwlnP&>%I z+A}uR>lB9l{8NnJYkY7UV&6}?r1g&u1J2qohUe>(H(k~rmv5R|-%=k|EDZVRiDG+tP<5~o z(OJW^xW0D_;s0>w3YAjcdIyc7-{DI_UM2>*Sf4_S)}oEi zS*jSwX~y7w)XP%PZ-fS^-)jT~H9o1m>_H)Z8@SZ+!=i9q?d)pND0RQcBS}y7^0)A!t86RLC=*8Zt!3gCklci zCWm;JEelpqZpMib)>0@$T6o&hMwL;38r(T`wNj0t*bXH`>4`_=8;)Ce1p}eMf_!&& zM;&EckhL~k#kMf|v|%ZDuXyBE#h?gsErc=rIFo@re{kVy&K2n{o6VKG3}4gh!VYnG z$T0Z}F(zjh8}9uHtKO4jLFt|bESh#3O@;GbEg3M*LBwNyi14t_fM>W zCHt(icYV<*C*4#5LG4lqo|u{WdFTkUAXiF^wuX?kMivYwG;cUv&>EMm`fy5DY_GEY znMK9$!aj|hES+Z<83-j~skhCI;Pu%H%+>^Bz%{6~ z>}YpJ(w89-R@khjmJ>;l(@rr{&Ut7oxQ4ROI#n&FwiPsAf@tHpW)*0}`|%SE1dzds@`VuDdk_DPCuFR*g))jsMvKP(mSkc zWo>iNM5cWmg-V{(>-AG8A$LSX1>5fmn2oSeZn%(_a0~7T8&|LOPYoH6-pY3oV-UAG z>$w!2>A-)ojaB{TUre9i&ZqFm*+jc}1Bec>7Hc z)^d17RH6xWWS;#2-z8jH`e`9#2wDjwYAJiX_=FtYQNC8?RGJ(08CKIcTk6nC8|x<4 zInb7n1cgS>8AxTxduWGD6vjTII3%LBmVM%od2z(4Bqn3?jQ1bxhKHi31_*Gjl)}nX z4;pA9m94^*U+@Y=>peo zNo?*$Q9)7N$K6y$*p)D{cc%Zu`DTK%K-lB8n^%GL_$`in;M$b1F~KI~*UTFnhF5T3{g*2a#kgwo$-jYFM0vcd$qn9xB+JyI$&7jpv=+%4#Xh0b<3g`3 zhdF_vA*~t{>b+!2>!`@0Elcl!sxDEbBBGAVD}8c_0)Eg^8|ej6d_yHd_!=@JmoYjU zDmq^-Q-Y^4Iaf={Hd9=CS%Sxv^kma=dNjj<{4BrZ7U*O{+?0VoYOUXdkY9f8yred3 ztpN8Hv^myWp5ub>K;@BkrH^pxge&e5<$+hC(#oJ)^}zggRWkm-)(xahJjQ~#K#-+7 zGw=m_285~N{?R4b0iN&}e;|6(xLHZ+m7tYXuBueU)KvCVwtw4faK<;_SvQmp^E)m# z2wuc@7d(|cOp;f4&di7t?qK&Dvz;BHR`}wHayxGn2zepXZYY_=6nL{@%h-F`d>*0Q zn0&eInMklJ-Zc>ySO7s;tT*FI;YoqMpa<82%uwHk)3aqzTi)x`BaHh-o(A5M(-c%C z>xbJ`o88`;vnf)yqQ#QGGlmJtz6c!pA6!rAUmxa%Af&ZuA6pl#o} zi;9zZ1L7ofenGj%Fj-a-nA*sqtn6mXc1h9sIIkyAemj!SNmgZ#+E!dS6+GY#zxqki zb%?-K82eoIzTOgfj^;CH>KDy#wY5d9Nn_w1vmh!-Y{1}0ZU?PFVQB5w6L!hDS!2Nm zFFx3+QkhQ!b)#H!Qnv*Nwz8+e29Cm1DoBJxbWja$>oKZ_j?7xTXnI@*OLCl)7N~te z^!mihr{LC84r@yM>a9}%1)_{R)=SSrg^r5K4bCz@v=5aU^X*cuVK=&yFmrD^Pd;58 zc{+qj?;Dd4ZD*S#@P<3-KttrOkR6cMPKCYnlb`7~wXB2OAeMfC-YoxO7+49pwP7q54W_vapya&dxE)%o2JOs4eiSpYf}8%s{8$4xxWR-`$?m$BIN3E zQoj|Q14<@0uqS?Ca)WJUljz837q%vSxmIWj)I!Phz!PwNGaWWFenr)_BF$zXoB&x3 z5AKB{|Ju|RZhRi1@rkc3yozo8Lm%0PcvIh1fczcQot9UMmk7TY%I%%{Jv6J$ghe~# z;;NX{@iF*)kraz$Dwl~RuXp0)#U7Bd<}3rF(S-|X?-p-f{tJfT(EGh4lN?CxX?fMo z6r(nG$vg9Hdaau3(&+jEObS+Bh2&`$3W9=Vps}yJSHN7y8hLvl)nB19>mWMwUr(xB z`cjbCljUKIR{#?{A!NG}3S4ASfj_~%Te3Fc^2~0n0$T)oW%Z5r8}9Y4I6v?`CyjTc zW1ibtcRdK^@IHShDdckCr;LN_ihXbY*`Rwwplb1h2Iv^eU_*mB%Aq4r%EZRv717gB z#EEi&Sa+vLapDq6h49;lqPn6=`dFaMzOCt1X+b5q=+_c-Qxn1)-O#BL z?4w>BKo5Oh8p~w)#te?LkZ8AgTQ_-ON;EC+bWR0OfLCGR@EB%TLDu*uOzv5Tz6qzw zd;41Q!&&Wk=*46P=F>+7yRpw?PlVqlbeRx^nUAjur^`NQj3Pzy`#W8}LQ^~C z6SA6PQBuL;kvli0x~wfGh6oEV+a7p%`B+Nk8B&T7fhVfnSP`m+nY$9nqsg%d@Wc~# zM_=SMx$)~a9@i=R+US4X?quG8RXz`Q(BSkHT zzZL%YCBF43|FhH0ZxHz=-vbP*)PRj}mVaTYA`>(M#1r_#4Z{8pyH(k{|M*PzKQ-VU z?L**Fg?xhb6M#*~$qivr@PHZ=L3IdiWpXyhpDAZ^7Ck4u)%G9`mC6kz>_jwt*--Mn zI@XU#*6kD5&GH)8-m3jlR!93cpjUli5a?4l)yW5xlB!H4#F4J>6c|bg%5==zhh8;< zyd|41?AFey>+YZ$aGVh4n4mH6TB)adN zYK+1QTX*9O8#m{gG5r@PhWvu{NdH>wcM?lXya3nbw$IihoXQR|ZpYSOOQuu&(b}=T@mEqbXCPJXqLOIyr3b!WAfOaIeN;` z-7LGUIB@FC#~qNa$tcN zX5}GhmZSvke@P$`rZPL;e>k2T2(loybV{`eh9reDyw4RH3)L+0{_^!}LY`8%ZSv(X zrUV3G;^Cqgrl1<+E#$K7eJT&}`=%UkF)5?@n**RqY!GS7IfP0i;CzQDvcT<&My#AO zF*fr!GY?U*wL&wyAZP=1+Qi~nVslG71q?od^V8Vqh^&c7-pUnF?Di1DTMGeFr%jIJnq3Q=CtEJBZ_*;5gE(xykw7*}hyR9nX9@#V3r8;{bg= z9k7}3uloES-TjX}gdbiYiYId4y`QHL>l5m+*$u)umJtKqa7I!B zrxZZFnHOBz$eMEvzmLF>{uWRmRIz35G5{1bhdJrWmsOl9dOYRFs%(mZ4TJfS3GUoY z8%!{#xmMt_#(CV~1B|S-+k=;OYwgZ>zdOx{bV!sg7PlQ%>~3rn9h{y{k%j&?l)QCzmpt zHxxOZhMRAamdjbDet~6M;}H_T<)05-)m@PTUhZTgox1US@22rBt{ z-J$qMz3nMW5lg-gSZYiiq_NTjy6dn7UP4QJVp2i=3C6W}#NW{4pV))OEIfENSc%N5 zbbE!*rXyV#@6d8HG0N;1ObGGvQsuc!K@Wsp*` zj5Feo3Uo)L`d(V)_%z0LSuP*wifuNhm>MU`mP|5@&}jDLHOFDCobqak;&7M-#@@sJ zeFwsYVpHiYKvWb576^#v*J1t7STX-*6BG~_&>2S=?GXV^hB;-|FNckyOcD+q1iq}^ zl%7>;YWnSDtZ(>_2##64LRp>! z*9*pH*4_{4p0`^SZ@eLG!1*OOgO^AW$vzk|~XXb*oA8s&FrVSWL}fi3lc5@!SyzO+s%&`?Oe4 zkR;*QmG35AhhA2SfnDHMst&Gbu8v2CbziPhoC3y$#s^&wx>%s65fF~LQNgVrv@mxR zUq4}YMRP*IJc$7-3xj3VGR{d6j^BbJ%WPJ!5>-JxBeLC-&aBEt1x+U_>6S}iMZJ^S z{JJ8w*pMp*q;3Av#JW3}di-p3a0qT$C*YE=iO|fgLuudeK7z#s& zNv`(!Jnv6-HodjGbW|B!i*4|(Ys5zBJ~*>wcA0o%dipNK0kmzf#N-#>oxdnB9C@qJ zqmO+ij2xS!AfV7k_eP~hiZHVz>MJGKryBS=Jt9|x%z6QXsaUE)u?4@0oV~6XtKPH!#1vpK* z)jy-Us#vP8CUT(F$k5$gTT-(9BIt=Sr2MEW=D5G!{LNB#?;~qb<#@@alHw*Y(rGCW z_UNXnRikDXV|HCJ)w{NGc>Ec*^~BeMOa69Pq~D7xb5A}Z)SD|_#A(FgMFKXqvAD$jkUTW(k@wWxE*Am-Wev1Tj7 zx-)GB`2%V-{OznB{Oz0`D>^RL{NtQC5J+7-m_u0m9x_QZ-baTi4OCwin2G4vRVd|b z#uKAfml%wjM@rWl#jJ3<4tO|rB7EHt-EWh~20>YsLjEjp3>!_lsi{&A`x9vIP&8bv0rcuU>SbE>yjv-bF98a?n5Z z<&ufd_I~;=Do5YX@x^DfqPGI(W+K4cO!w=+^@~&HUr$>^Dr){Gv`Bpe{Rv6F7MdUx zW)A&O7-s9M5-=uW)1EvCe`{eL^`P&QGCi_4%qNQW+w4%O7!gmvmj0*X7x`$C9w`}I z4;g13Yg{ZHPaaReZ;@%K4PL!ckRkRHLcC|p%H0VHOOvF_i}A_qqo<>=)$}_1&`8^H zvt6CH*&YT7gW^77VXG=wP(6H|y!s`eNHySyo_4s*1 z=f!O8)JUBi;6FWAAFRSw+n8{(KDOC1urUIu`~PH6oZe|(s}4- zFv%F83l3UnrivGQB0Y$6A3I}%UjsuSS5lrv!I7+fmE1qKI@|b3G{lEhwV^4k#)6@t z``RT+{ekO=nQlNxk%qv`1uX6*PM4%>`2;iX1goo$1GGk$e)G!}Iktpzk#0q5lk2BG z*C0cE_oTTG!AWCLL8l?KtKe;#h3|rB#j94i?up(xh?~CE_J?^IY%l53HfL?~l$lH$ z9QV)l9egqA>?4N^MgwjDis2eFsWsyx@54jl)hHan_28D?XK6RS)qj$rJ24+Lc?&)l__&M~~Eks$7NHRVjM8kB= zM2*Q*(ao%}v5^$}0_WY+$FEo4Abng*DPwxz;km27m4?S>tJ|l^o=!_A>N7_qX-!bEf>kUTDZId`-pC}2>~VC`+6y6aEnRr{w%^6Ww_2d$q?v1ErT)c*nRj2k zD4R5}HmXK#oGFBb+e%-lH#dr$u0(C3DvT{C@YPPFaXWQA-NxOH3Y!EA+zqUB9S+RLG+TB1#P43ER_cn=mVZa!pWn!m&=jF`ZEUFd8K z)@|48>!9(1S2XvPNpz4&_1*w2GKmzsrOcT>_AxDqm>1!MydeFD#4mJ+k)uStModCg za62D@jzzP-k+G5VxI(`NzJR$yL1!uG5V#xP2p23@t_r7|-3Z-aE`OD|LDL74$qQGj zAiBkpE5as_=qY@R&kXs>IiHec+Vv~%Ye

i1O>I_eGKr^&y;^hm-V94&lC5U~@}s zTa5yXP^(8uV=BE6Dt+3? zLaE_7J031uGhDE7|b&w1(^tMyMTy4Ezl&wqN)pF3smtPe#|`O&uE&FOFx;Lv>2r%dWXcsINN|d3 zzSwY&aG}}M#$$|eJ8DI()+D|nV{jSt3X)A(IZ~?lt!A^rpo2z1YmxIU1%lU1G1LL~ z*NFxe{FkBI2+QE9yNI%iZ4g#m1ZMFxYZ10u>`lpV;=aIeh-*?bt~GKz2fOafq^Y$g zM%cM+$NDj_M%^@usUdiHr=2peP3Y;}A%428!?wZ1t|ZvKyDc#F=ivQGFk$n$Auf_1 z)(P76l5Zj$=3GpYqa$Aox}1I>2ghWjQ1cu{;^*$HRyLhx?bh*pJ)4^KC^Ti5mx3`! zvq54tCQ8wFggOQH(f{j65`l?f+7=HujTDJq%0}^CEPFKPxXlgtM$%#TJB|{_HY7WM zd$9`r31iNC3?q53j|V!gg-2DPTYgyV?leMeIyCC^Np)_#+#JG5N3RiYT!$iODzD5A z6YnL^J_$q67@&_DHO|XTjV7qNFUdl}?SlKC>tG$*#ANykA3n~@zH4~pwrK89kqYs~ zSO`7+bxlDW-G#QxDvwT)N*#jp0`2Cv3kbvct-1AYh_@X*vgtjvB$wGRQ#=vi{<*DF z1Bh84t}k!gqK!CAPYZQId%Xi0FyxOvKCz5?x@0X*=|yS_6KV?iX!k=J;1;lT=_hz} zV=L5=<93TT&R^bhK#YsG)gu;R$yf4TTK0PRo?+jke){Nn3knK58FEQES$;Py4-j^G z(PxSif~@1(9Q{#mzxI-64DH)s7Yo=8Z4%`WBkwkcvPME0UF!K%y(a+o`}UBc@#`*3by8Z}nAp3?M6pH77=*ri%#`G z1QtkLqKP)tWpQF`2<_A6X&3Kj6}edvVlZF^2D?8dS3ObqWNwC zOIenicvYA65wda#o|w6m)lF2wmrdDsn*hSj7*D6e69S?j~z zk(0vUKo34jm&;RR0=U1()qpAHGe6dN0{eTD(xaRBK?0hy1z;><{QXNjG8ek|8)3)_iqnw}LZMK$v> zcP>Dw%~pt!=i@kesT1!3flkCj`90M=pS#q`TcWS1IWH~c(~TT<0|S<;Ugeay#fHqM zJQ3xWaJNGVCn0LC2t=mGq6n`!LbUHW_?@c_v%j(>0%{`@K^ll2$dZF9Bt}qbpR27XI#7 z;U+c(3^sbw9~!92lm}5O2ccjQ zWP+fnk`Y);H#FHdqQ1i4ozrAYV;T`diRyj;tH8UHfkT-Ix!6wNNo0_jkx3`w-eb56PVT}m! z1Lohml~_I04KHA|F$Dcv`{S?O>Ob2`|7Edp^m^(Q(xaRvR35Mtcn^=BVK#j&IC7;6 zwepMCRBECVZ*qEYsG#E^&e1)9)tfu>s&RL^@i2C- zY2oU4^msrZ3KB&g4Pl)`L7(sok*F4A2(ppxtn^vZVxbZ0O!nac=@WIal@ADc03m>q zV4O~~o)#h+8qb)F)FLw;C~+l^8!d^=Yp)_UFQHae#4vcDjb&R+pvpGW+Rr$1Wbs%L zNUlR)6%L3w?Gkri+%?_>exZp=8+NXwGZ^2KHCbclCCRST&H%%^tQ}O}W3I?Vq5S@> zQN2o74BTWy0ks%2111cNdL-vP7AHdR=A{uLeYB*kD;b<@b0U*rw~TdaaXGXEbM0cQ zCpxukm)k;`M*P}PY-{Nb3HH>hI;~c%$-QG&Z1%hL#7I1OEo=f$u9a%aLMb6>Ov9P> zHcM2@&oRJoN5)?^GP=XndpF!Lccdd2{Na2x#+X8|J%iMsr zBE*WMg3U}K_*z&RWY`E(;yLc7IZ3h+#;7TT>kAZAX&28F;W7`=bw*n`uO3woyC?-kF~99-?(=qf@bQg@*mw8t;f@}$ zFUn4>$|}JG@pYra1v+m$O1NV#np*C1*N1RlgXts+LJo3Eh+lI?hx_zS7?5Chqc@L= zee;2E38C=ITE7R8zok9Yy7j%kd(8TLCa8uGTR=SEG|^ZCX`8Zi^05UECSKxjO_luw zuW@tF=a@@l0k*cr5Nw`yI3wG0!j}hHOPr(WV-s#q6V*NyNF#JZ^!rN;Z5c(AuX%5K zM-cSO_ktIMpc`#~USvnFC-;Jvec-|rnjbRj2xRhs6``a89lhK}}SIMA??Kul-qE0}hn{(s5eKb02g3JKp&&xim%gn-Wv834dh_x}$G z07k`MB>;B*7Wf|+1Pv4*Tn+^Vgafef0eJr}>OEj|_;Dctod|yaDl4iaNG~ZT1~{ny z_hiPcx7GrHe^3T|&;Xx5uRP!HZ;StzOjb}%QcP4)>4mJ=Ul9kQyS`Xl-G zTdglXVEy$yuRoao1o%@H>d#2)&vVqf8aRsvnCEQ(CI8<1e_pk|z<|I06X5ez9bkRu z;wtFqU;>DJYpVYkP=lZx|*2EW;l!na}aSiriMT z0C`maq^AEHo-V*v{2M%RJ6!_{LuEktc?oMvLm6vpb4Q!sL*s2~FZBXI?g72@`-bqJ zSFJCg&;AY?5cAi<(H?NFOw826P*7jr(B58F*Gkv;Po;(Dc^`q-5&IrN`wn2w`+MFA z0Gg)1$-C+w${YYwhmonVBOrV>Ae8*?^RD2DZ@L4h5^O-;8UM^U;9l!{kMvvU=j|y0 z5R0(*ZyC>bWo%XfWE==kJKsl)Kd)L}dW_$q%UB!#*IF2}I|@Sw_?cJ%%KZIU)Sp+a zZzn(l@aFu)k;e_Z9J6 za`#W@THgeKhRkm&;zzaqx8dix#Nkgo2f%>-8@#_Y{eKfYd|n&R<@0{Bc&YxD<)25K z=c+kB;b!0b-{kpY0RHU+`q%t@-`skxdh-*|`0an!z@pBQGKad+*WUQO z-kwWv{6s}D{1>SIAjt7N|Ieikelnq%{0pYPItl?wXY*I#gXdJwBhCM$0y6(6s{ayq z{yF^fkm*0+6D|G;{zpsvQvmhnRL^6t{-kPj_!m@vRNDWluIJ$^f08UZ{|l0T4rckh zM9-uB{UpP3|0lA)?nXbyd>(7+C+4ZoKVkl8kUyaR`0*sP<1z(dhrp`TODO zxlip+%+ERh8}m0Fw$C&D+=1~Y!FJxiAo%Mj_4^V$cOUyn&|mlq!Jo6lzcKtiFUfOv zkDo+7rN0pU(p~@QaeKan{u4Q<;uqv!JJTP~zu!=QUWn&gmp`dotN)4Wf6S%NckF(` zF4X)J?2iHCANKE_7vQ;F|0ffE<1b9l-yi-`cmJb6&uz^=X+AgoLh~2c9|ij3_77v< zbED%=eDT(Qf&br4kk9MuxgF&v(F(vY@W-1uU_tnMtI8{BaKKyOcXb&iAS6KOxwoC) GfBiq \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000000000..b5adeb2fde6e8 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,90 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS=-Xmx2g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args +if "%@eval[2+2]" == "4" goto 4NT_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* +goto execute + +:4NT_args +@rem Get arguments from the 4NT Shell from JP Software +set CMD_LINE_ARGS=%$ + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/graphx/build.gradle b/graphx/build.gradle new file mode 100644 index 0000000000000..64ee2e856d38c --- /dev/null +++ b/graphx/build.gradle @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project GraphX' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile group: 'com.github.fommil.netlib', name: 'core', version: '1.1.2' + compile group: 'net.sourceforge.f2j', name: 'arpack_combined_all', version: '0.1' + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') +} diff --git a/launcher/build.gradle b/launcher/build.gradle new file mode 100644 index 0000000000000..22a32f5227a2a --- /dev/null +++ b/launcher/build.gradle @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Launcher' + +dependencies { + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + testCompile(group: 'org.apache.hadoop', name: 'hadoop-client', version: hadoopVersion) { + exclude(group: 'asm', module: 'asm') + exclude(group: 'org.codehaus.jackson', module: 'jackson-mapper-asl') + exclude(group: 'org.ow2.asm', module: 'asm') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'org.mockito', module: 'mockito-all') + exclude(group: 'org.mortbay.jetty', module: 'servlet-api-2.5') + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'junit', module: 'junit') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'com.sun.jersey') + exclude(group: 'com.sun.jersey.jersey-test-framework') + exclude(group: 'com.sun.jersey.contribs') + } + testCompile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion +} diff --git a/mllib-local/build.gradle b/mllib-local/build.gradle new file mode 100644 index 0000000000000..c4183a09ba733 --- /dev/null +++ b/mllib-local/build.gradle @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project ML Local Library' + +dependencies { + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.11.2') { + exclude(group: 'junit', module: 'junit') + exclude(group: 'org.apache.commons', module: 'commons-math3') + } + compile group: 'org.apache.commons', name: 'commons-math3', version: '3.4.1' + + testCompile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' +} diff --git a/mllib/build.gradle b/mllib/build.gradle new file mode 100644 index 0000000000000..0bcbd130afece --- /dev/null +++ b/mllib/build.gradle @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project ML Library' + +dependencies { + compile project(subprojectBase + 'snappy-spark-mllib-local_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.11.2') { + exclude(group: 'junit', module: 'junit') + exclude(group: 'org.apache.commons', module: 'commons-math3') + } + compile group: 'org.apache.commons', name: 'commons-math3', version: '3.4.1' + compile(group: 'org.jpmml', name: 'pmml-model', version: '1.2.15') { + exclude(group: 'org.jpmml', module: 'pmml-agent') + } + + testCompile project(path: subprojectBase + 'snappy-spark-mllib-local_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion, configuration: 'testOutput') +} + +// TODO: netlib-lgpl profile + +// fix scala+java test ordering +sourceSets.test.scala.srcDir 'src/test/java' +sourceSets.test.java.srcDirs = [] diff --git a/repl/build.gradle b/repl/build.gradle new file mode 100644 index 0000000000000..1e51600f39b04 --- /dev/null +++ b/repl/build.gradle @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project REPL' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' + compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion + compile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion + if (scalaBinaryVersion == '2.10') { + compile group: 'org.scala-lang', name: 'jline', version:scalaVersion + } + + runtime project(subprojectBase + 'snappy-spark-mllib_' + scalaBinaryVersion) + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') +} + +if (scalaBinaryVersion == '2.11') { + sourceSets.main.scala.srcDir 'scala-2.11/src/main/scala' + sourceSets.test.scala.srcDir 'scala-2.11/src/test/scala' +} else { + sourceSets.main.scala.srcDir 'scala-2.10/src/main/scala' + sourceSets.test.scala.srcDir 'scala-2.10/src/test/scala' +} diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000000000..70ea595ee61a5 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +def scalaBinaryVersion = "2.11" +rootProject.name = 'snappy-spark' + +include ':snappy-spark-tags_' + scalaBinaryVersion +include ':snappy-spark-core_' + scalaBinaryVersion +include ':snappy-spark-graphx_' + scalaBinaryVersion +include ':snappy-spark-mllib_' + scalaBinaryVersion +include ':snappy-spark-mllib-local_' + scalaBinaryVersion +include ':snappy-spark-tools_' + scalaBinaryVersion +include ':snappy-spark-network-common_' + scalaBinaryVersion +include ':snappy-spark-network-shuffle_' + scalaBinaryVersion +include ':snappy-spark-network-yarn_' + scalaBinaryVersion +include ':snappy-spark-sketch_' + scalaBinaryVersion +include ':snappy-spark-yarn_' + scalaBinaryVersion +include ':snappy-spark-streaming_' + scalaBinaryVersion +include ':snappy-spark-catalyst_' + scalaBinaryVersion +include ':snappy-spark-sql_' + scalaBinaryVersion +include ':snappy-spark-hive_' + scalaBinaryVersion +include ':snappy-spark-hive-thriftserver_' + scalaBinaryVersion +include ':snappy-spark-unsafe_' + scalaBinaryVersion +include ':snappy-spark-assembly_' + scalaBinaryVersion +include ':snappy-spark-streaming-flume_' + scalaBinaryVersion +include ':snappy-spark-streaming-flume-sink_' + scalaBinaryVersion +include ':snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion +include ':snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion +include ':snappy-spark-examples_' + scalaBinaryVersion +include ':snappy-spark-repl_' + scalaBinaryVersion +include ':snappy-spark-launcher_' + scalaBinaryVersion +include ':snappy-spark-assembly_' + scalaBinaryVersion + +project(':snappy-spark-tags_' + scalaBinaryVersion).projectDir = "$rootDir/common/tags" as File +project(':snappy-spark-core_' + scalaBinaryVersion).projectDir = "$rootDir/core" as File +project(':snappy-spark-graphx_' + scalaBinaryVersion).projectDir = "$rootDir/graphx" as File +project(':snappy-spark-mllib_' + scalaBinaryVersion).projectDir = "$rootDir/mllib" as File +project(':snappy-spark-mllib-local_' + scalaBinaryVersion).projectDir = "$rootDir/mllib-local" as File +project(':snappy-spark-tools_' + scalaBinaryVersion).projectDir = "$rootDir/tools" as File +project(':snappy-spark-network-common_' + scalaBinaryVersion).projectDir = "$rootDir/common/network-common" as File +project(':snappy-spark-network-shuffle_' + scalaBinaryVersion).projectDir = "$rootDir/common/network-shuffle" as File +project(':snappy-spark-network-yarn_' + scalaBinaryVersion).projectDir = "$rootDir/common/network-yarn" as File +project(':snappy-spark-sketch_' + scalaBinaryVersion).projectDir = "$rootDir/common/sketch" as File +project(':snappy-spark-yarn_' + scalaBinaryVersion).projectDir = "$rootDir/yarn" as File +project(':snappy-spark-streaming_' + scalaBinaryVersion).projectDir = "$rootDir/streaming" as File +project(':snappy-spark-catalyst_' + scalaBinaryVersion).projectDir = "$rootDir/sql/catalyst" as File +project(':snappy-spark-sql_' + scalaBinaryVersion).projectDir = "$rootDir/sql/core" as File +project(':snappy-spark-hive_' + scalaBinaryVersion).projectDir = "$rootDir/sql/hive" as File +project(':snappy-spark-hive-thriftserver_' + scalaBinaryVersion).projectDir = "$rootDir/sql/hive-thriftserver" as File +project(':snappy-spark-unsafe_' + scalaBinaryVersion).projectDir = "$rootDir/common/unsafe" as File +project(':snappy-spark-assembly_' + scalaBinaryVersion).projectDir = "$rootDir/assembly" as File +project(':snappy-spark-streaming-flume_' + scalaBinaryVersion).projectDir = "$rootDir/external/flume" as File +project(':snappy-spark-streaming-flume-sink_' + scalaBinaryVersion).projectDir = "$rootDir/external/flume-sink" as File +project(':snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion).projectDir = "$rootDir/external/kafka-0-8" as File +project(':snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion).projectDir = "$rootDir/external/kafka-0-10" as File +project(':snappy-spark-examples_' + scalaBinaryVersion).projectDir = "$rootDir/examples" as File +project(':snappy-spark-repl_' + scalaBinaryVersion).projectDir = "$rootDir/repl" as File +project(':snappy-spark-launcher_' + scalaBinaryVersion).projectDir = "$rootDir/launcher" as File +project(':snappy-spark-assembly_' + scalaBinaryVersion).projectDir = "$rootDir/assembly" as File + +if (rootProject.hasProperty('docker')) { + include ':snappy-spark-docker-integration-tests_' + scalaBinaryVersion + project(':snappy-spark-docker-integration-tests_' + scalaBinaryVersion).projectDir = "$rootDir/external/docker-integration-tests" as File +} +if (rootProject.hasProperty('ganglia')) { + include ':snappy-spark-ganglia-lgpl_' + scalaBinaryVersion + project(':snappy-spark-ganglia-lgpl_' + scalaBinaryVersion).projectDir = "$rootDir/external/spark-ganglia-lgpl" as File +} diff --git a/sql/catalyst/build.gradle b/sql/catalyst/build.gradle new file mode 100644 index 0000000000000..d2deeb94b9b93 --- /dev/null +++ b/sql/catalyst/build.gradle @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Catalyst' + +apply plugin: 'antlr' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-unsafe_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion + compile group: 'org.scala-lang.modules', name: 'scala-parser-combinators_' + scalaBinaryVersion, version: '1.0.4' + compile group: 'org.codehaus.janino', name: 'janino', version: '2.7.8' + compile group: 'org.antlr', name: 'antlr4-runtime', version: antlrVersion + compile group: 'commons-codec', name: 'commons-codec', version: commonsCodecVersion + antlr group: 'org.antlr', name: 'antlr4', version: antlrVersion + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') +} + +compileScala.dependsOn generateGrammarSource + +sourceSets.main.antlr.srcDirs = [ 'src/main/antlr4' ] + +// add generated sources to scala compiler path (plugin adds it to java path) +sourceSets.main.scala.srcDir generateGrammarSource.outputDirectory +sourceSets.main.java.srcDirs = [] + +generateGrammarSource { + arguments += [ '-package', 'org.apache.spark.sql.catalyst.parser', '-visitor' ] +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 741d2aaa3061d..2b2aa60f2538c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -54,6 +54,11 @@ class AnalysisException protected[sql] ( val cause: Option[Throwable] = None) extends Exception(message, cause.orNull) with Serializable { + def this(message: String, cause: Throwable) = { + this(message) + initCause(cause) + } + def withPosition(line: Option[Int], startPosition: Option[Int]): AnalysisException = { val newException = new AnalysisException(message, line, startPosition) newException.setStackTrace(getStackTrace) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala index 64b94f0a2c103..98153efd6710f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.{GenerateSafeProjection, GenerateUnsafeProjection} -import org.apache.spark.sql.types.{DataType, StructType} +import org.apache.spark.sql.types._ /** * A [[Projection]] that is calculated by calling the `eval` of each of the specified expressions. diff --git a/sql/core/build.gradle b/sql/core/build.gradle new file mode 100644 index 0000000000000..c6b3f5038db8d --- /dev/null +++ b/sql/core/build.gradle @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project SQL' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-sketch_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'com.univocity', name: 'univocity-parsers', version: '2.1.1' + compile group: 'org.apache.parquet', name: 'parquet-column', version: parquetVersion + compile group: 'org.apache.parquet', name: 'parquet-hadoop', version: parquetVersion + compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile group: 'com.h2database', name: 'h2', version: '1.4.183' + testCompile group: 'mysql', name: 'mysql-connector-java', version: '5.1.38' + testCompile group: 'org.postgresql', name: 'postgresql', version: '9.4.1207.jre7' + testCompile group: 'org.apache.parquet', name: 'parquet-avro', version: parquetVersion + testCompile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' +} + +// fix scala+java test ordering +sourceSets.test.scala.srcDirs 'src/test/java', 'src/test/gen-java' +sourceSets.test.java.srcDirs = [] diff --git a/sql/hive-thriftserver/build.gradle b/sql/hive-thriftserver/build.gradle new file mode 100644 index 0000000000000..6dd72cdbd08a1 --- /dev/null +++ b/sql/hive-thriftserver/build.gradle @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Hive Thrift Server' + +dependencies { + compile project(subprojectBase + 'snappy-spark-hive_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile(group: 'org.spark-project.hive', name: 'hive-cli', version: hiveVersion) { + exclude(group: 'org.spark-project.hive', module: 'hive-common') + exclude(group: 'org.spark-project.hive', module: 'hive-exec') + exclude(group: 'org.spark-project.hive', module: 'hive-jdbc') + exclude(group: 'org.spark-project.hive', module: 'hive-metastore') + exclude(group: 'org.spark-project.hive', module: 'hive-serde') + exclude(group: 'org.spark-project.hive', module: 'hive-service') + exclude(group: 'org.spark-project.hive', module: 'hive-shims') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + exclude(group: 'log4j', module: 'log4j') + exclude(group: 'commons-logging', module: 'commons-logging') + } + compile(group: 'org.spark-project.hive', name: 'hive-beeline', version: hiveVersion) { + exclude(group: 'org.spark-project.hive', module: 'hive-common') + exclude(group: 'org.spark-project.hive', module: 'hive-exec') + exclude(group: 'org.spark-project.hive', module: 'hive-jdbc') + exclude(group: 'org.spark-project.hive', module: 'hive-metastore') + exclude(group: 'org.spark-project.hive', module: 'hive-service') + exclude(group: 'org.spark-project.hive', module: 'hive-shims') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + exclude(group: 'log4j', module: 'log4j') + exclude(group: 'commons-logging', module: 'commons-logging') + } + compile(group: 'org.spark-project.hive', name: 'hive-jdbc', version: hiveVersion) { + exclude(group: 'org.spark-project.hive', module: 'hive-common') + exclude(group: 'org.spark-project.hive', module: 'hive-metastore') + exclude(group: 'org.spark-project.hive', module: 'hive-serde') + exclude(group: 'org.spark-project.hive', module: 'hive-service') + exclude(group: 'org.spark-project.hive', module: 'hive-shims') + exclude(group: 'org.spark-project.hive', module: 'httpclient') + exclude(group: 'org.apache.curator', module: 'curator-framework') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'org.apache.thrift', module: 'libfb303') + exclude(group: 'org.apache.zookeeper', module: 'zookeeper') + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + exclude(group: 'log4j', module: 'log4j') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'org.codehaus.groovy', module: 'groovy-all') + } + compile(group: 'net.sf.jpam', name: 'jpam', version: jpamVersion) { + exclude(group: 'javax.servlet', module: 'servlet-api') + } + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-java', version: seleniumVersion) { + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'io.netty', module: 'netty') + } + testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-htmlunit-driver', version: seleniumVersion) { + exclude(group: 'com.google.guava', module: 'guava') + } +} + +// add generated sources +sourceSets.main.scala.srcDir 'src/gen/java' diff --git a/sql/hive/build.gradle b/sql/hive/build.gradle new file mode 100644 index 0000000000000..a3c22973264b2 --- /dev/null +++ b/sql/hive/build.gradle @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Hive' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'com.twitter', name: 'parquet-hadoop-bundle', version: hiveParquetVersion + compile group: 'org.apache.derby', name: 'derby', version: derbyVersion + compile(group: 'org.spark-project.hive', name: 'hive-exec', version: hiveVersion) { + exclude(group: 'org.spark-project.hive', module: 'hive-metastore') + exclude(group: 'org.spark-project.hive', module: 'hive-shims') + exclude(group: 'org.spark-project.hive', module: 'hive-ant') + exclude(group: 'org.spark-project.hive', module: 'spark-client') + exclude(group: 'org.apache.ant', module: 'ant') + exclude(group: 'com.esotericsoftware.kryo', module: 'kryo') + exclude(group: 'commons-codec', module: 'commons-codec') + exclude(group: 'commons-httpclient', module: 'commons-httpclient') + exclude(group: 'org.apache.avro', module: 'avro-mapred') + exclude(group: 'org.apache.calcite', module: 'calcite-core') + exclude(group: 'org.apache.curator', module: 'apache-curator') + exclude(group: 'org.apache.curator', module: 'curator-client') + exclude(group: 'org.apache.curator', module: 'curator-framework') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'org.apache.thrift', module: 'libfb303') + exclude(group: 'org.apache.zookeeper', module: 'zookeeper') + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + exclude(group: 'log4j', module: 'log4j') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'org.codehaus.groovy', module: 'groovy-all') + } + compile(group: 'org.spark-project.hive', name: 'hive-metastore', version: hiveVersion) { + exclude(group: 'org.spark-project.hive', module: 'hive-serde') + exclude(group: 'org.spark-project.hive', module: 'hive-shims') + exclude(group: 'org.apache.thrift', module: 'libfb303') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + exclude(group: 'org.apache.derby', module: 'derby') + } + + compile group: 'org.apache.avro', name: 'avro', version: avroVersion + compile(group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion) { + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'org.mortbay.jetty', module: 'jetty') + exclude(group: 'org.mortbay.jetty', module: 'jetty-util') + exclude(group: 'org.mortbay.jetty', module: 'servlet-api') + exclude(group: 'org.apache.velocity', module: 'velocity') + } + compile(group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion, classifier: 'hadoop2') { + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'org.mortbay.jetty', module: 'jetty') + exclude(group: 'org.mortbay.jetty', module: 'jetty-util') + exclude(group: 'org.mortbay.jetty', module: 'servlet-api') + exclude(group: 'org.apache.velocity', module: 'velocity') + exclude(group: 'org.apache.avro', module: 'avro-ipc') + } + compile group: 'commons-httpclient', name: 'commons-httpclient', version: '3.1' + compile(group: 'org.apache.calcite', name: 'calcite-avatica', version: '1.2.0-incubating') { + exclude(group: 'com.fasterxml.jackson.core', module: 'jackson-annotations') + exclude(group: 'com.fasterxml.jackson.core', module: 'jackson-core') + exclude(group: 'com.fasterxml.jackson.core', module: 'jackson-databind') + } + compile(group: 'org.apache.calcite', name: 'calcite-core', version: '1.2.0-incubating') { + exclude(group: 'com.fasterxml.jackson.core', module: 'jackson-annotations') + exclude(group: 'com.fasterxml.jackson.core', module: 'jackson-core') + exclude(group: 'com.fasterxml.jackson.core', module: 'jackson-databind') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'com.google.code.findbugs', module: 'jsr305') + exclude(group: 'org.codehaus.janino', module: 'janino') + exclude(group: 'org.hsqldb', module: 'hsqldb') + exclude(group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm') + } + compile group: 'org.apache.httpcomponents', name: 'httpclient', version: httpClientVersion + compile group: 'org.codehaus.jackson', name: 'jackson-mapper-asl', version: '1.9.13' + compile group: 'commons-codec', name: 'commons-codec', version: commonsCodecVersion + compile group: 'joda-time', name: 'joda-time', version: '2.9.4' + compile group: 'org.jodd', name: 'jodd-core', version: '3.5.2' + compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version + compile group: 'org.datanucleus', name: 'datanucleus-core', version: '3.2.10' + compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + } + compile(group: 'org.apache.thrift', name: 'libfb303', version: thriftVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + } + + testCompile group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion, classifier: 'tests' + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion, configuration: 'testOutput') +} + +// fix scala+java test ordering +sourceSets.test.scala.srcDirs 'src/test/java', 'compatibility/src/test/scala' +sourceSets.test.java.srcDirs = [] diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala index 35f13a0626666..3af8af0814bb4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala @@ -14,24 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ package org.apache.spark.sql.hive diff --git a/streaming/build.gradle b/streaming/build.gradle new file mode 100644 index 0000000000000..fcfb62199e447 --- /dev/null +++ b/streaming/build.gradle @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Streaming' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile group: 'org.eclipse.jetty', name: 'jetty-server', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-plus', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-util', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-http', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-servlets', version: jettyVersion + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-java', version: seleniumVersion) { + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'io.netty', module: 'netty') + } + testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-htmlunit-driver', version: seleniumVersion) { + exclude(group: 'com.google.guava', module: 'guava') + } +} + +// fix scala+java mix to use scala first for tests +sourceSets.test.scala.srcDir "src/test/java" +sourceSets.test.java.srcDirs = [] diff --git a/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala b/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala index 41f7c60e1cc35..9618ec2507c76 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala @@ -14,6 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package org.apache.spark.streaming.rdd import java.nio.ByteBuffer diff --git a/tools/build.gradle b/tools/build.gradle new file mode 100644 index 0000000000000..05b48719a0d3e --- /dev/null +++ b/tools/build.gradle @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Tools' + +dependencies { + compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion + compile group: 'org.clapper', name: 'classutil_' + scalaBinaryVersion, version: '1.0.12' +} + +// TODO: anything special required for deploy, install and source plugins in maven? diff --git a/yarn/build.gradle b/yarn/build.gradle new file mode 100644 index 0000000000000..f8eb23d2b5566 --- /dev/null +++ b/yarn/build.gradle @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project YARN' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile(group: 'org.apache.hadoop', name: 'hadoop-yarn-api', version: hadoopVersion) { + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'asm', module: 'asm') + exclude(group: 'org.ow2.asm', module: 'asm') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'com.sun.jersey') + exclude(group: 'com.sun.jersey.jersey-test-framework') + exclude(group: 'com.sun.jersey.contribs') + } + compile(group: 'org.apache.hadoop', name: 'hadoop-yarn-common', version: hadoopVersion) { + exclude(group: 'asm', module: 'asm') + exclude(group: 'org.ow2.asm', module: 'asm') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'com.sun.jersey') + exclude(group: 'com.sun.jersey.jersey-test-framework') + exclude(group: 'com.sun.jersey.contribs') + } + compile(group: 'org.apache.hadoop', name: 'hadoop-yarn-server-web-proxy', version: hadoopVersion) { + exclude(group: 'asm', module: 'asm') + exclude(group: 'org.ow2.asm', module: 'asm') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'com.sun.jersey') + exclude(group: 'com.sun.jersey.jersey-test-framework') + exclude(group: 'com.sun.jersey.contribs') + } + compile(group: 'org.apache.hadoop', name: 'hadoop-yarn-client', version: hadoopVersion) { + exclude(group: 'asm', module: 'asm') + exclude(group: 'org.ow2.asm', module: 'asm') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'com.sun.jersey') + exclude(group: 'com.sun.jersey.jersey-test-framework') + exclude(group: 'com.sun.jersey.contribs') + } + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile group: 'org.eclipse.jetty', name: 'jetty-server', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-plus', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-util', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-http', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-servlets', version: jettyVersion + compile group: 'org.apache.derby', name: 'derby', version: derbyVersion + compile(group: 'org.spark-project.hive', name: 'hive-exec', version: hiveVersion) { + exclude(group: 'org.spark-project.hive', module: 'hive-metastore') + exclude(group: 'org.spark-project.hive', module: 'hive-shims') + exclude(group: 'org.spark-project.hive', module: 'hive-ant') + exclude(group: 'org.spark-project.hive', module: 'spark-client') + exclude(group: 'org.apache.ant', module: 'ant') + exclude(group: 'com.esotericsoftware.kryo', module: 'kryo') + exclude(group: 'commons-codec', module: 'commons-codec') + exclude(group: 'commons-httpclient', module: 'commons-httpclient') + exclude(group: 'org.apache.avro', module: 'avro-mapred') + exclude(group: 'org.apache.calcite', module: 'calcite-core') + exclude(group: 'org.apache.curator', module: 'apache-curator') + exclude(group: 'org.apache.curator', module: 'curator-client') + exclude(group: 'org.apache.curator', module: 'curator-framework') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'org.apache.thrift', module: 'libfb303') + exclude(group: 'org.apache.zookeeper', module: 'zookeeper') + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + exclude(group: 'log4j', module: 'log4j') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'org.codehaus.groovy', module: 'groovy-all') + } + compile(group: 'org.spark-project.hive', name: 'hive-metastore', version: hiveVersion) { + exclude(group: 'org.spark-project.hive', module: 'hive-serde') + exclude(group: 'org.spark-project.hive', module: 'hive-shims') + exclude(group: 'org.apache.thrift', module: 'libfb303') + exclude(group: 'org.apache.thrift', module: 'libthrift') + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'com.google.guava', module: 'guava') + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + exclude(group: 'org.apache.derby', module: 'derby') + } + compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + } + compile(group: 'org.apache.thrift', name: 'libfb303', version: thriftVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + } + + testCompile project(subprojectBase + 'snappy-spark-network-yarn_' + scalaBinaryVersion) + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + + testCompile group: 'org.eclipse.jetty.orbit', name: 'javax.servlet.jsp', version: '2.2.0.v201112011158' + testCompile group: 'org.eclipse.jetty.orbit', name: 'javax.servlet.jsp.jstl', version: '1.2.0.v201105211821' + testCompile(group: 'org.apache.hadoop', name: 'hadoop-yarn-server-tests', version: hadoopVersion, classifier:'tests') { + exclude(group: 'asm', module: 'asm') + exclude(group: 'org.ow2.asm', module: 'asm') + exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'commons-logging', module: 'commons-logging') + exclude(group: 'com.sun.jersey') + exclude(group: 'com.sun.jersey.jersey-test-framework') + exclude(group: 'com.sun.jersey.contribs') + } + testCompile(group: 'org.mortbay.jetty', name: 'jetty', version: '6.1.26') { + exclude(group: 'org.mortbay.jetty', module: 'servlet-api') + } + testCompile group: 'com.sun.jersey', name: 'jersey-core', version: sunJerseyVersion + testCompile group: 'com.sun.jersey', name: 'jersey-server', version: sunJerseyVersion + testCompile(group: 'com.sun.jersey', name: 'jersey-json', version: sunJerseyVersion) { + exclude(group: 'stax', module: 'stax-api') + } + testCompile group: 'com.sun.jersey.contribs', name: 'jersey-guice', version: sunJerseyVersion +} From ecb7f7ff72fb94dcad8ce6bda0a924d9f3d6d01b Mon Sep 17 00:00:00 2001 From: ymahajan Date: Mon, 30 Nov 2015 12:46:19 +0530 Subject: [PATCH 016/142] [SNAPPYDATA] Dynamic CQ changes in spark streaming - Allow registration of output streams on active StreamingContext - Added flag to DStream.initialize to allow initialization of newly added output streams with zeroTime - generatedRDDs made thread-safe --- .../spark/streaming/dstream/DStream.scala | 41 +++++++++++++++---- .../streaming/dstream/FileInputDStream.scala | 3 +- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala index e23edfa506517..479864f9d7051 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala @@ -19,8 +19,8 @@ package org.apache.spark.streaming.dstream import java.io.{IOException, ObjectInputStream, ObjectOutputStream} +import java.util.concurrent.ConcurrentHashMap -import scala.collection.mutable.HashMap import scala.language.implicitConversions import scala.reflect.ClassTag import scala.util.matching.Regex @@ -82,9 +82,17 @@ abstract class DStream[T: ClassTag] ( // Methods and fields available on all DStreams // ======================================================================= + import scala.collection.JavaConverters._ // RDDs generated, marked as private[streaming] so that testsuites can access it @transient - private[streaming] var generatedRDDs = new HashMap[Time, RDD[T]]() + // private[streaming] var generatedRDDs = new HashMap[Time, RDD[T]]() + private[streaming] var generatedRDDs: scala.collection.mutable.Map[Time, RDD[T]] = _ + + initGeneratedRDDs() + + def initGeneratedRDDs(): Unit = { + generatedRDDs = new ConcurrentHashMap[Time, RDD[T]]().asScala + } // Time zero for the DStream private[streaming] var zeroTime: Time = null @@ -184,12 +192,24 @@ abstract class DStream[T: ClassTag] ( this } + /** + * Initialize the DStream by setting the "zero" time, based on which + * the validity of future times is calculated. This method also recursively initializes + * its parent DStreams. + */ + private[streaming] def initialize(time: Time) { + initialize(time, skipInitialized = false) + } + /** * Initialize the DStream by setting the "zero" time, based on which * the validity of future times is calculated. This method also recursively initializes * its parent DStreams. */ - private[streaming] def initialize(time: Time) { + private[streaming] def initialize(time: Time, skipInitialized: Boolean) { + if (skipInitialized && isInitialized) { + return + } if (zeroTime != null && zeroTime != time) { throw new SparkException(s"ZeroTime is already initialized to $zeroTime" + s", cannot initialize it again to $time") @@ -213,7 +233,7 @@ abstract class DStream[T: ClassTag] ( } // Initialize the dependencies - dependencies.foreach(_.initialize(zeroTime)) + dependencies.foreach(_.initialize(zeroTime, skipInitialized)) } private def validateAtInit(): Unit = { @@ -221,9 +241,11 @@ abstract class DStream[T: ClassTag] ( case StreamingContextState.INITIALIZED => // good to go case StreamingContextState.ACTIVE => + /* throw new IllegalStateException( "Adding new inputs, transformations, and output operations after " + "starting a context is not supported") + */ case StreamingContextState.STOPPED => throw new IllegalStateException( "Adding new inputs, transformations, and output operations after " + @@ -535,7 +557,8 @@ abstract class DStream[T: ClassTag] ( private def readObject(ois: ObjectInputStream): Unit = Utils.tryOrIOException { logDebug(s"${this.getClass().getSimpleName}.readObject used") ois.defaultReadObject() - generatedRDDs = new HashMap[Time, RDD[T]]() + // generatedRDDs = new HashMap[Time, RDD[T]]() + initGeneratedRDDs() } // ======================================================================= @@ -651,8 +674,12 @@ abstract class DStream[T: ClassTag] ( private def foreachRDD( foreachFunc: (RDD[T], Time) => Unit, displayInnerRDDOps: Boolean): Unit = { - new ForEachDStream(this, - context.sparkContext.clean(foreachFunc, false), displayInnerRDDOps).register() + val dStream = new ForEachDStream(this, + context.sparkContext.clean(foreachFunc, false), displayInnerRDDOps) + if (ssc.getState() == StreamingContextState.ACTIVE) { + dStream.initialize(ssc.graph.zeroTime, skipInitialized = true) + } + dStream.register() } /** diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala index b8a5a96faf15c..ae97ba0a5d823 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala @@ -316,7 +316,8 @@ class FileInputDStream[K, V, F <: NewInputFormat[K, V]]( private def readObject(ois: ObjectInputStream): Unit = Utils.tryOrIOException { logDebug(this.getClass().getSimpleName + ".readObject used") ois.defaultReadObject() - generatedRDDs = new mutable.HashMap[Time, RDD[(K, V)]]() + // generatedRDDs = new mutable.HashMap[Time, RDD[(K, V)]]() + initGeneratedRDDs() batchTimeToSelectedFiles = new mutable.HashMap[Time, Array[String]] recentlySelectedFiles = new mutable.HashSet[String]() fileToModTime = new TimeStampedHashMap[String, Long](true) From 40a4ea0dea8a5013a44cb2e299044fb4d9f4d676 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Mon, 8 Aug 2016 14:47:58 +0530 Subject: [PATCH 017/142] [SNAPPYDATA] Fix cluster startup due to executionId format Allow for non-integral values as executionId (SnappyData uses full DistributedMember representation) --- .../cluster/CoarseGrainedSchedulerBackend.scala | 10 ++++++++-- .../scala/org/apache/spark/sql/AnalysisException.scala | 5 ----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index 4d75063fbf1c5..74a19b91b253b 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -195,8 +195,14 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp // in this block are read when requesting executors CoarseGrainedSchedulerBackend.this.synchronized { executorDataMap.put(executorId, data) - if (currentExecutorIdCounter < executorId.toInt) { - currentExecutorIdCounter = executorId.toInt + // [snappydata] skip toInt used for Yarn since snappydata's + // executorId is not an integer + try { + if (currentExecutorIdCounter < executorId.toInt) { + currentExecutorIdCounter = executorId.toInt + } + } catch { + case nfe: NumberFormatException => // ignore } if (numPendingExecutors > 0) { numPendingExecutors -= 1 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 2b2aa60f2538c..741d2aaa3061d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -54,11 +54,6 @@ class AnalysisException protected[sql] ( val cause: Option[Throwable] = None) extends Exception(message, cause.orNull) with Serializable { - def this(message: String, cause: Throwable) = { - this(message) - initCause(cause) - } - def withPosition(line: Option[Int], startPosition: Option[Int]): AnalysisException = { val newException = new AnalysisException(message, line, startPosition) newException.setStackTrace(getStackTrace) From 37ee5141e309a907cde2d4808bc5d86a8ffbb2b4 Mon Sep 17 00:00:00 2001 From: nthanvi Date: Wed, 2 Dec 2015 17:05:06 +0530 Subject: [PATCH 018/142] [SNAPPYDATA] Accept Spark properties with "snappydata." prefix - in addition to properties with "spark." prefix, also accept "snappydata." prefix in system properties, spark submit/shell Conflicts: launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java --- .../scala/org/apache/spark/SparkConf.scala | 21 ++++++++++++++- .../scala/org/apache/spark/SparkEnv.scala | 21 ++++++++++++++- .../spark/deploy/SparkSubmitArguments.scala | 20 +++++++++++++- .../scala/org/apache/spark/util/Utils.scala | 2 +- .../apache/spark/launcher/SparkLauncher.java | 26 +++++++++++++++++++ 5 files changed, 86 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala index f53b2bed74c6e..d6ece44c7173f 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark @@ -72,7 +90,8 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria private[spark] def loadFromSystemProperties(silent: Boolean): SparkConf = { // Load any spark.* system properties - for ((key, value) <- Utils.getSystemProperties if key.startsWith("spark.")) { + for ((key, value) <- Utils.getSystemProperties + if key.startsWith("spark.") || key.startsWith("snappydata.")) { set(key, value, silent) } this diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index 72123f2232532..2ff379cced1a8 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark @@ -431,7 +449,8 @@ object SparkEnv extends Logging { // System properties that are not java classpaths val systemProperties = Utils.getSystemProperties.toSeq val otherProperties = systemProperties.filter { case (k, _) => - k != "java.class.path" && !k.startsWith("spark.") + k != "java.class.path" && !k.startsWith("spark.") && + !k.startsWith("snappydata.") }.sorted // Class paths including all added jars and files diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 9db7a1fe3106d..c962b4df84e80 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.deploy @@ -138,7 +156,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S */ private def ignoreNonSparkProperties(): Unit = { sparkProperties.foreach { case (k, v) => - if (!k.startsWith("spark.")) { + if (!k.startsWith("spark.") && !k.startsWith("snappydata.")) { sparkProperties -= k SparkSubmit.printWarning(s"Ignoring non-spark config property: $k=$v") } diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index be5becf145d0f..5706a7ba35ff3 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -2105,7 +2105,7 @@ private[spark] object Utils extends Logging { val path = Option(filePath).getOrElse(getDefaultPropertiesFile()) Option(path).foreach { confFile => getPropertiesFromFile(confFile).filter { case (k, v) => - k.startsWith("spark.") + k.startsWith("spark.") || k.startsWith("snappydata.") }.foreach { case (k, v) => conf.setIfMissing(k, v) sys.props.getOrElseUpdate(k, v) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java index f86d40015bd22..fd6763e13ed35 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.launcher; @@ -170,6 +188,14 @@ public SparkLauncher directory(File dir) { */ public SparkLauncher redirectError() { redirectErrorStream = true; + } + + public SparkLauncher setConf(String key, String value) { + checkNotNull(key, "key"); + checkNotNull(value, "value"); + checkArgument(key.startsWith("spark.") || key.startsWith("snappydata."), + "'key' must start with 'spark.' or 'snappydata.'"); + builder.conf.put(key, value); return this; } From 53f2d15fd6f5d640c8d881dab4de0fc8bba792db Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 16 Aug 2016 13:07:56 +0530 Subject: [PATCH 019/142] [SNAPPYDATA] More fixes for SnappyData for Spark 2.0 - make AbstractDataType.classTag as lazy to optimize DataType object creation (e.g. DecimalType) - allow for more than 16 bytes in serialized Decimal objects (precision has been increased to 127 in SnappyData) - minor updates to build.gradle's including a proper dependency for generateBuildInfo to avoid its re-run (and thus all dependent projects) every time - add modification headers for touched files (and remove from a couple of old ones which have later been reverted) - updating dependencies as per latest merge from branch-2.0 - fix scalastyle errors Conflicts: sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala --- .gitignore | 2 + assembly/build.gradle | 2 +- build.gradle | 11 ++- common/network-yarn/build.gradle | 16 ++-- core/build.gradle | 19 ++-- .../CoarseGrainedSchedulerBackend.scala | 18 ++++ external/flume-sink/build.gradle | 4 +- external/kafka-0-10/build.gradle | 2 +- repl/build.gradle | 4 +- settings.gradle | 2 +- .../codegen/UnsafeArrayWriter.java | 20 +++- .../apache/spark/sql/AnalysisException.scala | 18 ---- .../sql/catalyst/CatalystTypeConverters.scala | 1 + .../codegen/GenerateUnsafeProjection.scala | 28 +----- .../plans/physical/partitioning.scala | 91 +++++++++++++++++++ .../spark/sql/types/AbstractDataType.scala | 23 +++++ .../apache/spark/sql/types/DecimalType.scala | 3 +- .../parquet/ParquetSchemaConverter.scala | 3 +- sql/hive/build.gradle | 1 + streaming/build.gradle | 2 +- .../spark/streaming/dstream/DStream.scala | 26 +++++- .../streaming/dstream/FileInputDStream.scala | 18 ++++ yarn/build.gradle | 1 + 23 files changed, 239 insertions(+), 76 deletions(-) diff --git a/.gitignore b/.gitignore index 6e8668c46a350..ae8da5c979079 100644 --- a/.gitignore +++ b/.gitignore @@ -86,8 +86,10 @@ spark-warehouse/ # For R session data .RData .RHistory +.Rhistory *.Rproj *.Rproj.* + .Rproj.user # gradle specific diff --git a/assembly/build.gradle b/assembly/build.gradle index 5a0f18c45e3f6..63db32e3e41f6 100644 --- a/assembly/build.gradle +++ b/assembly/build.gradle @@ -113,7 +113,7 @@ task product(type: Zip) { into snappyProductDir } def releaseFile = file("${snappyProductDir}/RELEASE") - String buildFlags = "" + String buildFlags = '' if (rootProject.hasProperty('docker')) { buildFlags += ' -Pdocker' } diff --git a/build.gradle b/build.gradle index c235dcf712cc4..dddce0ab57942 100644 --- a/build.gradle +++ b/build.gradle @@ -80,6 +80,7 @@ allprojects { commonsCodecVersion = '1.10' avroVersion = '1.7.7' jsr305Version = '3.0.1' + jlineVersion = '2.14.2' scalatestVersion = '2.2.6' pegdownVersion = '1.6.0' @@ -326,19 +327,19 @@ subprojects { def eol = System.getProperty('line.separator') beforeTest { desc -> def now = new Date().format('yyyy-MM-dd HH:mm:ss.SSS Z') - def progress = new File(workingDir, "progress.txt") - def output = new File(workingDir, "output.txt") + def progress = new File(workingDir, 'progress.txt') + def output = new File(workingDir, 'output.txt') progress << "$now Starting test $desc.className $desc.name$eol" output << "${now} STARTING TEST ${desc.className} ${desc.name}${eol}${eol}" } onOutput { desc, event -> - def output = new File(workingDir, "output.txt") + def output = new File(workingDir, 'output.txt') output << event.message } afterTest { desc, result -> def now = new Date().format('yyyy-MM-dd HH:mm:ss.SSS Z') - def progress = new File(workingDir, "progress.txt") - def output = new File(workingDir, "output.txt") + def progress = new File(workingDir, 'progress.txt') + def output = new File(workingDir, 'output.txt') progress << "${now} Completed test ${desc.className} ${desc.name} with result: ${result.resultType}${eol}" output << "${eol}${now} COMPLETED TEST ${desc.className} ${desc.name} with result: ${result.resultType}${eol}${eol}" result.exceptions.each { t -> diff --git a/common/network-yarn/build.gradle b/common/network-yarn/build.gradle index 323463acad43c..bbb6d8c7f81ae 100644 --- a/common/network-yarn/build.gradle +++ b/common/network-yarn/build.gradle @@ -75,19 +75,19 @@ shadowJar { relocate 'com.fasterxml.jackson', "${shadePackageName}.com.fasterxml.jackson" relocate 'com.google.common', "${shadePackageName}.guava" - String createdBy = "" + String createdBy = '' if (rootProject.hasProperty('enablePublish')) { - createdBy = "SnappyData Build Team" + createdBy = 'SnappyData Build Team' } else { - createdBy = System.getProperty("user.name") + createdBy = System.getProperty('user.name') } manifest { attributes( - "Manifest-Version" : "1.0", - "Created-By" : createdBy, - "Title" : project.name, - "Version" : version, - "Vendor" : "SnappyData, Inc." + 'Manifest-Version' : '1.0', + 'Created-By' : createdBy, + 'Title' : project.name, + 'Version' : version, + 'Vendor' : 'SnappyData, Inc.' ) } diff --git a/core/build.gradle b/core/build.gradle index e1b52bfec0799..9395a129dac33 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -57,6 +57,7 @@ dependencies { exclude(group: 'org.ow2.asm', module: 'asm') exclude(group: 'org.apache.zookeeper', module: 'zookeeper') exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'jline', module: 'jline') exclude(group: 'commons-logging', module: 'commons-logging') exclude(group: 'org.mockito', module: 'mockito-all') exclude(group: 'org.mortbay.jetty', module: 'servlet-api-2.5') @@ -142,6 +143,7 @@ dependencies { testCompile(group: 'org.apache.curator', name: 'curator-test', version: curatorVersion) { exclude(group: 'org.apache.zookeeper', module: 'zookeeper') exclude(group: 'org.jboss.netty', module: 'netty') + exclude(group: 'jline', module: 'jline') exclude(group: 'com.google.guava', module: 'guava') } } @@ -155,12 +157,17 @@ sourceSets.test.java.srcDirs = [] // generate properties using spark-build-info and add to project resources String extraResourceDir = "${buildDir}/extra-resources" -task generateBuildInfo << { - file(extraResourceDir).mkdirs() - exec { - executable 'bash' - workingDir = buildDir - args "${projectDir}/../build/spark-build-info", extraResourceDir, version +task generateBuildInfo { + outputs.file "${extraResourceDir}/spark-version-info.properties" + inputs.dir compileScala.destinationDir + + doLast { + file(extraResourceDir).mkdirs() + exec { + executable 'bash' + workingDir = buildDir + args "${projectDir}/../build/spark-build-info", extraResourceDir, version + } } } sourceSets { diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index 74a19b91b253b..7ffae9b68cb0e 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.scheduler.cluster diff --git a/external/flume-sink/build.gradle b/external/flume-sink/build.gradle index 99a0357048896..3a0060cca4897 100644 --- a/external/flume-sink/build.gradle +++ b/external/flume-sink/build.gradle @@ -41,9 +41,9 @@ dependencies { compile group: 'io.netty', name: 'netty', version: nettyVersion } -// for compatibility with maven generated code, though default "string" seems +// for compatibility with maven generated code, though default 'string' seems // more efficient requiring no conversions -avro.stringType = "charSequence" +avro.stringType = 'charSequence' tasks.withType(JavaCompile) { options.compilerArgs << '-Xlint:all,-serial,-path,-deprecation,-unchecked' diff --git a/external/kafka-0-10/build.gradle b/external/kafka-0-10/build.gradle index c898dfce495c5..d5dc6611a97f0 100644 --- a/external/kafka-0-10/build.gradle +++ b/external/kafka-0-10/build.gradle @@ -21,7 +21,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) - compile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.0') { + compile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.1') { exclude(group: 'com.sun.jmx', module: 'jmxri') exclude(group: 'com.sun.jdmk ', module: 'jmxtools') exclude(group: 'net.sf.jopt-simple', module: 'jopt-simple') diff --git a/repl/build.gradle b/repl/build.gradle index 1e51600f39b04..018602803e4f2 100644 --- a/repl/build.gradle +++ b/repl/build.gradle @@ -25,9 +25,7 @@ dependencies { compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion compile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion - if (scalaBinaryVersion == '2.10') { - compile group: 'org.scala-lang', name: 'jline', version:scalaVersion - } + compile group: 'jline', name: 'jline', version: jlineVersion runtime project(subprojectBase + 'snappy-spark-mllib_' + scalaBinaryVersion) diff --git a/settings.gradle b/settings.gradle index 70ea595ee61a5..ca33d18d94bf5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,7 +15,7 @@ * LICENSE file. */ -def scalaBinaryVersion = "2.11" +def scalaBinaryVersion = '2.11' rootProject.name = 'snappy-spark' include ':snappy-spark-tags_' + scalaBinaryVersion diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java index 791e8d80e6cba..c58e90cad50d9 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.catalyst.expressions.codegen; @@ -190,7 +208,7 @@ public void write(int ordinal, Decimal input, int precision, int scale) { } else { final byte[] bytes = input.toJavaBigDecimal().unscaledValue().toByteArray(); final int numBytes = bytes.length; - assert numBytes <= 16; + // assert numBytes <= 16; int roundedSize = ByteArrayMethods.roundNumberOfBytesToNearestWord(numBytes); holder.grow(roundedSize); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 741d2aaa3061d..50ee6cd4085ea 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -14,24 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ package org.apache.spark.sql diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala index 86a625a6c8272..c40d712817791 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala @@ -188,6 +188,7 @@ object CatalystTypeConverters { convertedIterable += elementConverter.toCatalyst(item) } new GenericArrayData(convertedIterable.toArray) + case a: ArrayData => a } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala index d9e9c866aeb5b..b8ccf6e8e4ad7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala @@ -14,24 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ package org.apache.spark.sql.catalyst.expressions.codegen @@ -62,7 +44,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro } // TODO: if the nullability of field is correct, we can use it to save null check. - private[sql] def writeStructToBuffer( + private def writeStructToBuffer( ctx: CodegenContext, input: String, fieldTypes: Seq[DataType], @@ -195,7 +177,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro } // TODO: if the nullability of array element is correct, we can use it to save null check. - private[sql] def writeArrayToBuffer( + private def writeArrayToBuffer( ctx: CodegenContext, input: String, elementType: DataType, @@ -231,13 +213,13 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro $arrayWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); """ - case a @ ArrayType(at, _) => + case a @ ArrayType(et, _) => s""" final int $tmpCursor = $bufferHolder.cursor; ${writeArrayToBuffer(ctx, element, et, bufferHolder)} $arrayWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); $arrayWriter.setOffset($index); - ${writeArrayToBuffer(ctx, element, at, bufferHolder)} + ${writeArrayToBuffer(ctx, element, et, bufferHolder)} """ case m @ MapType(kt, vt, _) => @@ -276,7 +258,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro } // TODO: if the nullability of value element is correct, we can use it to save null check. - private[sql] def writeMapToBuffer( + private def writeMapToBuffer( ctx: CodegenContext, input: String, keyType: DataType, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index 77b40e6452880..e488281de0dc1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -209,6 +209,97 @@ case object SinglePartition extends Partitioning { case ClusteredDistribution(_, Some(requiredNumPartitions)) => requiredNumPartitions == 1 case _ => true } + + override def compatibleWith(other: Partitioning): Boolean = other.numPartitions == 1 + + override def guarantees(other: Partitioning): Boolean = other.numPartitions == 1 +} + +/** + * Represents a partitioning where rows are split up across partitions based on the hash + * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be + * in the same partition. Moreover while evaluating expressions if they are given in different order + * than this partitioning then also it is considered equal. + */ +case class OrderlessHashPartitioning(expressions: Seq[Expression], numPartitions: Int) + extends Expression with Partitioning with Unevaluable { + + + override def children: Seq[Expression] = expressions + override def nullable: Boolean = false + override def dataType: DataType = IntegerType + + private def matchExpressions(otherExpression: Seq[Expression]): Boolean = { + expressions.length == otherExpression.length && expressions.forall(a => + otherExpression.exists(e => e.semanticEquals(a))) + } + + override def satisfies(required: Distribution): Boolean = required match { + case UnspecifiedDistribution => true + case ClusteredDistribution(requiredClustering) => { + matchExpressions(requiredClustering) + } + case _ => false + } + + private def anyOrderEquals(other: HashPartitioning) : Boolean = { + other.numPartitions == this.numPartitions && + matchExpressions(other.expressions) + } + + override def compatibleWith(other: Partitioning): Boolean = other match { + case p: HashPartitioning => anyOrderEquals(p) + case _ => false + } + + override def guarantees(other: Partitioning): Boolean = other match { + case o: HashPartitioning => anyOrderEquals(o) + case _ => false + } + +} + +/** + * Represents a partitioning where rows are split up across partitions based on the hash + * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be + * in the same partition. Moreover while evaluating expressions if they are given in different order + * than this partitioning then also it is considered equal. + */ +case class OrderlessHashPartitioning(expressions: Seq[Expression], numPartitions: Int) + extends Expression with Partitioning with Unevaluable { + + + override def children: Seq[Expression] = expressions + override def nullable: Boolean = false + override def dataType: DataType = IntegerType + + private def matchExpressions(otherExpression: Seq[Expression]): Boolean = { + expressions.length == otherExpression.length && expressions.forall(a => + otherExpression.exists(e => e.semanticEquals(a))) + } + + override def satisfies(required: Distribution): Boolean = required match { + case UnspecifiedDistribution => true + case ClusteredDistribution(requiredClustering) => + matchExpressions(requiredClustering) + case _ => false + } + + private def anyOrderEquals(other: HashPartitioning) : Boolean = { + other.numPartitions == this.numPartitions && + matchExpressions(other.expressions) + } + + override def compatibleWith(other: Partitioning): Boolean = other match { + case p: HashPartitioning => anyOrderEquals(p) + case _ => false + } + + override def guarantees(other: Partitioning): Boolean = other match { + case o: HashPartitioning => anyOrderEquals(o) + case _ => false + } + } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala index 3041f44b116ea..f405f1ea2e128 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.types @@ -115,6 +133,11 @@ protected[sql] abstract class AtomicType extends DataType { private[sql] type InternalType private[sql] val tag: TypeTag[InternalType] private[sql] val ordering: Ordering[InternalType] + + @transient private[sql] lazy val classTag = ScalaReflectionLock.synchronized { + val mirror = runtimeMirror(Utils.getSparkClassLoader) + ClassTag[InternalType](mirror.runtimeClass(tag.tpe)) + } } object AtomicType { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala index c19067c73955f..117158aa500a2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala @@ -66,7 +66,8 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType { } if (precision > DecimalType.MAX_PRECISION) { - throw new AnalysisException(s"DecimalType can only support precision up to ${DecimalType.MAX_PRECISION}") + throw new AnalysisException( + s"DecimalType can only support precision up to ${DecimalType.MAX_PRECISION}") } // default constructor for Java diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala index 6eeaa02388379..530ee180218a3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala @@ -612,7 +612,8 @@ private[sql] object ParquetSchemaConverter { } // Returns the minimum number of bytes needed to store a decimal with a given `precision`. - val minBytesForPrecision = Array.tabulate[Int](DecimalType.MAX_PRECISION + 1)(computeMinBytesForPrecision) + val minBytesForPrecision = Array.tabulate[Int](DecimalType.MAX_PRECISION + 1)( + computeMinBytesForPrecision) // Max precision of a decimal value stored in `numBytes` bytes def maxPrecisionForBytes(numBytes: Int): Int = { diff --git a/sql/hive/build.gradle b/sql/hive/build.gradle index a3c22973264b2..25f6d76d4d7eb 100644 --- a/sql/hive/build.gradle +++ b/sql/hive/build.gradle @@ -46,6 +46,7 @@ dependencies { exclude(group: 'log4j', module: 'log4j') exclude(group: 'commons-logging', module: 'commons-logging') exclude(group: 'org.codehaus.groovy', module: 'groovy-all') + exclude(group: 'jline', module: 'jline') } compile(group: 'org.spark-project.hive', name: 'hive-metastore', version: hiveVersion) { exclude(group: 'org.spark-project.hive', module: 'hive-serde') diff --git a/streaming/build.gradle b/streaming/build.gradle index fcfb62199e447..bcfeecb82f8ff 100644 --- a/streaming/build.gradle +++ b/streaming/build.gradle @@ -40,5 +40,5 @@ dependencies { } // fix scala+java mix to use scala first for tests -sourceSets.test.scala.srcDir "src/test/java" +sourceSets.test.scala.srcDir 'src/test/java' sourceSets.test.java.srcDirs = [] diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala index 479864f9d7051..039c401ace9af 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.streaming.dstream @@ -193,10 +211,10 @@ abstract class DStream[T: ClassTag] ( } /** - * Initialize the DStream by setting the "zero" time, based on which - * the validity of future times is calculated. This method also recursively initializes - * its parent DStreams. - */ + * Initialize the DStream by setting the "zero" time, based on which + * the validity of future times is calculated. This method also recursively initializes + * its parent DStreams. + */ private[streaming] def initialize(time: Time) { initialize(time, skipInitialized = false) } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala index ae97ba0a5d823..cb8a2443bf1f1 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.streaming.dstream diff --git a/yarn/build.gradle b/yarn/build.gradle index f8eb23d2b5566..5fb389cba38c0 100644 --- a/yarn/build.gradle +++ b/yarn/build.gradle @@ -91,6 +91,7 @@ dependencies { exclude(group: 'log4j', module: 'log4j') exclude(group: 'commons-logging', module: 'commons-logging') exclude(group: 'org.codehaus.groovy', module: 'groovy-all') + exclude(group: 'jline', module: 'jline') } compile(group: 'org.spark-project.hive', name: 'hive-metastore', version: hiveVersion) { exclude(group: 'org.spark-project.hive', module: 'hive-serde') From 3e82acba88ab28c493392ab8be5cb709b5b69739 Mon Sep 17 00:00:00 2001 From: nthanvi Date: Thu, 25 Aug 2016 12:39:02 +0530 Subject: [PATCH 020/142] Snap 293 (#1) * Using the scala collections compiler was not able to differentiate between the scala and java api * Minor changes for snappy implementation of executor This is required as we need to have a classloader that also looks into the snappy store for the classes. * Revert "Using the scala collections" This reverts commit c2ab0c5aa3974337277560fb1c8a9d0c3661ec09. Conflicts: core/src/main/scala/org/apache/spark/executor/Executor.scala --- .../spark/executor/CoarseGrainedExecutorBackend.scala | 5 ++++- .../src/main/scala/org/apache/spark/executor/Executor.scala | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala index 8cc65f6e4efa7..fab79c8904c07 100644 --- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala +++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala @@ -71,6 +71,9 @@ private[spark] class CoarseGrainedExecutorBackend( }(ThreadUtils.sameThread) } + protected def registerExecutor: Executor = + new Executor(executorId, hostname, env, userClassPath, isLocal = false) + def extractLogUrls: Map[String, String] = { val prefix = "SPARK_LOG_URL_" sys.env.filterKeys(_.startsWith(prefix)) @@ -81,7 +84,7 @@ private[spark] class CoarseGrainedExecutorBackend( case RegisteredExecutor => logInfo("Successfully registered with driver") try { - executor = new Executor(executorId, hostname, env, userClassPath, isLocal = false) + executor = registerExecutor } catch { case NonFatal(e) => exitExecutor(1, "Unable to create executor due to " + e.getMessage, e) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 05ec6565dfe25..b3b107e5a49fd 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -37,7 +37,7 @@ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager} import org.apache.spark.rpc.RpcTimeout -import org.apache.spark.scheduler.{DirectTaskResult, IndirectTaskResult, Task, TaskDescription} +import org.apache.spark.scheduler.{AccumulableInfo, DirectTaskResult, IndirectTaskResult, Task, TaskDescription} import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.storage.{StorageLevel, TaskResultBlockId} import org.apache.spark.util._ @@ -64,7 +64,7 @@ private[spark] class Executor( // Application dependencies (added through SparkContext) that we've fetched so far on this node. // Each map holds the master's timestamp for the version of that file or JAR we got. private val currentFiles: HashMap[String, Long] = new HashMap[String, Long]() - private val currentJars: HashMap[String, Long] = new HashMap[String, Long]() + protected val currentJars: HashMap[String, Long] = new HashMap[String, Long]() private val EMPTY_BYTE_BUFFER = ByteBuffer.wrap(new Array[Byte](0)) @@ -697,7 +697,7 @@ private[spark] class Executor( * Create a ClassLoader for use in tasks, adding any JARs specified by the user or any classes * created by the interpreter to the search path */ - private def createClassLoader(): MutableURLClassLoader = { + protected def createClassLoader(): MutableURLClassLoader = { // Bootstrap the list of jars with the user class path. val now = System.currentTimeMillis() userClassPath.foreach { url => From 27025d1bd767caf70f6bd480fdf4c24409dd8dea Mon Sep 17 00:00:00 2001 From: ymahajan Date: Fri, 2 Sep 2016 16:53:33 +0530 Subject: [PATCH 021/142] [SNAPPYDATA] fix a scalaStyle issue Conflicts: core/src/main/scala/org/apache/spark/Partitioner.scala sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala --- core/src/main/scala/org/apache/spark/Partitioner.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/Partitioner.scala b/core/src/main/scala/org/apache/spark/Partitioner.scala index c940cb25d478b..bdfecd08f27c8 100644 --- a/core/src/main/scala/org/apache/spark/Partitioner.scala +++ b/core/src/main/scala/org/apache/spark/Partitioner.scala @@ -109,6 +109,12 @@ object Partitioner { class HashPartitioner(partitions: Int) extends Partitioner { require(partitions >= 0, s"Number of partitions ($partitions) cannot be negative.") + require(buckets >= 0, s"Number of buckets ($buckets) cannot be negative.") + + def this(partitions: Int) { + this(partitions, 0) + } + def numPartitions: Int = partitions def getPartition(key: Any): Int = key match { From 9ac66618630b6f0456c4cb0b83a67966a9a30478 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 7 Sep 2016 18:46:12 +0530 Subject: [PATCH 022/142] [SNAP-966] Prefer conversions to date/timestamp and not strings (#7) - for all cases of implicit casts, convert to date or timestamp values instead of string when one side is a string - likewise when one side is a timestamp and other date then both are being converted to string; now convert date to timestamp Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala --- .../sql/catalyst/analysis/TypeCoercion.scala | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index e8669c4637d06..8921683a0e033 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -345,6 +345,43 @@ object TypeCoercion { if findCommonTypeForBinaryComparison(left.dataType, right.dataType).isDefined => val commonType = findCommonTypeForBinaryComparison(left.dataType, right.dataType).get p.makeCopy(Array(castExpr(left, commonType), castExpr(right, commonType))) + // We should cast all relative timestamp/date/string comparison into string comparisons + // This behaves as a user would expect because timestamp strings sort lexicographically. + // i.e. TimeStamp(2013-01-01 00:00 ...) < "2014" = true + case p @ BinaryComparison(left @ StringType(), right @ DateType()) => + p.makeCopy(Array(left, Cast(right, StringType))) + case p @ BinaryComparison(left @ DateType(), right @ StringType()) => + p.makeCopy(Array(Cast(left, StringType), right)) + case p @ BinaryComparison(left @ StringType(), right @ TimestampType()) => + p.makeCopy(Array(left, Cast(right, StringType))) + case p @ BinaryComparison(left @ TimestampType(), right @ StringType()) => + p.makeCopy(Array(Cast(left, StringType), right)) + + // Comparisons between dates and timestamps. + case p @ BinaryComparison(left @ TimestampType(), right @ DateType()) => + p.makeCopy(Array(Cast(left, StringType), Cast(right, StringType))) + case p @ BinaryComparison(left @ DateType(), right @ TimestampType()) => + p.makeCopy(Array(Cast(left, StringType), Cast(right, StringType))) + + // Checking NullType + case p @ BinaryComparison(left @ StringType(), right @ NullType()) => + p.makeCopy(Array(left, Literal.create(null, StringType))) + case p @ BinaryComparison(left @ NullType(), right @ StringType()) => + p.makeCopy(Array(Literal.create(null, StringType), right)) + + case p @ BinaryComparison(left @ StringType(), right) if right.dataType != StringType => + p.makeCopy(Array(Cast(left, DoubleType), right)) + case p @ BinaryComparison(left, right @ StringType()) if left.dataType != StringType => + p.makeCopy(Array(left, Cast(right, DoubleType))) + + case i @ In(a @ DateType(), b) if b.forall(_.dataType == StringType) => + i.makeCopy(Array(Cast(a, StringType), b)) + case i @ In(a @ TimestampType(), b) if b.forall(_.dataType == StringType) => + i.makeCopy(Array(a, b.map(Cast(_, TimestampType)))) + case i @ In(a @ DateType(), b) if b.forall(_.dataType == TimestampType) => + i.makeCopy(Array(Cast(a, StringType), b.map(Cast(_, StringType)))) + case i @ In(a @ TimestampType(), b) if b.forall(_.dataType == DateType) => + i.makeCopy(Array(Cast(a, StringType), b.map(Cast(_, StringType)))) case Abs(e @ StringType()) => Abs(Cast(e, DoubleType)) case Sum(e @ StringType()) => Sum(Cast(e, DoubleType)) From 8455f59f3d57b219e30b23384d57ceb95f919684 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 7 Sep 2016 21:11:26 +0530 Subject: [PATCH 023/142] [SNAP-1034] Optimizations at Spark layer as seen in profiling (#10) - added a aggBufferAttributeForGroup to aggregates to be used to avoid nullable checks in generated code in aggregate buffers used in HashAggregateExec (if aggregate is on zero rows, then there will be no row in the map); accompanying "initialValuesForGroup" added for initial aggregation buffer values - use OpenHashMap in DictionaryEncoding which is faster than normal hash map; added clear methods to OpenHashMap/OpenHashSet for reuse - minor correction in the string in HiveUtils Conflicts: sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala --- .../spark/util/collection/OpenHashMap.scala | 17 +++++++++++++++++ .../spark/util/collection/OpenHashSet.scala | 6 ++++++ .../expressions/aggregate/Average.scala | 8 +++++++- .../catalyst/expressions/aggregate/Sum.scala | 12 ++++++++++++ .../expressions/aggregate/interfaces.scala | 8 ++++++++ .../execution/aggregate/HashAggregateExec.scala | 17 ++++++++++++----- .../compression/compressionSchemes.scala | 3 ++- .../spark/sql/execution/joins/HashJoin.scala | 4 ++-- .../sql/execution/joins/HashedRelation.scala | 9 ++++++++- .../spark/sql/execution/metric/SQLMetrics.scala | 3 ++- .../org/apache/spark/sql/hive/HiveUtils.scala | 6 +++--- 11 files changed, 79 insertions(+), 14 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala b/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala index 10ab0b3f89964..00cccd33daf97 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala @@ -149,6 +149,23 @@ class OpenHashMap[K : ClassTag, @specialized(Long, Int, Double) V: ClassTag]( } } + def clear() { + // first clear the values array and value for null key + val bitSet = _keySet.getBitSet + val nullV = null.asInstanceOf[V] + val values = _values + var pos = bitSet.nextSetBit(0) + while (pos >= 0) { + values(pos) = nullV + pos = bitSet.nextSetBit(pos + 1) + } + haveNullValue = false + nullValue = nullV + _oldValues = null + // next clear the key set + _keySet.clear() + } + // The following member variables are declared as protected instead of private for the // specialization to work (specialized class extends the non-specialized one and needs access // to the "private" variables). diff --git a/core/src/main/scala/org/apache/spark/util/collection/OpenHashSet.scala b/core/src/main/scala/org/apache/spark/util/collection/OpenHashSet.scala index 60f6f537c1d54..835fec1320c37 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/OpenHashSet.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/OpenHashSet.scala @@ -212,6 +212,12 @@ class OpenHashSet[@specialized(Long, Int) T: ClassTag]( */ def nextPos(fromPos: Int): Int = _bitset.nextSetBit(fromPos) + def clear() { + _data = new Array[T](_capacity) + _bitset.clear() + _size = 0 + } + /** * Double the table's size and re-hash everything. We are not really using k, but it is declared * so Scala compiler can specialize this method (which leads to calling the specialized version diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala index 708bdbfc36058..70331bb183cd2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala @@ -53,10 +53,16 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit } private lazy val sum = AttributeReference("sum", sumDataType)() - private lazy val count = AttributeReference("count", LongType)() + private lazy val count = AttributeReference("count", LongType, nullable = false)() override lazy val aggBufferAttributes = sum :: count :: Nil + override lazy val aggBufferAttributesForGroup: Seq[AttributeReference] = { + if (child.nullable) aggBufferAttributes + else sum.copy(nullable = false)(sum.exprId, sum.qualifier, + sum.isGenerated) :: count :: Nil + } + override lazy val initialValues = Seq( /* sum = */ Cast(Literal(0), sumDataType), /* count = */ Literal(0L) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala index 86e40a9713b36..6dacb7148072c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala @@ -53,10 +53,22 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast override lazy val aggBufferAttributes = sum :: Nil + override lazy val aggBufferAttributesForGroup: Seq[AttributeReference] = { + if (child.nullable) aggBufferAttributes + else sum.copy(nullable = false)(sum.exprId, sum.qualifier, + sum.isGenerated) :: Nil + } + override lazy val initialValues: Seq[Expression] = Seq( /* sum = */ Literal.create(null, sumDataType) ) + override lazy val initialValuesForGroup: Seq[Expression] = Seq( + /* sum = */ + if (child.nullable) Literal.create(null, sumDataType) + else Cast(Literal(0), sumDataType) + ) + override lazy val updateExpressions: Seq[Expression] = { if (child.nullable) { Seq( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala index d6d394f5b8ae9..d060fdc2b3c3c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala @@ -194,6 +194,9 @@ abstract class AggregateFunction extends Expression { /** Attributes of fields in aggBufferSchema. */ def aggBufferAttributes: Seq[AttributeReference] + /** Attributes of fields in aggBufferSchema used for group by. */ + def aggBufferAttributesForGroup: Seq[AttributeReference] = aggBufferAttributes + /** * Attributes of fields in input aggregation buffers (immutable aggregation buffers that are * merged with mutable aggregation buffers in the merge() function or merge expressions). @@ -375,6 +378,11 @@ abstract class DeclarativeAggregate */ val initialValues: Seq[Expression] + /** + * Expressions for initializing empty aggregation buffers for group by. + */ + def initialValuesForGroup: Seq[Expression] = initialValues + /** * Expressions for updating the mutable aggregation buffer based on an input row. */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala index cfdda6a046f80..7e09a40f01cdb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala @@ -70,6 +70,10 @@ case class HashAggregateExec( aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes) } + @transient lazy private[this] val aggregateBufferAttributesForGroup = { + aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributesForGroup) + } + require(HashAggregateExec.supportsAggregate(aggregateBufferAttributes)) override lazy val allAttributes: AttributeSeq = @@ -316,7 +320,7 @@ case class HashAggregateExec( private val declFunctions = aggregateExpressions.map(_.aggregateFunction) .filter(_.isInstanceOf[DeclarativeAggregate]) .map(_.asInstanceOf[DeclarativeAggregate]) - private val bufferSchema = StructType.fromAttributes(aggregateBufferAttributes) + private val bufferSchema = StructType.fromAttributes(aggregateBufferAttributesForGroup) // The name for Fast HashMap private var fastHashMapTerm: String = _ @@ -336,7 +340,7 @@ case class HashAggregateExec( */ def createHashMap(): UnsafeFixedWidthAggregationMap = { // create initialized aggregate buffer - val initExpr = declFunctions.flatMap(f => f.initialValues) + val initExpr = declFunctions.flatMap(_.initialValuesForGroup) val initialBuffer = UnsafeProjection.create(initExpr)(EmptyRow) // create hashMap @@ -405,7 +409,7 @@ case class HashAggregateExec( val mergeExpr = declFunctions.flatMap(_.mergeExpressions) val mergeProjection = newMutableProjection( mergeExpr, - aggregateBufferAttributes ++ declFunctions.flatMap(_.inputAggBufferAttributes), + aggregateBufferAttributesForGroup ++ declFunctions.flatMap(_.inputAggBufferAttributes), subexpressionEliminationEnabled) val joinedRow = new JoinedRow() @@ -473,14 +477,14 @@ case class HashAggregateExec( } val evaluateKeyVars = evaluateVariables(keyVars) ctx.INPUT_ROW = bufferTerm - val bufferVars = aggregateBufferAttributes.zipWithIndex.map { case (e, i) => + val bufferVars = aggregateBufferAttributesForGroup.zipWithIndex.map { case (e, i) => BoundReference(i, e.dataType, e.nullable).genCode(ctx) } val evaluateBufferVars = evaluateVariables(bufferVars) // evaluate the aggregation result ctx.currentVars = bufferVars val aggResults = declFunctions.map(_.evaluateExpression).map { e => - BindReferences.bindReference(e, aggregateBufferAttributes).genCode(ctx) + BindReferences.bindReference(e, aggregateBufferAttributesForGroup).genCode(ctx) } val evaluateAggResults = evaluateVariables(aggResults) // generate the final result @@ -775,6 +779,9 @@ case class HashAggregateExec( val hashExpr = Murmur3Hash(groupingExpressions, 42) val hashEval = BindReferences.bindReference(hashExpr, child.output).genCode(ctx) + val inputAttr = aggregateBufferAttributesForGroup ++ child.output + ctx.currentVars = new Array[ExprCode](aggregateBufferAttributesForGroup.length) ++ input + val (checkFallbackForGeneratedHashMap, checkFallbackForBytesToBytesMap, resetCounter, incCounter) = if (testFallbackStartsAt.isDefined) { val countTerm = ctx.addMutableState(ctx.JAVA_INT, "fallbackCounter") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala index 00a1d54b41709..a06511c753418 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow import org.apache.spark.sql.execution.columnar._ import org.apache.spark.sql.execution.vectorized.WritableColumnVector import org.apache.spark.sql.types._ +import org.apache.spark.util.collection.OpenHashMap private[columnar] case object PassThrough extends CompressionScheme { @@ -393,7 +394,7 @@ private[columnar] case object DictionaryEncoding extends CompressionScheme { private var values = new mutable.ArrayBuffer[T#InternalType](1024) // The dictionary that maps a value to the encoded short integer. - private val dictionary = mutable.HashMap.empty[Any, Short] + private val dictionary = new OpenHashMap[Any, Short] // Size of the serialized dictionary in bytes. Initialized to 4 since we need at least an `Int` // to store dictionary element count. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala index 0396168d3f311..b73a409f5e58a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala @@ -106,9 +106,9 @@ trait HashJoin { joinRow.withLeft(srow) val matches = hashedRelation.get(joinKeys(srow)) if (matches != null) { - matches.map(joinRow.withRight(_)).filter(boundCondition) + matches.map(joinRow.withRight).filter(boundCondition) } else { - Seq.empty + Iterator.empty } } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 1465346eb802d..9af1712618dca 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -140,10 +140,17 @@ private[joins] class UnsafeHashedRelation( // re-used in get()/getValue() var resultRow = new UnsafeRow(numFields) + private var mapLoc = initMapLoc() + + private def initMapLoc(): BytesToBytesMap#Location = { + val map = binaryMap + new map.Location + } + override def get(key: InternalRow): Iterator[InternalRow] = { val unsafeKey = key.asInstanceOf[UnsafeRow] val map = binaryMap // avoid the compiler error - val loc = new map.Location // this could be allocated in stack + val loc = mapLoc binaryMap.safeLookup(unsafeKey.getBaseObject, unsafeKey.getBaseOffset, unsafeKey.getSizeInBytes, loc, unsafeKey.hashCode()) if (loc.isDefined) { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala index 77b907870d678..d88763037b3bf 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala @@ -31,7 +31,8 @@ import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, Utils} * the executor side are automatically propagated and shown in the SQL UI through metrics. Updates * on the driver side must be explicitly posted using [[SQLMetrics.postDriverMetricUpdates()]]. */ -class SQLMetric(val metricType: String, initValue: Long = 0L) extends AccumulatorV2[Long, Long] { +final class SQLMetric(val metricType: String, initValue: Long = 0L) + extends AccumulatorV2[Long, Long] { // This is a workaround for SPARK-11013. // We may use -1 as initial value of the accumulator, if the accumulator is valid, we will // update it at the end of task and the value will be at least 0. Then we can filter out the -1 diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala index c448c5a9821be..0f2b35e9afbc0 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala @@ -303,9 +303,9 @@ private[spark] object HiveUtils extends Logging { if (builtinHiveVersion != hiveMetastoreVersion) { throw new IllegalArgumentException( "Builtin jars can only be used when hive execution version == hive metastore version. " + - s"Execution: $builtinHiveVersion != Metastore: $hiveMetastoreVersion. " + - s"Specify a valid path to the correct hive jars using ${HIVE_METASTORE_JARS.key} " + - s"or change ${HIVE_METASTORE_VERSION.key} to $builtinHiveVersion.") + s"Execution: $hiveExecutionVersion != Metastore: $hiveMetastoreVersion. " + + "Specify a vaild path to the correct hive jars using $HIVE_METASTORE_JARS " + + s"or change ${HIVE_METASTORE_VERSION.key} to $hiveExecutionVersion.") } // We recursively find all jars in the class loader chain, From 58af69003ef76a0a98c165542e5e9bce021bac75 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sun, 11 Sep 2016 13:10:23 +0530 Subject: [PATCH 024/142] [SNAPPYDATA] Updated Benchmark code from Spark PR#13899 Used by the new benchmark from the PR adapted for SnappyData for its vectorized implementation. Build updated to set testOutput and other variables instead of appending to existing values (causes double append with both snappydata build adding and this adding for its tests) --- build.gradle | 12 ++++++---- .../org/apache/spark/util/Benchmark.scala | 23 +++++++++++++++---- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index dddce0ab57942..53d3684a80bdb 100644 --- a/build.gradle +++ b/build.gradle @@ -245,15 +245,15 @@ subprojects { def result = new StringBuilder() extensions.add(com.github.maiflai.ScalaTestAction.TESTRESULT, result) - extensions.add('testResult', { String name -> result.append(name) } ) + extensions.add('testResult', { String name -> result.setLength(0); result.append(name) } ) def output = new StringBuilder() extensions.add(com.github.maiflai.ScalaTestAction.TESTOUTPUT, output) - extensions.add('testOutput', { String name -> output.append(name) } ) + extensions.add('testOutput', { String name -> output.setLength(0); output.append(name) } ) def errorOutput = new StringBuilder() extensions.add(com.github.maiflai.ScalaTestAction.TESTERROR, errorOutput) - extensions.add('testError', { String name -> errorOutput.append(name) } ) + extensions.add('testError', { String name -> errorOutput.setLength(0); errorOutput.append(name) } ) // running a single scala suite if (rootProject.hasProperty('singleSuite')) { @@ -295,8 +295,10 @@ subprojects { test.configure { onlyIf { ! Boolean.getBoolean('skip.tests') } - jvmArgs '-XX:+HeapDumpOnOutOfMemoryError', '-XX:MaxPermSize=512m', '-XX:ReservedCodeCacheSize=512m' - maxHeapSize '3g' + jvmArgs '-ea', '-XX:+HeapDumpOnOutOfMemoryError','-XX:+UseConcMarkSweepGC', + '-XX:+UseParNewGC', '-XX:+CMSClassUnloadingEnabled', '-XX:MaxPermSize=512m' + minHeapSize '4g' + maxHeapSize '4g' // disable assertions for hive tests as in Spark's pom.xml because HiveCompatibilitySuite currently fails (SPARK-4814) if (test.project.name.contains('snappy-spark-hive_')) { jvmArgs '-da' diff --git a/core/src/main/scala/org/apache/spark/util/Benchmark.scala b/core/src/main/scala/org/apache/spark/util/Benchmark.scala index 7def44bd2a2b1..7576faa99c96d 100644 --- a/core/src/main/scala/org/apache/spark/util/Benchmark.scala +++ b/core/src/main/scala/org/apache/spark/util/Benchmark.scala @@ -69,12 +69,17 @@ private[spark] class Benchmark( * @param name of the benchmark case * @param numIters if non-zero, forces exactly this many iterations to be run */ - def addCase(name: String, numIters: Int = 0)(f: Int => Unit): Unit = { - addTimerCase(name, numIters) { timer => + def addCase( + name: String, + numIters: Int = 0, + prepare: () => Unit = () => { }, + cleanup: () => Unit = () => { })(f: Int => Unit): Unit = { + val timedF = (timer: Benchmark.Timer) => { timer.startTiming() f(timer.iteration) timer.stopTiming() } + benchmarks += Benchmark.Case(name, timedF, numIters, prepare, cleanup) } /** @@ -101,7 +106,12 @@ private[spark] class Benchmark( val results = benchmarks.map { c => println(" Running case: " + c.name) - measure(valuesPerIteration, c.numIters)(c.fn) + try { + c.prepare() + measure(valuesPerIteration, c.numIters)(c.fn) + } finally { + c.cleanup() + } } println @@ -188,7 +198,12 @@ private[spark] object Benchmark { } } - case class Case(name: String, fn: Timer => Unit, numIters: Int) + case class Case( + name: String, + fn: Timer => Unit, + numIters: Int, + prepare: () => Unit = () => { }, + cleanup: () => Unit = () => { }) case class Result(avgMs: Double, bestRate: Double, bestMs: Double) /** From 1c4ff5a42441fb107a101d47f15cf03cdc76d5ba Mon Sep 17 00:00:00 2001 From: Amogh Shetkar Date: Tue, 20 Sep 2016 16:40:35 +0530 Subject: [PATCH 025/142] [SNAPPYDATA] Spark version 2.0.1-2 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 53d3684a80bdb..fed74f727af00 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.0.1-1' + version = '2.0.1-2' ext { scalaBinaryVersion = '2.11' From 5acb3599bbce373880ff495e159be6641c204417 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 22 Sep 2016 18:38:06 +0530 Subject: [PATCH 026/142] [SNAPPYDATA] fixing antlr generated code for IDEA --- sql/catalyst/.gitignore | 1 + sql/catalyst/build.gradle | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 sql/catalyst/.gitignore diff --git a/sql/catalyst/.gitignore b/sql/catalyst/.gitignore new file mode 100644 index 0000000000000..42b6ce41f8a60 --- /dev/null +++ b/sql/catalyst/.gitignore @@ -0,0 +1 @@ +src/generated/antlr4 diff --git a/sql/catalyst/build.gradle b/sql/catalyst/build.gradle index d2deeb94b9b93..cc0e9bbf2822e 100644 --- a/sql/catalyst/build.gradle +++ b/sql/catalyst/build.gradle @@ -38,10 +38,13 @@ compileScala.dependsOn generateGrammarSource sourceSets.main.antlr.srcDirs = [ 'src/main/antlr4' ] +// use an output directory that IDEA can easily find +String antlrOut = 'src/generated/antlr4' // add generated sources to scala compiler path (plugin adds it to java path) -sourceSets.main.scala.srcDir generateGrammarSource.outputDirectory +sourceSets.main.scala.srcDir antlrOut sourceSets.main.java.srcDirs = [] generateGrammarSource { arguments += [ '-package', 'org.apache.spark.sql.catalyst.parser', '-visitor' ] + outputDirectory = file(antlrOut) } From 26adf26d012d62dcc60574e641c4ebf03d09fe57 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 18 Oct 2016 04:35:21 +0530 Subject: [PATCH 027/142] [SNAP-1083] fix numBuckets handling (#15) - don't apply numBuckets in Shuffle partitioning since Shuffle cannot create a compatible partitioning with matching numBuckets (only numPartitions) - check numBuckets too in HashPartitioning compatibility Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala --- .../plans/physical/partitioning.scala | 11 ++--- .../exchange/EnsureRequirements.scala | 45 +++++++++++++------ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index e488281de0dc1..dd12426d17bf4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -265,9 +265,8 @@ case class OrderlessHashPartitioning(expressions: Seq[Expression], numPartitions * in the same partition. Moreover while evaluating expressions if they are given in different order * than this partitioning then also it is considered equal. */ -case class OrderlessHashPartitioning(expressions: Seq[Expression], numPartitions: Int) - extends Expression with Partitioning with Unevaluable { - +case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int, + numBuckets: Int = 0) extends Expression with Partitioning with Unevaluable { override def children: Seq[Expression] = expressions override def nullable: Boolean = false @@ -291,12 +290,14 @@ case class OrderlessHashPartitioning(expressions: Seq[Expression], numPartitions } override def compatibleWith(other: Partitioning): Boolean = other match { - case p: HashPartitioning => anyOrderEquals(p) + case o: HashPartitioning => + this.numBuckets == o.numBuckets && anyOrderEquals(o) case _ => false } override def guarantees(other: Partitioning): Boolean = other match { - case o: HashPartitioning => anyOrderEquals(o) + case o: HashPartitioning => + this.numBuckets == o.numBuckets && this.anyOrderEquals(o) case _ => false } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala index e3d28388c5470..6b398640e1e56 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala @@ -179,19 +179,38 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { numPartitionsSet.headOption } - val targetNumPartitions = requiredNumPartitions.getOrElse(childrenNumPartitions.max) - - children = children.zip(requiredChildDistributions).zipWithIndex.map { - case ((child, distribution), index) if childrenIndexes.contains(index) => - if (child.outputPartitioning.numPartitions == targetNumPartitions) { - child - } else { - val defaultPartitioning = distribution.createPartitioning(targetNumPartitions) - child match { - // If child is an exchange, we replace it with a new one having defaultPartitioning. - case ShuffleExchangeExec(_, c, _) => ShuffleExchangeExec(defaultPartitioning, c) - case _ => ShuffleExchangeExec(defaultPartitioning, child) - } + children = if (useExistingPartitioning) { + // We do not need to shuffle any child's output. + children + } else { + // We need to shuffle at least one child's output. + // Now, we will determine the number of partitions that will be used by created + // partitioning schemes. + val numPartitions = { + // Let's see if we need to shuffle all child's outputs when we use + // maxChildrenNumPartitions. + val shufflesAllChildren = children.zip(requiredChildDistributions).forall { + case (child, distribution) => + !child.outputPartitioning.guarantees( + createPartitioning(distribution, maxChildrenNumPartitions)) + } + // If we need to shuffle all children, we use defaultNumPreShufflePartitions as the + // number of partitions. Otherwise, we use maxChildrenNumPartitions. + if (shufflesAllChildren) defaultNumPreShufflePartitions else maxChildrenNumPartitions + } + children.zip(requiredChildDistributions).map { + case (child, distribution) => + val targetPartitioning = createPartitioning(distribution, + numPartitions) + if (child.outputPartitioning.guarantees(targetPartitioning)) { + child + } else { + child match { + // If child is an exchange, we replace it with + // a new one having targetPartitioning. + case ShuffleExchange(_, c, _) => ShuffleExchange(targetPartitioning, c) + case _ => ShuffleExchange(targetPartitioning, child) + } } case ((child, _), _) => child From 5cacaa1fe787ab0e561d434a24b9bad7ff718de2 Mon Sep 17 00:00:00 2001 From: Amogh Shetkar Date: Thu, 20 Oct 2016 21:39:25 +0530 Subject: [PATCH 028/142] [SNAPPYDATA] Spark version 2.0.1-3 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index fed74f727af00..5f3c2fa470c40 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.0.1-2' + version = '2.0.1-3' ext { scalaBinaryVersion = '2.11' From 2142c81fd98df16ab6bc2b785d80daa189c3a5c8 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 25 Oct 2016 00:44:58 +0530 Subject: [PATCH 029/142] [SNAPPYDATA] updating snappy-spark version after the merge --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5f3c2fa470c40..e7dd3cda9e7e9 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.0.1-3' + version = '2.0.2-1' ext { scalaBinaryVersion = '2.11' From ce30bd98ec0e63f0e226e10016b67b86f3b6550d Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 24 Nov 2016 19:45:40 +0530 Subject: [PATCH 030/142] [SNAPPYDATA] Bumping version to 2.0.3-1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e7dd3cda9e7e9..402593595ce2a 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.0.2-1' + version = '2.0.3-1' ext { scalaBinaryVersion = '2.11' From 46c3807cea572540ca41262e6a7735b1be4133a5 Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Sun, 27 Nov 2016 19:54:49 +0530 Subject: [PATCH 031/142] [SNAPPYDATA] Made two methods in Executor as protected to make them customizable for SnappyExecutors. (#26) Conflicts: core/src/main/scala/org/apache/spark/executor/Executor.scala --- core/src/main/scala/org/apache/spark/executor/Executor.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index b3b107e5a49fd..4f1b35415e5af 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -126,7 +126,7 @@ private[spark] class Executor( // Create our ClassLoader // do this after SparkEnv creation so can access the SecurityManager - private val urlClassLoader = createClassLoader() + protected val urlClassLoader = createClassLoader() private val replClassLoader = addReplClassLoaderIfNeeded(urlClassLoader) // Set the classloader for serializer @@ -748,7 +748,7 @@ private[spark] class Executor( * Download any missing dependencies if we receive a new set of files and JARs from the * SparkContext. Also adds any new JARs we fetched to the class loader. */ - private def updateDependencies(newFiles: Map[String, Long], newJars: Map[String, Long]) { + protected def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) { lazy val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf) synchronized { // Fetch missing dependencies From 1c255df34050d7a33e6ede5dc7784ff2c665acbc Mon Sep 17 00:00:00 2001 From: Soubhik Chakraborty Date: Mon, 28 Nov 2016 15:53:28 +0530 Subject: [PATCH 032/142] [SNAPPYDATA]: Honoring JAVA_HOME variable while compiling java files instead of using system javac. This eliminates problem when system jdk is set differently from JAVA_HOME --- common/sketch/build.gradle | 2 +- common/unsafe/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/sketch/build.gradle b/common/sketch/build.gradle index a5e5efff08b5a..28936ece117e9 100644 --- a/common/sketch/build.gradle +++ b/common/sketch/build.gradle @@ -24,5 +24,5 @@ dependencies { tasks.withType(JavaCompile) { options.compilerArgs << '-XDignore.symbol.file' options.fork = true - options.forkOptions.executable = 'javac' + options.forkOptions.executable = "${System.properties['java.home']}/../bin/javac" } diff --git a/common/unsafe/build.gradle b/common/unsafe/build.gradle index 69d29942f5f1c..ee2347c9eb872 100644 --- a/common/unsafe/build.gradle +++ b/common/unsafe/build.gradle @@ -34,5 +34,5 @@ sourceSets.main.java.srcDirs = [ 'src/main/java' ] tasks.withType(JavaCompile) { options.compilerArgs << '-XDignore.symbol.file' options.fork = true - options.forkOptions.executable = 'javac' + options.forkOptions.executable = "${System.properties['java.home']}/../bin/javac" } From 455f3281edcf4001d85d69de1c67895445bf1b9e Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 1 Dec 2016 21:10:12 +0530 Subject: [PATCH 033/142] [SNAP-1198] Use ConcurrentHashMap instead of queue for ContextCleaner.referenceBuffer (#32) Use a map instead of queue for ContextCleaner.referenceBuffer. Profiling shows lot of time being spent removing from queue where a hash map will do (referenceQueue is already present for poll). Conflicts: core/src/main/scala/org/apache/spark/ContextCleaner.scala --- core/src/main/scala/org/apache/spark/ContextCleaner.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ContextCleaner.scala b/core/src/main/scala/org/apache/spark/ContextCleaner.scala index 4d884dec07916..04781a2945786 100644 --- a/core/src/main/scala/org/apache/spark/ContextCleaner.scala +++ b/core/src/main/scala/org/apache/spark/ContextCleaner.scala @@ -64,7 +64,7 @@ private[spark] class ContextCleaner(sc: SparkContext) extends Logging { * have not been handled by the reference queue. */ private val referenceBuffer = - Collections.newSetFromMap[CleanupTaskWeakReference](new ConcurrentHashMap) + new ConcurrentHashMap[CleanupTaskWeakReference, java.lang.Boolean]() private val referenceQueue = new ReferenceQueue[AnyRef] @@ -171,7 +171,8 @@ private[spark] class ContextCleaner(sc: SparkContext) extends Logging { /** Register an object for cleanup. */ private def registerForCleanup(objectForCleanup: AnyRef, task: CleanupTask): Unit = { - referenceBuffer.add(new CleanupTaskWeakReference(task, objectForCleanup, referenceQueue)) + referenceBuffer.put(new CleanupTaskWeakReference(task, objectForCleanup, + referenceQueue), java.lang.Boolean.TRUE) } /** Keep cleaning RDD, shuffle, and broadcast state. */ From 17b6b3b34ee55a19963c5b27642e0494f4cd4628 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 3 Dec 2016 17:35:11 +0530 Subject: [PATCH 034/142] [SNAP-1194] explicit addLong/longValue methods in SQLMetrics (#33) This avoids runtime erasure for add/value methods that will result in unnecessary boxing/unboxing overheads. - Adding spark-kafka-sql project - Update version of deps as per upstream. - corrected kafka-clients reference Conflicts: sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala --- assembly/build.gradle | 1 + core/build.gradle | 2 +- external/kafka-0-10-sql/build.gradle | 33 +++++++++++++++++++ external/kafka-0-10/build.gradle | 3 +- settings.gradle | 2 ++ .../execution/basicPhysicalOperators.scala | 6 ++-- .../sql/execution/metric/SQLMetrics.scala | 5 +++ 7 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 external/kafka-0-10-sql/build.gradle diff --git a/assembly/build.gradle b/assembly/build.gradle index 63db32e3e41f6..0d81f789eb05b 100644 --- a/assembly/build.gradle +++ b/assembly/build.gradle @@ -27,6 +27,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-sql-kafka-0.10_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-yarn_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-mllib_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) diff --git a/core/build.gradle b/core/build.gradle index 9395a129dac33..1caee72201e40 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -124,7 +124,7 @@ dependencies { } compile group: 'org.apache.ivy', name: 'ivy', version: '2.4.0' compile group: 'oro', name: 'oro', version: '2.0.8' - compile(group: 'net.razorvine', name: 'pyrolite', version: '4.9') { + compile(group: 'net.razorvine', name: 'pyrolite', version: '4.13') { exclude(group: 'net.razorvine', module: 'serpent') } compile group: 'net.sf.py4j', name: 'py4j', version: '0.10.1' diff --git a/external/kafka-0-10-sql/build.gradle b/external/kafka-0-10-sql/build.gradle new file mode 100644 index 0000000000000..45108783e735d --- /dev/null +++ b/external/kafka-0-10-sql/build.gradle @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Kafka 0.10 Source for Structured Streaming' + +dependencies { + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + provided project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + provided project(subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion) + provided project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) + + compile group: 'org.apache.kafka', name: 'kafka-clients', version: '0.10.0.1' + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.1' + testCompile group: 'net.sf.jopt-simple', name: 'jopt-simple', version: '3.2' +} diff --git a/external/kafka-0-10/build.gradle b/external/kafka-0-10/build.gradle index d5dc6611a97f0..9b6db7851f1d4 100644 --- a/external/kafka-0-10/build.gradle +++ b/external/kafka-0-10/build.gradle @@ -19,7 +19,8 @@ description = 'Spark Integration for Kafka 0.10' dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) + provided project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + provided project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) compile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.1') { exclude(group: 'com.sun.jmx', module: 'jmxri') diff --git a/settings.gradle b/settings.gradle index ca33d18d94bf5..7150a225b3de3 100644 --- a/settings.gradle +++ b/settings.gradle @@ -40,6 +40,7 @@ include ':snappy-spark-streaming-flume_' + scalaBinaryVersion include ':snappy-spark-streaming-flume-sink_' + scalaBinaryVersion include ':snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion include ':snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion +include ':snappy-spark-sql-kafka-0.10_' + scalaBinaryVersion include ':snappy-spark-examples_' + scalaBinaryVersion include ':snappy-spark-repl_' + scalaBinaryVersion include ':snappy-spark-launcher_' + scalaBinaryVersion @@ -67,6 +68,7 @@ project(':snappy-spark-streaming-flume_' + scalaBinaryVersion).projectDir = "$ro project(':snappy-spark-streaming-flume-sink_' + scalaBinaryVersion).projectDir = "$rootDir/external/flume-sink" as File project(':snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion).projectDir = "$rootDir/external/kafka-0-8" as File project(':snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion).projectDir = "$rootDir/external/kafka-0-10" as File +project(':snappy-spark-sql-kafka-0.10_' + scalaBinaryVersion).projectDir = "$rootDir/external/kafka-0-10-sql" as File project(':snappy-spark-examples_' + scalaBinaryVersion).projectDir = "$rootDir/examples" as File project(':snappy-spark-repl_' + scalaBinaryVersion).projectDir = "$rootDir/repl" as File project(':snappy-spark-launcher_' + scalaBinaryVersion).projectDir = "$rootDir/launcher" as File diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala index a15a8d11aa2a0..ece8bcd9681d1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala @@ -202,7 +202,7 @@ case class FilterExec(condition: Expression, child: SparkPlan) |do { | $generated | $nullChecks - | $numOutput.add(1); + | $numOutput.addLong(1); | ${consume(ctx, resultVars)} |} while(false); """.stripMargin @@ -308,7 +308,7 @@ case class SampleExec( s""" | int $samplingCount = $sampler.sample(); | while ($samplingCount-- > 0) { - | $numOutput.add(1); + | $numOutput.addLong(1); | ${consume(ctx, input)} | } """.stripMargin.trim @@ -322,7 +322,7 @@ case class SampleExec( s""" | if ($sampler.sample() != 0) { - | $numOutput.add(1); + | $numOutput.addLong(1); | ${consume(ctx, input)} | } """.stripMargin.trim diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala index d88763037b3bf..67bb66da5f38b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala @@ -64,6 +64,11 @@ final class SQLMetric(val metricType: String, initValue: Long = 0L) def set(v: Long): Unit = _value = v + // avoid the runtime generic Object conversion of add(), value() + final def addLong(v: Long): Unit = _value += v + + final def longValue: Long = _value + def +=(v: Long): Unit = _value += v override def value: Long = _value From 98b2f85eb5523cde9236ce13bffd50e5d0d71b3b Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 24 Nov 2016 14:44:51 +0530 Subject: [PATCH 035/142] [SNAPPYDATA] More optimizations to UTF8String - allow direct UTF8String objects in RDD data conversions to DataFrame; new UTF8String.cloneIfRequired to clone only if required used by above - allow for some precision change in QueryTest result comparison --- .../apache/spark/unsafe/types/UTF8String.java | 30 +++++++++++++++---- .../sql/catalyst/JavaTypeInference.scala | 4 +++ .../spark/sql/catalyst/ScalaReflection.scala | 10 +++++++ .../org/apache/spark/sql/QueryTest.scala | 1 + 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index b0d0c44823e68..712129391cd80 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -1183,7 +1183,27 @@ public String toString() { @Override public UTF8String clone() { - return fromBytes(getBytes()); + UTF8String newString = fromBytes(getBytes()); + if (isAscii) { + newString.isAscii = true; + } + return newString; + } + + public UTF8String cloneIfRequired() { + if (offset == BYTE_ARRAY_OFFSET && + ((byte[])base).length == numBytes) { + return this; + } else { + final int numBytes = this.numBytes; + final byte[] bytes = new byte[numBytes]; + copyMemory(base, offset, bytes, BYTE_ARRAY_OFFSET, numBytes); + UTF8String newString = fromAddress(bytes, BYTE_ARRAY_OFFSET, numBytes); + if (isAscii) { + newString.isAscii = true; + } + return newString; + } } public UTF8String copy() { @@ -1194,6 +1214,10 @@ public UTF8String copy() { @Override public int compareTo(@Nonnull final UTF8String other) { + return compare(other); + } + + public int compare(final UTF8String other) { int len = Math.min(numBytes, other.numBytes); int wordMax = (len / 8) * 8; long roffset = other.offset; @@ -1219,10 +1243,6 @@ public int compareTo(@Nonnull final UTF8String other) { return numBytes - other.numBytes; } - public int compare(final UTF8String other) { - return compareTo(other); - } - @Override public boolean equals(final Object other) { if (other instanceof UTF8String) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala index 3ecc137c8cd7f..a43381c05ad7e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala @@ -240,6 +240,8 @@ object JavaTypeInference { case c if c == classOf[java.lang.String] => Invoke(getPath, "toString", ObjectType(classOf[String])) + case c if c == classOf[UTF8String] => + Invoke(getPath, "toString", ObjectType(classOf[String])) case c if c == classOf[java.math.BigDecimal] => Invoke(getPath, "toJavaBigDecimal", ObjectType(classOf[java.math.BigDecimal])) @@ -421,6 +423,8 @@ object JavaTypeInference { Invoke(inputObject, "floatValue", FloatType) case c if c == classOf[java.lang.Double] => Invoke(inputObject, "doubleValue", DoubleType) + case c if c == classOf[UTF8String] => + Invoke(inputObject, "cloneIfRequired", StringType) case _ if typeToken.isArray => toCatalystArray(inputObject, typeToken.getComponentType) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala index 9a4bf0075a178..cd34a18754377 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala @@ -263,6 +263,9 @@ object ScalaReflection extends ScalaReflection { case t if t <:< localTypeOf[java.lang.String] => Invoke(getPath, "toString", ObjectType(classOf[String]), returnNullable = false) + case t if t <:< localTypeOf[UTF8String] => + Invoke(getPath, "cloneIfRequired", ObjectType(classOf[UTF8String])) + case t if t <:< localTypeOf[java.math.BigDecimal] => Invoke(getPath, "toJavaBigDecimal", ObjectType(classOf[java.math.BigDecimal]), returnNullable = false) @@ -534,6 +537,12 @@ object ScalaReflection extends ScalaReflection { inputObject :: Nil, returnNullable = false) + case t if t <:< localTypeOf[UTF8String] => + Invoke( + inputObject, + "cloneIfRequired", + StringType) + case t if t <:< localTypeOf[java.sql.Timestamp] => StaticInvoke( DateTimeUtils.getClass, @@ -745,6 +754,7 @@ object ScalaReflection extends ScalaReflection { val Schema(dataType, nullable) = schemaFor(elementType) Schema(ArrayType(dataType, containsNull = nullable), nullable = true) case t if t <:< localTypeOf[String] => Schema(StringType, nullable = true) + case t if t <:< localTypeOf[UTF8String] => Schema(StringType, nullable = true) case t if t <:< localTypeOf[java.sql.Timestamp] => Schema(TimestampType, nullable = true) case t if t <:< localTypeOf[java.sql.Date] => Schema(DateType, nullable = true) case t if t <:< localTypeOf[BigDecimal] => Schema(DecimalType.SYSTEM_DEFAULT, nullable = true) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala index 9fb8be423614b..73949c682780b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala @@ -290,6 +290,7 @@ object QueryTest { Row.fromSeq(row.toSeq.map { case null => null case d: java.math.BigDecimal => BigDecimal(d) + case d: Double => math.floor(d * 1000.0 + 0.5) // round to three digits // Convert array to Seq for easy equality check. case b: Array[_] => b.toSeq case r: Row => prepareRow(r) From 9a566cc796dc91f21c7a0957db0de62e590a50c2 Mon Sep 17 00:00:00 2001 From: Hemant Bhanawat Date: Fri, 9 Dec 2016 15:41:03 +0530 Subject: [PATCH 036/142] [SNAPPYDATA] Adding fixed stats to common filter expressions Missing filter statistics in filter's logical plan is causing incorrect plan selection at times. Also, join statistics always return sizeInBytes as the product of its child sizeInBytes which result in a big number. For join, product makes sense only when it is a cartesian product join. Hence, fixed the spark code to check for the join type. If the join is a equi-join, we now sum the sizeInBytes of the child instead of doing a product. For missing filter statistics, adding a heuristics based sizeInBytes calculation mentioned below. If the filtering condition is: - equal to: sizeInBytes is 5% of the child sizeInBytes - greater than less than: sizeInBytes is 50% of the child sizeInBytes - isNull: sizeInBytes is 50% of the child sizeInBytes - starts with: sizeInBytes is 10% of the child sizeInBytes Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala --- .../plans/logical/basicLogicalOperators.scala | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index a4fca790dd086..253a5d5e81223 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression +import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, RangePartitioning, RoundRobinPartitioning} @@ -130,6 +131,19 @@ case class Filter(condition: Expression, child: LogicalPlan) override def maxRows: Option[Long] = child.maxRows + override lazy val statistics: Statistics = { + // Expected filtering by expressions + val expectedFilterDivisor = condition match { + case EqualTo(_, _) => 20 + case StartsWith(_, _) => 10 + case LessThan(_, _) | LessThanOrEqual(_, _) | + GreaterThan(_, _) | GreaterThanOrEqual(_, _) => 2 + case IsNull(_) => 2 + case _ => 1 + } + child.statistics.copy(sizeInBytes = child.statistics.sizeInBytes / expectedFilterDivisor) + } + override protected def validConstraints: Set[Expression] = { val predicates = splitConjunctivePredicates(condition) .filterNot(SubqueryExpression.hasCorrelatedSubquery) @@ -341,6 +355,18 @@ case class Join( case UsingJoin(_, _) => false case _ => resolvedExceptNatural } + + override lazy val statistics: Statistics = joinType match { + case LeftAnti | LeftSemi => + // LeftSemi and LeftAnti won't ever be bigger than left + left.statistics.copy() + case _ if ExtractEquiJoinKeys.unapply(this).isDefined => + Statistics(sizeInBytes = children.map(_.statistics.sizeInBytes).sum) + case _ => + // make sure we don't propagate isBroadcastable in other joins, because + // they could explode the size. + super.statistics.copy(isBroadcastable = false) + } } /** From f27ef92a5b24ebf939200a217c6669e322a575fb Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 10 Dec 2016 00:55:19 +0530 Subject: [PATCH 037/142] [SNAPPYDATA] adding kryo serialization missing in LongHashedRelation Conflicts: sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala --- .../spark/sql/execution/joins/HashedRelation.scala | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 9af1712618dca..678262ada8bee 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -789,7 +789,8 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap private[joins] class LongHashedRelation( private var nFields: Int, - private var map: LongToUnsafeRowMap) extends HashedRelation with Externalizable { + private var map: LongToUnsafeRowMap) extends HashedRelation + with Externalizable with KryoSerializable { private var resultRow: UnsafeRow = new UnsafeRow(nFields) @@ -831,6 +832,11 @@ private[joins] class LongHashedRelation( out.writeObject(map) } + override def write(kryo: Kryo, output: Output): Unit = { + output.writeInt(nFields) + kryo.writeClassAndObject(output, map) + } + override def readExternal(in: ObjectInput): Unit = { nFields = in.readInt() resultRow = new UnsafeRow(nFields) @@ -838,6 +844,12 @@ private[joins] class LongHashedRelation( } override def getAverageProbesPerLookup: Double = map.getAverageProbesPerLookup + + override def read(kryo: Kryo, input: Input): Unit = { + nFields = input.readInt() + resultRow = new UnsafeRow(nFields) + map = kryo.readClassAndObject(input).asInstanceOf[LongToUnsafeRowMap] + } } /** From 6951cda7449b9f89f0032ebe9f990131078d5466 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 10 Dec 2016 21:58:00 +0530 Subject: [PATCH 038/142] [SNAPPYDATA] Correcting HashPartitioning interface to match apache spark Addition of numBuckets as default parameter made HashPartitioning incompatible with upstream apache spark. Now adding it separately so restore compatibility. Conflicts: core/src/main/scala/org/apache/spark/Partitioner.scala sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala --- .../scala/org/apache/spark/Partitioner.scala | 6 ++---- .../catalyst/plans/physical/partitioning.scala | 16 ++++++++++++++-- .../execution/exchange/ShuffleExchangeExec.scala | 1 + 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/Partitioner.scala b/core/src/main/scala/org/apache/spark/Partitioner.scala index bdfecd08f27c8..7d0e7591a1954 100644 --- a/core/src/main/scala/org/apache/spark/Partitioner.scala +++ b/core/src/main/scala/org/apache/spark/Partitioner.scala @@ -106,14 +106,12 @@ object Partitioner { * so attempting to partition an RDD[Array[_]] or RDD[(Array[_], _)] using a HashPartitioner will * produce an unexpected or incorrect result. */ -class HashPartitioner(partitions: Int) extends Partitioner { +class HashPartitioner(partitions: Int, buckets: Int) extends Partitioner { require(partitions >= 0, s"Number of partitions ($partitions) cannot be negative.") require(buckets >= 0, s"Number of buckets ($buckets) cannot be negative.") - def this(partitions: Int) { - this(partitions, 0) - } + def this(partitions: Int) = this(partitions, 0) def numPartitions: Int = partitions diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index dd12426d17bf4..08841c2419c35 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -265,8 +265,10 @@ case class OrderlessHashPartitioning(expressions: Seq[Expression], numPartitions * in the same partition. Moreover while evaluating expressions if they are given in different order * than this partitioning then also it is considered equal. */ -case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int, - numBuckets: Int = 0) extends Expression with Partitioning with Unevaluable { +case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int) + extends Expression with Partitioning with Unevaluable { + + private[sql] var numBuckets: Int = 0 override def children: Seq[Expression] = expressions override def nullable: Boolean = false @@ -337,6 +339,16 @@ case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int) def partitionIdExpression: Expression = Pmod(new Murmur3Hash(expressions), Literal(numPartitions)) } +object HashPartitioning { + + def apply(expressions: Seq[Expression], numPartitions: Int, + numBuckets: Int): HashPartitioning = { + val partitioning = HashPartitioning(expressions, numPartitions) + partitioning.numBuckets = numBuckets + partitioning + } +} + /** * Represents a partitioning where rows are split across partitions based on some total ordering of * the expressions specified in `ordering`. When data is partitioned in this manner the following diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala index 4d95ee34f30de..297f46142f0cb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala @@ -214,6 +214,7 @@ object ShuffleExchangeExec { // `HashPartitioning.partitionIdExpression` to produce partitioning key. override def getPartition(key: Any): Int = key.asInstanceOf[Int] } + // case p@HashPartitioning(_, n) => new HashPartitioner(n, p.numBuckets) case RangePartitioning(sortingExpressions, numPartitions) => // Internally, RangePartitioner runs a job on the RDD that samples keys to compute // partition bounds. To get accurate samples, we need to copy the mutable keys. From b0400cd545c014f0e6b250a2a9be44b08c6c4abe Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sun, 11 Dec 2016 16:32:33 +0530 Subject: [PATCH 039/142] [SNAP-1233] clear InMemorySorter before calling its reset (#35) This is done so that any spill call (due to no EVICTION_DOWN) from within the spill call will return without doing anything, else it results in NPE trying to read page tables which have already been cleared. --- .../collection/unsafe/sort/UnsafeExternalSorter.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java index 66118f454159b..4d682dccff9c6 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java @@ -226,7 +226,17 @@ public long spill(long size, MemoryConsumer trigger) throws IOException { // Note that this is more-or-less going to be a multiple of the page size, so wasted space in // pages will currently be counted as memory spilled even though that space isn't actually // written to disk. This also counts the space needed to store the sorter's pointer array. - inMemSorter.reset(); + + // temporarily clear inMemorySorter so that a recursive spill call will return + final UnsafeInMemorySorter memSorter = inMemSorter; + if (memSorter != null) { + inMemSorter = null; + try { + memSorter.reset(); + } finally { + inMemSorter = memSorter; + } + } // Reset the in-memory sorter's pointer array only after freeing up the memory pages holding the // records. Otherwise, if the task is over allocated memory, then without freeing the memory // pages, we might not be able to get memory for the pointer array. From 1f46a571fbee08493a04bd70c900831bc404ff53 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Mon, 12 Dec 2016 05:37:07 +0530 Subject: [PATCH 040/142] [SNAPPYDATA] Adding more filter conditions for plan sizing as followup - IN is 50% of original - StartsWith, EndsWith 10% - Contains and LIKE at 20% - AND is multiplication of sizing of left and right (with max filtering of 5%) - OR is 1/x+1/y sizing of the left and right (with min filtering of 50%) - NOT three times of that without NOT --- .../plans/logical/basicLogicalOperators.scala | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index 253a5d5e81223..652405c955b3f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -132,16 +132,27 @@ case class Filter(condition: Expression, child: LogicalPlan) override def maxRows: Option[Long] = child.maxRows override lazy val statistics: Statistics = { - // Expected filtering by expressions - val expectedFilterDivisor = condition match { + // Expected filtering by expressions based on some constants for now. + def expectedFilterDivisor(cond: Expression): Int = cond match { case EqualTo(_, _) => 20 - case StartsWith(_, _) => 10 case LessThan(_, _) | LessThanOrEqual(_, _) | GreaterThan(_, _) | GreaterThanOrEqual(_, _) => 2 - case IsNull(_) => 2 + case In(_, _) => 2 + case StartsWith(_, _) | EndsWith(_, _) => 10 + case Contains(_, _) | Like(_, _) => 5 + case And(left, right) => + math.min(20, expectedFilterDivisor(left) * expectedFilterDivisor(right)) + case Or(left, right) => + val leftDivisor = expectedFilterDivisor(left) + val rightDivisor = expectedFilterDivisor(right) + math.max(2, (leftDivisor * rightDivisor) / (leftDivisor + rightDivisor)) + case Not(e) => math.max(2, expectedFilterDivisor(e) / 3) + case IsNull(_) => 3 case _ => 1 } - child.statistics.copy(sizeInBytes = child.statistics.sizeInBytes / expectedFilterDivisor) + + child.statistics.copy(sizeInBytes = child.statistics.sizeInBytes / + expectedFilterDivisor(condition)) } override protected def validConstraints: Set[Expression] = { From 66d9d42b36fed73f2ee9085c466ab0dc91162275 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 13 Dec 2016 23:02:01 +0530 Subject: [PATCH 041/142] [SNAPPYDATA] reduced factors in filters a bit to be more conservative --- .../catalyst/plans/logical/basicLogicalOperators.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index 652405c955b3f..5e41a0a6edc5f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -134,19 +134,19 @@ case class Filter(condition: Expression, child: LogicalPlan) override lazy val statistics: Statistics = { // Expected filtering by expressions based on some constants for now. def expectedFilterDivisor(cond: Expression): Int = cond match { - case EqualTo(_, _) => 20 + case EqualTo(_, _) => 10 case LessThan(_, _) | LessThanOrEqual(_, _) | GreaterThan(_, _) | GreaterThanOrEqual(_, _) => 2 case In(_, _) => 2 - case StartsWith(_, _) | EndsWith(_, _) => 10 - case Contains(_, _) | Like(_, _) => 5 + case StartsWith(_, _) | EndsWith(_, _) => 5 + case Contains(_, _) | Like(_, _) => 3 case And(left, right) => math.min(20, expectedFilterDivisor(left) * expectedFilterDivisor(right)) case Or(left, right) => val leftDivisor = expectedFilterDivisor(left) val rightDivisor = expectedFilterDivisor(right) math.max(2, (leftDivisor * rightDivisor) / (leftDivisor + rightDivisor)) - case Not(e) => math.max(2, expectedFilterDivisor(e) / 3) + case Not(e) => math.max(2, expectedFilterDivisor(e) / 5) case IsNull(_) => 3 case _ => 1 } From ba84de4e2143711733481feae6106127ca4218f0 Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Wed, 14 Dec 2016 10:32:41 +0530 Subject: [PATCH 042/142] [SNAP-1240] Snappy monitoring dashboard (#36) * UI HTML, CSS and resources changes * Adding new health status images * Adding SnappyData Logo. * Code changes for stting/updating Spark UI tabs list. * Adding icon images for Running, Stopped and Warning statuses. * 1. Adding New method for generating Spark UI page without page header text. 2. Updating CSS: Cluster Normal status text color is changed to match color of Normal health logo. * Suggestion: Rename Storage Tab to Spark Cache. * Resolving Precheckin failure due to scala style comments :snappy-spark:snappy-spark-core_2.11:scalaStyle SparkUI.scala message=Insert a space after the start of the comment line=75 column=4 UIUtils.scala message=Insert a space after the start of the comment line=267 column=4 Conflicts: core/src/main/scala/org/apache/spark/ui/SparkUI.scala core/src/main/scala/org/apache/spark/ui/UIUtils.scala core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala --- .../spark/ui/static/snappy-dashboard.css | 119 ++++++++++++++++++ .../snappydata/SnappyData-Logo-230X50.png | Bin 0 -> 5208 bytes .../snappydata/cluster-status-error-16x23.png | Bin 0 -> 597 bytes .../snappydata/cluster-status-error-62x90.png | Bin 0 -> 1639 bytes .../cluster-status-normal-16x23.png | Bin 0 -> 787 bytes .../cluster-status-normal-62x90.png | Bin 0 -> 4656 bytes .../cluster-status-warning-16x23.png | Bin 0 -> 611 bytes .../cluster-status-warning-62x90.png | Bin 0 -> 1749 bytes .../static/snappydata/error-status-20x19.png | Bin 0 -> 795 bytes .../static/snappydata/error-status-35x34.png | Bin 0 -> 1346 bytes .../static/snappydata/error-status-70x68.png | Bin 0 -> 4842 bytes .../static/snappydata/info-status-20x19.png | Bin 0 -> 847 bytes .../static/snappydata/info-status-35x34.png | Bin 0 -> 1419 bytes .../static/snappydata/info-status-70x68.png | Bin 0 -> 5154 bytes .../static/snappydata/normal-status-20x19.png | Bin 0 -> 612 bytes .../static/snappydata/normal-status-35x34.png | Bin 0 -> 1098 bytes .../static/snappydata/normal-status-70x68.png | Bin 0 -> 3892 bytes .../snappydata/running-status-icon-20x19.png | Bin 0 -> 830 bytes .../snappydata/running-status-icon-35x34.png | Bin 0 -> 1955 bytes .../snappydata/running-status-icon-70x68.png | Bin 0 -> 4683 bytes .../static/snappydata/severe-status-20x19.png | Bin 0 -> 616 bytes .../static/snappydata/severe-status-35x34.png | Bin 0 -> 1105 bytes .../static/snappydata/severe-status-70x68.png | Bin 0 -> 3883 bytes .../ui/static/snappydata/status-20x19.png | Bin 0 -> 866 bytes .../ui/static/snappydata/status-35x34.png | Bin 0 -> 2218 bytes .../ui/static/snappydata/status-70x68.png | Bin 0 -> 5709 bytes .../snappydata/stopped-status-icon-20x19.png | Bin 0 -> 796 bytes .../snappydata/stopped-status-icon-35x34.png | Bin 0 -> 1227 bytes .../snappydata/stopped-status-icon-70x68.png | Bin 0 -> 4790 bytes .../snappydata/warning-status-20x19.png | Bin 0 -> 805 bytes .../snappydata/warning-status-35x34.png | Bin 0 -> 1362 bytes .../snappydata/warning-status-70x68.png | Bin 0 -> 4938 bytes .../snappydata/warning-status-icon-20x19.png | Bin 0 -> 665 bytes .../snappydata/warning-status-icon-35x34.png | Bin 0 -> 930 bytes .../snappydata/warning-status-icon-70x68.png | Bin 0 -> 3533 bytes .../scala/org/apache/spark/ui/SparkUI.scala | 9 +- .../scala/org/apache/spark/ui/UIUtils.scala | 51 +++++++- .../scala/org/apache/spark/ui/WebUI.scala | 2 +- .../apache/spark/ui/storage/StoragePage.scala | 2 +- .../apache/spark/ui/storage/StorageTab.scala | 50 +++++++- 40 files changed, 224 insertions(+), 9 deletions(-) create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/SnappyData-Logo-230X50.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-error-16x23.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-error-62x90.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-normal-16x23.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-normal-62x90.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-warning-16x23.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-warning-62x90.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/error-status-20x19.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/error-status-35x34.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/error-status-70x68.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/info-status-20x19.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/info-status-35x34.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/info-status-70x68.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/normal-status-20x19.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/normal-status-35x34.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/normal-status-70x68.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/running-status-icon-20x19.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/running-status-icon-35x34.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/running-status-icon-70x68.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/severe-status-20x19.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/severe-status-35x34.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/severe-status-70x68.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/status-20x19.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/status-35x34.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/status-70x68.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/stopped-status-icon-20x19.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/stopped-status-icon-35x34.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/stopped-status-icon-70x68.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-20x19.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-35x34.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-70x68.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-icon-20x19.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-icon-35x34.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-icon-70x68.png diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css b/core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css new file mode 100644 index 0000000000000..1ddd1b5263e45 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css @@ -0,0 +1,119 @@ +/* + ========================================================================== + SnappyData Custom Styles + ========================================================================== +*/ + + +.keyStates{ + float: left; + padding: 5px; + margin: 5px 10px; + border: 1px solid #DCDCDC; + box-shadow: 0px 1px 10px rgba(0, 0, 0, 0.4); + background: white none repeat scroll 0% 0%; + overflow: visible; + width: 100%; + max-width: 223px; + min-height: 100px; +} + +.keyStatesLeft{ + float: left; + padding: 5px; + margin: 5px 10px; + border: 1px solid #DCDCDC; + box-shadow: 0px 1px 10px rgba(0, 0, 0, 0.4); + background: white none repeat scroll 0% 0%; + overflow: visible; + width: 100%; + max-width: 223px; + min-height: 100px; +} + +.keyStatesRight{ + float: left; + padding: 5px; + margin: 5px 10px; + border: 1px solid #DCDCDC; + box-shadow: 0px 1px 10px rgba(0, 0, 0, 0.4); + background: white none repeat scroll 0% 0%; + overflow: visible; + width: 100%; + max-width: 223px; + min-height: 100px; +} + +.keyStatsValue{ + padding-bottom: 10px; + font-weight: bolder; + vertical-align: middle; + text-align: center; + padding-top: 25px; + font-size: 24px; +} + +.keyStatesText{ + font-weight: bolder; + min-height: 25px; + text-align: center; + padding: 10px; +} + +.clusterHealthImageBox{ + float: left; + width: 94px; + border-right: thin inset; + height: 100px; +} + +.clusterHealthTextBox{ + text-align: center; + float: left; + width: 200px;} + +.statusTextNormal{ + color: #87B025; +} +.statusTextWarning{ + color: #FDB406; +} +.statusTextError{ + color: #FD063A; +} + +.divClass2{ +} +.div-width-100{ + width: 100px; +} +.div-width-200{ + width: 200px; +} +.div-width-300{ + width: 300px; +} + +.progressBar{ + height: 19px; + width: 100%; + border-radius: 5px; + border: thin solid #3EC0FF; + background: #A0DFFF none repeat scroll 0 0; +} +.completedProgress{ + float: left; + border-radius: inherit; + background: #3EC0FF none repeat scroll 0px 0px; +} +/* +.remainingProgress{ + float: left; + border-radius: inherit; + background: #A0DFFF none repeat scroll 0px 0px; +}*/ +.progressValue{ + float:right; + width:20%; + text-align:center; +} \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/SnappyData-Logo-230X50.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/SnappyData-Logo-230X50.png new file mode 100644 index 0000000000000000000000000000000000000000..f4520e17965f3e22cb4c0abe60f20d6e24f17bac GIT binary patch literal 5208 zcmai2XIK+kv>l2N0RcfJND&YOLJ=f}9_cOg-VqQ9z1IY!NbjhCAf14M5u(BcX;Opq zCOshi(tDNia^KJQ=k0IKob#O@=bKr3_FijEoUXR&Evoxe007)lS5wj_&MU<61Vl=_ zsIgk~a*T+Ol#qHN@Y~lXN`!DU2A2jnmMR zzi~(K`AwxpP!XxWsD>dFW+&(AN6Eanxhh&xEa^X>A_@M1wq zT2=?i|GS=lMP*&_)Z1Buy_vzJn}?vxmMiob#bV}J+&TZ4eAMf_`tQ)d+Jb`Qrw9N5 z0c0Gnxtdy}OqxkvX3R`@ouc&oQP$4EW}I{*3zDhOYTK^YDN>VOZTR_}g&y}ltvX0( zk4bu|5mZ@Bvsf&AKF8zo)bco+1y1`a)5`xMiomT!0wAN=hx$VGb(p9Ne=UwL;}gDK zA6W1dd5fo=h0ll2fV@vP+A68<-hjlvrY8ktu;-zYu2jJ_u&pKAEpVW1er*#4pgZ}~ ziez#3q+qkL+zwy~83Rz_p!xF+Ua|J5o^w>+zl*TW&NaDE3W~xH`pdtKByxO*Y$o4j zDh~x^&URt$DAI+@!JI!n6DMK40&HrGImrN0;)CH}+kCu6pE`oT?fsY8)Iimfpp=)$ zw+ey3bt{@nWBisab}mW^)E+j4-{cHDEJb&B$htYZ9Ah`z+uIp=wU^*smvf2z2de{% zi;D;ijVu0)W4b~w(WdzAA2;%4g1+(w4`4SfXr7DE zkO8`PqHIkots5H}8dmyp%d92&`JGPk2!7Rm+Wp!$_imcoWZ9%PdoOFmDLoPQbl26@ zMFa8X<+(&?c+;;{$=UHL56u69hSF zcz&ga=>)}5Yf({=WVnx7I=A}p?CjO)34WR}6k8Vpx%QJJ2LO=>S2RUBaw?Ej z%ZU?(bCz?yrM)?(tS4IU$=RL*fkV0#1?oo%*|zr%*(5uH3us#w78b(701Bi@iwF(H zd=iWJWIDe=Swr9+X~=J!J^rl^e;)>oM$e8mgwtzqr(eXJna^38nXNq|*kJT=@n1b& zx?qE@F3ys2VE4h`?-K#hw?X74&R-(sc#;^N|5Lo!~_x0aWe69Iz0THO$@)RX&UkHdV>(Sy8O&oNLaI+m@s zHmA$UfGX?lKjM3*aTp3J)O?g9;bb?hw6q_`$V&?R@Lg-I8PVmltVP-$z?4)~SC^Ly z9Y4g-u}bLb2!W%8Y;ZDMb_vP+633Uge|%f2c$Jlpj*eX8Fc?f64yRD43nS6GRgo!V z!gnXbYECj7!p|RMU{GwGj{e~{SA4f8SzfSge>~@L-jlEvj&{yyMNLBOx5$0RKQnbc z`k&qotcuatySkzm{w^AijR9d77Z-&3W`Xl%CX9ZZt!WCo?Dvx-e7)yhQT|+@iOG0? zAi^NW4+<9+rj_KRX)AF4_3PI+_O@#RRPBvJ=qpa{%7_QrLL|%QmTbw;a+-{ zs1wTzBGr5G9gnoIm=5^-KKmZX_NZc=SS7>5`gNAHoKPrKCNtg6V@HJ8JyseKT3Xoh z@+VNJ;oP1650YZ2W7isPfsOODhpKAvIwyaYMELV$arvlRhyKQbQPS!Dp|y3MDdzLJ^eEG@2YHXn^!|g3HQUr^{%vQ zXZ83}CrVVz+DO?qp2EVyG`%!RI(BYQ!~n0bRYUEP)wo%OnxUcLOiQgw8iFZP*la

4!BciR`lG+zd`#o$Z|*<3&EJ{qG(NkdV{2Mp7=0@>2s@(kDVpcQur) zt=W?oxz%Z}k+sdQ5`qdJgr0_TfJPfyTNP;VuN7>O$bWym;ho+AUsUUUGDO}~0Qo#X zo_mgsjS&+z(qiH-VD;^#R?cCbhLnRGn4=i|=O;;a5dbI@diG?12odYB+wVV$90WGR3MG}I7fX{|cNnYlbMs6nwqwD zKL(MhTd2epH-jT<^SYdVi9~|g8Y!_!GGVXL~yr2J()9d1Nni zadJpg!`Z@pUw$^SyE}6eGOWsd67HY24I_}fq~J5s_&zk-+r-S;K;yDMNu)+`WGccU zXz@u`M{8@uEh_%_V0@Zveta+igJ9z2Z+>)8e(^<# zK0&6=e1bdirj$(Z{QMAgO7P!3=`(Ij6W%FlgopdnJ@JZ5rKiVc)k>emL7A4 z629)95tpqWk4^1=Y)o5Wk8!oNr~dtJAWp53NQxha+XNQi&)Qnt(;K-`8B}c|_e(U@ z)IjD9;IoK3%13L2vNPv#dUZ`r8Y+FpwS(lYBIN{h=Xn9UEjU`du8OT!`m=d;Ioqhb zllj=qPai)%Z#HN9F{S$)1R#;fi6%Gc^39En1fMc5gH=2P=JKKvs74 z`Pq5l>l97l1Om%n6hA`rP~UMNrJl|owiO(BND7Z81Jtr>U_p4jDb}zR@L+{{?rrlwI7o09XtS;@$+`f(-UtkVX?JS&#Ea8n5eQA!)b|ezsv7m6Y1}6#=Q{ncm@$}!#_xV4mHmS< z>NBN++Kx5^W1}ouR)n3PdbWF46|0P>6LN z`qt>^Xt(Kq)&`GicSVa5>gs~zI_=OCnly&D%`8%@ggH5!+t`gbKBe7O$##= zU$X8UGK&X@8#{+kBh++sDoa}+fVV@wR)0~NSvn~1-^Kan$&{K@QKFc=?RHGQetWjvbh!K`+7IyCC^T+wX1 zv3u3R@(~#zeD=j%hK~zlWBCl%>4)|x)}Euy(fg9_yUaocGzrx9w(Vh#*|i^8h2~Xt z((ePnz>CU=P&gF_he%%DTZj4O|01e3Co5|=zf4E(?45v#ukJmdDqwEb?Ay!2#(DgM zSM3(DdfItXJZWn5PdC=rVWD+;mj#i1THP8e{{9$AHEU_tOUR-p_4GzuX2ygp-)md2)viyw(Jh$eW^R+T( z8&TrY(iXBFvC$v})5D%O{(K=*P%jzw$5K+}1Uq{2mf+Q!H}|&61cR6RBM_MgDh?o= zIVL&z;_dRrf`IDM+b7%yrn_N7_Ha)*Gy#3RiOn1UQ^-_?+thosrTa|Y{?U?O6CqhK#oq+Lu zsHDQ@ijaAQsh-TolRWW@fr!XQX`{TnJX%K5!+0_+wJj#{$Ggl z5yFG=Kc$@`Cj}MCaez`Lyq9*5mfF2lTYBy9_bPT!?l@*7OP~T{;rll~g`HvEdarc&%mdzWhN{OtJ3X|NJPXbSGo#APod%LezF%JeTL~o2|R+_pak$iWk?CtHLd=`41{t=?Q68INEeWx=bhE z+T1w)iw-vruU1YulS{zXRMhr~?w#uAR}mRwc3dO)fY2`77MaFI=v`5W@MIrtNLUJBr72kJK6JN}{8aZiCxI-aOJCUNyW4 zq^5g^Zcd24&EDx+smWt{rdsc3gKsiFqXIkw0*K$MYDWSUAm?CY9HK=3+6!(FnMyw7HS%6WKr9PSMkDsrpafmQTIOVr<(2d|D{ zEgUI&+r2(;#FiKc=^Gd4j(`1{S^;5*^PR{bI=MW!&;9rHL`&YA|Loes>-@JKhjVp* zU4H-j_u07dWTMN&;c!H&gTsm1-PIa-fZh#14-)TzVZSrRDOD&c{_b_KKQS25*P|)X z<$F1hjV)GhYHHG=yj9-H8wCIl3vWZ@`#-`%n5d#Y$9{V6B(0LtxEZs5VZMp)j>)nIW!(Xi?-7BKtqUi-n({#st|@Vis#VF`V}pJ9VTpO^D*Fx?EaCa*Cxufa7P~beYJWS9K$ziXh$}8$>}3`^VkYrR zBWkoQaGNqU)BEgf$h&SGQn=HvM}a6*3K*^+oH_pc_iy$4uhusIqP)e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{00F&8L_t(I%bk+3Ym`9{#echN zCMhCVdWgH0Mm z1d|X!P_pNKYvF#fcYGJ_ZkpMBZ{G}Wc3_BzqU}MP8A99wHh{grmJ7az-_;<%1y2;c z22261OkH)s->dmS-Rk@*il%%%97R(UU5uhPhVlC>idJ=jjJ?S@MbX(EEdWammKF-= zTgltv;h`ux(IKk8O8KpPGcLEO+vwN#Of{HF;9}!tH-E*cWal z!=|lRFMn)@U7O2!rnHcNx4=n8i(@8FbQK7Ij390Uvlcj)0U5FTZ=hv?GmIDKfrl38 zFn+wYz{@ekYcBZb7We>oAdPAvieiC3F8DUld~@Pk1qKCdw>g`U%O8&Om3U& jz@-MU2VS_~Ze zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{00qxUL_t(|+U;CTY}-^E|J&|D z5t}9o5*!fX1`@}7V1_y2w`0q~&cp)hiL z4bn@1g+iQwpo$&}qrfrV4me(%c5N-f%XUpQ>O>$IHxo{!gU7g=tdm{_J@n3vRwoAP zu|1{gM!@ar+i5f0PV?_rX&G83~>3mquW+v zvcg8!^NPi&w}G!Dwvhg#wQ9exJl|>-8gFaE&1ZS8g4JxW0hctvQCRwT?(!7?Z`G~J(0N$ zE#v)~nCBZ2ZaF)0kZj}J1g;hvu0}+!n>BU2@%Xv|q@={5I9VQ?jnnD`Xv{?{K=|BU zgvwOCTb;?Ou$QE?gYHLErV7b|>^-;;HwdA ztf%8ekSmPoT+ies)5C}jrUvX&Ejv2p}`F;ujU|yLw4}Tbw zizYBpWxD!I;ymvTP_C%+4;wnf*j&S;pdMJ!$P2SCM z(%DI6&!!3TYwB(&unKa!L1727R|na1BjSxD+9|LK61CQ&9HBZ5boCa6Tjih6zlgPI zVmg_e6CA0m9txwZ_!|NMU`kH|06-zK30h=S$8GN)roPhw08Zy-b`zaUq_0j7mbg5T zy>_^9m&(6H(9&n>w@hSrfqq>&%b~;aUYU?D< zG*l0Tk}B(>E^DUs@dzL?uW{?`oHf8H!sMb}R)-Dciu9QoGjOH^|} z`=ERz4|19J9P7>gTVIU7DlP6DD<3&xoD?AhiSlllmx(EGvN9TiAr>fRa z|G9I=Nt70TM*skGwOP14UfTwr3E3&VTb?`*eiYIe?{>zv_qBHPP#DQ;lINUUzkXy7 zcxq!74qd-GYBne{ll3p->j`_6X0%Cv0=JW~`cX=yYrX&+S7`HG-LBC%ELoY89G`@M zN}zzVrP!WE?*WhGCVUNAUW-GCEt}Wl!)(J&?AwShFsj~8YG9ok5d<~2tj=Z>{(r`U zz&6#W6TY@_J2Hdq-2Cd;tkj=Vg-GiIW(hY2fH3$ zTs;#Aw#BLtoLt>Z&0AF6&z_xX)CsA?n`dqbmGw3FEUTCU-(X6s~0wzO)@@ z`-2vX>C-?ZSBC1$?84(eIB!Hak!ytq?t3Kp;Pqj2WR9GhmAnQgwx~v(c&hN%JzfLM zs&{tnjC=kz@jfOWe2pHS*KY#HyG4Qdmb>n$!Vz^4mKea+Q(yyh!WP@)yr lw7_V)aV-Kl8qasm{{@dD(?G5%!Ycp(002ovPDHLkV1l690n7jZ literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-normal-16x23.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-normal-16x23.png new file mode 100644 index 0000000000000000000000000000000000000000..f3bacd7ae3f93aff710baa1dcb98ed030a733abf GIT binary patch literal 787 zcmV+u1MK{XP)e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{00MhSL_t(I%axN~XpM0g$3Ne9 z=ZqP1)jSfTg&n$fWxm>{u%JO?kfN_>CfkohWea;lqAU`B$U{PD0bT~HNKh| zu%|yuzGaBWy93BNF+KwHx8_HLqVXVkhuQTROK8jQ)?)B0Et-@u~7VSs`5@qy_M894RF)tiQ>$X)L8B5QVQy+h%$9Y zK5Q;!R-9v&G?u31ruHD{LMi~%pfjL~J`3phtQ^8=br?LY+DqEYY@lE*=#oKDv+Am z@kGgKjQaApyGkXPMloH}kJJ?t-^;$MfV!yw8S5%54Q#0Z3Sjj%dn_yC&EN0R7J(L( zamDOF%klsvbASen^`NG|12muVnu0x3kZuW(mI4dGy1;6v+JcAJcynOb#_~_9`e44I zq!aWq6`+?kkyh2#fRQ#O4+1kAh0^K%mr28YVSXgYx%d;}@_$But0}U3VwH1Yd=hk` znoxsLWE*<2zR*74N@o}OkerK&VzxPOP%d?{ezQ%zX1A;^GY+r RIyC?Q002ovPDHLkV1l01W>f$G literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-normal-62x90.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-normal-62x90.png new file mode 100644 index 0000000000000000000000000000000000000000..f5a95585a021317e2294308a282bace687d1fd5c GIT binary patch literal 4656 zcmV-063^|4P)e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{01=o;L_t(|+T~n%n4ML5|Gn=y z-@SKcGGrl03|6dAux=n$P%#*iS?)G-XURrlv~{Vdv=3Tp75`8`v07`b_|!*v)EY<- zlUeUNGf7yY7R4@V)q;vbKm-D0o4IrE_nr6cA9u1$X3b>M%%ta^dG7u0`Of*BcmJJp z2AG4bC7rzCbdSMv32;+7MF0?3dG+ZY%g!a>>82iSPB;IWwfx-wD~ZSZ;(rTUyhzEs z830n*y3;lm%^ADSC6_9Ky$l->5aDxl20V1+vl@p`sfyUpHS=24qxowA0Bh6rp~it| zc#PVGA|EBV>Z6~d>#la=$O@!3e?yLR97bp zNjq zY3pn3hIGqX_52^J@jnK5@RYlEv~)btnr<2O`x3eTV3p$}&Tz)T`u34>Fy!V1@U^*C z;8I}u;+JhX6dJx)Am>@NQ1}S>=92)>)zm$D5>)jQ*3tN^cqn$Z(Hi)f~@ zU`p>(_CY>DE#QHT8N{QbHN)1yxy=C3mW|I1;8*N@g&nRA{zVyYCI^I_Mac{weQv-* zwLwJ2O;>xH(3{BKC2BK70Es~$et*Z>?1Swc2>`%XZ~L+*RDJ^oa#L$Ac6(hixFB6pAF@RxI| z`$l=YnxMS;Se~s5JUvi-MuBgS_v&IgrqjR!%qx#+56<(+E<2+{L@Lu#6HswA>EvsuEH58A|Jg8rV8ZwxD*yoQ zOx-PGt0_9_DTz=r27_l*;w3vABO&M%bsi}%wMCCwcCI^RA*te!E7RdK3OwTYMr8x& z6!9Hc>LTG}cineqRzHjRHh~;SprA*(li4TFC~)B*P^D><_6q09m&|mWoE@TPhA&nGpeaPI1b9NXoXU7k-Hl3|&eRwB61yAE9PnZ54k_G00`Jo) zl%18}z)0-6W=1L3>TDerXyhB0dizqXnG~|C`u^{zr)d6K6k_*{&bkVT2UTT-DugCk zNFfCiz>c0SdZpKsGyC$ffiuSXOeyiUd=pZAv6sj4tKM`X=#i50w??LZ7b^iuaHNab z*>~k*187gT%{}04g-VWWgDOLga}ihbiz5`$2_!;?gFQGc)lq;zk&v1%wD-1N+0oQF ztx2|ITSonzjW+3gQ@dgdykhtk-xN0z;g7qceZwaL0RgS~xQZV6Vj51WQhh#QH2j** zc=zD+z_+$`C%U@gg?i+o1E){oiOeQMhUG0j2Okj%esC}}T%K^*ME~ogG7(`?!i9kr zuLgkDd~C`-MSkJ2M4{npx#$4eva3!H@RoclWs_SmOy*mJ0EDW&yH`CR6UTDBWxCP^ z0Ad1E0p`B;d@R)!%TMW4`(h9L#3-+B%dWn?GqD@VLTvVdcgMQ4CExM|q2K~XRzu>Z zvX_Sd9GLE)WeQ-$fR#{!?r6)$-URAcSue3V%FVh~1Hi=Fyso%^vX9 z%$gDVE&q*@Ibg*d4z&7($zIYOEBwKV_dPKHO(Q~DAvh8!6a+pD;eXncYro>&c=v>T z3Jv#zi^~U`@%I|x8(Oll&vYg8Xv){k4)Ctz4z%}PcST?vJf(#~sCDF5O=*MGhd*U~ zZ^~k2oT_V)uv~e74tD}TdrxA*-*&ZqwX}G+_R~TN5Z~OCUwvmfwi7M+#O$cLc!QT1 z$8c~JdxX1aWB+v%I>O{YJ#wpd|2VMnbJdfafH(*UM$Lt(O#CYysVv(1W8*I0J9=BO zCDr+LQv8U7pvK$SmThY2j%CqSh@Lud3Q9}GBSeBQ%H+e4tR3?A*GyPP*N=o3$G0d4 zC&$VXfmAD@(CCNS`eR+44f%1G@9yTVk#HFNbvZozGaH1e9S41D`l7Gyj22EExHDcg z`HEu5mrxLbf^dkQ3jx5!%*OGF^Jzow4+4gePZk#CfHrim%b>Zp zdFH@}!UqnK5RRvld@Vh{x8}dM06?OUn$U&q=hBau2ntq$KQoU@{1R5ZHPP3Y3>J(Z zP>SCEgepn|Dg~r4Mm~|uCU^8U_s$%6q5k_1TVlv@Nr^|Eh`6?7p?Oy}+M}(R)R=zp zLL&wm2Yw?(+DN3NGp%}+03y!_-MRa!htZl%j9KV>-9vkcGz_&NhJ&hsCBLC9+xEJ3 z>dq%HjQFck+0@@(ru(QVmmD>rT(nXnYi+3-j!)sZg+LXSl$NFP z@!xkPvN}O1{9YyVXOJoaum#HBE<3ky#=wp9wg|fkRI&Vb|85uUOvf|YTo}zOG!4{Y zIP}7|%3iQ%Hg+pSsJgkY{@d^Fy?)H=jPG{PC^nKqfeqdu)_ibAz}tG7(9xXkHqJh( zq+^P>fYdjRR;BXst>l+q0syhx+W`RS`kkmfw46h}c~SJ#kJwm}g{q-y&XROLic!Bcin$VRf>@2<&S zaehAjz_I?xOh82cN#@jnJ0=(xz^^Vn{GqMyU;Lfab^t&}+W=I^9$y}7J62{h0jt7L<;b~w{*$SE(?tM~9ErW8fI}dsOc*$az*>X@d+y^) zo}4zjY7GB*Y0=(ACk;F~P>-E&`?u%8^79iVek|coC>)8B1>CMgK6Yznq{Q?M-|x9u zI88wgPwKpB`34O_u=wnRz}My)msH4r__%4P3z_wkx#vnXrVzzBVbxEp z$);9RYV^`Y0y_)-?U*hz%hH39!HcIAr`4cDrRyhrUS?h2nu~p;D>in&xp|-&>D4{I zARs+;(jkk54Et2d0f6VQTxR>ch4SH$GYr28TzEzdV!UAEPaHNV!(1Mn1>lYo&GAlBxxP@Jx8tS>LNf;^n!7v&NsDgmfutd4Z`9qmG{JRv42!mwDeWVyw1i9_C|_7 zAxM$I@O0`8R-)nujp|ha=`2ee5GX{0O8`JEm8Q{aNg~tC<2ttshs~gI=28`NIHL5C zmcG~nRYQx;0f6gvufsztcl%*mbF+#<#7$XumKoL{w18*%q%47<>cJ4d6#$%SH6T;; z(LV{9q0qR%v$32OYX_hAn72A8pmfb(xY61eCyC$8wV^iTeYTMq5t`$Ez z45rVBn58(wO-XB#x!6q>7SPE>uB=0T*y)78*L1JL_O@*WM~;=8g@DSoe8cem;a7I9 z?HZ{~@dVGd-Oi+JRdykzhBEmT{hyTDqD5ZWcoPEO(Nay#%)h`7AV)I62 zsfEzIflZ*tnLQr>;<-jd)U5C!uR1F6`krf#c+7VluOqAs(Lo+rk-FoLY0e9ILxx3% zv~7C|xp*JSjJp8+$|%vT+lNWD$O7?i`;R+Sb!k81yZpJktQ=8}1Y z{p&sP{QCi*b!*GTW}*94K>k0Bvhmv5rR|{+&;TIh;3zfTbeU8iSJ^(23 zfqG)S%mfAjmmWN->TTJ^B@W)tl*`NMs2vlG5Pa*v^T?qciQftx>WSxT8au8sk?N&S zLuCmuKPi6s&$yxnNf0lHD9dslRdy;I>%0SfWbOv`y#QcqE_fIkoX5(>sT&NJZIJXv~3T#e769ibp%+>3iC;ZP!JDL$4FI;t0y5&unkc587j#w7DLn5GjvN zb>?RyF#T*@I{K$YQ`|;kK%!7mL{zq)kg~d^P&pewS0s+o-9^SWR#h6Yc%y^^sQ&&@ z;PG9nUxu@*DZ#EWYCTdNm9F;1dVZS7H&qN;)V^+19GIVWJ*w(7u$zKV027!ZpdLVQ zOrNYUPHAv#pb>?-zC0^K3}d1@CG$??Dlh>7O69-?qLr%dn@vZKs^;q}Tf1d1D0@DI z#8e!?S<}cZG|rZory+*ZM6OVm=Jz zN9WN0%Q$5n_iY&f!4vyA0O+kxV~na78Z*8l+ya%eMaL0wRe;JpUGYq|H8=VU+)>~y z`8d+C`~#}^;n||*fdr+9t-TWfy5gDffp^DpXwJsSGv=+zv|r|V>Z(v-jof^1 zwn@&hPksw2+tAMJea?r!+^DIV>t;?q`fb_IrInK_s@p-dYwZO7nU!PXFt}v zI(t$Anf8IKXE9VxC-NV7&OCK4DGey-X(fPwAtgF(nXW>i1SnU|)K*S(n->F??^l_c ztJBfG50@}pnu#Nvc02wyakNmkC@kJrl{h6BeuOo0Ih_Vga0FN#Ik0DfpGHbmx3T9Q z+#c_J768^1TCgM9jnfMJ_|}w*QaWBx0BGuO_!I|pIf$=>k`VNQky{p^qhV$=rA3Mp zu4>V7Jrp2%3aV^~9t&U|AR7K1%Z5rNG&|OSIUmk m@$554!)u`)J<-{@bN;_mef@U*M61mJ0000e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{00GNML_t(I%bk+Fixfc+#edVS zH7i`Cm>hluf*9ywAc7*8C?YKe3W6XAf?%TTKOiPzU~Hg)i3aADavW-~!5Nr{;$d=$ z9|+~1AZSM__ZtLniR5mXk|s}*B_w*fzat-vqq{7~JD0(x53 z!0SP!4YX6~nsuI_vY{FPeUsw(P>Fq&xu9}$oPM7Nm8vGlXAo3kP}!Bg-_*cy zX7pFq`BPcyd*cj(%ATOIc?A4Ezgy=Epf^OY`m9AzIRtd9^F82URzCpj$OwZ zlXc!lxY$7Q&N{ygd;^w-oVcnoBe0Tk|A1Mb+dy&|xSQs3)jI#k^t60Zf<(BJ9+kk4 zb!_rD+rn)C(~Yfmgl+*}^NnrbwcY95IcOod?k#Yb$>Lyx6%5d~&L0C7#sJkKC#~}z xt@8sPV4dfWwvLokH-JkG;6>n-b^cbK{10C2rM!(YI0XOz002ovPDHLkV1ld!2b}-_ literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-warning-62x90.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/cluster-status-warning-62x90.png new file mode 100644 index 0000000000000000000000000000000000000000..65e0522077a1549ab2c97e1f625445d0edca44b9 GIT binary patch literal 1749 zcmV;`1}gc9P)e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{00uotL_t(|+T~kYj1yHD{$^$= ztU%C(L=$-+@}N-?l}Jcbgk9S0OnZ8n?skD)7o$Yuli|T9jlS?^VkD>`V$c{|k;`sx z)6TTr?GkSxA(EItd@!0AeE^NC2FOB}+3}%tZOcqMZD+dEng3;zozrvXoA3KC=RY%m z1x*%QirH(>RstMIg(x`HWWl94j`3Q+p|snywJ0n5HA9M&BN$gRzsNd|aWh$`tqi*B zgBvUMcQj&KO6rw>YvtSMVVEIzQy3>ncgORxC4dXTX>6N#tVb(F@u{%=fJZYt@Gx+G z-A@ZzV`720?)em81^|NTL2Cm)u_&IC-_@4(fs>=vZXuDnKHNi!ZU9L7bB4N&VZeDQ z09kM;E3mk;-nirYzNmH!R%mIxNZMx&;9Gy&)YIy3!^wau>kKJULz8|0z|liTR#XPe zDSyTq=}fN#>&d!yj>@q{*YdOxfQ|4w@3gN9h?yG{Lx4APgNjm$&}x=-XFx<23@K9o zD7_nob9<~^xi!mnt4vx>>P-i-=@I)@ZuKngK5fW6?ZyDhHzLdmc62BG+sKFDFL=94 z>a9Mk-FF&~Z@QkMc_D=KK(=#HofZ2bakGX1A~ni=1wiWY|0+=>c%Vot~4{0@M~U+MULL8eOM`O8e!3IjMQD!;)5!T{H$4w#o%=8l3SWw>ztw^#fs>fR-B(Z&4Q(Q$9fnO4Ua>fxH{++AWOHl7A`r zs-aC2)yX1>fG)|(f=h|ro>l;W%ao4HQl4o@*DlFM`}@cACo%x+O+-tD&XlLEPLCmZ zxu4qdz*qi@SIv|=0KE3^i+vxCee}29HC~u|h5CH|2hB@e27c9ZL$k`IE9ILYA_ueV z$huGGni>CD-!Y|WA18SaT%@?ESziz@_^bwNly}yEU_Hx<^Ee+jK9v0FFwAye{JQYilQK#Ep79r zyaJwC^Q;S9Y-&W%0Pw?&*=v>mTuiT2CKn0{0PHJ_;QX$_0&HetQqr5bl$&tDjMsWh^?n~U&`ad@e!;KT-&Phn}?>NG1yY-0B|;3 z|G=J;2J8vEp7njVjuF{{EdYlERzL7R=jLzQh3N#5APX*qhC|I!zyhc$;6~XwuxZKPF9w_m8~|u8@3m|T*$fNZc~#ZcQN1qX zP6}w1%64|iI(}-uBiIh+1J0=2v}9f-bvs9Ph7>72L)0s`u;{Zj_?S;n2Oj4#w%w>i zr3f^4XLj{4Ed7kCh#*){HVVXp~o(#Hs1dT^~aTN3}m{3o0h$S?N=F6 zq@G*x=z6OG_Vw;)IvBV7HSu!Gzs&Hk6vkOL4pV{G2EEF!|2heFd}0hI#wOaDtL=PE ra)S^CbEc`F(sx6W2MN}gZ_fV%l_ubB=3V$y00000NkvXXu0mjf|LaBO literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/error-status-20x19.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/error-status-20x19.png new file mode 100644 index 0000000000000000000000000000000000000000..68ba8591f824be911b11b534bd7806eb1f4aa8c9 GIT binary patch literal 795 zcmV+$1LXXPP)i7BQVSrvp#R4X+=xw1XZ$xmPP zp%+6IT9__I3CBFUaBMs5Bx|8}PZEF|{XR~j7FUJlMds`3 zjI5k4YG?q8B^`((Tb>sB&o5x#)&x5?7y=uO5rN5q&HVVKhx@Vt=2(Z*Hty->`oJKB zzFrqwQJ`h9ur4F9S}O2kN^b#}(Mh{qV&d+faxqf-!1bdpah(7ZN=nbJ#AaiRY^w)T zuCR2`JX@-vEOvtfkIuMwh3Ll#@O_FkCbTT82OC}H#2=kd4qcw5RGHuACgDntTQi?k z^8-yARh_N}X8iK+xk)&8zr;jAx%|vWwLC&?YYj;!N*GuQM&W%uPzsb;AKrenjCqOx zuiuB9zFopimfMTd-AOQR{D4<+6%LJr1YvFB$Y;yYh-GqG@q^gSd26!67YE}6zWUrJ zof6WidB&NNvOF!Ey;q8m8R5>|!C*=NcDI?1z>kBqS)P2>}TYL!vwzqo|Oe zJZOxGiNr!u5($Y|1xYk$VNq)$X@H28CTQgvq1<-a?RK}jb2&b=v}||TIj!(L&YYR^ zJKy*JE;B+($yobsWeer3E!jB(mI9iZK?o1k;);=c{!mD5(JRR!C*dotrx(hco5Cx}hAFYjWe|7Pu zng+`j>bRadj06&Cxx%r{+7Xy!M!qZ+!n3}`DDued_R`Y^58Nm4#}Tf9oQ(PbO8_*- zHXlq6%GqHL)lC+xY%br*Dl6c_wH7s%0?$_$b0#N;U|f&+p2A6%%-7Kr5~j9Fu-&nk zHdS5F**uK^THcp)RcgvJ^K@m`S5FdbeZ^94BZDq;hhRKv!xrlr%&aTcN+K=Uyw|1e zNU>^X)(O_P7)svpd>M6Y)6#sO&?8$$F-R&Sd25f$v44C{o-4iM@`VNwQz+?1N}?K& z4tJv@Kj!c|4tZR>F$B@u7qGvpSg{orG|f`0%sURbNlgg)R2HAPGJ5)}Ny*PWg`4j@ z?6RvwxJuQe680~EW*VGUS^dZPQJ52Ff|BBFKkD(>4~0M~D7rSppcxun!-I5LJKXJ$ zh%uhNbDDgA!sFD%QSb}b!xd~9;#UA|_b*;n6U0LO7?&J5<&R8l$r)zR z!#Yzd1zYyIDiyf5rixu92lAFi{U_jA)w2KSOpWu`B>jV<@0eaANTi2qlfr<$^v9F_ z!ICp@)Do+2<-Cghc`+ye0l1uyN+7^{w}X`YRI!`%1%5y$5ar%kd#u}E75b^tane7o z36=rzdj(Pz5o=3+7MHrXdBzhlP4E=ZdzV2r#k&1f2ab7JWf;A3GFxUf_Zk|H1KaNc zL@%&3*6o+g*gLjV&5E#P&Z6`?Q|7z|JO=DhX&Ym#2iRa}JQC|J6zhb3i2wwHS#sB} z@00006VoOIv0RI60 z0RN!9r;`8x010qNS#tmY3ljhU3ljkVnw%H_000McNliru;0yv1A{ZVOJ-`3}5_d^N zK~#9!&75hBU1xd6f6sf)-DY3BOzb#bve=33NrD3*X+j8?B^094po*$Um5^E$fl#4T zR9(~$mHM%&P@&}uO24$UrG=IXoCJtd5|hS>;}FM7?8NKXW6yZ@yPWNP`r$oi?%eTs zW_<5W$s=9eJ9q9q=e+;-|Lo7XLL2136ORJ$;J-h6-OqgWQ@&RT=m*xRvQ}XY&;#Uw zE?^LdfDvFCP=Q%sRD_HF@N|56rSoU9+;yy0Hl*;NgYVBz&KC_6sQ^_6Tq17G3ptnfk2g-2o-_qx1Ne?&1}?4;?rjN@t+ms zzg|w(sY`czpTTXrUiK4O*o6U%U%AVCaK0#U3DEV6koUm%h<%SY1^&kvIp`UV zsBTf^{R)p+tGjCv{k4dmFh=4;8`L9vHp^^-$jUos6=DlXs05Kao-yL*eYSNJSX1!z z{hpCiR(U}aegK?lD`V!FVy#2?`_Ik)@5`U?yQ?9$CYA@R)khM`M`KHWC4eAO!q{EB zELf~a`;4M$+$3sYx(XU zelm9msJ70SPe0PYlt2GW&P1vo`R&(O1vJNJeC#>>&wTMs5)wpxEs-|31 z5_JsuRe{2S4H0py4MJtQqD)uV8YO(*Dqm3LzPB&y;9qm703t|UKV&`p%w1Xv9p%%^CdOQQAyi-qY!3M<^ zrYlMzCxhK3o)FP~W8`WE%!)Q(1nB9|D{d_IrLo-#jhLQTQt?z8Jpkxe5 ztg{V9ppbKbWcDK4`Bs3>j+OVJvnZ7EoXq=tyOft_Ji}Yx+!vPtzMFpjvnzR}uRZQ} z9D3Vsj}v{k9B_XSb9)#mv2E1pX5#||DnUK?8|)Ugt}_gFyP$BY43{P?bG614p;-@> zq+Tnra#L>ypG_=-s(c5RYc&AU0L)^mJY8YHDi4G)9|&T$1)*DjwOMH1JX5k5r?&%& zc|&&*HVghhS@44F^FhaEtGpft2vKXzWfP)va2XlfBhd( zvRHGLWK1>GA#3??81qOFYIl^RHpR3?>mY`?s?ysjY+G;GwMn@1X2Zq-p{pd+Lgm6$ zC5V)~UsowlS_da%&Rk8sD3LAQCHBS%I$PEKz{E9-Iq<}z^Njh+-^%&rny$Gxp?3u_ zcZM;&ahm6}YEz;F3O@9&G2F67xM!-K?)%&A__r{6# zEHt()F(n5Z`h{J$7~Xfg$Bqqx?=8Kn-6f&BB=~0O@!dM9-?E)SQVje69^j?z^{OND{`L8{m+hmp7f|$)Ink_}LVGOzFeF2TAU80>N}P87G7J*p5VkXSmSnBiKe_f$jeU08SWQ}^7RSngKk z?jY1sZ0ku`GjU3sGG(w^*gNcT->ruAy{kW1Det`2WEoChwp7Ekv?=XlO&a`_f|%P9 z<*aY$yM{59Ks#!o?n*2>txCaPL;mC&^JDx>p1qeMOgq09o3NSQNnh0ZI7ndV$)cqqX4@%8}{64 z80x#eO}UuJ3 zL1{fd=`IO7HyL(qGW2%b_%Vz)BfMp;uxpbLCx#1GEwfdpQl?EsFs*keN~&xv2W+iH zi~<#N>hFraDt%$BeNn6}0V23EV}GZxV}o$}O+qQ({Dlv83m?13X~?(id;pjYzZRWS`D-a;D6_5@j4L(;)G7C%^BS+bkBRx-z97q5Qh4s<%qSAgN*}1 zVh!iV6%osoX{8#rJ+vuUWkV2glL!R>GgDTi+`@ z{$7vA@AKF=APs8J4x6KpixLKcn86?-2jB-0Lm~_SezPt*>l+nwuz9Vpd96^)wQ)OB z=73>jLW!+qbW$k?uGZ29dy})4!8l<+#MxaF#kx+FK3eEXW~$^CVAwcdDCAnx`=zA6 zQ~1zsj}Pwj=qw7gi1`{(i(;(Ro;YD3ic`kKmR_I}@LDxug`BXa8`k!$E+YM-E3*~ zMt8Cd2Tml6O)Gt!!o~rYKCI;8*YyhT+3HSm@{%$+r!A|LRS?A!aRl+LC9jH4le04O zpcu&cuI93m)FK$4QI3r$ubr`s%_xK2f?~L1!^+?9DZyQv47CU@P9%)axP_VKjc?W@ zHwzgfs?CFZcEs}98Oy~3|>qx5&y(}G$A7blg`No_E; ziw$2E*hLAvHEOAauE#XB*|7T-vt+R=o6&2?@^5e*0S9}F@N$K{+T$@QFR8mX*Dte)j&BrrkovD!uov9{4_&#mAOsUPey!C0swQzmupF4nFOx30*q(eqQ_^P+Z~^6 zTxb@A=`xd>i;rh5%vND++8OMz8K;W#tBG7XFkN;5VhgR7zzFAZKId}2BS?0riojWc zb3o9nok=WILKoSd9kX1T)cJmqZ|ai^tKC&&oZX2M7@0`>+6#cG1nyiDb4rxdC%c*g z#&Eir|9T*)KKZ5mNvj;w zRm`WY#(jGRCgx!O88uC`J;TAAN4bIhPo^)rK37$%j0hYD&S(C-6)#l{mA5Zjj-0a`yI`4^ zbJJ7XHwe8Q^~iXsh}CM=#EwFJ`d}e*vZi}6 z2V2&;{?yHbLKwsI#}b}-osAtAdCKiRCTe1>j(sSkbClfyA9Hb`8_FYZw+X=8tu)K7+eawcs;pl^XE`_Z9fbR`s*l#dLX;l%O> z3VU%kd=_YfWVR;@3YS}~ZNt#@0(k@Y9`I9ZIrQ97TLyS~cnE-x{<4nKO19&i2>> zFSH@PSv6As{P^Ux9&x3k{_y3b3LF*qM{yo)^j!|b z;&AZ-a6sUneZzlu6l6Z-`Fh-Y-K3;p=QE}}wafXfj~%&^^>8l72`?s=1n37I05;%y z-tRWaw4jW;S@dOKzi0U0XMSnN0lwekKCl#1mIUT{^5a*nK6bB}u*yM!tE#-N@Hyb_ zWnzV2rG&ss!2VQyj^+G1Q~a(hnpvfdSu9W8HRzagw6VDN;oatLtNae|5U?q=JEeCC zATz*G;3?oa;H77e*eGL)-vu2vZ*}W<>2zz1Swc=4;oGXbsPGZsL16e@0L*FN=`=V! z44ltEX;vSXm8W*PO!H%Jj4ftPpH&_OehqjZum$%EA07Ws465OpWD}`VI1KCqUVZk6 zop%z;d7pMOCadMn*o6JSu{1u|1>B#q=XTmKbF+*@3u2R3fx`l))qym(@Po^FpOzSt zNfseiFOKp@K+^ zaKARR_&NfY<1&{5GnKOAZQvwu5tvNNuyerNVo=)U^P7^DcI3mmP0lJifV)!lx;14{ ze+ozs(3i4kq0FRJ#HloL9ZS{qb>I+i`q?Ab0`bEkFvTLdp1x~xpBt6`1MO^Bd$Ir5 Q*Z=?k07*qoM6N<$f-f;u?EnA( literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/info-status-20x19.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/info-status-20x19.png new file mode 100644 index 0000000000000000000000000000000000000000..d339d24d6cc28de8825cd7539664060b5cd45171 GIT binary patch literal 847 zcmV-V1F-ywP)#Fs~CPt>6swImoT4+^t93_;An!Jgf{CMxpd-q-!P9~YWCwNx(-gEBv ze9!lua|Q3Pp23k`ql5h_@OEKTCkRIf`7)MB=Mm!1=l9OfZSFt8)Wm4RJ+Zu4c6JY( z%xd4c>O21*^xZNTQx7v3WXvLwZYO4?evBtuetP!DSw9503T)V`OkeZe*SwD}1elr_rDx#O$dX;W=DTy@n?|9ki=kt;GTr1=N~L7;hC~Cyu8)sr-AZXj z2VMl2p?lyvK=Azjd4!CI{8l>is;RAKqpHp;37~vuEDCI0a|fS%J5ER65bHnOkI_Lm zp%C~>t9(O(Iis4fsx z7iPMp(W-r|E1pQ##dj+uCa-*rR&O;JbgtH)ji$BVDvdD1=)NeyJPKt}9y;)-l>Ww>p9TcwXLtGM`Uo;^A)rCul?RW% zkVg5xhX4Z?yO_tQVD)se@bX{E#k=^<99mVwlhfMymz;S#o{ti^m_JRtsTC{Pwjzb^ zl4$PW!Oia@*vZTPOp7jj<$*P}+Ou46RV$z`xQ;Ovue zvRe;g7jlPzVcyRiV7pz&UFhgL6#igqS~C0iHksZ7>00^2z&>DOE#xUM2Hda1bY-KAHSQriMusk9{}R0x7u9wd=qK#_={(u82dkh(GW;iST3nSz+$^aY0J`5wimYTUgq}sK-o}s>3%DCo@Qsx zH}l)`oy&hl5khdUwQYKu>8v4CJqM@(766679H0Y;0UbSUP3`;#bI3Y2gsSHQTY(M2 zHka9^j<8LHun?As>c{~=)m-@efnlH-I0zi=X=}1%o}pxlOhc&J3%qArMxz;zXqG;X zZRykBR8i4fxj1t2(VR0nP?ViLZB2(C2&N%a{fe-ykB#V1o)M1#BJ0qrniE&L5=mDQUuwYsk zpFgl8F+RA}GNKX?0!xn~giTr9CW@=q$gvBX#8|)Awu~b_f1ofejL#n^HDV*X%|t}v zA6i&uoO#Q)vHaEJJi77?X8Kl2Z8PVZ984%6srmlV;bD!9T zs<{Ag=a)eiiEdj4Ms%pr=MU6Qf${kR^Nr|mW5%ixHmW0sy88X(&Z?SnX-07ss?(EI zLL2djW}EukQ($b%Xf)yxO(wjx8$QI9TavcXJyN>?VVPOr2#nb9MxQ_6O~RP5(ao6} zny@YAt=OItk-M$@@dn&Qa}l;BjbrH(9${N+0aTwqP+{uh^D`}SJj3qJ)OBOzhqNo}DoE|TAn?55azf?M*Ej_vzKoyo=Bhyp7 zm8;~XfV+^IwI4{25!;N{Oh!icW^Useo}!A>YKV={b#fPd=k}*u^AuGuZ^d@0=m^{N z`1}EvDl9W&72WA2cbl1r@YnIT80|Ytq-~BkkmPu@&+M1DlndLPtACWhUq=s zL~OJdfQfLBflH}!MRm9-C|@eIRXwNo-cnK20U77$(3Rw|J~7D9l~YsU&g`Y{+~=uP zm{%fm`w)PtYR*m>?@0e2$>Uqs+S4Xy|Ap_k`r9Y>G(#n`v{1B8098>PXD5a9v=$Ej zoCMILL-+L?ynKwY8<V^#w&Rs(BJ&z3tiFJof^7pH&##3R?GK!A7d21)y>GQ0YjWkOiL z0NKg5?CfAT*ixVIRpn;a$#z9i*8{`<7l^}^_j1}~PIvO~W23E#qCN?9KV*=d;b2Ss zz@JBA(lACz(!y(P&)ac2*3i)PfC(z;M;Jp<)3WG Z{{=-=R;`|b^|b&1002ovPDHLkV1n$QvTXnW literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/info-status-70x68.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/info-status-70x68.png new file mode 100644 index 0000000000000000000000000000000000000000..41b4679310ca14f336b400898e11370df02ab79a GIT binary patch literal 5154 zcmV+-6y58IP)00006VoOIv0RI60 z0RN!9r;`8x010qNS#tmY3ljhU3ljkVnw%H_000McNliru;0yv1B0IctOtJs~6R$}` zK~#9!&750|T~~R>e{1c%&uz|SuJ-sA#`oBc?LYz{Ay?vriz;ZFmq_r0gw#Gzp@NW# zDiyUaeQX7-L{*_y2$~8iphYMbNK8XQnvmE8d`axs;-lpavQK2k+N8;#m0_^bwC_M6o?jUTtJj|Ba}7(N+}C&thD2U z)DRg5&I9itat7oqaE|xD)Os*w$3tIsY#daqiG~onNyP3DYY!yA$PTe)D^L=#AO^99 z~Y}2d(N1fiQ33VzK|1ZdIi}iHa;N6JcQVLM9d+vv9fW1 zH4)Z?OIfnQ$Ep+%k?TQ|HaJQ<345Fj+VRj%2BqCtY4>eV|EsiqO=;)1+R0o%IWwcb z_=m7l=B$}9PhQ%s-~|EL|EYge^~twK1o2v0t_(RMtev1GQZxbrYp; zvP5bn$QF^xf{hVtu(6-GmvUU?WOg{2!k{MmVWpk3BK9Rjeh7@ND`VPBvB?A7_k*7= zFy;O)mTcVIDc0O8);=U+9v2(;nXrM4eXI!)8!x!PvhLA^rHpDZOKAt~csO1G?PgKh zn^xL=RcZZwrQKJQb}o&+^o=>S=pNR>9CxiZ0m~t%)8G6Yhz_dRa`k zNK9pc8S4qiip&zk#>8O_6*Wex?+omI>}zX!X7^^Qeej8YQMK{Yl{lQgON=>UP56L~ zn!{#6OGGKHx&=h6T_n~?XioG)-?ED(DS=IOPO{eVwtzgKw09X16U6>&z~tiRT#h;J z&7x0j{B%WZ^a(NMhzV;BMnL}8x#4nXg4?D zW=l_?RG-p%ss+rd8nYZ?hoAhmGjsmMyR0>zv$6lt*spH4CUC{Zh_zk0PKzO8+W@28 zEGQ>sjx30IH^8UbqumT{rik|PXS9>~rsL(GQ`&iF^rf%Y0iJ4+VzsEWmraj@H(Af+c=>Pf-^6E%=HLd4(|1}OCJz$*+OHX>@y;#bEoL2XHi(2@24 zVhwRv!?9+Q%2l2eF#}@dTfo`t0a*gf^{O2EbNwRrUTeY!Z5-^dv7cnF)+T=61yjVv zD5dan6||ej&6dbjHdEZN4<}n9_N(}_7YXZA#4?A%={~%R#l{9uMIC3~4>XnIy*_gP z7iZjTY4+^T{?1(QIm2@kAshRpRwjE(Vz>;gtqCwe4L4VzFtC%-#)A|$ z?k88-f}1U2q9*>_C2Sm^-Avo1lKk-PL zF?$~WTQ?4CgZ|vuertk#5{DH@Q}AvzMQnt2vSfPu$@SevW%K(fZ9YtK!+x}rNp8j( zY#iX`$|&U|pooQTS(Ad3loSy*W>CD`y(Vnbl-4I(z%2HhB@_r7Qxy|7?g#F-VWZDP zjdtSHv7d_|Ho`x)-+F-j(^4~Vt*h>bpIZB%ZFZP!e3v*gNKC=DN^_qKZ|ZrF#? z?y|z{=E@0Uw2RzosJ?~tKuoADGNQD5V0!cymv%n<>S(KMc=?G(78oNUn8@F3qWTAH z93DcX&{e%70%8)T6b9I^?~fQh_@@;5x32(7lD{~aB3_|C(U|1Tb@FJz?-LVhx>zl-7HdM3cFFg?kMi(ADqHWOFt7_J zQ(QeA&z8vcY#>t@Kq-xwXh|`<9<`MtqU)Hyxs~^=`7IO)F~3imsIDp}668N*fQ5=@~?6uU-4B8#qcKX~Yy07Eo$O zWBT0oi?4kv-_Cq(927-ltJrvRQoD3lpSamFrQtg$4IjkIRXVt7jgu*o?;oKwd=RlQ z&Do2XU_NPzqt~g$A*~5T9+B-PtZ%p4zX~+;BY!pOib$_DL2uG<>ZawW(2j>!=%Z(7 zUy`SI`HnA{DGV~S_u~xh`52j=Aw&#fB3gy)y0uD0aw2xb#=&+IHgf>(#aI7zgHozr zY~0fxZ`|r?q^Y}hGPv1(a^)>#%fm?l*oA2~ORlmN5kqtKB1$QO+BIy{WSuRG*v&Q$ zHlb7&K*xS{3owXSN79~cr%99c^5pxsQ|RA*T}IpGT^EPhc;M4)Jn(69<*f^26_It) z9PzA)`mG5DY#exLc@hkxR6pR-VTevDZ-e9J$oGwq?;An8xpf1k#T)>C?lNLyg83_$ zu#R#v-DzTC9%2W?#QiAc0%#jY8xZNGtEpt_N^X{XWh=SL)}-;&<%r~lyD#)Jxa&~{ zc0GcVDPm&3-6QH2dl6F>8~2Ne62^#)D?m^8C9I&`EMCtbnVz9F>5DZXu|J13Av*2u zJDHvp!@J`ZC=Bk#L=EPzyh$8Xm*zK}FeeS$slN88lrgS|$&1+RS|4ypvzE7ZnM(U?BBlsPLzDeY#-mWIiehEa}}e0H~ajhFzyRZ2GjAF$n# z#v;~W;+3;f6U&(fT|fIoMs4T{n_S!E}KiJDZe zyh+rYAr5PlH>J(;R!(%Ct0<+>UI8al0Hu>_b_YzYg@db*?&t0ZDSBLf`fzlfIrg$-X?4_E4RS za~B0lpblydrBYzDGXZK2Fr82Qbo;@=+9Y9Z5-;E1E(>hz<5w>eRIem0&t*Gt34=5s zCZaxd78Ci{I3U_IOL^0wWt&}!r4x+yRO8kjAXZAv0<$E0jd8V`9Rs;dM8a;^f>Kx$ z;#V(|Ee})ZA6XQQV&di;QF9i=CK|F_XjiH&qQZmo*RS>_MHlb1)6E$egT%b8~0UHNPgxSPCCT^~&wrLIdt%+#P zj#8gEO>_2ABG`+go>=0bN*vCo`*c*N*>8YSV_JKcwe|u4ee~~l7{tr~Q(ZNem4n!r zus)Gwqm9WW58A3}76x1Drq@&2NgsTkur`4|cZskzjx1`Xnb=P>t2vhd-ub8{P)fh6 z9PeG_cs`4A=mw|@px#aLnoeXUiApAF5Y?w>PLC2)$J+CvBmz0NHq}fwKU)*wS1;3? zyM#3hw`alzVSSRYF_q>-9k)CKls>QB+&L%H(`2F5pz2DgvmiS}sDO0dGY}#2XQ_>! z#>p0OvL&?RB@@PM8MKo|9ldI8(=Ic_VuE>0FyC%z#=$(n{8ggnEGBB8wTE&#?wrga zGMX(8UGxfrK^qwDW*fjsYr<`a*#}fQ^C%74#e_AgW5)m?-}^rDeIsN`o5&V7B*U&a zyoC^}mbjR9Gl(_RC*Gz$@fIem-4K~>(G1AA*yz&eOJAQ`GFNgk#fIY*-cm}RA~_P% zvDPRjp?)xbmHOmaYS&KS&t1gH6v&o0lPUBk`*B;kvs!0R3@C+@E#YQMNut=8qBedq z1*V>AONTCmIuGiMSToUj^Mwa)Pk_1d>!+HBpZxY25qk<`lCCWg&!{=g)H^RQef~vk z9FQ#yQyADuu6G+sJJ>k5Nw+^8h(*Z`@LYKd+0rm#4Z(a;Q`INWVB%(a2&1E^>AX_v zHKm=`l;hQx)M5XYzIghpSwZY&P^W>5t$5yP_(nOHs6l<|9p0hR0Lgy}(R?T8Unll%;cJfC|pL>x^&miSZhsl*Up}hhs z-F>ixIg;%5cDx+qS_mHn_MdFa@b1yRW&e3!?W@|%Z$C2VHBF8IR?>;ep>c>kM z<5uo`KRkD=RjpPL`?}b;7dQa)ca__uv+SrjLmbRw!X|F6gm$uMFOQe25czYk!k|wn z2Pcz9DTQCXOzc;wT{}T@W;6}|Zl_EMNzOeT2RVP{xv#~rbob_x_%?C+Ck+vKb7;?F zMc@Ho3(2`f-7$08?qUh%$Cy0xQ=Ci@H&ehX^dktdUu_@VX+Nmah!{-Rq&|6u#`HN% z*d(Z3yY+l%p@Y`?gQ`;cSx`@-RD1tjena|V9P~ra9pl-98yVd5xN7NxSo03>67X7D zSkV=8(g!eMo#xCa&6!bRKS|?RuIN(=NLHG=tT)FWX?;VHez2gupX3Tf9zm~@rI%6{;_J@eAiP()@xzu%KLo%V1WaX8{+KA;w z!>vp@4r>b=qLtI_nT1BW*gpmN_4uhDUtL4?bfXRKdHj%A^JgGOfZKp9>tP}b#o#Ij zG+Ls(PKP*70N)3GCf2?&eeP%-;MqgTrl!Z=T)=5H?RZxy^<7Z^$r|?G->UYkMJtqA zm*YMbco+B;@Eq_;3zRi6=BIaU1K`N(7u)Y`-2dskiR$+zM_YJ+x;|Z;t%r$KMidbWSKHy$pGs(>R56xO$ zP$xMo`YYh5($4=)zWwYZzz@pFFD%EDWr11ACQkoEhHn4lG-6)^H6g@ATiz#jq3R|}W@aU`OyJO5Uc19`vO%ZtoKd zZOFZ`<;Zxt@{$&^?~;61%o$(|m`U5PHK2AqDC-sTJF?YQWN`NpPsH91>`&cmXUd|! z6p%8|o3d!JRnsowY?`@Fr0)6~;0@sX)Y<260OEg!LFaOK<=z`=pIf#62e>Gq&2lLk QSpWb407*qoM6N<$g3H0_rT_o{ literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/normal-status-20x19.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/normal-status-20x19.png new file mode 100644 index 0000000000000000000000000000000000000000..3872e2ec8292664a2de0abb7004db90e75a5ddaf GIT binary patch literal 612 zcmV-q0-ODbP);}MvK}cL6Ez! zP2{S9DYQjx+X!uHIb$hfp)yW0)3lg3&glCLFFg2o_kQP|bKbo#vRh0#qvDE^^Q!tq zief)>S`hic&zw9mg z6?HkLqR?O(lYref;k#L@fs0kFZ_`tcyaRG_`m}Fncvp{{Wu*YvU))P7pQ3F?zJ|ko zI%&X~YMo(GCJ1&E?=ga_s03;v6!ckf+cdE;=j08h3M-gN7(h2p|Hsc2FX|rVTuyld zueTg-N_&pYT>v+5$6zdSDV6L@7`4iZ(2hBRQxIX`<&`J(Vc64DmOISoc0se^bmRXDgfNhpyW%H@TCgxGz zG0cnjj8*&zZqAoj#zN5GDRd=IBZ@eJ$pBbv#!GAU{==@R$Ocz#7$1`JahxJRf&Drx zDK-TSd6Vo!dbJv?Yk_^P$4yytO>1u;>lWSyM)wTGoqKTsL~0000$N+BsqVPq0gM3KQof}TK}H0s2r&=$_uL?$UFb=axT7Jp;Zt)qZB&d7VJT24(OlN6JB z!66r|b;a#P9`_m4U|nbnot1WXJc7N>)j(tv1GpApJA$(c27H_plLb?fNs7r97d&Sm z!g0XWl*}`Dyaw`rYBucCq?j~SB$E`AIWF6wFT!Cd?I?f4 z@upt0qKGtLA6^{mE8zYb6MYz$Y)*>F;&Eh>VlrQ^&6&}q^rVT`z~6lG`c6#e;kW|Uh9)$(tMN56&o`fS_v;#h)6k}#RI8rEUqdjSq-_Q zn9S2}e&(XPu`$D@5j$~LTE_9MVT_Dn4@WIrTd^xmMl;^SoEj2(z>+aCLl#Bt?aCTL z--NeoAaiYuL(Y)qOyfGttML70Y{lLRbMw+1iy<_kkQ9^poRNqss;^j<@!-Ix`pifv?I`+f+DKR&75_wZr>iNZ$6S=$yuDSXD*CtiYXVGt6#y zJ(}&WD9Dpl$=9j7I4r?$X~dL?tlmgXoVO-?8QLog@`6n_YCa0zPap#4B8jlAoJi%X z%Iaj9k|Wd5ZMEQF=m;A!uPUKEoRG6F4fg*+#4OKOyA@Buk=!Gla+ag#HWS$zI>Lr< zI1EOUscW3==*-q0c}vqH@mA4;HK8MH&w5`_d7^%%V}%v{odXMtcnBXx9UJ4(jaTG7 z7&^u$>$F~p!05F>*Dqf1Skvi(QL9p=vLqFkbIT00006VoOIv0RI60 z0RN!9r;`8x010qNS#tmY3ljhU3ljkVnw%H_000McNliru;0yv19}0&=1U3Kw4#7!8 zK~#9!&750oT*rCGe{;^+3zzqcB#M%3QNBplg;rv#j?~DC5JkQqC}JoNe&|zy0xg09 z0gN^X(4s|)J{EA$KD0#zycg)A$!71yd8Srhue-in_8$9nn$bK+bE8*2F*@j zE3gI4L%)egO3MjJW4qDvTOk!WnA)TGH9lNao{SNF*H|z&v*b$ zwS*}@!lghrCDstI9-l1)>;Sf+*@DkTppyXbr3caxZkX4uH7XZyWceJxB@2wB_zW1s z;S)SA<1qoGahT^-3awyJYZz0Y7oTT=y=a~yU=u~!D%~eh0Ygi#ZMx1%g^^GN!_m zAe_;@qo=ALzCtHC9-+)$0uJDFm@>U&Ww1pVyt1yYTBwS;=3iIXlL8Vzj2S$BgTpU~ z@jfmS`X@TAlv)*YR%VR0xcz6eQ{Y=@zD0@s6nUI^x+q#=sx4-yFTq`CS%?gf17h@$ zWDgyz19lVen)*|9PCkt&zO_vGYaC|jkQ z^yAV`KpQTz_$2rNqip93etB*PQT;gFF*IbR|m^*}N` z5R<+Kq#35f9}w`zXrAUO{rpX1bM}^1?VqB!OgHy=iZX{NvY#A-mX>H7RAfK`ORSZ( zTuu`V3M`S88XBlsQC(rzr?1enm@EOITu7 z*$(pBWU@MfiIs&pN&4yJ2#Q`@?iyfLWK0cW;ZS~#%ddHyG6yKIpB$Sg$jyR?rehf5 zTVmZMfM|<7K}7R3+5;XQ8-cyV`G`LukIPkFWgg(h5*o9#`2Ls_AF!1Y`O{xE6mc#J^m_8 zT5Zc&nlV`h3HUZe4v?p_63IFd2B?7oa3AOa9tCy*J3vy*Fw(dwdK0%ysukcE=G@1n z#AZ4bK-?*KE47<2l_!75|^7LN=A++sTgo}fr~B?OPEDJ6+a`+$dmuLHY5SU~fmhZzV1OihnJ0t8r~#(FSGr3#KHmhsNtRxUHA!5vOeqN))&k!VK!yaQ zc+Gd)fTT5%RySX=^gO!=m?h2$j&fChPnYbp!CRm&8Ttu$h7x-y(X~KqqcUX!Faqoa zo~R@HO1x8-e6D#gEdYu*Xg1@qgLj!w|EsQ8=J`5BM$n`pfnwZo zPQ1|>tp#VNsG5?4P~n+SN60uKwCQCB1c z;9+6P(;~PX4g9`S%&uQXSZXnQbftXJPKgn6>?T_Urs|c_bmH?QnkUKACgF5*Y*BL4 zti&hJ0lxzbuHqUg0dW?%Dy(x_B*JzqrD>zUqxf9Mqh~Q==1GxbCq6p}NL8ZSX3fVF zVI3Clv%Nv4*uZQRE$Xn`wr;UWX`qNpi9s}j{39u@g>-NvZb^}2DVFvjP0g? zCxJcUeO6mpZWmB?0&AoYMI|w8EihsEHHBgepE1n8&{hSeKnHm?Qf3HEEK+`=#jETR zZRu{}8jdh!Sgx~8!1i0roN7+ihGv*~hM6cCLxG-!rQ;@UqGWaZWW^tsxTM)O z0K5piAk6V)g_||?dgIn43D``YVX`E{0Pmk!i=vM*?UmI~O(Gu`P&NSTTOgJRNf&&{ zpA_)LYKxQ&-YH;+0_#y>px`ja24H{yHxf+)VaFEf@fHqy5O_)Y#sw?g8ne>GDA7lO zehQ(Yc@!8#(Fb@7q;@CBv99-QpnK$Rd^y!Ccee)rwMc zF;Q|-M&d0K-#jM4LX?#}nyP=jHV2#o z&Ikx4>-TenI|HMHXQYA#yi`JBoIx>TzXi^|os2iax^V7aLDK5$ujS|;IH z^^rTkMav|IsBXM1)jX-$^vSiON5lpEuqy1y$;jLaYu%Xm`h+`w8JSS6ce+P--f6-D z#gW%>8O4XItPo0N5(XwB@#yFOYm}HoFFr`TDqtK5)vrk} z)DShlirmH#T`9@6J|^6uOxGBEAOIR6Xny~zfZ{eWZV?Md?yw0TpIK3c=7Di#($nI0 z@6=4R_9&}r;Cg{fdP)R)N*H2C&@;lua-^iDfWr-9+#nVb#9S(i;yQ{OHEX?kuap%~ z#swggLwQypr&e)IPna_ytg%OD?sZqb<3wPL!#GJkqdj!`o~+P3iYsWg0Nv;aMFTA> zcv&=|Q*@)7>KT}FgDHe#vF*O}z>0}J7v0LrJ(@3^q|sbsEjJl30bUqEW`R+PY{KV> zCQI1W70yV6tF6M-dU4_A)I*~#R@J}obx%60po~{#%n!?u_Nfg^e)IXFM zCmmUIXK*-2oD0Osm^Tl;`!P=6ngtq>#^E#$|0D7iE#NZ&*frpF;FJ`k%=wrU)zb&H z*>u@T2OgI?a#61FmI!cXeFV|M7r<^iVJAQ`HL{@d`&uw9NaOoTI-M0bE*T_ zFU0R^`u56N(Lmyw6XM3(MTku;#unPzxM)mjG+C+RxJ`K?JBymb;RK4~sOtH;jKX2@uaxsu}EG>St;se z6z`GXJgKU-^z0m=XXnU5#+xiy~3oW}fu>Y<>ZPOZh1 zE9}#o(PEaYEcQt>AM#5k8wsBlx91tb=a0}F0v;mTHrhf``xXGpO*W`gUTCge0)7Gf zihy&x6$U0;g2R+7~s`!H&FZx#lJ%f;+tCbG{S(^Jn9pOgZ}_{6L{Z%!ZKCU zN;~<~CIAkdy&1l7oO`^)S>}nO*(QSAPb(fWRW1JB@}OJ?P6O`%qnwl&?GWntM6b4) z(kNr#C^=vh#lN9=O{DOv+-k$c3E(t}pW|_i_G-#=e44ASd>2KT zeth2T{ISE`QUS*2cQHf?&2aI9L8{7NC|Bv2Tb$ZsQVLzXx0Z zZdQc$LHLF@c7&PcbC;$UesGu`0uBPdOL)wAGu17ywl4vQyo7)=;uJ0b$AR;lw48)^ zpXWcGZTP~`+^)+U0q;nP_$J}ujAw*BkJ0)k0qjeI+nB$~VE)dB9p=^6?n+iRRhF_=UizHq-4)<0 zaDh0N_^*YVGsm~}h5`I4n9>@?EEc|TDLkGMSH4EXc{AZz&LP6HQ=I}P)YBzyU z6cr^QQ8%U#B^9ZKSV0u@1tTl_a;clPy?b@<-uHCjHdk4W{#S?RJm>d(oaY?DUv#x& z*FsAfYYbCgZYeWgAax<6*PfEmE`;dZyZV~{a@#h_*EaofPQ(_m&4!L0(~O9IEi^K) zUK{C0DJ$xx0I=C6k>SY0o|;jeHYvMqcS(cs2QUFvHmLPn4~_UpMpGxx(bMcXvn|W; z#WIa&c(r|JrT53{dc)Kc%@Zo!o;@-vyC}1?_)Itsy!2r6G=K0zU(zrUZGr+&I30Q4 z^D+x(YGJ=A0m4SG)i8&Xh6M%$T3=SYzro?Y+(G@@{_?d=$SZBzm-)3%E0QfS&@XWB ze9Vc8rx41I=AZ}+_g(~$KJ8G6n9wFcK=;m-mNKWu4jE>M zicO8o&MTny$aBKJe3UX7?3L6UdP#QH3;=F)9iXTG2~ya8slc#?v!qfg7&D~EH158s zB4UM+Qc}IY9jCT|lNB%GW!~sINVm6x#5TobSrJWw^)}HssC}aTb}^&34wZY(k(y!$ zprNCbZg2ZvFU)33?UfNL-1hrMIHFO1rX9M$=I8~EyfF~Q53`&=lcsBJD#<2g7T zxvVNU!n5~{^aniuoKq=mSE{zYEf)Z%T9&LcE#uyJj;3W`lc^KVXNt{{9?-t&@sKq) z7c~Jc{!d%oE>8=91gL9SjN9ee1~mT%>}U)Y2Ucx)Lv3>*ZkNXeqAW>}4lH)NJZDee zD*E$-9qr{(j%|5G&CNxVqve+YMZnCc%R8VIXmPtdt>Z=c39h9nwTs4qTmS$707*qo IM6N<$g0!x8`v3p{ literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/running-status-icon-35x34.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/running-status-icon-35x34.png new file mode 100644 index 0000000000000000000000000000000000000000..a3074c57faa3da4aa63058a17219718024debcad GIT binary patch literal 1955 zcmaJ?c~lem9gj6ZE)ADf%Ob->sa0}ug+u}ZNl3azF$OmRt0b8r37AYshE0gzQv&5E zAZkR=r+^@8ta&W%x(mvBi>{VMMWlFel_`+Pp%>+{|e zCv1pc==wWX91gcoz~_jtm4v;ZvlI4xSdco6Ez3}@1l^3Jp$4fA#>GI03QPj7kBcdWS~OGP1$`LL_F>%E?OM5wRZu>hr5Zmm`h|2nW@}Z4{qZ@zSoM} zD2sE9Od^6@Gw$2`8HOK6pRN6uqW9C33p*!AdiR{0>}Wfu-!*x+x;jK&YuR5cvg>|Z z9B{7oMklEJtmz41qABAa)|-p==T~{1O2$1UK5bm}?*4AjbL*IQ!MD@tOYYmli`GQl zEO@*{Ie*-xf5_;*bntblpGmMKCvkaS1uzv~y!|I-czUSwb=moSs6Jf4fU`W!10U@^ zEGjrHR@QDAfA>Y1K;xqtv@sezk7PMrZv3K7GH;clYj4q;l-=E?oYBW-Qq8=+2fE?I|3}><;J5&+8efWHhPw(?LRR5x{5He#CVE)Ua_ie zc=^afzkC-r)OhQKTM_Z~J?>%JDJ|6V;8G1?7;-t`+WM-{Z~h?tC4A;;KX>`XOah6g z_`5wy{$Nk?^tRa3Io!ogSF(0>txbL4y`ukQ!{mXzrLFM8%YSNH9Z|R9H%>%Xj;g7o zC1GPl-Mb)NV{^gKiL;vsSJ41GG`s`vex$Rv$n$Td{=(pfak#}^Z*QQ@u zygQl-rMPCs-?BMwQ1|%Mm6cc5r{lyC|LpsNllewTaTcj?-oa}l{*=yOVA1U-{9qtp ziuH4eySu?!PN~^G?PjJVxtFEyo|p)g5*Wt`*JFO^iuop@=0NAd_wr|2!Efvb6Ex}Id_F<9HGHl;=O zAHEZoSfRBwx2k#|+R~#*ZoV04d$TXJZQE9S=Z$OMiR}m6of~S}-76bM{X?*y_T>ocQ37ILBkW85)MGKw}TzH{f*Y5o>z4f{2x;0n%RPQhQK?j!6 ziZBAZdbnl6>N54P{q5Fue*x_Rd_DZ&_SvIntxTHiBpCX=-JG(Nc)T^Sr8`ArUA5;? zkR{NwVPO2H?=HwSi5t|fUR+M-%S(O)?MZ#S6BsSCcdd{2J@0*I-;L;n$3ma4{Gor} z9Tzo14)m?AOzbh^lJjIo7kRKdg-JltPW6)E6^t|dyZm#HYQJ_iF}j=Uz?XNw2YQom z72TE&g~wW`BrEWCQ%|&P&E_ln6+_EM#?pUpIZ={*#SBLI$3{000006VoOIv0RI60 z0RN!9r;`8x010qNS#tmY3ljhU3ljkVnw%H_000McNliru;0y={4hT=f$MpaJ5!gvY zK~#9!)tqT;T-SBSf9Jk8D=F?GDasNld66vHlw(;^Y)iJAIzd)1NP;@Co1!rSBq-7% zO`630k`FDAI6=`iX+Jbwf)t6ZENCp(K;1YooJHPb+43eYvMq~}D2Wnxa%SFg@9Bs4 zhNdWr8p@Q!9$-k~&2Zj(|NA@3J@?KC3t&HZ>~;YD^rl~&_BGp|`n(qg!_BI$9asw5 z1~dR!pdM%iBH%DE3ZlR`a8yM4KXb!RkDclKRh#QNYBf0*Lj34=cV?Z&bwClwtGaqs z+oI~49IgpmqbiM{MIfV4K(zrO7UTrTB&dRvMPx|CJ8r~3YQ!4_0-z#L5;5hww%uPj zn}Ir$`7E&Q-oHa$K6u|!MXvx|1F{-)C1{u88dPKB)OxBed7$Fd8CC0psAv30FcuUM z#i(IMExp=8+yv(Ux?bNMT+!~bC zin~bFEm5^NwIshCih?>E4s{L$Jjozp00j)jco^~VjE~@zfc*$>31**&dCe3570+Z2 zfsBeVlXq>sFIWhI7KE7Cm77)F4WQR5Y;Gg@3JTSq zn*1#yCykegAP6D`BYwdcf0bvl%RH0a=$qWI@w|Rj_Y%l&)af4Jz&R&ouAug#-`$ya z8a1i94y)06=j@HDc8gQnYh5I<3$QjIb|KD1SQn$X^n^0UrRa>~I`y0}KAy?qd0Bju z!#7zx$;O`XUq|G(BHk{~WM1-3?tn3w;ZI-x!|-g$@#iU`0{r^L@5tfd-93uFSJ4lt z+ABjl*&M~?hS&z;T!iA#)Lm2+5dbsCNXQwFt2%5dbSz~=JUsDz&tz9;{kk^aFTB^N z|Ate$Q=L8lyuF~rOp)STqVW56Og(qs&OccaM&lP-jjnZSZ&Y<3ur6#4<4Gc0CWMmKi>9}(Zoe-C+Ez> z=sRd{f8w)sQCz-BaW`2PZHsLAvM`=#h;2wUu%_AM90>BxSWde~6veubN@R)R5}wI4 zWxT?se5Rqplgw41+qXaQ*=K?ByC>$+TTdco$5Ve{Cd0wzz)oJ}G`=Oa;SI4Z_eQpC zf_Q@12Ga;R7h;@|Ay7p`Fp?qT7szKC$awizMf{(6Ci9qxxA&svO~-ET{hHQ7j;ZCu z6XA(gtMSLx+0O;h?ohMeX0e^pA3AM|Wc+e#KGXDZgJ~9{gH>V9NX!hy6kz9@|K`2k_t0fd z?TwKwZ;Rs6ipW;5E=1LxQiv@^b82xe2GsXW_98*A^~k^QpTGU1iZR}scWu3I5}>c| z7qw-jpFHuo;(?>TTVpla7DVG)B3rq%D#~ot>0&iOPyy#+%HbH+iUDgkvy9?IAU8tkow40-h-I3 zCz;*%@BGYgKGQIA`{sYLGd*WkVj^4WbZY-Fwv`)WTUip>V5&Y@R7A`IuIlM)AFC1N zaMX)!&{N1Z-)vo|<#6ONVCWQLe(>1sQ^f4p^(DU&jkb-H4z7#h^5wCsG+Gy-Y8RjT z2*j}?3@j-48H48=Of?_qssEtrh;13rwu~!W8^jY6g4tUYX1eE`gup~)pfQXmx2o#a zAf9Lr;z`^h478HodswlYI8x%+;R(h^1HvFiRWWk%z2XTU?~r*b#;f7Zzx%TY)+cr=z&AOl)E_V-s>_ zxo*`YTgA3wRJF%sGV5RKfB4XwM<0E=I&JvDW4FIUjH(jaa#s+KUmd%kS8+vjF_tJw z5Lku=D)@0H*IoH(-gnL2bTwXsgslW6sFVYoQ)Vl(L=u;ZY=ziX>QTKeip$pp(Rkx2 zS?Rx?{$lp0Pu$(1YA=c6O1E{PNm{Oph?XiTfyIRdnhV|Zv~Ol@&sK`LMw}J0NAesQ zdyYzQgxCj>)5S)`xfm;6opn)fXeW*d=J&wSld@7N99*I<-k{2c$W{v01xXKO5s3my zn3B>`T*>CkKhE0jtt@TnCZDZi)0H2lt+k6MUvwNAjEO3PI2Vx3W$-+ik*%caCWXkA zFVV;yFvf2>g_s~3FRJPlPUH2?#l>V|xJYQ3K}t(;B|Yt1SkrwyJzc$co(I5^x@J~& zUB%Gw0G?+U3ZEz7D3UhHX5NcP5{Rt}JyqA4m15_kFWp=0UHOS}Tf@bg2op{efi>1e zYe4JHx~7=w-=0fSKEayq>si)z@suc4t1mROY5j*;*4f2NhYkHByD3LQSXY^)Fcnml z*oJsUiy~(Av9Z0Y9^HHIexT$h&Fr9F)m>y=)CKd5EoseQW0&;xO|=DZ?nqhHwc!*| z0K`CD-4a^0gZy9)F;y~7*L+e`EvlxVs;h!%VwI8dQJ`c-%l*FMnj%+evMxZ;+8s#1 zh~S(ORAPd1oCu`4prn{^DVs0<1lz8@V^&h8+hlNpfzj6)n%K`|Fo1Po+EJ9#Hn|kd ztGb>jF0BgV$vlAns~7&Z9cXbbs!P^F0f zA|OK0yEQ z3k1;^p2^hc)|#;{Y7sH5KnB2%Y`GJpMZr(psD?L6Nr^+nR&o^mPL{W9#HXIYv4c!R zhX}*Nly#D9&cpM~nZ=nRCB_9sT8rIu&x@4jdHeWI`iGxqWU`NpSHu{4? z*aG4M_^}JyQEieM*O(-oyj98}F3e$~Hrktd*|hFfmNs=WGI5MI4?WFOul$U`;Xzv3 z^AvoUo17_9Vup?dwAS^odEF;j+kHJ9t*hrj${s4wC>gJKX7@SyeT=GYuufW}N{n++ zqb4cZlN#R!^$=I2=y%ZG*vqBMucv!iFZIPn+FMp)jK|Rv2T)a9aGWyb(~~okl)4^z z+PAQ_`+9noZ%8JDHIcGsUZl)q#;HSy>!eDI)3~7Oa<#1(C{7)o6e#*_bTz$)t1r8i z?#>O=6&j|_v#jl6wq1P(?FUxy^xmH{H2V4!IWv7;PeV#)+v+(*SqM_5&l4RM!Fhu5 zQekXXQD19FMVjXQtY4tHa4{=8uA--F1NDXGlU`R>Xk=B_1^@;I4anfa~`RE$XSCPtkKs#2@6*2NU^^{iTPC95vFl6-ddLfP{?mbG2NjT=A3 zmUW-Ri`p3+2&j}|thHEcsgz;{2LimPoz3e$$qgGnH7`gJPEl^Dwyp&O(W4zeWQF>BkiuFycw@`RjYgMDxmY;=Oscq!?Uxh1p}+vw@IP^8Qj zKa=Xz7YIP(T4p##3}UiO27|nE=zkcU>}TDIh>Kb_QlD?1HPsstI@?yW?dm(|>|4d| zS0CcYz-s_3YwKa-6(8l&m0QkhOqt89MnQFUT?Q(Et7$<(JYrj6C>X}6rI>34g|6le zG!(mL6{nbQNM^z&!`S!;MZX<@f7cN<$RG+ZF^UB@I|~~ zknwp5gCNGo+K7>`FPZY^mjZK>lg}2(x2&u=Pn<_m954wo3X%xpnapv4(L|W}wF?q* zit{3;R#DC`DH4mwh=`1k>@~*E`i1_TC4|x>Gvdam}G-Brm5I%t&4_%!Lt`eFx8kd z*+>8I9^O8_lS6~M87m!4x31Pk%AvvCCy`=pgfafYbOleZOkl+Hc_!26nQUdM84i?0 z8w*_r8wwo(JemlT@$w~wz1D^+oVv>u z8s8O};!o8%uO0a%T<1V<9d+eY*k?69UOo7| zhp$V7`LhrFrmU*_zVqMfFH-cxg2goBk&z;mXpDVF9${eYRXQ8jQI}hq3Ui#Jqk9>h zJV+%PBjcY}QUDn@Vs?p`T_}?$twi4WJ^1k#K}7UhJJuWqc@?x9=&Uw|&pz0vF~@}r zmkv@2hY7+l^4VlHv=WXmH2F3`JVD0GW5nk?+mK+M^-XS%U?!>uzxct&@mICCP~A{u z#2Z!7$5mZ}s=FCzS&(rNkBpa30&V386T$JcofxAi883gHNtp`$M-|<3aodeA-}>Hv zh)*JBdCOJ6qp2|d80c=RQIo>@1rsJEM^VCfa%w>?iACOpASDF$gX{s_Ulrxg|2=u~ zyLZo68TiV_UMQ*R3->;_wjS69EGPNgqFE?`5D%#qeCKyJ`Sg{-hGA>!do0s`gD_C$I|`S=b^|(k=Gpd~Y+Uo-6`h5zNkv$v>Cz z3RAJAtF@P|)?TJpzD)1n`CegI)ji-eo=|li3*}Oaz)I<0|4MPs|K&%YJ$fb;XSHuX zxUSc_`17DQ11ryVs<4RcW#H$)uT|Z%e|`I#lgWLq2jHu>y!=i}_6+HyPsH3OB0r&q z{r8L59MA_m13UseSrw%wV*c)-s{r`&NB2w}`}>DC6@p;w+T^1xY$N&Lxc~o5lmo!y zz^{NE_uRH`7~q=^Tq=OxbKAaC{^^X_=1U*kRR-BBB0mJ#PP&0_5!=ij$06WxLGJfV z=Am4sVRVkya?)I>YE@G1eeiMszIMwi$&38lF>9kIA-1YC1J?pwK;0q~rs@TgBp()i z26)UE|EaIuviAhQ*Y7U?FdHc~69e#-k36r&^M^ooiI{%|`9A3#~J{>e(P&9k8e1&C zzUQ`mQI(XOUr$r9I|&mXd4`%?X&yz|KE(Wssy+$&e&9NwcM%A)ANXZjobCY*Rz*2m zf1K0q-LW=_Ik&z#^LUd}`v<`Fz-FL_1JUhd~_ zzC4^H!god`2V9b7tPSbk9Vs~lAW!loIvK!AMf1|f%*AE6=2Vad zplOy);TUi*6~qC@Q+It4cox`y&u#lo5#k#^TZ(V;$I z1{)(C#5H0;ZJ~*cmJ13243cHtUDje|advnZPBNJ{@7{NE&zXB)WE(u-bd}3x&eiCS zD8>9}XC`8Hl;t6e<<3QoMI(5h&JwSFadipxOi~@Ya0qQY-pi9`RJIUi$Oh+_Sg|v7 zQ|jPom8C|dv6O6tk?%|_2M*ToJ1Zx}9D9lsyJ&5y@E)B-7Rmv#jihoBM$wXs*Kv@a zC+JzF7DI`t5ZMvD#{jON8pw&TMAssB#t;v2c8=bBVin_Qg>V3;{^RE&FCi{+DGDUK zZWx@A{sQaU5U$}ay>Z0Fe6St+Gv9G4tre44*+%*T$KzVkR!wNjm|w(glrz#t)1Q$? zg?({7>4+wj(*rRcC8m3t?rq?F+Ta1-*Wx-~Lxg2~PI^s*j~LGQ8V3kln2#%b$OqOX zo0VKMf59x4@h3bvpWA8{e`5X5^t}gz%9HD$&d1PT{DIoWSfN3xR#I(U~q@G zR|#hdJns)(u&I}(68Y`^kzYVhn({48nJgslE%*z70*9umgi2)q0000Dg-^SS4D&bi-vjfhaM?_SzWGYw?(tATaE8lVkW4GaS%U^pXkkwqx39XOE9cLIBX z=PR(G3{-%s{akLxB>>l$0+0jF0H0<=s+#{WVUZcg=F`A?Gtg6n*aT1xS=0eqfEB<4 zFl?YVBXaJ(U~OfiM*~P=HnqND}xEc-{6DfX5;x`V24*J+3k9=fSwfbQYkernsWz7G!PR zxwW=pqNoV5GQ2wvX11Ye!g~aGxIv?P!CtXe0u-RzH6}d=a~F1tVtd{Uh=^aW0FT?T zTHS+G6}AC5t}z`G&>1%AAz*jNm=@qLu+o-sgx27}%)k?xi93RtQh?IHaj)B|6M7Q5 z0GzUC1w!|BQ-sjB0f!>MOj!n}>|IP##ej~0-|qmuz_EaH9X7`_Kov;2#w47IqKXp1 zWy#yX1>na9^SXULCoFFSQVuYo2;%||lL4;V<14_~fO$#aN#$03Ao7pnz)j8ItG?qg zAYZ?jP274tqGP!M;4Fj7n$eeh$3Ha4g;T)MdXJ)5J7yW2xVP=4unAxK0LGh)`waNm z_x%qAOai_Ia6S?#Ii4kKsy~6Re6FpT&iktE7X?fbKA*cCB;lkg$$(onDW^mF686!c z9KP)3(;^q;ZlB8*!Uof#C_75b?1g~&w*cG_tFFCeY(YNN;E1v;?psUgQJ zijgG-=^Yf=^|44v4HIiwLEm#5N8(;f$t(fH2(W!nq<`^mA!;?IZjI7^{z@0{j&%i5 za%mVi5{DBxk00006VoOIv0RI60 z0RN!9r;`8x010qNS#tmY3ljhU3ljkVnw%H_000McNliru;0yv1CL_yo5#Rs-4!B7~ zK~#9!&750oT*Vp3e{=SDZLjZ_B+d=ua7oC;gi8uZ2_Yg47on;sec+)_l`6GDl~AEY zNUd76>SHBP`_NVuYE=oS1PY=N3Ly;yN=gYN0TKrY!3l9f?v5RMy=QmNnLf-n-pS&% z*LzMVBaOWF?w&LApa1v&E;Gl91lsP+8vt1Q!fO>j^Ejlk&;iWWFiS%>&lg%hBggt@_rrA zs0~O(1?PYkM8o0`762^~X2u~9Nk^z*U#FZwr2tP>&ojSgfdOT}IPAg4J{`4fcU{E5A37{i{JApMCZi!%F9vW>Aq27X_rI$BdVWpIG&-BJR=&ggfX}Bc~ zLmEzMH~{R5U_Wr|iZf;^QF{a$LZcURim<#0>m#@=guWuEK$J9J#*#`mRb>FVfLU-- zvXW^LrnLFJ=))%p-tb_L4+qk4ECoZqGq}?=QTj?705`>JwO`;dTx8g|IRQ z9XV*tgDRAnhyCf*utM(NBcLcC=St?~r(tmhy6fR~;L`}+((od1G*QM(FeNfmTDxV} z#CwlI%Q#$9gf$VY4`EXgIy= z9#%@th&j_UMwiI_GiVCnE)91TU|k-rA2TeO5L4|iLwyC_Ld!yo0^MGz?}i~1>lVfOH^6E(#^d`=&-a1o`$XzbVbmZ zf}s%VABO&AaQb1;mMK#)$E#)#L(mn%<`906gZoF}h7s^aWl-K%2uLN6Y>&jGFPiYV zw*cRd;71y6Jpx_7sdCQ#RI2?ns6)_p32rIE#yqSWhaN{uvdxGpH&Bo-)^1z9Kf z>jtiI((57MqqFD6s0 z_h=JyX#wT~%jES4@HvL>!vaiPVYWG%g64X-2^g<~_a23z7P$B;Nm^aYnUpbE=!xL& zJgm<_OQ9@2sWmVQFa?%&;96iMCevIXpo{~@MQ_rMNhJY}WzHo?6<|>dY%GEvf!Bcx zUt-Me%^M~d^IPyoq5A|Z3*d%4v=vKe$6`uBBGXRbYT#yIl`tjk3_K7vr5v3}B_Qz1 z83)ka0Bgoz)PsHtOhvDB9@;{<1Gpm#9r>~(u2!ZLgblNSZwVmt1f+D?XB&+dnV3k@ zP2#S`9IT9BC=J`Uz>)80U2xL|-$iNEM2oi-V08gni!nocmnrjsK41-SLj~1W;+( zu-qh|tN><7A&OFBm?$t(e$6OY{2BEA#W+R-OaRR}SWq-u9Ca$D{kBT*)<>D5RoOgY zO0QgJsem0O%p4~tYt+y?2E8K|m=oY@Xb+%0AM++kR<~1D{CbH?YHi)X{lI;~oKRM{ zR#UGuZf!ON7v-QgYr-h}Kf`PVokeJhY7||Id|E)656n$KEbAp*2qk|~n0?xD5^wbp z%nM+yl1RZDG4tq-%#G>wEEi>0R%9CZ)yp!5ZouXG}2rJ=3>odI+OB58gAJ<2S! zU#p9@`$i4GBIylDE~f*?pqSDZ#+#B(rleXIL01tvmG}xTZ%!Yy6JuA>no>QmPv>2-zpVbj!*ut>XKw68z;br;N@z)y0^m>$!|k zItD1VvggfUVCR4x0?G*|Hd#<*d}4E{Vc_2h1%2{;92SlE zNc1J;#3*MZvz~6H((<`ZClAvxX)p88)&c?@BMnhb@@T96mD&if7qjDaT9{(LKZ4mA z=*OH2imLEZ^%CRsh#7|oFej9QN4{sgUM_BwZFc~sib^j5hole7Bz&hnat?C>$eQFl zN*iyJHD78ropS92BH|#Fvaqj8*9_x|Z;09X8w8`u>@ zpskG$3459=EQp#U%b($*7L_6NMXV_`n8STn>9X@`Gi8gCNuQ9+y4QK%ZmthG@rGC7 zI#xvOj-&;C-Xy?C1#UthV|b*zf-}mjX#v3dIUFs(DLu(S;+X*BNvM8QdQisG+zfIX zPjsaq*ZK&vLpfY#?~wp#bs)z6SD@fz9h|6>p?eD<49#Mmj{{6fCVf`i?zyswmLFwR zRa`HUNuLqHKEvg-&WUDSlzYd>X#x*EuQNS|bFHj`W6H2Mh~t&A0?L2@WOFFr3FOoa zuIUSN28A{5;OxE5%V%O7I0!r#Xn@a}#OwRALSqU()uzVOrejnMw5;GE(S#P!jW$YW zV2V|y5T3)fOVT4JCi+5jD=YViahiKr!_nDryxRtN5k zws199T(~{;P_2tqWv|o1%#THTMu~m?FVt|R0E15&8Jn03&5+H&$6hHYtyya<^-qb* zJs>8K5zo>kZrrOP)D9g{ZIOHCB=*T~<`33Jj@xNc2p7jl7oKNJ({ms*V7;H$AnbxFY<5B8+w*x3YQ z{#v4rj^KnUdB;LrV6vFATps`j1(5ktb7^-1JzXIbwS}))k|;KcsJ|~U&TxFuz2L## zH0)0ss{~-fn;+qI&bmw^hCSHr!GA>GVg-CQ06U5~*uP5(QucgIhSKSS@@%^3qyyJW z9XT)8*eL>EAG02-P5ichH5`1YoH1>yZgR8LkrccKye*84WlmNnhQv#KC~kLLR=>?L zB~_tRYQY;6^En_u=ba?9mPi-)Siyb`1HX{C^Ev61d$_HT)mndWk5bLRI$?fmt@l^X zibfLGY!^4)BtmR!F|N>-$3=&wMw68~PO3Ifl%N6eF&6qJu zRu+4QhWB4Gm`@u&eW~3GeIfi%!$!;lpYUI0lP7Wo%?r4jJ6TfV@o@<%am#v!xrOR z`xX368E=JJ$ibFY^MMCPsx+laZ7b*p;6em@6r9)ap@#2CDVCTe*DKNT zz&m0(JAgxJhbg|TkN()tCbLO5TioA5(VfPx0VsKyZodGs z3)o{dWlO68C9^;((kBPvF=mo>*n>Z5cw3tzD{CST-6G{ zvfN~v_Z&QX+21#r|L(B?_$KC@^CC)nV2xh^5IKnfFNjmvZ=5Onb~sMLdY^kg8mjuj zvD|J7Mj~U!ha^5&0o*3+xsK!?1#nxziB0wa`xOl6(w-jGc%MYV1REiapFG8*=j6G$ z(yG*BPG_tUQNC8HnT?d5wz2<~!TzHUH_j_B-W8m5swj1>ob(aVyH7FIfcI&$V{jC zB$L9IB!NH6q_B|?DuCl$Dt+1{2(?zf8u3|M>rgW=e!!1J4^icGM`M=6``-k;p8B9? z6HJyt2*6mxj%u_>$uUBuPw!P5GM`rU4TThdF|hWAHo>vlAM|=~7&%f{ET|1x1uPu$ zMkA#FP%|)&na-w1yGx7x!7aCff+Wju`-Ocvd&1jSQU%BIy4F~zIS}F0_1Y}KzU-C+ z9v~zRjKQJS;au+%sF;lP$K4=m%|TSgpv^IE>Q9}?3;o%WC;&+Ou0;w631EQdL>cbQ zufcS?3&$I-AtcF4-h@DOyRt>++TEIgaZqb(T{-VqH6_~p=2Fn{?sM#}H)Md7RVZjj zqYStDC9LoGlT&xD7f~jKk0t6OqnZDa>Wc;6`774 z(|I^wJVUZsrz%bNaBkeT87(mk0%%b-tsKc3A(3;E+yS^@w)uu*%fV8X4YSR66Tn*~ z7rH-^Nx{g&C|C@zX<-wV)0r^ad@UfP4Tlo}K&jd0eLhxItX!}F@S!{xjq^^7mKbz< s^xFVb0La@j`31lNVAyQ)S(5$o4?9;coJ$+U`~Uy|07*qoM6N<$f+-<)p#T5? literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/status-35x34.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/status-35x34.png new file mode 100644 index 0000000000000000000000000000000000000000..e1594511a0fb17c8ccc44e4faf699e4ebedf57f3 GIT binary patch literal 2218 zcmaJ@X;f2p9t}hm1u6kqi}*}XC`$4|*b+7&2@(+uVJQe|wg(B2#pDG-Xb})p97I%> zU>yVj9hXLtC4=VoYyQ?Ff|Ge6UD(PED410gtqk^}RI3fX-wTQvW1!4^wf$;K`i&>mR2nN_tJYVRIetoqG4e+_%=Kxw z6lOqw=KDW|rQxY!2*-e=BAJAv8C)D@F_f4@mOw06BncOZl9swi=ZRpElqV7cN|Am-2M3dALMhA=av&PT8?E7E`Ft*khzBVYB7qt}aVOyMRJcsaDtFNc04pSZgH&tHRh!&jdvEeUrehxe6c+w)OcN@F}0Dd<0Jy11JEdb;qv~vsI;vy z5ta)I(aE|UL2Hp##jQXRX3Tt)vn3on=ESjvd%p9}+s$(`y>r>OE7lIxY?|_X zIuX8JU%RWsz;k?)e`G~ud2xmH$c)>=OY#A3&CaZ=#&hVTAo@?CC(ZkKu4_uQ%2=C* zh(nd3D=UMe1?@ri9R z1>G`FzJo)XhZZX5Fa>ZK&+hikC;e;CCzj`mC-5%ZTDgZnnCx%a8q&;IQBBWk ziM2WXDB7s4AmX@{->jF*sGaVOk?paDE6W^kcNm441w9qRr;)p!o$7SX^4sfvQ&kh^ zHdIn(DJvXi1_}^Ulj?G2YbRw$r)+mP1U`6jcmp-g@ibm%^;x@hX5!Z=yNsu^_IPb3qd^6^Y_Q2YgYZOWSNF}+h` z27dt@H{PvzoAz_yjVpHGY`L8?e$bJjqoP`;VKa@3h~qOwfegj#gucG{?E&9bM;hE)Q`;k|ZB(%T*pmQ6p`TITw^w-x{q`9t;Yz-Fzu-M>@wM_QM#x!}I>mX?$_U>RaWg zPUCf#HaZ=m9vLV5;N~A+UcLVAOvb{Exw>a=FB3Aw#rbU_FvDe)c}IC6-r~6KHxnJP zR`1^+mMt{2rhm9-)ZCW4EzF{#che-Mo)S}k_WUoKE5feO^8T3|Gcr^6L}#+Lm@8hb z$D9?Xb$AH;*0xW(vwg_zX2~WKXMNMJ5q_Q7c>#C!uT@jEhp+cu^LTyy=zRjy! zqdx{n`t8%LUT(_>wG3uI_i=Pxu}u%Wq+UL)grp7LB@Mc%sH(=(=0C8rS1)8&v3yU2 z*RPpq>|*YDueIDWv*LXB!iag)({BCokzWg|A`h1KD+O5_l!iUTIj`M+lSJBXds1*R zCeegEy8A($NHG+uZ}^0rw=UsW<0`beh0{noYF3>vG1??dKBF_$6ENkNOzN}nRbkb2 zk5-1%$9tWd4*z!q$Kd_>sMp1)13H;63%$;H$z<~;`xJYX&x;KXuybm1W+|l*wq_)C zZteZ+#g=V;M~+;%Ho;00006VoOIv0RI60 z0RN!9r;`8x010qNS#tmY3ljhU3ljkVnw%H_000McNliru;0yv16cnDn;GO^g71~Ke zK~#9!&74_~T-SZxf9KqLdz*d1%m9OB0FVF(fB->|1V9p8WSXK}N+K1dn28mYEk~~S zC8<>M=Ahoq89B^Bo>Rjx#?k|jEkBRZBG5|SlSGHDP1u@EEx5&#GL?94n+JOwv3iJRma1$s4f?@_3RZ8DF`1)I8OWl9A-S0V;+tlX)#QWd)az;$j0#X2# z6Kh&ArdzD-6l*#W=|H4IgaU{M<>j~|8qsnp1AR$74QgxK} z4}R}2gJuS5sphiEZRE#4R#(6KU)G4VyMgT>TM*eO$U4NfV~iG)IAV+=);fqN5v#=* z7nEq%jnR$)0j;fa-9&p{fJhaw<01?Lml3(Blw1Nn<8x?gHJEbzcfaPCpxz;Iyhcp2 zR*cyq*6c!LJ0e@e+VzNS6JwZrSp(LB2qG3S22g0%1+CK`)*9uwDAz^19!l#vFpN^4 zq0~jColmsuo>yMx22e&jwS#~1<8U9^ z6k?MY<=0V;Mmeslv$-v9OKY#rHxBuB<9!=9pq_itrELnK!$?{dd}u z>daP2;zMH0^J2}*Hj27Tt%~vMn7~Jp4Ov99Y3gu{YHiz5NhLg*qGudh7 zddHOO{6Kq|qsnn^9QfXw<%NT?9Ok&odyrX?eYM5-^%oF%!N$>H>-*c1>P)+hf;mw| z1T`z3s|m;=2C3OG6_6+*HYH5GiuN*{PQLJ@+uqTS=RJnCub=pxubl+^l{4n6&)>$B z)BogmbTmELmDH<`Ns|0+8->rAz~5v2x{j+AY#7cnq)9LqUL{SiNW`L)MtL4?zJS}_ ziIdA;P_FwwmFvB&9Oqp7-lxZQ{lUMMo=>!HMg zi6QdABr6hdF$SB&AmZvwW<797XLBDOeeH{Za^1@ZzWd`Ez_q(*%w4V0iQoIXg{i;( z*>)R6hm&gMfC>DEZQx_0s4*zbgQArpHimK-G80>BC^CKB^zV9%mfl{7Le%6K`m;}v z@-#RODBZLLQ)q0G5Le3BI9{9Sc3-n5?ond@8*mBWwRiQE2ZZDoqnzI!Acx8fMz0rhg?? zUn#H_69k}CtMUDPDCH|J^TDy-{z}nn?=0>6!|#~IGv}_1G4N)RWE)_kP5pT6Du>3{r5);;zFotw6hZEq*3*O|U@nNqb(lF5)*3$ij? z=9wc%91~ZHj*WsLue0YR8-)^;rr!pp?_kV(uf90Pm^1(U+iqMg_13Qq?=*h>5gP^_ zHjWUJG|XW$k0>No2&^S2w31yrM9agEuc$tQyZHdE>psR<9>!l18 zVO5$+Lkbi4AoM!<;E<`#RB@bhv%uWbDuv@C9g-x^f;?+#GhH?e8baGVC}M@cSjJ*Y zcGCuS{H?Ds_~aov)^DPvw|5SdlvFl#&I+xdHO5*Lf?6$P0E|VFByarsqhf7QXLCn} zzxzj*hkomKq6NW1Q?i6ei0W9scUugr!G z@4fosJYz(JU%v9hIvYig+tA;G72Ry|QvphZ$OsdWp|EuWyI=bz8xB2B$Kd+Af|BN? zG=;Wya=m>-g%*6rBiqmi8h=+#9Gy-nzYf;6I<0L_o4}v3QCM6UPq_dm|M537C%*ml zelg}D+@*4-~eexML9D0tv?T;+$H?o~wc=)=*w_(}8TK-P+EY1J5$>?2EK-*o5moSZX=f+RFMzA7kM0r!a$?n8~*hDuppg z>QAjwUPr{H8Nnzlp!JrS3uiZ9{15*sKc_J^j9SH-^%6(x(i=CYEdoSnvR%FOJ^mE^ z`wx)s>1{$EHQClyx*yt3-xE)#!QiFy)QXejmBrI~b@WCAF$tn|9da>4`=dZz zSFc}l#oA6A1f3>|Ky1@DvDzWZ6>$33F|g-x);{tmnOwf<=Wgp8V9!^+&W^ACJ+YB47*TI9sqAycn!PHNR0fP3-Z{Xrj!Zkxm{Vj7wJ2OSL&um+VY;CA<5b?e#xoBx14fB#$Ptve{YIbzb_a8*JE z!LxDPZ9~7uhQ0^j8s8s4(GB9#3|pX4NleHT3bZ}6osNgL;pB3w28^4{(!FUjN@*sq zUST|rNrpcms7&K|9*$b6^CzB|q{l|88>L(TZKI$Mkz`I&s4{vA6pwHZcE{0&iY3W@57 ziIHXlU8Pb1PGzKJmNBj*aUN^4E7LNCD24X2e1g%NlujLE^1YwqkKDu?7z7czc0ab{_4%$I2KGIPuFWuc`2xYn4Km_3 zKb=Iy6i$KKcu4S1zY%(gJ9$a`L%=hjqDZRc^TT)4w6~Zk81BsD+^6#1a5OXBpn$o z5DJns=fKP+n+1uZMrp;|^VGOfV(PuOnfS#|@o!#B(+N5q=eS&?e)Te?Q}0u`a27Gi zl2%IR@)Xt&(XxIBC!0fzU2#TB;y6qhqm&N>fL)PQvVx6c;(G0#=S8JyN@q?`K69M7 zIMpy^>GZR4L@+)|@q@o598$X_p~2R=k#0tm8CI`UR~zi-W5irwD3hCBT46fr%r6 zi7|qSF=L#nGA5vED9R|60;Ao`gi6=dnx^fYQlK?93<*bW5RBZU z(Ax(MV3HtU=K2+8Zd}DgQJV8zOm`8Kwl z2(x)TSzOCn{_`g&T~bPwNY@(UI{Ex95xY@DqNdecDU^1wfscRnGXDBa*ljz2v?Df- z38p9Urzg{8V)`EWTcuF0hnR%8I7Q7QX?L|le&ZI}76F5`5cr5+2V+)Zks_t_xYBw8 zR1BaW|4;w@ijAXDgrIqiQ7Fg3)N9l~JxBfGIjrw5s23R$zfR)U=hoxhPs)|n=v*Ef zMN~gMPwC7FD#I7?OGWGgYb7SZl%}wyDM;dU;ZT#JtF*qQJ@2OW+%UCxLtBl+NfBgf zMagAHW22C8^d|nbVS=%d+e#M33U)!sRV~Fp#{p}JOGW%!H<-C{ncBz=OgO)c*9JcE z#2E49I5v*b^}$Wmz*MNTzNS6zn$Bbb=Hnx&E2XX~rLF;CQ{t;MVhl;GN;o#c%%|t5 zUmH%p^*r!0i0d`#$jfY6<76_}B&Kq8n97yQX>oqmFRE1uM{g32k04RhSOCyO+5`#F zdf3UgeCD>c`E$T1t!v70E}``$lnPd2@5CCy=}F3Gj#D~wf}~tTdl|A_J!Cq%Fxnwn zxaN1sDVjRxLAgXwE@2zjMU@i%^(%x^#eAs~vJ3 zU1U1CKqg`*@Hhnyd#}TFZ}GR|H9iJU&NU6 zh)lB5rW80X;rIxnKl%aVZ~bpn;M2NcGc7y!klnHklgSaqF=FqY?N335;f_W|Y>)wd)>NnapjmjQ-C2d=@e0CQ4lZ zb$!;+Y2JKmlIjeV^B+I4Y)RY4O6&iz<_j+$f9B8Lxy>589gppU zpPWkR<9Zbl`9R{R6W9fGuV^}nAl48~O%Rt$m|B%wS2s>BPqxrP)YXkGS_tZQX4FK` zj!U++4NpOMSMJ)K5mc;A2g;{@RK#+R=xt_-|%>PB5e zK0LCkw-w|tFi5&@Q8&$;MhXFIsb3rB#{c>rZbv6hTLdAGh1dtnmn zOquFOr>UL)kfb_;f9raKB@aA(7N!F$N;^lC)^DQq-1@tGU*p3#xX=IO6h|KI2T(7c z9TTM#Z*K3rj8gBCZsOd(V&*hh5YLneC&uvFJ80dqE!}g3q*yGBi_>+l>B^@{iKJX4 zDota<0HvK~HD&gK7}R;RdRONQr$tN$!o5&sovNmCa#pyY=SlOdeXe6)Pdt5KgHTJFCD)!x)h3c zZATvM-(!>bA0hG*uo1|vN@=m7J+pDty*ryOZk)Ar>C>*LwDvLZ1K_X3nv-Apv{C~& zvS$E*mp`1xNiQ+V*Ob=ZL#aPy8SC#KRC|KH3Ia1b3u>gBMNa^4YuA1M<+J0H06%O` zPq-UX?h4Gk+KVSg<*)V)OeaPkS4yTt{!b4G+=3HOfDmZ0wtxAC)Zd zP{U&FZG7%;FeeY>NdFTZk95U`ZoJ=Oy3+;Ixy=fBfsG zZzbIROiNmy&xJ8}4dr2~A$l#qnua(J0b3d-z6)q;fK_M&s_q>rM;LER69GyMc6^VlVKdkKku_S#~SP4OecmOB1l+4R6_+X72>Id;MF<}$vVXhA#a2EsE7;2 zLobAv@Kkvbu|pw*MTf9UAe|c8zxP$P#=CcWecvu^==Mf`zNd%h`#hiDpXZOlZ?v$m zaLTsrKGQVMIgS$n3?al9zu*5-QIuCBBO`n7(^rY!TR*Z-sSGOS{QP{_G|k6_Lg7k2 zpRWt#@puS@Liqju`+-1UGLcA>>wt$~L%OanyMf%))YN1IgTeN`{8PP>sdA{q+dDfu znNq35zbF_CnjIY-q2b}-QWZEmI~&$C?OQEOQ4}1<`QuzD6#ClQ+S(TTzI-46Ow(Me zg%t_~ip3&|QZ0gFvB<%}L0y))T<*mF{{D^Vy>&z;lR2yF`uW2PEX!hIVuDmEg|6$> zxq*QJq?EX|C7;hf1Q430_12m#EiE)RHxmd1n30NSSpo{3){ATxS57w(A#^o8?jgn+qQpug>2g{2;f7lVObW5M1r249sstt zw<(v)_TtTd{5CzHu1K%D>6H7Vr^fB<-XyB#Uzb>P{5z;0DwRbcEv zmd#rcq?B=B=r2JAa7IeG@b>QM`VSW1&7G56%6uZ3Odk7H{t$2hX#2I~Bd`XnNGaFc aqWlDFB_VXg5%+um0000URLt?Czy;uK zDdou^2nd1zDJ4=$G)+TM6f{jk*Y(H1Ja7m2!LqDyS1=WGJPmvjhT+88+FGK~XwYmn zV-KpTDu!WTnkM`9?ORk7<%VThcXx!TnByNvDL;1{=YZ?FbXwahx~_BZ;6V(-r~v0J z%c>285ke$@&pgk&w6e1DOo;6<41-iERaI5>-EP-Z8%79`jH2i}$8p|r9EaVErfH

ho3hTjVz4t2x`Ar9B;^;>IeYwUHTlq@eVACpqv>WGn2ezdx}dLZr~zVB13)sRxg z{!pz}!}oo99m{c?2_eL>Ef^uhQO9v6;;P2?eSF_1m&;+>c1Mtu65F=P<#HSA0%K!iOifL-fm)EMsVT0)Ox)+)oPV; zxlBK8qS0ua08oA3&-PP}El{;uZ979dfvh*fEPxsWLAu}m&<(T`$a;a8LWra)rA!Rc zuw2(=Zf>sQ7}0DtnVXx#b=`rxauh|T3e*SboVIN6%6{R5yT5{U;pfmEy2wrglmP|Kidns#q}kdB?! zxt9h1=4lBpK0(n0QJh1E1se#mAJ*JREpu@;jUHFt;&Xnhe@SUJu%5-@~4g4 zfngY5#VuZ1TB1}cVcT}s^4$)yZJSc5#M07IFUx~`K7Vfmvw#2oyFj%c(89t(*FMq$ zEi5eb3qjZQPuphTd_F%B1i_a)-#kpEQpc}dyVh(Q$gf_#`U~*-^O9(qc6xSpcD~~b zY#7G*R?R<$6gERTZdI;dzkV_d!(Rb&4rWbXD>sV8;%wJFddK!`yui5wzdzJ8?G@mQ-KZMibR48B=KuRd zl5;QKtDoh^4}q6}uY27#I%E;ZX_|JtSS-%(%JUiNH}F^g<+l2c8AZ_>z!~5KaCrSe p5kLt1u|AT_1K$>l#lHuU{|CQyvUL!rQ^EiM002ovPDHLkV1lQyMwS2o literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/stopped-status-icon-70x68.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/stopped-status-icon-70x68.png new file mode 100644 index 0000000000000000000000000000000000000000..0dee5f01bcbacd5d53c7622ec15a40accef2cee8 GIT binary patch literal 4790 zcmWky2Q*u47>-iZR-<m2n&>mlvN1F9u!4(E z%t4@wcMbJ)EuMcL%XW`LETTyfTOk7t7>z z27Kv+;~}Wa?A=0fUWt-2ECEb62fe{(Le*n=0p;~UF`XCeo_Kjdt@9<5f=d5$*I%L` zzn&2eSKn$bySQ#ll5pt4@{qt^4Q|IRf8A&;TMaR&D$eEz9nhzsCZ&08CQh+ByKyhx zq^zhcJ(u%lOofp`d4k(f;j_x(t)F2}%)qMK6UapNuYqS1u3r+4Gl^oiA+(KJFgK2| zEHVx=`pRj-CC;5Ii80S!Z>?ZUl)Lwy_<1OfJC7LyD`Q*0!a4|EoKNry^w-VkpitcI z*(psnR4%qVC2^QuSr2@qz%9C$;Q4jQI zYbeB{m4ZYfxh)Un6t|3ym)O0=Jt;3i)h`@7JoV&am{eq|brL8>Dn>4Z?Jgfp?KLu_ z3uGS^)en3pG}(JJ^$|8V30}+f6ncqWk6DaTF_T8nl%NXSjJA^)k$s_~N>|S|bf(2b z??4E7Q*R+bku7D5#-Mwfi{X_MMoF^axK~{}&EzZ|#zOL~l4GS3=B@JTf!DP(DDEkh z5CNdn#0Q=w-51;8CugHomLh67WuD&!*`pWv0_WOa2TFW%k3)k#1HkUh9Il7XSSj6d zR^+Geap^W@1T+5FYnAdoB-FP1W}9-a@+CREb~<969YLXt+p#@c-VwWIyHoyy zT2|}CU)_pqLHFg3JH(lVv<5p4!ytnl0=3dy{LT`}uk}|0gs22x5vh{;a4Yb>3duJC zIOkt8Eq`|x)-yQ$H_&#vyF0Hh*_R$byx(IsSUY?Yw`eU^IE%Fx;s)m)hqjGVfBJXI zv2<{eYpUeR|4D&v_PZ#{x2ka;ud*sM`iMv?aP=018;!W`IM zes;0kWmPLn%P>oisZy*^d{l-?w;gMB$6HRgmX;P(e<3S(c?vF*LZLN#GwHT2_v$fF z3AnN{i4Xa zXj}Rme$ z9DLfCiQ!_PgW`n71GmogrJ|Y+|LhGe*G_Iss5kG#`w*8WRDNsA>MxLWxi;8ep`!a42odYaZ2kolN({WTn5$F5VnO%=YFP{79FR4BZVfH zG-wguacFq?IuRZ^^D!~@Q&%V8IG#g)5w~G?gkGtIjNzf^%eq&d^?-V(D%bfV&6O0B zwcPpyEA1L!Tj=Ra`U|ls&>}!7*T%rP%Sq(Lp5?uxW}Vl%tWMcmw1qviUxQsJ(#WqI zFfzcX{q?jU2Hl~P1FH9}5u3*k?&0jNtH&C=-J;!!R|7z>>fj)V1gI5J3;i8kI36Q z9~56}vZ4YQ*mSu@K|!?Uh)?Up{p7>ye0%A!si`KSh0LZO;;>DeIa7+&Xs=$J*#~in zT=Uq|(`CLp-jhm*Us`hQCXted6NQ^Io*L+3ralvinStw+>M+`~N0oNb>qNti7+rfd z|DWf^2%BtXEFIem>mOURS++N_+@xw?AIwVml5R!adMV!1O&9bb&q9VTrc0vy5#D() zSIcv<<~6VYWbjG~)OT$ZJdIu!^=l9@F)?vb&M0rSg3b}2)(lo6zDy7UW$}1?Aq;@^tWAqQ@)lCErMN>aA5A(CMdF%^Yg&# zC>B;$BSS;2#U@!3H7HguFE0=fNj5Fv=Dv;~Vu#Sd6&T^$NH$>U3Wt_lc{riUeMGXX ztn3|@9N`+bx*gV{ z>*7qi^V3Ks#rbQd>W{F)sULykh#vtXCK?IJ8$7Xwz#2f7VPs_NS@Z+ca5aTw3|KI~ zpx|ZBt)Ood$~^>P3cFVX1KZl#K7RVNI}jn0(wEsjAG_fa__sQH+CoLO(0fNG;D zsA7PBDcR##x)L*KpPirj-&SwP%gY-=(xWwf`=e=92L}h`&V6j{f_LxUwQGEybmv2r zg&9{;9j1cgg<}d#hTRd!A*!u0kDvJ%N&z|-(FxBU9)9@j*N@)s^&ULq8PS?wg0_|@ zaBmY(RxG%3U0FlJASg(Ec_0TciQN77tAH;VZpapifNn9rNVz@YXWZQNE??=*n>W!L zpZpg41T)MX90t4Ng?*=@dpiPdnuLX&Ru~J4h%EGHsQljF_g(#VO@w3w6aPYHQuq^TkGK*AWWE#&nETb(BZ_=~&E!oFBg5u3u10%v9#l z;GXrDXPc@SD(1!w&v%^Lv2$~C*H)GGO5Fl3BK7p-)ul%=lF5r2ehbW}r>BtZR{A|g z*MI;lP#;F9)GXsJ`{n2hDW5Mo=8p$*f6yAd!gjZgB3EY`0~S-SPeom88iRSHWekuy zFRIsJKHui@wl>{^Bm(ClN6xRUP1XuIsB-R#LH`Z!is9RstbIE$U@Av;2Bgx$~p8cD=(k?>^mQaI1coYwbJ_|&9<_`obA(K zCrt^irvyBERs)FbJCzC;#kOh*ZfU6>5^{@nuvt)B%YzThKecCaWKrTEP;PR@DKIHk zy9lgrXlsG2km#u3Rub_%&q|5<;uU!LHmdpCN(SAC|)*Ue(2k`J8(EP+n7!O__U?vy{uIw$jr)$l&R|Bx{)TS z;`aEl^l*F{(b^Ea2rjzrj(?kB{VM)+yMr_0q~r3LPY%W+Rm1A8xw*N7a25Xh2(Uv{kviOabmE1R2(zgI^V{2FY!_Kz75`29$f&8e|m zRd@68QdUQ0WhF$Bh>a|-FRrXi_?o3&>(uc^b06E*rW+g*a{PPbfuK;PHTxDVT$xi9 zZZ$nL^sh_N*;#mZcXz4~22nJMCf5w8RIi2R>s(BF2U`>I(Z1*Tq18oM@X3=WrFeYK zr(D0EozaDd89(5GK|v}S8tFaUDyy3H_4RxAImc+-$Us_vV zM>yl=rbD&@<;Z;)gC>mUxhI!`7Zlli5ow{?4;t)>aX9xrc62VOwG}iqHI=O$cJtx1 zSrz>fK~d4giE3o%(cD{rU4`IO@#cWpaf#hso(GVf-oc17 ze?x)fUKgmuZnaXeJX;63wXADYvCBCLhQl&AbAM=V&J|}C5E9D&^eM`7XY#Yy3jNQhoz>&>G^_0ZD=vnwIHHj1Zmu!otY^Flu)3bGnBOL zXNhrFE3OYG*Hkz>!9Qr(r@-|i?;q4Twh-yXAdI#it7~g(R?x7k5}U1k)=Tix6fz2^ zJlkQHNxsY-|28)_7k#sr|_Lx-fG35p+xt`78Fu(0LW9XLx}42M;(U?o?ab`QSfb+iTvkdcjW{7E?J}&h|e(uyK4mUNOVbC`|9K?{R9+#lYU`6-(z@V=2SO z^7% z3-W7F=kwmU0%7_uo%20ci*{pRYk~NE4=B#%6jAn2N~@@O9T$XNhI^oQd!;Nxow<)0 zj`)Vy%1gY7-c#u}C9ERq(POWJ&Cv1+~zo+8WFni?xmyl7UWeTiV9~+dlH%-%*<(oKfUr z(!9=58)pn?E4-X967kn(?vs-v$4TzEZA&5cgmbC-?2J*3XFrM*sUBINB)#Wsd^qFD z;J7>!@y;`34OdcA1atqSIZ&m|G;xz|1vjIA9dhc@zmo<&15JfAmzS)u#7Y9aRKlei1|h1?VD6^v!xw;i1oq?LE7R+MI|C?`2?xoB60*`9I9 zXP^1am%a)_0)cz^p)+{w3HLT+I9b>~j`BAR{c!B${cwGAHniwna93YpwX*w?D6*J3 z+>-sc=2pb-N$lp#*}WqHko(*6^9V{qCwOS)1IKZrC)$hwOKn z4Y0TzO1FZUsLepx`TMi)NsUQLMT!+!yvHm^8ZrB$-d|IaBz(Rjx6F7%jbPPOTm3@1 z906hXj8UfK3@@i*G;PE?I^VBoBo&qw1g#5=bKg@JyGN%r`{}Q+dGya711xBoJ{X1@ zefvP?^f&M7>?UFDL{_}sa){G_zf zB(1^Dh^}lp#>mjC&X*$F1I$uomlgV!gpw*NuagA)Wvf*EKE`v$3SGIqmToPei2ygA zv%`RtT74EOFM~)?&GbwhcIP|c5xMtDlr<)a<-zY?fB#EYam9YVvs7ad_`?J;)Hl_u IyyN)dKj_G1-~a#s literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-20x19.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-20x19.png new file mode 100644 index 0000000000000000000000000000000000000000..0aa770cca4b8e749fcf928e3afea9f4e3fa5a003 GIT binary patch literal 805 zcmV+=1KRwFP)bD86c;TYM)TGyvCyhnJd1=(?b6x&gUe@7K zQPDNQs^B=#+iOTBxwCE^W514jUI@4bHc}|)^zyRuJ+^V9sdRU<)%RIyRB8aGCOt+k zJ6GSm)9F$v46_!>D`8-v5F~OreV0ce4!3VrDHfG`cU@E=u&`^l!EwSs-=}Oq{%d#CYKrT^n|}lH z`2Z1k{mSS1wF>b#h}0!$w^)q4@jT76JE~P!c#RwYt9;8-?eXPTeG;h_$)8G8bF8r6OS z@(tLP2CNC3pR@dUxPxrQ9tTGFKC1vbvl%<_(~(&CgI(2Q5-KRMiP)~8iIJp{KvGjv0v4o&!fxxfdzstuLrcrD%g$8b{W$xc zv(LQe_g>BjDJ6H?Z|hf5%pw(CwZKAP0Z!0VR9 z62k!72HTcc7L=3-0DL|Oh4nbl4jcncMk9vp%L0Orgw3Hqp*b$8wZTAHespxv!w%Het%d%urBIWc$@6!ac9uWkCbO0Np5o2&# zFrl!n0-q<7Y`=Na!Ev~6^ZNzW)uLb1Sa&Z?<-vr)x-KQhZ{3n>Mn)tL(lkv_T`kgn zKWn2Aqa*Kl?|vX9k%0m4AdUm+6l62*zLq5!7;plq6yJry`hub`p|HMvXh?2Or`WJJE@R?uu4nws>yh}>!G*ZZigRj_UEK&g}r*fy^M zD50?a*w`5J-4@N7C1`xpSJbC8C}`R1r=mh&S>DB*N=?9+CQn(Gdj`GyiiV=NZ(dm` z*tb7Gb+vavr_*u~fMOUd^m4Y#mwRsT%q$n|`pEAYCYu>&P)t)+xjCyU%$=7Xmf?hC zPjiMtUmJP9&7G&v)TDa?vTPX$g>_xAEL^il_vNQ-@?Xi$4>NRk+gvzbSg~tf(iBD4yy)=5NwZ+5Zg^7hbX2WTV?Z;zn@c+?VzMY9QJZl2U`tp+;C`UP%*N_>k`0;6Q=9%4)lKCW}cmd znK@Im0sU^PMn@%|ev+lKQn2hfwO|i2vasZ7mA<~h8CGCm|176#AN|%aG)?UBlvS4< z-hVH{iWM4j<|uhWes;M^dAVRubH+=->#M91Ulkn4Us$2VfakrI{rY;9{(gti(WyOZ zY7~YOl1$dU-)Wjyefnp!z2FS&_lu3UbKZ;f^w>Ed0E2^$ClFxQogf9TDyv@9me@9D zfB+BAj>S5y4X#&}Urt-yio#kT{;)uTL9wOawYbp3kM+eJio#N$?*W7Ch;>>UPW@=6 zJ;SKM88io%_Gy|}3>5PiVvSf^DqWAE9;6upGz8Oug@8IQjUGyq4uw2iwp4s6pj z@kFfCYQOJKA^=V%P4?yub_f3sI1VoXjld$HX5vE;fDN3R7)jcJqp?ow(saoG16=AQ UrO7$QK>z>%07*qoM6N<$f=g_KjQ{`u literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-70x68.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-70x68.png new file mode 100644 index 0000000000000000000000000000000000000000..7b4def1da8e289fefb84d9fd4d11fb62ca718741 GIT binary patch literal 4938 zcmV-Q6SeG#P)00006VoOIv0RI60 z0RN!9r;`8x010qNS#tmY3ljhU3ljkVnw%H_000McNliru;0yv1EH^7hEf)X)64yyY zK~#9!&765`9M^rvKkv=#E|*;1qC`>FX9Hh=VkB zkTy*k^v|LfiULL2*fCPHY0;)_f+R@N_(~$#mF(KFB|EZY-8V&16t6w!c<#YkuYnPiDPpMS6kdA`FPI4Cn%~ zKmq6nBH#ir1&C4(j%m#i-{Y|FtMkB=)|8+6Mi{h$K`miS z7|?5sJP6z;!rj)gIts~$q2!~8yfNfvgCQv*vqrj0&6_Hv6iP`7Qa+Pmz3;O!n^kvb zvP_6(SVRs2d#qIlfHUtoW3DG^pM4_RW{mWRup*Ax8XIWuc7_=m5DVWZ4h zGGm_U{vVYHB7gbUZR*5vyH11;itsUG*&fvB4Fc&3LsS$wR!9PbmAYwMbB5I7`>g=6 zP&H7ifz}Go6F-|}Lq4yTWpjKGkK@*|SA-t`r(4RH8KxvteE;|N%)Ixx&u2SpRk=Af z+-EI2t>GhaOmDR+)M^lgLTu`@e7&1)WX#qja4hmOiuQIACaH1H=QXYQZpPU>lh-ln9)i=uEi_?Oe~a2 z;CYHnMtX8N?&&C~6~GwaD|JtxeI<0lA!dS9|=I@3C8J4sF`(UHbAL)?_Kn(M$Q_ zsY&TKhR2QN_ktQ9FPCLYv4}301Zy2beqErXEJ`=AP^$@3Q!qU(>*I*eSj!V4eBjh6 z+yB+C<(72M{`FM*@4nuqjvlq$rJ~#&8y<~99;nr1aF&*c)~IFyN!_@$?&eZc)cVFv z%nD#!ydzAvjp)B}Eo3nF4J)#PaA;g~fR^HpU zH@-VId?u*zaIq-sgFyT=P;16?Qid4g{%Q@mtOG|eAG6$IX zsysa{eb#bc6!KsgvcZ`TbA+~0ro^#3zN-t`@|r?Hv2sYUX_I?hEDGn(3lrmp>1pR> zniXa%V4UYHWV3Q7sH&ggjn92PJ5ea8$-nxGz|6Ot1sPMVGH5N2grV%L)#!{PnG>Hh zO6wqSGuPd%SieDY+g6Xwn>9m2nnEYks&L_gB?v5;OkI#$0ghviP%0@ej%8h;pdK|b zQY^}DVEih^?0bCI3}e3br#Zh|l4Tb!%56a)TcU_=6T3X8Sxt!|$mO87SF?JxV#nN=ty5&73$AYN%zooZMHgP|})1DKOXcO5@|wZ4Dm+ zK2)vJ8-_9`iEET8Q3SQBFfgEbXqV5n?V6Q?nx0-Y14{bYwl)V~8k4ko7z@fsJ_yuy zYh@yvRnLF>U-Kt^_tTYVwu|*=qf{-NzLfm^-L-Zgdm^*+#d#V zf2AVbSEU4B{^M-+$uH(sTgxqB$l5SO)5L69HzW8ytX|`oa>t#T4I4Ct!s5plIu!#0 zirzj~xpQTV*5+2Ri;g>@NOsg}+)=I4J-hDYJNHbdF>n!Mxr#C6inLd2 zY%_+f)=GP9>cyoSHXrMl(%Yk0vsSa?4(EB6!jw8Nid$~=*tp4~qoeM&Tge)g)NI+JS+-1($+-P-+&XYl*2IwzhO{fihSR6*`hWR$n4fXiVMsxQo8p)uT9BV~ zcEWA9X>Qx9>FT=ilWFZlc+FZD1V%AjI4_h+Gez;1lsF`i)AhcL#q_Vc$`0P zQ3^&!Ewx(9Lz|okLqWhwrN{y3Vv$wAGO>78XxK|z>$GL!>{z}+@z`S- zKJl2(&?-Hrq|g%05i(IkUl>Y%5J(2V4+0rfiax+^)I}TPwpG^t8-G z$!$4Bdk5Ea@9phy^Ua!S1r8px6sH9%*KY6Lh}*VS5K$c|V|-)CTT5=K9fRu)PeW&6 z>7qM1DZKiM;qqmnr$;d~q}aOEqpSP+#|FOx$Yd1RC1^{n23N*}BS$Q6zhxL36Z-lUA|AJHltnM4vs1BUi>6i+E)HA9 z#_H9@M&lU-6li8KMu{{IN?OcLx225nE5h#GF~54-aPgwd#3n;SvZVN=z1`(f)6>GM zuPVe!)7~ioM|oQL31hG#*k<*m-c!Ezj2q)L<-`fgI|nVtj#QXoQ1gCR^RUdBKe);kFcjbahrddsqZzhQLL&a`$# zpjZ?}Mx1$_K0TK?*J4913(J=)mM?etSlW(hW>_NufUh-GpaxB{#=06u?K-XVQc*a1 z#BlV8WqR7tKdscoFa(ueckrNPa?&M?+qSzdLK+jzHx;d2 zw4HBPXf3UXf=ek5zUQe4kYdxoD3{<(Ob8R>g1@}(wvB;u3C6AnlM_O@1i3b5$_Yw= zRuD!|t_X4L8f_xbmUCHlTieylNsCt*ziyI^S`biHFs0NCFkDuODZn%;T&YJ2L*dF5 zVeE>~)2nD}tGnlm7cG}YEMX{UZ(-p}1G_ke6DKUynyc?@n-sQg^A;?2&F^3|BESk_ zrI-XJAu**~j$skjh!8fzmXt9Uhb=2sDAul3Gv3L>&Wt7}>&r$Ka_wo^!Wbvi7tYxk z;0IT_PUV8FnV{ZJZ*YVtMwQ}n3XJ~JALUL*5hLORxM5(lhDt>^dsaAm))LHGRBFYs zrdH#+UTZc3b2*4&ICs`^?5O3!dCTzd$YfLipm%I7 z8Ea$$xY7(Tp64!lY|Jt;B8*?D8@x_h7P7>=COx0EP%6UZ%a-9`;qsV^duHyY<}__; z%IQ)Qt(HJ*bvBdXY$oFflAfxf;EaN^K+vep7z5R+i)_!Fv0S{Eu6F_7bGg)w%&OMT zCC4!g4_hvrmpQ;xs=}3VC-!mNRLL{Ynp3$PC)?Xqb*AQvqO26h5jG%n3p89b5OI2P z^r)qypePhPG8yRVQgn8?K2drTi_LKQ8OI#w!rkH`MX?B%Mx1U{s_B~Fh8Z&@!s&ql zeXg(HnUe1cL8<~A3PV{fhAlvMqpb0LD3^q{-!h1}!qu8JiWP&di6-K_S0E~O}!q}la&V&@41CANX z<#gvS9@y+O=6m0btIvEZf6Q7rEK69Qwp1~mm=IomDQ4fkm>?7eS18u6*Q{FQg1;cR zcC|lUc~z@9Q5_o6tQb^S<4owt5zF!8mRe2l{f4`yMWuLMd%W)Zl;^PjpOP2LdW#~K z3ku%Fp%^s_OtmVUJSn{Mj^*9MuAsDXNV9RHrn|cy87~yET1lJyivE6=DCXM~XU?v3f|S8Us!CWx-QU@cc=({b!-$ZOYXZoWmc>Lx`P348a(y!?`3YD&oFn(Jo^ zjEHdH&bz#Kp8aV&hcSNl9e2VD``tgjnzoiVOe{UXR-msLaME(bF{<*UV_nt60*_Cs80 zTy*k;<;-cJQWZu=>nk~L5R};!v}vt)9>udt)z{zUmnSbqLGOHFKhJMp!NZSu%H52@ zSWWM6syGnQf!#xIogrN>z7mb8wJ3dfEbPM)w-D$=N?q(?-+VHA6F zIkmsNy{;`i+jh{iZAaZ*1AJw4nf#nfN!d_MSSvpeOUVi=nlY*Q?8#=`)QyfxvM4Bd zYGr|1OS(Q+q%Ouy#w?X&vR_1a>)D^iBa5N9;CT2EZ-+H}R^U-!4NXME^P}T58C>iE zjoDAdwV>np4)9&zXV&uO%P*NS!1Fr>0r<$qX$O;$Yj-}IF?tf!2rF$eZUZ|=ly;gsTGuQ4~za1 z*sVSO_vb$|V*uapavxZTDNANd65ojLt8t}xT`T@x!8b{s+*o9`eqAF3UIYFUcdb9n zX4Oo*;uqYuWR00Wp1ZBzG3U_S;@+-Z-tAWSH1Hu{b)q}@_X!}=z#-r{U=Q%xp4}!& znc^3q{igK*Jo4HpmSjpdFpg=((;~bo@L}M7V8{Cam{Y*>NpN}qIG2Lbs6AeHJh#SYa3No4{`X>u|sD(flF!e@<1EzM|3i{)mu9G>n(5{Cb@Bw2zs0p>1)Y0odlf6fj58y znT&ep#TVu-JbQL?@1ox4Jz>m3sXVR?vnO%o1Bp1V12!Zsz8C0Bz-q_++R*waX! zDFJ3OVaG|}7%&V>;C`8?1eE53(kh?d&{%9ncJ1;qR@e;OmYCPg35$9YK)QgQghjK* zbW%l}Od{9IiMjp?coR6aXScZuh#w7tZfhgc)pJ$ubED(`0O#eV1ha5h&Hw-a07*qo IM6N<$f{0O#X8-^I literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-icon-20x19.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-icon-20x19.png new file mode 100644 index 0000000000000000000000000000000000000000..7ec3b56dead2eddb86c2fc6642c23efeb60fdba0 GIT binary patch literal 665 zcmV;K0%rY*P)vxQHG1c9&p9o0oP5lr=67`--}5=&ALk(`qxAIc;~1~Y zsBDWt9TrFzAst}_CqN`a&4WJ|KHebQGPG`;C^m|d>FNH6QSLMCE%xYq8qg)1F(Ca| z{wTr<-WPWD!%)+s!X{u5wlLM-r*kvUbzzb1$f`Qx3cIErN=vON0&*BJ(Bm*)xl&fJ@((gk#qoiBmjT^{ho zSUfSZc7)E`ZesDoC17|Lu)9Ur8Zg!|j+5*n7Ekm7ecJ@NKw~VP7)-stu=#@pNWD8p z?U~m&$D1 zN^b%{=}m-z9|GgR4d8p%tEBDxfoj7U7dVH+qbB*sl4KS!^XpW#(AE}IZY#>#QUi6` z^(r?v3iAb|VBxzhup8Z)c@zah# z0K`P3^V6jB4?aYa-@?Ezwe3t8_<4};O!V-GiJ#C-DP>~~yg7#+IO=+pi8h>(N;*H? zbr1lNm}KsOBkA*R!@%FOrZZvSe`OYC#!S*^tN2g1!(wg`j7?}~}(rzCy84?v4q*h`=Fl0l1!-2THvP8j%kjieTRmjT4l zz$~wUT<>B-qx8&cz%mVxQ!012(+XL#Xzid{WXPfHsb(YXWUbM0YA+AX+e@ikpFH#f zm?K$>Y$UZ?nyUQZ$oA5$DZSWUI9L1lzTDkjx;3?wP1@de6S&_G5~cIsw50an{_Bxa z_W3@r+|#yuWyQ9$K6&sL;85>eu@CPUnHXQOAy?MslP~vt4|MjwAS;e4^}d3Pk0$e%|)rZw>64NEiMdLDC0-HhKGS%s ztj*+{p?gZ(`+*DH*fqf6wLsRIwL#ff&hA-lzi|rifNRaz$f6qXrPB60#d3AL8%H8w z7BF$xY@W?Wz%gJSu(Nqk1dsy1H(Qc%;CiuK{VNOk3nzhFCvd@kWB>pF07*qoM6N<$ Eg3&L&H~;_u literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-icon-70x68.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/warning-status-icon-70x68.png new file mode 100644 index 0000000000000000000000000000000000000000..da206305618863f281b45bfe2176dfcd09549b6c GIT binary patch literal 3533 zcmV;;4KnhHP)00006VoOIv0RI60 z0RN!9r;`8x010qNS#tmY3ljhU3ljkVnw%H_000McNliru;0y=~0x-j@qci{j4O&S= zK~#9!<(vDDRo8XLKWm?J=gte}!DBEO7l;Y9G2q0GF_?#~+L2OK?GO2->f?vhs??}e zTe<0PtrRK!2U;a*s;aFjHEG(WRT4$5UyUKcK#U*Qn87yKW5CcE9`m02I%n^1e>msP z17=`6%mo5VjOO0MfwT8_f7f@dwf8#4BG~1(p9NsqwttxWp5uQW;m5i_D?uxO<-k&) z31|nF0SRyp7y$%~f%Su(U$*r-1J`?g-L5r``IO6{r9uGgt23^6kn(7V3Hyv&c5k z_hV!1zgGx71Z+ZV0PX{=Mp}x36Fx#d0!1j82o8l9Ot?4!=Rp`2usDUdf<@y}xB&D4 z9|9+(a0>W{2AC}dQwEPc7l}#-N>`w85BLqpZP6qY~$d89!0IRKjcZWve$ zW*{&#uqeM40j>nrXYdhbC&9gs#V5?gpMZ^ko9N#1t?FVhXi*p=Rk|ST0qrpKh~(EQ zR9h5kErg^+N{h&6V4(WH%KuD#7nXrjzhd>ZGcX4hwU`ywV}*Ngg-6Ua4MB8X(z~Gj zprgR)hBM|`qIUV%A2l0kCxjk_bTdl#N?|+rt%TI18VWTCH53xS=M1q~o<4mTn)M?& zd&~-$YXaAVSpi%jh3Eq__cMr&;No}9noeW!Q1?TBU0Wz~;!g>p0ERyJriIG!bwZDu z=}X|ZD3rV8D@zq>hSF*7qMRvYtNv-$$FBIIfJl%}F<(I$i(+$48;Dw$n`?d?MW?{O zD)d9(<3(l66jMTu@IOuI$-95DM1JBkDBS^muN1zBk}mlY3Kit5WNwqM^1_4PT#j4! z-~9h~m}|nKR?IbnD-2@sYew_~bA`j;J~_SRZ%007_NkZ?&wC)i;PL0$2$d%xJc*Ls z3YE`>w6qk}&~y!H2q0IirA>`bN>Dn6Ss7fR(_HZj7A@(4_+g>%-*^1EH?z=x!;JaL z45kdf|CcT#!(H;FhmrqvskVnu-6~&k!H+{orZJ=uV9aCj>I||Nb1k^I9dpI^u=qc) zc%PZ;we~FoT^s&b^I?vg&tpR}v<#(R2LJo=rC(J#_CQERGEn660o^K4^7XAhUmk@r zAsHqlBkKvZ-!sBjL63&gsbz!je|=taj<2QKV`u)}lCk3*N=F|->C+0;-3sMZ3KbNp zAg?^x`1Dh$&qY_yQ8@t?CYK@;e-EgrP*@|h+pKU7$;+3I0GFoEb2jG0*NQ&L*t;Es zIh6)LL;Riu<@8Q6kAiJWCYYBpd3_@D)VGgBQD5<)Tzu6db@mwwsIbwPBw z4$O5Ka|L1o7&-ghC>{LiXOZ7asO*+6ugzGJ7$`=XdPd6$Ng9|At@2<~h#jph z1>!hD3qvP2k3tF%D>U5;>@e5#&fu{>s9@2D-4A_hBD?Rv{G!re@7G%W$ZMNWvRmoI zc7^IjCPA6EG|dW16d9xADJT1e85*e&MTX0xSdj2ubagbdaYGkf9WmZ>1K{MSD<$|* zOvydYb^fNwFGKoA;8aG4_h$#@N>x@e+znxeLUoTqWrL=>&kYG|mhic#jlz<9D7qrf z31`ZIF+wm3ZB<+}3aU<(1UD4k1B4o2?S#tbgq5-Qjltu8I%M&Z;jT^J@K<`y?0HNm zuR{J;2(`Tml_d(bMF0g9JgKkJwsbvP*Z&sQ@<29urTH9Uup%m-K$pKqTImHVX4h{x z_YqP`NXH^`wRIMEJZ-8;>Bv4{@G8b!e*4);#teV(^;qfHa$h<3AinZ|RO^t^sgQi5 zz(~laG#A_HT(cDl_s)J}Qeb%e$5bX#ENa3QB@4()3SR-V+(PpX`H69Gy>(!wd(I58 zQ0-EOd}$Z3EBI1Z@TDw7ELM(USxAxsKCR3xnQ0}fwH6liuJ6vlw?)PzOr`o z%nu4z>R3D-rI3yS|RXL8`UN+y*~!YJKVScUmShUxivbgnJdz)if{~+@4zFk*vj}^-3q! zW0iiO-uBgDANtcd(- zh1xwpF)PRY$QrQaQi!rhbT^s`DD5Uxmw^>BjS1CNKsP8}j6(4j)C!Vc21$3lVvQAQ zD}l~MC}Dq*8RJk`D$?#r#z;vAs11njPEhhqW;_`VlZ;7Qq)=S&w%nb<83m9c%tcv( zr_hw^i`~VrWZMx0SI_NY2zGaaG94VuEORB7)!=t+oSp@oY{$w0RUp0lQr@|=Cavbc zSedM+?WVD2vU6(A8{!!aL^si@$R`;v=Eg9}dAzzwbTuq0&W%e{WZHq~&9rea4-lKV zA;H+q0H)3qg%as_FU~CMsmc3k%p%M+laBYIP{M8=a{|}|SsQH~n8HP1B-C9bk8au7 z^*n$v>mzFU$Sf*P1IA4Nryy*|`}`XN%q=~;zNU?UK8~7u&bp=*)k$D1DwdFXrDQGA z135Sisqm+tT}Or4Z)@z?_0^99T~J6q>E8V9k*P2NSkzXssO5b#cOsAF4NmcH?b-EA zZLvPElPJAdzj*QW-^duV;?e&qOFDVs@Kg7KE^*hGjbZMHnLC0sF(a|)`_FnPH9?Hg|77 zzT&axXD}vi>evFD$XOGQg5H3nQ(`mF-H=+zmq^BXb3K_Et}QB(jP)Yl&|0Ym^no1( z`j%|`MhbB5rEI!uo*A6kgSX=Gmr9b}xo~K2JFpvAL-yX{;uI`-k?5=al+OJN7PZ!; zcnUZS`4Ya`pS?7(c(u~>v!PMUy=b`SF`Mkab4{!29!9+U^Z(9;Lr($N${nxB%osTM z#D`$7lD)+F$l@`l&X@_3@#A?Xs9iOFvp5GOUQBnYKPUlC81||~#kb7dq_z~=AIv}X z&S1>s2+TDPfqp2Z<0$kju1g8bVXjE_0_|Kb=Nh3gCEy6~ve4TrAA6yHt{Y>3l{;RU zadGA2|62n-960n;8)zG-1-NhVGUEI#k((c%`+$AGenNV-e(}QLrvO;~$Pbv2;9a8} zK7+aMf&Clv=)b>>Eo5~duLCauhkKqlI58i;%aHr|7c<6ee{u5Kz-zzOf*;+1!dG)a zUPc36OKvbIr-6gO%fOF%b{-f47&y4g0QKxVFw4%-b*$&HpOlgHg8eht_ksS~9EdB0 z>5~i;`&TUfiN)ePiCO(*q38o9Ka=qA@x8Pume~PwB3$*e*Sxc z>}Ao{fqhu?^PV01E&=@CEB9yfn}sQ}0y8ITcD<;9SDzWg{1MD9BOL?%4zTSSxx%fq z8t_^!v@ZiEEGmvnrBiM5Mt4>lIJhTA_GdG%AKJ4Gh2I8t0c&&JX}L`R83lTQ7l8f1 zYdt#;Bz2}()P`$boq5P(|8rxEnFY_2nEf5%Fz6XzC$RN40HzOk5qK3i3Y@KjvQU4# zrVSk0n`_Sg8P_iy+S@7lr+_Db$AEP}YyNXH7*qkq$=)96&%t;ec&lgUf&MAx&mDah znK6?o@+k8A@+$Nt;2~gp&Yq1lw&y09mYcklfnt52!5l2+6Mb%oF;`}YY54*#{Mm87 z^5%3s+EoPZ&m-2h{BuvvoED%tuK@~V-CHjI9nCThpHJN9lS;GQHY)kgXr6GM0^R_Q zTGVuW<@TSZGc|J8b)!#X7&B`r&*fb7+JF_gIIjaX)C81v+yUO}Ei} z7jZbxTrcMCdJK3I= JList, ServiceLoader} import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer import org.apache.spark.{JobExecutionStatus, SecurityManager, SparkConf, SparkContext} import org.apache.spark.internal.Logging @@ -36,7 +37,7 @@ import org.apache.spark.util.Utils /** * Top level user interface for a Spark application. */ -private[spark] class SparkUI private ( +class SparkUI private ( val store: AppStatusStore, val sc: Option[SparkContext], val conf: SparkConf, @@ -66,7 +67,7 @@ private[spark] class SparkUI private ( attachTab(new EnvironmentTab(this, store)) attachTab(new ExecutorsTab(this)) attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static")) - attachHandler(createRedirectHandler("/", "/jobs/", basePath = basePath)) + // attachHandler(createRedirectHandler("/", "/jobs/", basePath = basePath)) attachHandler(ApiRootResource.getServletHandler(this)) // These should be POST only, but, the YARN AM proxy won't proxy POSTs @@ -95,6 +96,10 @@ private[spark] class SparkUI private ( appId = id } + def setTabs(newTabs: ArrayBuffer[WebUITab]): Unit = { + tabs = newTabs + } + /** Stop the server behind this web interface. Only valid after bind(). */ override def stop() { super.stop() diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index ba798df13c95d..a80ace578f5a3 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -165,6 +165,7 @@ private[spark] object UIUtils extends Logging { + @@ -231,8 +232,8 @@ private[spark] object UIUtils extends Logging {

- - {activeTab.appSparkVersion} + + {org.apache.spark.SPARK_VERSION}

@@ -256,6 +257,52 @@ private[spark] object UIUtils extends Logging { } + /** Returns a simple spark page with correctly formatted tabs */ + def simpleSparkPageWithTabs( + title: String, + content: => Seq[Node], + activeTab: SparkUITab, + refreshInterval: Option[Int] = None, + helpText: Option[String] = None, + showVisualization: Boolean = false): Seq[Node] = { + + val appName = activeTab.appName + val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..." + val header = activeTab.headerTabs.map { tab => +

  • + {tab.name} +
    • + } + // val helpButton: Seq[Node] = helpText.map(tooltip(_, "bottom")).getOrElse(Seq.empty) + + + + {commonHeaderNodes} + {if (showVisualization) vizHeaderNodes else Seq.empty} + {appName} - {title} + + +
    +
    +
    + + + {org.apache.spark.SPARK_VERSION} + +
    +

    + {shortAppName} application UI +

    +
      {header}
    +
    +
    +
    + {content} +
    + + + } + /** Returns a page with the spark css/js and a simple format. Used for scheduler UI. */ def basicSparkPage( content: => Seq[Node], diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala index 8b75f5d8fe1a8..83548006b7f9d 100644 --- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala @@ -47,7 +47,7 @@ private[spark] abstract class WebUI( name: String = "") extends Logging { - protected val tabs = ArrayBuffer[WebUITab]() + protected var tabs = ArrayBuffer[WebUITab]() protected val handlers = ArrayBuffer[ServletContextHandler]() protected val pageToHandlers = new HashMap[WebUIPage, ArrayBuffer[ServletContextHandler]] protected var serverInfo: Option[ServerInfo] = None diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala index b8aec9890247a..5e1aa05977b0f 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala @@ -32,7 +32,7 @@ private[ui] class StoragePage(parent: SparkUITab, store: AppStatusStore) extends def render(request: HttpServletRequest): Seq[Node] = { val content = rddTable(store.rddList()) ++ receiverBlockTables(store.streamBlocksList()) - UIUtils.headerSparkPage("Storage", content, parent) + UIUtils.headerSparkPage("Spark Cache", content, parent) } private[storage] def rddTable(rdds: Seq[v1.RDDStorageInfo]): Seq[Node] = { diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala index 688efa24ade0c..1eb361c1e0cff 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala @@ -22,8 +22,52 @@ import org.apache.spark.ui._ /** Web UI showing storage status of all RDD's in the given SparkContext. */ private[ui] class StorageTab(parent: SparkUI, store: AppStatusStore) - extends SparkUITab(parent, "storage") { + extends SparkUITab(parent, "Spark Cache") { + val listener = parent.storageListener - attachPage(new StoragePage(this, store)) - attachPage(new RDDPage(this, store)) + attachPage(new StoragePage(this)) + attachPage(new RDDPage(this)) } + +/** + * :: DeveloperApi :: + * A SparkListener that prepares information to be displayed on the BlockManagerUI. + * + * This class is thread-safe (unlike JobProgressListener) + */ +@DeveloperApi +class StorageListener(storageStatusListener: StorageStatusListener) extends BlockStatusListener { + + private[ui] val _rddInfoMap = mutable.Map[Int, RDDInfo]() // exposed for testing + + def activeStorageStatusList: Seq[StorageStatus] = storageStatusListener.storageStatusList + + /** Filter RDD info to include only those with cached partitions */ + def rddInfoList: Seq[RDDInfo] = synchronized { + _rddInfoMap.values.filter(_.numCachedPartitions > 0).toSeq + } + + /** Update the storage info of the RDDs whose blocks are among the given updated blocks */ + private def updateRDDInfo(updatedBlocks: Seq[(BlockId, BlockStatus)]): Unit = { + val rddIdsToUpdate = updatedBlocks.flatMap { case (bid, _) => bid.asRDDId.map(_.rddId) }.toSet + val rddInfosToUpdate = _rddInfoMap.values.toSeq.filter { s => rddIdsToUpdate.contains(s.id) } + StorageUtils.updateRddInfo(rddInfosToUpdate, activeStorageStatusList) + } + + override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = synchronized { + val rddInfos = stageSubmitted.stageInfo.rddInfos + rddInfos.foreach { info => _rddInfoMap.getOrElseUpdate(info.id, info).name = info.name } + } + + override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = synchronized { + // Remove all partitions that are no longer cached in current completed stage + val completedRddIds = stageCompleted.stageInfo.rddInfos.map(r => r.id).toSet + _rddInfoMap.retain { case (id, info) => + !completedRddIds.contains(id) || info.numCachedPartitions > 0 + } + } + + override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit = synchronized { + _rddInfoMap.remove(unpersistRDD.rddId) + } +} \ No newline at end of file From f796276e46efe0247884b6dce23eba72e37736ff Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 15 Dec 2016 10:56:41 +0530 Subject: [PATCH 043/142] [SNAP-1251] Avoid exchange when number of shuffle partitions > child partitions (#37) - reason is that shuffle is added first with default shuffle partitions, then the child with maximum partitions is selected; now marking children where implicit shuffle was introduced then taking max of rest (except if there are no others in which case the negative value gets chosen and its abs returns default shuffle partitions) - second change is to add a optional set of alias columns in OrderlessHashPartitioning for expression matching to satisfy partitioning in case it is on an alias for partitioning column (helps queries like TPCH Q21 where implicit aliases are introduced to resolve clashes in self-joins); data sources can use this to pass projection aliases, if any (only snappydata ones in embedded mode) Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala --- .../catalyst/plans/physical/partitioning.scala | 10 +++++++--- .../exchange/EnsureRequirements.scala | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index 08841c2419c35..446e156bd9dcd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -221,7 +221,8 @@ case object SinglePartition extends Partitioning { * in the same partition. Moreover while evaluating expressions if they are given in different order * than this partitioning then also it is considered equal. */ -case class OrderlessHashPartitioning(expressions: Seq[Expression], numPartitions: Int) +case class OrderlessHashPartitioning(expressions: Seq[Expression], + aliases: Seq[Option[Attribute]], numPartitions: Int, numBuckets: Int) extends Expression with Partitioning with Unevaluable { @@ -230,8 +231,11 @@ case class OrderlessHashPartitioning(expressions: Seq[Expression], numPartitions override def dataType: DataType = IntegerType private def matchExpressions(otherExpression: Seq[Expression]): Boolean = { - expressions.length == otherExpression.length && expressions.forall(a => - otherExpression.exists(e => e.semanticEquals(a))) + expressions.length == otherExpression.length && + expressions.zipWithIndex.forall { case (a, i) => + otherExpression.exists(e => e.semanticEquals(a) || + (aliases.nonEmpty && aliases(i).exists(a2 => e.semanticEquals(a2)))) + } } override def satisfies(required: Distribution): Boolean = required match { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala index 6b398640e1e56..a1381a23d47a4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.internal.SQLConf * the input partition ordering requirements are met. */ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { - private def defaultNumPreShufflePartitions: Int = conf.numShufflePartitions + private lazy val defaultNumPreShufflePartitions: Int = conf.numShufflePartitions private def targetPostShuffleInputSize: Long = conf.targetPostShuffleInputSize @@ -145,17 +145,19 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { assert(requiredChildDistributions.length == children.length) assert(requiredChildOrderings.length == children.length) - // Ensure that the operator's children satisfy their output distribution requirements. - children = children.zip(requiredChildDistributions).map { + // Ensure that the operator's children satisfy their output distribution requirements: + // The second boolean parameter in the result is true when a ShuffleExchange + // was introduced to satisfy the output distribution. + val newChildren = children.zip(requiredChildDistributions).map { case (child, distribution) if child.outputPartitioning.satisfies(distribution) => - child + (child, false) case (child, BroadcastDistribution(mode)) => - BroadcastExchangeExec(mode, child) + (BroadcastExchangeExec(mode, child), false) case (child, distribution) => - val numPartitions = distribution.requiredNumPartitions - .getOrElse(defaultNumPreShufflePartitions) - ShuffleExchangeExec(distribution.createPartitioning(numPartitions), child) + (ShuffleExchange(createPartitioning(distribution, + defaultNumPreShufflePartitions), child), true) } + children = newChildren.map(_._1) // Get the indexes of children which have specified distribution requirements and need to have // same number of partitions. From 718a20ba7c2faf3b42669577c7d9ef9d7dd92272 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 15 Dec 2016 11:44:04 +0530 Subject: [PATCH 044/142] [SNAPPYDATA] reverting lazy val to def for defaultNumPreShufflePartitions use child.outputPartitioning.numPartitions for shuffle partition case instead of depending on it being defaultNumPreShufflePartitions Conflicts: sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala --- .../spark/sql/execution/exchange/EnsureRequirements.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala index a1381a23d47a4..6fb19a67401c8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.internal.SQLConf * the input partition ordering requirements are met. */ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { - private lazy val defaultNumPreShufflePartitions: Int = conf.numShufflePartitions + private def defaultNumPreShufflePartitions: Int = conf.numShufflePartitions private def targetPostShuffleInputSize: Long = conf.targetPostShuffleInputSize From 0af421b8bf0b2f0ae8372e45cbd9b17ff118a515 Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Thu, 15 Dec 2016 16:52:29 +0530 Subject: [PATCH 045/142] [SNAPPYDATA] Code changes for displaying product version details. (#38) Conflicts: core/src/main/scala/org/apache/spark/ui/UIUtils.scala --- .../main/scala/org/apache/spark/ui/SparkUI.scala | 10 ++++++++++ .../main/scala/org/apache/spark/ui/UIUtils.scala | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index 979441fa33b68..c25bffa83b496 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -160,6 +160,16 @@ private[spark] object SparkUI { val STATIC_RESOURCE_DIR = "org/apache/spark/ui/static" val DEFAULT_POOL_NAME = "default" + var productVersion: String = new String() + + def getProductVersion: String = { + productVersion + } + + def setProductVersion(version: String): Unit = { + productVersion = version + } + def getUIPort(conf: SparkConf): Int = { conf.getInt("spark.ui.port", SparkUI.DEFAULT_PORT) } diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index a80ace578f5a3..8e07b43c1df7b 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -233,7 +233,8 @@ private[spark] object UIUtils extends Logging {
    - {org.apache.spark.SPARK_VERSION} + + {getProductVersionNode}

    @@ -287,7 +288,8 @@ private[spark] object UIUtils extends Logging {

    - {org.apache.spark.SPARK_VERSION} + + {getProductVersionNode}

    @@ -596,4 +598,12 @@ private[spark] object UIUtils extends Logging { NEWLINE_AND_SINGLE_QUOTE_REGEX.replaceAllIn(requestParameter, "")) } } + + def getProductVersionNode = { + val versionTooltipText = + "SnappyData Ver. " + SparkUI.getProductVersion + " ( Underlying Spark Ver. " + org.apache.spark.SPARK_VERSION + " )" + + {SparkUI.getProductVersion} + } } From d544f95e5b696d2793b7e064474a1fb53b654f47 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Mon, 19 Dec 2016 04:35:03 +0530 Subject: [PATCH 046/142] [SNAPPYDATA] Removing duplicate RDD already in snappy-core Update OrderlessHashPartitioning to allow multiple aliases for a partitioning column. Reduce plan size statistics by a factor of 2 for groupBy. Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala --- .../catalyst/plans/logical/basicLogicalOperators.scala | 9 +++++++++ .../spark/sql/catalyst/plans/physical/partitioning.scala | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index 5e41a0a6edc5f..28232ab47525b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -583,6 +583,15 @@ case class Aggregate( val nonAgg = aggregateExpressions.filter(_.find(_.isInstanceOf[AggregateExpression]).isEmpty) child.constraints.union(getAliasedConstraints(nonAgg)) } + + override lazy val statistics: Statistics = { + if (groupingExpressions.isEmpty) { + super.statistics.copy(sizeInBytes = 1) + } else { + val stats = super.statistics + stats.copy(sizeInBytes = stats.sizeInBytes / 2) + } + } } case class Window( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index 446e156bd9dcd..66fc8772b6773 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -222,7 +222,7 @@ case object SinglePartition extends Partitioning { * than this partitioning then also it is considered equal. */ case class OrderlessHashPartitioning(expressions: Seq[Expression], - aliases: Seq[Option[Attribute]], numPartitions: Int, numBuckets: Int) + aliases: Seq[Seq[Attribute]], numPartitions: Int, numBuckets: Int) extends Expression with Partitioning with Unevaluable { From 5bc6591358b5fb4cabe13e07e90ad8bd4db4af90 Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Tue, 20 Dec 2016 14:40:38 +0530 Subject: [PATCH 047/142] SNAP-1257 (#40) * SNAP-1257 1. Adding SnappyData Product documentation link on UI. 2. Fixes for SnappyData Product version not displayed issue. * SNAP-1257: Renamed SnappyData Guide link as Docs. Conflicts: core/src/main/scala/org/apache/spark/ui/UIUtils.scala Conflicts: core/src/main/scala/org/apache/spark/ui/UIUtils.scala --- core/src/main/scala/org/apache/spark/ui/UIUtils.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 8e07b43c1df7b..0ea856466a507 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -240,6 +240,7 @@ private[spark] object UIUtils extends Logging {

    {shortAppName} application UI

    + {getProductDocLinkNode()}
      {header}
    @@ -295,6 +296,7 @@ private[spark] object UIUtils extends Logging {

    {shortAppName} application UI

    + {getProductDocLinkNode()}
      {header}
    @@ -599,11 +601,18 @@ private[spark] object UIUtils extends Logging { } } - def getProductVersionNode = { + def getProductVersionNode(): Node = { val versionTooltipText = "SnappyData Ver. " + SparkUI.getProductVersion + " ( Underlying Spark Ver. " + org.apache.spark.SPARK_VERSION + " )" {SparkUI.getProductVersion} } + + def getProductDocLinkNode(): Node = { +

    + Docs +

    + } + } From 587a3f9aa4788ca793df415b38c3605f45a9d58e Mon Sep 17 00:00:00 2001 From: Amogh Shetkar Date: Wed, 21 Dec 2016 14:51:24 +0530 Subject: [PATCH 048/142] [SNAPPYDATA] Spark Version 2.0.3-2 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 402593595ce2a..2ab964fbd6eb4 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.0.3-1' + version = '2.0.3-2' ext { scalaBinaryVersion = '2.11' From f49a3ac3f7a0bdc9660b7023384383d6c1f531ef Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Tue, 3 Jan 2017 15:36:57 +0530 Subject: [PATCH 049/142] SNAP-1281: UI does not show up if spark shell is run without snappydata (#42) Fixes: Re-enabling the default spark redirection handler to redirect user to spark jobs page. --- core/src/main/scala/org/apache/spark/ui/SparkUI.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index c25bffa83b496..a4f8f2b798b01 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -67,7 +67,7 @@ class SparkUI private ( attachTab(new EnvironmentTab(this, store)) attachTab(new ExecutorsTab(this)) attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static")) - // attachHandler(createRedirectHandler("/", "/jobs/", basePath = basePath)) + attachHandler(createRedirectHandler("/", "/jobs/", basePath = basePath)) attachHandler(ApiRootResource.getServletHandler(this)) // These should be POST only, but, the YARN AM proxy won't proxy POSTs From 9f91917eddc623c85a5510e180eb3b3ed82308f9 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 30 Nov 2016 18:19:50 +0530 Subject: [PATCH 050/142] [SNAP-1185] Guard logging and time measurements (#28) - add explicit log-level check for some log lines in java code (scala code already uses logging arguments as pass-by-name) - for System.currentTimeInMillis() calls that are used only by logging, guard it with the appropriate log-level check - use System.nanoTime in a few places where duration is to be measured; also using a DoubleAccumulator to add results for better accuracy - cache commonly used logging.is*Enabled flags - use explicit flag variable in Logging initialized lazily instead of lazy val that causes hang in streaming tests for some reason even if marked transient - renamed flags for consistency - add handling for possible DoubleAccumulators in a couple of places that expect only LongAccumulators in TaskMetrics - fixing scalastyle error due to 2c432045 Conflicts: core/src/main/scala/org/apache/spark/executor/Executor.scala core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala core/src/main/scala/org/apache/spark/storage/BlockManager.scala Conflicts: common/network-common/src/main/java/org/apache/spark/network/client/TransportClient.java common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java core/src/main/scala/org/apache/spark/MapOutputTracker.scala core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala core/src/main/scala/org/apache/spark/executor/Executor.scala core/src/main/scala/org/apache/spark/internal/Logging.scala core/src/main/scala/org/apache/spark/storage/BlockManager.scala core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala --- .../spark/network/client/TransportClient.java | 27 +- .../network/protocol/MessageDecoder.java | 4 +- .../server/TransportRequestHandler.java | 4 +- .../org/apache/spark/MapOutputTracker.scala | 4 +- .../spark/broadcast/TorrentBroadcast.scala | 5 +- .../org/apache/spark/executor/Executor.scala | 27 +- .../spark/executor/ShuffleReadMetrics.scala | 16 +- .../apache/spark/executor/TaskMetrics.scala | 25 +- .../org/apache/spark/internal/Logging.scala | 63 ++- .../apache/spark/rpc/RpcEndpointAddress.scala | 2 +- .../apache/spark/scheduler/ResultTask.scala | 4 +- .../spark/scheduler/ShuffleMapTask.scala | 4 +- .../org/apache/spark/scheduler/Task.scala | 2 +- .../apache/spark/storage/BlockManager.scala | 364 +++++++++--------- .../org/apache/spark/storage/DiskStore.scala | 5 +- .../storage/ShuffleBlockFetcherIterator.scala | 20 +- .../org/apache/spark/util/JsonProtocol.scala | 5 +- 17 files changed, 324 insertions(+), 257 deletions(-) diff --git a/common/network-common/src/main/java/org/apache/spark/network/client/TransportClient.java b/common/network-common/src/main/java/org/apache/spark/network/client/TransportClient.java index 8f354ad78bbaa..5e8df06207b16 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/client/TransportClient.java +++ b/common/network-common/src/main/java/org/apache/spark/network/client/TransportClient.java @@ -131,9 +131,10 @@ public void setClientId(String id) { */ public void fetchChunk( long streamId, - int chunkIndex, - ChunkReceivedCallback callback) { - long startTime = System.currentTimeMillis(); + final int chunkIndex, + final ChunkReceivedCallback callback) { + final boolean isTraceEnabled = logger.isTraceEnabled(); + final long startTime = isTraceEnabled ? System.currentTimeMillis() : 0L; if (logger.isDebugEnabled()) { logger.debug("Sending fetch chunk request {} to {}", chunkIndex, getRemoteAddress(channel)); } @@ -144,7 +145,7 @@ public void fetchChunk( channel.writeAndFlush(new ChunkFetchRequest(streamChunkId)).addListener(future -> { if (future.isSuccess()) { long timeTaken = System.currentTimeMillis() - startTime; - if (logger.isTraceEnabled()) { + if (isTraceEnabled) { logger.trace("Sending request {} to {} took {} ms", streamChunkId, getRemoteAddress(channel), timeTaken); } @@ -169,8 +170,9 @@ public void fetchChunk( * @param streamId The stream to fetch. * @param callback Object to call with the stream data. */ - public void stream(String streamId, StreamCallback callback) { - long startTime = System.currentTimeMillis(); + public void stream(final String streamId, final StreamCallback callback) { + final boolean isTraceEnabled = logger.isTraceEnabled(); + final long startTime = isTraceEnabled ? System.currentTimeMillis() : 0L; if (logger.isDebugEnabled()) { logger.debug("Sending stream request for {} to {}", streamId, getRemoteAddress(channel)); } @@ -183,7 +185,7 @@ public void stream(String streamId, StreamCallback callback) { channel.writeAndFlush(new StreamRequest(streamId)).addListener(future -> { if (future.isSuccess()) { long timeTaken = System.currentTimeMillis() - startTime; - if (logger.isTraceEnabled()) { + if (isTraceEnabled) { logger.trace("Sending request for {} to {} took {} ms", streamId, getRemoteAddress(channel), timeTaken); } @@ -210,9 +212,10 @@ public void stream(String streamId, StreamCallback callback) { * @param callback Callback to handle the RPC's reply. * @return The RPC's id. */ - public long sendRpc(ByteBuffer message, RpcResponseCallback callback) { - long startTime = System.currentTimeMillis(); - if (logger.isTraceEnabled()) { + public long sendRpc(ByteBuffer message, final RpcResponseCallback callback) { + final boolean isTraceEnabled = logger.isTraceEnabled(); + final long startTime = isTraceEnabled ? System.currentTimeMillis() : 0L; + if (isTraceEnabled) { logger.trace("Sending RPC to {}", getRemoteAddress(channel)); } @@ -222,8 +225,8 @@ public long sendRpc(ByteBuffer message, RpcResponseCallback callback) { channel.writeAndFlush(new RpcRequest(requestId, new NioManagedBuffer(message))) .addListener(future -> { if (future.isSuccess()) { - long timeTaken = System.currentTimeMillis() - startTime; - if (logger.isTraceEnabled()) { + if (isTraceEnabled) { + long timeTaken = System.currentTimeMillis() - startTime; logger.trace("Sending request {} to {} took {} ms", requestId, getRemoteAddress(channel), timeTaken); } diff --git a/common/network-common/src/main/java/org/apache/spark/network/protocol/MessageDecoder.java b/common/network-common/src/main/java/org/apache/spark/network/protocol/MessageDecoder.java index 39a7495828a8a..a8cb9726f728c 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/protocol/MessageDecoder.java +++ b/common/network-common/src/main/java/org/apache/spark/network/protocol/MessageDecoder.java @@ -44,7 +44,9 @@ public void decode(ChannelHandlerContext ctx, ByteBuf in, List out) { Message.Type msgType = Message.Type.decode(in); Message decoded = decode(msgType, in); assert decoded.type() == msgType; - logger.trace("Received message {}: {}", msgType, decoded); + if (logger.isTraceEnabled()) { + logger.trace("Received message {}: {}", msgType, decoded); + } out.add(decoded); } diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java index e94453578e6b0..c1f28e98bff29 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java +++ b/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java @@ -221,7 +221,9 @@ private ChannelFuture respond(Encodable result) { SocketAddress remoteAddress = channel.remoteAddress(); return channel.writeAndFlush(result).addListener(future -> { if (future.isSuccess()) { - logger.trace("Sent result {} to client {}", result, remoteAddress); + if (logger.isTraceEnabled()) { + logger.trace("Sent result {} to client {}", result, remoteAddress); + } } else { logger.error(String.format("Error sending result %s to %s; closing connection", result, remoteAddress), future.cause()); diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala index 195fd4f818b36..9c8d7d8797a49 100644 --- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala +++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala @@ -693,7 +693,7 @@ private[spark] class MapOutputTrackerWorker(conf: SparkConf) extends MapOutputTr val statuses = mapStatuses.get(shuffleId).orNull if (statuses == null) { logInfo("Don't have map outputs for shuffle " + shuffleId + ", fetching them") - val startTime = System.currentTimeMillis + val startTime = if (isDebugEnabled) System.currentTimeMillis else 0L var fetchedStatuses: Array[MapStatus] = null fetching.synchronized { // Someone else is fetching it; wait for them to be done @@ -730,7 +730,7 @@ private[spark] class MapOutputTrackerWorker(conf: SparkConf) extends MapOutputTr } } } - logDebug(s"Fetching map output statuses for shuffle $shuffleId took " + + if (isDebugEnabled) logDebug(s"Fetching map output statuses for shuffle $shuffleId took " + s"${System.currentTimeMillis - startTime} ms") if (fetchedStatuses != null) { diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala index e125095cf4777..c393c82f54e22 100644 --- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala +++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala @@ -227,9 +227,10 @@ private[spark] class TorrentBroadcast[T: ClassTag](obj: T, id: Long) } case None => logInfo("Started reading broadcast variable " + id) - val startTimeMs = System.currentTimeMillis() + val startTimeMs = if (isDebugEnabled) System.currentTimeMillis() else 0L val blocks = readBlocks() - logInfo("Reading broadcast variable " + id + " took" + Utils.getUsedTimeMs(startTimeMs)) + if (isDebugEnabled) logDebug("Reading broadcast variable " + id + " took" + + Utils.getUsedTimeMs(startTimeMs)) try { val obj = TorrentBroadcast.unBlockifyObject[T]( diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 4f1b35415e5af..80da4e1a6f1dc 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -307,7 +307,7 @@ private[spark] class Executor( Thread.currentThread.setName(threadName) val threadMXBean = ManagementFactory.getThreadMXBean val taskMemoryManager = new TaskMemoryManager(env.memoryManager, taskId) - val deserializeStartTime = System.currentTimeMillis() + val deserializeStartTime = System.nanoTime() val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime } else 0L @@ -351,7 +351,7 @@ private[spark] class Executor( } // Run the actual task and measure its runtime. - taskStart = System.currentTimeMillis() + taskStart = System.nanoTime() taskStartCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime } else 0L @@ -395,7 +395,8 @@ private[spark] class Executor( s"unrecoverable fetch failures! Most likely this means user code is incorrectly " + s"swallowing Spark's internal ${classOf[FetchFailedException]}", fetchFailure) } - val taskFinish = System.currentTimeMillis() + + val taskFinish = System.nanoTime() val taskFinishCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime } else 0L @@ -404,22 +405,24 @@ private[spark] class Executor( task.context.killTaskIfInterrupted() val resultSer = env.serializer.newInstance() - val beforeSerialization = System.currentTimeMillis() + val beforeSerialization = System.nanoTime() val valueBytes = resultSer.serialize(value) - val afterSerialization = System.currentTimeMillis() + val afterSerialization = System.nanoTime() // Deserialization happens in two parts: first, we deserialize a Task object, which // includes the Partition. Second, Task.run() deserializes the RDD and function to be run. - task.metrics.setExecutorDeserializeTime( - (taskStart - deserializeStartTime) + task.executorDeserializeTime) + task.metrics.setExecutorDeserializeTime(math.max( + taskStart - deserializeStartTime + task.executorDeserializeTime, 0L) / 1000000.0) task.metrics.setExecutorDeserializeCpuTime( (taskStartCpu - deserializeStartCpuTime) + task.executorDeserializeCpuTime) // We need to subtract Task.run()'s deserialization time to avoid double-counting - task.metrics.setExecutorRunTime((taskFinish - taskStart) - task.executorDeserializeTime) + task.metrics.setExecutorRunTime(math.max( + taskFinish - taskStart - task.executorDeserializeTime, 0L) / 1000000.0) task.metrics.setExecutorCpuTime( (taskFinishCpu - taskStartCpu) - task.executorDeserializeCpuTime) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) - task.metrics.setResultSerializationTime(afterSerialization - beforeSerialization) + task.metrics.setResultSerializationTime(math.max( + afterSerialization - beforeSerialization, 0L) / 1000000.0) // Expose task metrics using the Dropwizard metrics system. // Update task metrics counters @@ -542,7 +545,8 @@ private[spark] class Executor( // Collect latest accumulator values to report back to the driver val accums: Seq[AccumulatorV2[_, _]] = if (task != null) { - task.metrics.setExecutorRunTime(System.currentTimeMillis() - taskStart) + task.metrics.setExecutorRunTime( + math.max(System.nanoTime() - taskStart, 0L) / 1000000.0) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) task.collectAccumulatorUpdates(taskFailed = true) } else { @@ -748,7 +752,8 @@ private[spark] class Executor( * Download any missing dependencies if we receive a new set of files and JARs from the * SparkContext. Also adds any new JARs we fetched to the class loader. */ - protected def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) { + protected def updateDependencies(newFiles: HashMap[String, Long], + newJars: HashMap[String, Long]) { lazy val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf) synchronized { // Fetch missing dependencies diff --git a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala index 4be395c8358b2..5a435f2f9a1ba 100644 --- a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala @@ -18,7 +18,7 @@ package org.apache.spark.executor import org.apache.spark.annotation.DeveloperApi -import org.apache.spark.util.LongAccumulator +import org.apache.spark.util.{DoubleAccumulator, LongAccumulator} /** @@ -33,7 +33,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable { private[executor] val _remoteBytesRead = new LongAccumulator private[executor] val _remoteBytesReadToDisk = new LongAccumulator private[executor] val _localBytesRead = new LongAccumulator - private[executor] val _fetchWaitTime = new LongAccumulator + private[executor] val _fetchWaitTime = new DoubleAccumulator private[executor] val _recordsRead = new LongAccumulator /** @@ -66,7 +66,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable { * blocking on shuffle input data. For instance if block B is being fetched while the task is * still not finished processing block A, it is not considered to be blocking on block B. */ - def fetchWaitTime: Long = _fetchWaitTime.sum + def fetchWaitTime: Long = _fetchWaitTime.sum.toLong /** * Total number of records read from the shuffle by this task. @@ -88,7 +88,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable { private[spark] def incRemoteBytesRead(v: Long): Unit = _remoteBytesRead.add(v) private[spark] def incRemoteBytesReadToDisk(v: Long): Unit = _remoteBytesReadToDisk.add(v) private[spark] def incLocalBytesRead(v: Long): Unit = _localBytesRead.add(v) - private[spark] def incFetchWaitTime(v: Long): Unit = _fetchWaitTime.add(v) + private[spark] def incFetchWaitTime(v: Double): Unit = _fetchWaitTime.add(v) private[spark] def incRecordsRead(v: Long): Unit = _recordsRead.add(v) private[spark] def setRemoteBlocksFetched(v: Int): Unit = _remoteBlocksFetched.setValue(v) @@ -96,7 +96,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable { private[spark] def setRemoteBytesRead(v: Long): Unit = _remoteBytesRead.setValue(v) private[spark] def setRemoteBytesReadToDisk(v: Long): Unit = _remoteBytesReadToDisk.setValue(v) private[spark] def setLocalBytesRead(v: Long): Unit = _localBytesRead.setValue(v) - private[spark] def setFetchWaitTime(v: Long): Unit = _fetchWaitTime.setValue(v) + private[spark] def setFetchWaitTime(v: Double): Unit = _fetchWaitTime.setValue(v) private[spark] def setRecordsRead(v: Long): Unit = _recordsRead.setValue(v) /** @@ -134,7 +134,7 @@ private[spark] class TempShuffleReadMetrics { private[this] var _remoteBytesRead = 0L private[this] var _remoteBytesReadToDisk = 0L private[this] var _localBytesRead = 0L - private[this] var _fetchWaitTime = 0L + private[this] var _fetchWaitTime = 0.0 private[this] var _recordsRead = 0L def incRemoteBlocksFetched(v: Long): Unit = _remoteBlocksFetched += v @@ -142,7 +142,7 @@ private[spark] class TempShuffleReadMetrics { def incRemoteBytesRead(v: Long): Unit = _remoteBytesRead += v def incRemoteBytesReadToDisk(v: Long): Unit = _remoteBytesReadToDisk += v def incLocalBytesRead(v: Long): Unit = _localBytesRead += v - def incFetchWaitTime(v: Long): Unit = _fetchWaitTime += v + def incFetchWaitTime(v: Double): Unit = _fetchWaitTime += v def incRecordsRead(v: Long): Unit = _recordsRead += v def remoteBlocksFetched: Long = _remoteBlocksFetched @@ -150,6 +150,6 @@ private[spark] class TempShuffleReadMetrics { def remoteBytesRead: Long = _remoteBytesRead def remoteBytesReadToDisk: Long = _remoteBytesReadToDisk def localBytesRead: Long = _localBytesRead - def fetchWaitTime: Long = _fetchWaitTime + def fetchWaitTime: Double = _fetchWaitTime def recordsRead: Long = _recordsRead } diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala index 85b2745a2aec4..dc6dc6878f567 100644 --- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala @@ -44,13 +44,13 @@ import org.apache.spark.util._ @DeveloperApi class TaskMetrics private[spark] () extends Serializable { // Each metric is internally represented as an accumulator - private val _executorDeserializeTime = new LongAccumulator + private val _executorDeserializeTime = new DoubleAccumulator private val _executorDeserializeCpuTime = new LongAccumulator - private val _executorRunTime = new LongAccumulator + private val _executorRunTime = new DoubleAccumulator private val _executorCpuTime = new LongAccumulator private val _resultSize = new LongAccumulator private val _jvmGCTime = new LongAccumulator - private val _resultSerializationTime = new LongAccumulator + private val _resultSerializationTime = new DoubleAccumulator private val _memoryBytesSpilled = new LongAccumulator private val _diskBytesSpilled = new LongAccumulator private val _peakExecutionMemory = new LongAccumulator @@ -59,7 +59,7 @@ class TaskMetrics private[spark] () extends Serializable { /** * Time taken on the executor to deserialize this task. */ - def executorDeserializeTime: Long = _executorDeserializeTime.sum + def executorDeserializeTime: Long = _executorDeserializeTime.sum.toLong /** * CPU Time taken on the executor to deserialize this task in nanoseconds. @@ -69,7 +69,7 @@ class TaskMetrics private[spark] () extends Serializable { /** * Time the executor spends actually running the task (including fetching shuffle data). */ - def executorRunTime: Long = _executorRunTime.sum + def executorRunTime: Long = _executorRunTime.sum.toLong /** * CPU Time the executor spends actually running the task @@ -90,7 +90,7 @@ class TaskMetrics private[spark] () extends Serializable { /** * Amount of time spent serializing the task result. */ - def resultSerializationTime: Long = _resultSerializationTime.sum + def resultSerializationTime: Long = _resultSerializationTime.sum.toLong /** * The number of in-memory bytes spilled by this task. @@ -126,15 +126,15 @@ class TaskMetrics private[spark] () extends Serializable { } // Setters and increment-ers - private[spark] def setExecutorDeserializeTime(v: Long): Unit = + private[spark] def setExecutorDeserializeTime(v: Double): Unit = _executorDeserializeTime.setValue(v) private[spark] def setExecutorDeserializeCpuTime(v: Long): Unit = _executorDeserializeCpuTime.setValue(v) - private[spark] def setExecutorRunTime(v: Long): Unit = _executorRunTime.setValue(v) + private[spark] def setExecutorRunTime(v: Double): Unit = _executorRunTime.setValue(v) private[spark] def setExecutorCpuTime(v: Long): Unit = _executorCpuTime.setValue(v) private[spark] def setResultSize(v: Long): Unit = _resultSize.setValue(v) private[spark] def setJvmGCTime(v: Long): Unit = _jvmGCTime.setValue(v) - private[spark] def setResultSerializationTime(v: Long): Unit = + private[spark] def setResultSerializationTime(v: Double): Unit = _resultSerializationTime.setValue(v) private[spark] def incMemoryBytesSpilled(v: Long): Unit = _memoryBytesSpilled.add(v) private[spark] def incDiskBytesSpilled(v: Long): Unit = _diskBytesSpilled.add(v) @@ -299,9 +299,10 @@ private[spark] object TaskMetrics extends Logging { if (name == UPDATED_BLOCK_STATUSES) { tm.setUpdatedBlockStatuses(value.asInstanceOf[java.util.List[(BlockId, BlockStatus)]]) } else { - tm.nameToAccums.get(name).foreach( - _.asInstanceOf[LongAccumulator].setValue(value.asInstanceOf[Long]) - ) + tm.nameToAccums.get(name).foreach { + case l: LongAccumulator => l.setValue(value.asInstanceOf[Long]) + case d => d.asInstanceOf[DoubleAccumulator].setValue(value.asInstanceOf[Double]) + } } } tm diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala b/core/src/main/scala/org/apache/spark/internal/Logging.scala index c0d709ad25f29..a1105b5ce1a8f 100644 --- a/core/src/main/scala/org/apache/spark/internal/Logging.scala +++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala @@ -34,6 +34,8 @@ trait Logging { // be serialized and used on another machine @transient private var log_ : Logger = null + @transient private[this] var levelFlags: Int = _ + // Method to get the logger name for this object protected def logName = { // Ignore trailing $'s in the class names for Scala objects @@ -49,17 +51,54 @@ trait Logging { log_ } + private def setLevel(value: Boolean, enabled: Int, disabled: Int): Unit = { + if (value) levelFlags |= enabled else levelFlags |= disabled + } + + protected final def isInfoEnabled: Boolean = { + val levelFlags = this.levelFlags + if ((levelFlags & Logging.INFO_ENABLED) != 0) true + else if ((levelFlags & Logging.INFO_DISABLED) != 0) false + else { + val value = log.isInfoEnabled + setLevel(value, Logging.INFO_ENABLED, Logging.INFO_DISABLED) + value + } + } + + protected final def isDebugEnabled: Boolean = { + val levelFlags = this.levelFlags + if ((levelFlags & Logging.DEBUG_DISABLED) != 0) false + else if ((levelFlags & Logging.DEBUG_ENABLED) != 0) true + else { + val value = log.isDebugEnabled + setLevel(value, Logging.DEBUG_ENABLED, Logging.DEBUG_DISABLED) + value + } + } + + protected final def isTraceEnabled: Boolean = { + val levelFlags = this.levelFlags + if ((levelFlags & Logging.TRACE_DISABLED) != 0) false + else if ((levelFlags & Logging.TRACE_ENABLED) != 0) true + else { + val value = log.isTraceEnabled + setLevel(value, Logging.TRACE_ENABLED, Logging.TRACE_DISABLED) + value + } + } + // Log methods that take only a String protected def logInfo(msg: => String) { - if (log.isInfoEnabled) log.info(msg) + if (isInfoEnabled) log.info(msg) } protected def logDebug(msg: => String) { - if (log.isDebugEnabled) log.debug(msg) + if (isDebugEnabled) log.debug(msg) } protected def logTrace(msg: => String) { - if (log.isTraceEnabled) log.trace(msg) + if (isTraceEnabled) log.trace(msg) } protected def logWarning(msg: => String) { @@ -72,15 +111,15 @@ trait Logging { // Log methods that take Throwables (Exceptions/Errors) too protected def logInfo(msg: => String, throwable: Throwable) { - if (log.isInfoEnabled) log.info(msg, throwable) + if (isInfoEnabled) log.info(msg, throwable) } protected def logDebug(msg: => String, throwable: Throwable) { - if (log.isDebugEnabled) log.debug(msg, throwable) + if (isDebugEnabled) log.debug(msg, throwable) } protected def logTrace(msg: => String, throwable: Throwable) { - if (log.isTraceEnabled) log.trace(msg, throwable) + if (isTraceEnabled) log.trace(msg, throwable) } protected def logWarning(msg: => String, throwable: Throwable) { @@ -91,10 +130,6 @@ trait Logging { if (log.isErrorEnabled) log.error(msg, throwable) } - protected def isTraceEnabled(): Boolean = { - log.isTraceEnabled - } - protected def initializeLogIfNecessary(isInterpreter: Boolean): Unit = { initializeLogIfNecessary(isInterpreter, silent = false) } @@ -163,6 +198,14 @@ trait Logging { } private[spark] object Logging { + + private val INFO_ENABLED = 0x1 + private val INFO_DISABLED = 0x2 + private val DEBUG_ENABLED = 0x4 + private val DEBUG_DISABLED = 0x8 + private val TRACE_ENABLED = 0x10 + private val TRACE_DISABLED = 0x20 + @volatile private var initialized = false @volatile private var defaultRootLevel: Level = null @volatile private var defaultSparkLog4jConfig = false diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointAddress.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointAddress.scala index fdbccc9e74c37..21061fb11a946 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointAddress.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointAddress.scala @@ -37,7 +37,7 @@ private[spark] case class RpcEndpointAddress(rpcAddress: RpcAddress, name: Strin this(RpcAddress(host, port), name) } - override val toString = if (rpcAddress != null) { + override def toString: String = if (rpcAddress != null) { s"spark://$name@${rpcAddress.host}:${rpcAddress.port}" } else { s"spark-client://$name" diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index e36c759a42556..2acac37dcff94 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -72,14 +72,14 @@ private[spark] class ResultTask[T, U]( override def runTask(context: TaskContext): U = { // Deserialize the RDD and the func using the broadcast variables. val threadMXBean = ManagementFactory.getThreadMXBean - val deserializeStartTime = System.currentTimeMillis() + val deserializeStartTime = System.nanoTime() val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime } else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, func) = ser.deserialize[(RDD[T], (TaskContext, Iterator[T]) => U)]( ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) - _executorDeserializeTime = System.currentTimeMillis() - deserializeStartTime + _executorDeserializeTime = math.max(System.nanoTime() - deserializeStartTime, 0L) _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime } else 0L diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index 7a25c47e2cab3..ef2b3e8764b71 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -77,14 +77,14 @@ private[spark] class ShuffleMapTask( override def runTask(context: TaskContext): MapStatus = { // Deserialize the RDD using the broadcast variable. val threadMXBean = ManagementFactory.getThreadMXBean - val deserializeStartTime = System.currentTimeMillis() + val deserializeStartTime = System.nanoTime() val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime } else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, dep) = ser.deserialize[(RDD[_], ShuffleDependency[_, _, _])]( ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) - _executorDeserializeTime = System.currentTimeMillis() - deserializeStartTime + _executorDeserializeTime = math.max(System.nanoTime() - deserializeStartTime, 0L) _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime } else 0L diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index f536fc2a5f0a1..2a9b0628c1131 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -179,7 +179,7 @@ private[spark] abstract class Task[T]( def reasonIfKilled: Option[String] = Option(_reasonIfKilled) /** - * Returns the amount of time spent deserializing the RDD and function to be run. + * Returns the amount of time spent deserializing the RDD and function to be run in nanos. */ def executorDeserializeTime: Long = _executorDeserializeTime def executorDeserializeCpuTime: Long = _executorDeserializeCpuTime diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index e0276a4dc4224..54780b380e26d 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -26,8 +26,9 @@ import java.util.concurrent.ConcurrentHashMap import scala.collection.mutable import scala.collection.mutable.HashMap -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.{Await, ExecutionContext, Future} import scala.concurrent.duration._ +import scala.concurrent.{Await, ExecutionContext, Future} import scala.reflect.ClassTag import scala.util.Random import scala.util.control.NonFatal @@ -125,7 +126,7 @@ private[spark] class BlockManager( val blockTransferService: BlockTransferService, securityManager: SecurityManager, numUsableCores: Int) - extends BlockDataManager with BlockEvictionHandler with Logging { + extends BlockDataManager with BlockEvictionHandler with Logging { private[spark] val externalShuffleServiceEnabled = conf.getBoolean("spark.shuffle.service.enabled", false) @@ -214,14 +215,14 @@ private[spark] class BlockManager( private val maxRemoteBlockToMem = conf.get(config.MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM) /** - * Initializes the BlockManager with the given appId. This is not performed in the constructor as - * the appId may not be known at BlockManager instantiation time (in particular for the driver, - * where it is only learned after registration with the TaskScheduler). - * - * This method initializes the BlockTransferService and ShuffleClient, registers with the - * BlockManagerMaster, starts the BlockManagerWorker endpoint, and registers with a local shuffle - * service if configured. - */ + * Initializes the BlockManager with the given appId. This is not performed in the constructor as + * the appId may not be known at BlockManager instantiation time (in particular for the driver, + * where it is only learned after registration with the TaskScheduler). + * + * This method initializes the BlockTransferService and ShuffleClient, registers with the + * BlockManagerMaster, starts the BlockManagerWorker endpoint, and registers with a local shuffle + * service if configured. + */ def initialize(appId: String): Unit = { blockTransferService.init(this) shuffleClient.init(appId) @@ -290,25 +291,25 @@ private[spark] class BlockManager( } catch { case e: Exception if i < MAX_ATTEMPTS => logError(s"Failed to connect to external shuffle server, will retry ${MAX_ATTEMPTS - i}" - + s" more times after waiting $SLEEP_TIME_SECS seconds...", e) + + s" more times after waiting $SLEEP_TIME_SECS seconds...", e) Thread.sleep(SLEEP_TIME_SECS * 1000) case NonFatal(e) => throw new SparkException("Unable to register with external shuffle server due to : " + - e.getMessage, e) + e.getMessage, e) } } } /** - * Report all blocks to the BlockManager again. This may be necessary if we are dropped - * by the BlockManager and come back or if we become capable of recovering blocks on disk after - * an executor crash. - * - * This function deliberately fails silently if the master returns false (indicating that - * the slave needs to re-register). The error condition will be detected again by the next - * heart beat attempt or new block registration and another try to re-register all blocks - * will be made then. - */ + * Report all blocks to the BlockManager again. This may be necessary if we are dropped + * by the BlockManager and come back or if we become capable of recovering blocks on disk after + * an executor crash. + * + * This function deliberately fails silently if the master returns false (indicating that + * the slave needs to re-register). The error condition will be detected again by the next + * heart beat attempt or new block registration and another try to re-register all blocks + * will be made then. + */ private def reportAllBlocks(): Unit = { logInfo(s"Reporting ${blockInfoManager.size} blocks to the master.") for ((blockId, info) <- blockInfoManager.entries) { @@ -321,11 +322,11 @@ private[spark] class BlockManager( } /** - * Re-register with the master and report all blocks to it. This will be called by the heart beat - * thread if our heartbeat to the block manager indicates that we were not registered. - * - * Note that this method must be called without any BlockInfo locks held. - */ + * Re-register with the master and report all blocks to it. This will be called by the heart beat + * thread if our heartbeat to the block manager indicates that we were not registered. + * + * Note that this method must be called without any BlockInfo locks held. + */ def reregister(): Unit = { // TODO: We might need to rate limit re-registering. logInfo(s"BlockManager $blockManagerId re-registering with master") @@ -334,8 +335,8 @@ private[spark] class BlockManager( } /** - * Re-register with the master sometime soon. - */ + * Re-register with the master sometime soon. + */ private def asyncReregister(): Unit = { asyncReregisterLock.synchronized { if (asyncReregisterTask == null) { @@ -352,8 +353,8 @@ private[spark] class BlockManager( } /** - * For testing. Wait for any pending asynchronous re-registration; otherwise, do nothing. - */ + * For testing. Wait for any pending asynchronous re-registration; otherwise, do nothing. + */ def waitForAsyncReregister(): Unit = { val task = asyncReregisterTask if (task != null) { @@ -367,9 +368,9 @@ private[spark] class BlockManager( } /** - * Interface to get local block data. Throws an exception if the block cannot be found or - * cannot be read successfully. - */ + * Interface to get local block data. Throws an exception if the block cannot be found or + * cannot be read successfully. + */ override def getBlockData(blockId: BlockId): ManagedBuffer = { if (blockId.isShuffle) { shuffleManager.shuffleBlockResolver.getBlockData(blockId.asInstanceOf[ShuffleBlockId]) @@ -402,9 +403,9 @@ private[spark] class BlockManager( } /** - * Get the BlockStatus for the block identified by the given ID, if it exists. - * NOTE: This is mainly for testing. - */ + * Get the BlockStatus for the block identified by the given ID, if it exists. + * NOTE: This is mainly for testing. + */ def getStatus(blockId: BlockId): Option[BlockStatus] = { blockInfoManager.get(blockId).map { info => val memSize = if (memoryStore.contains(blockId)) memoryStore.getSize(blockId) else 0L @@ -414,28 +415,28 @@ private[spark] class BlockManager( } /** - * Get the ids of existing blocks that match the given filter. Note that this will - * query the blocks stored in the disk block manager (that the block manager - * may not know of). - */ + * Get the ids of existing blocks that match the given filter. Note that this will + * query the blocks stored in the disk block manager (that the block manager + * may not know of). + */ def getMatchingBlockIds(filter: BlockId => Boolean): Seq[BlockId] = { // The `toArray` is necessary here in order to force the list to be materialized so that we // don't try to serialize a lazy iterator when responding to client requests. (blockInfoManager.entries.map(_._1) ++ diskBlockManager.getAllBlocks()) - .filter(filter) - .toArray - .toSeq + .filter(filter) + .toArray + .toSeq } /** - * Tell the master about the current storage status of a block. This will send a block update - * message reflecting the current status, *not* the desired storage level in its block info. - * For example, a block with MEMORY_AND_DISK set might have fallen out to be only on disk. - * - * droppedMemorySize exists to account for when the block is dropped from memory to disk (so - * it is still valid). This ensures that update in master will compensate for the increase in - * memory on slave. - */ + * Tell the master about the current storage status of a block. This will send a block update + * message reflecting the current status, *not* the desired storage level in its block info. + * For example, a block with MEMORY_AND_DISK set might have fallen out to be only on disk. + * + * droppedMemorySize exists to account for when the block is dropped from memory to disk (so + * it is still valid). This ensures that update in master will compensate for the increase in + * memory on slave. + */ private def reportBlockStatus( blockId: BlockId, status: BlockStatus, @@ -450,10 +451,10 @@ private[spark] class BlockManager( } /** - * Actually send a UpdateBlockInfo message. Returns the master's response, - * which will be true if the block was successfully recorded and false if - * the slave needs to re-register. - */ + * Actually send a UpdateBlockInfo message. Returns the master's response, + * which will be true if the block was successfully recorded and false if + * the slave needs to re-register. + */ private def tryToReportBlockStatus( blockId: BlockId, status: BlockStatus, @@ -465,10 +466,10 @@ private[spark] class BlockManager( } /** - * Return the updated storage status of the block with the given ID. More specifically, if - * the block is dropped from memory and possibly added to disk, return the new storage level - * and the updated in-memory and on-disk sizes. - */ + * Return the updated storage status of the block with the given ID. More specifically, if + * the block is dropped from memory and possibly added to disk, return the new storage level + * and the updated in-memory and on-disk sizes. + */ private def getCurrentBlockStatus(blockId: BlockId, info: BlockInfo): BlockStatus = { info.synchronized { info.level match { @@ -478,7 +479,7 @@ private[spark] class BlockManager( val inMem = level.useMemory && memoryStore.contains(blockId) val onDisk = level.useDisk && diskStore.contains(blockId) val deserialized = if (inMem) level.deserialized else false - val replication = if (inMem || onDisk) level.replication else 1 + val replication = if (inMem || onDisk) level.replication else 1 val storageLevel = StorageLevel( useDisk = onDisk, useMemory = inMem, @@ -493,19 +494,19 @@ private[spark] class BlockManager( } /** - * Get locations of an array of blocks. - */ + * Get locations of an array of blocks. + */ private def getLocationBlockIds(blockIds: Array[BlockId]): Array[Seq[BlockManagerId]] = { - val startTimeMs = System.currentTimeMillis + val startTimeMs = if (isDebugEnabled) System.currentTimeMillis else 0L val locations = master.getLocations(blockIds).toArray logDebug("Got multiple block location in %s".format(Utils.getUsedTimeMs(startTimeMs))) locations } /** - * Cleanup code run in response to a failed local read. - * Must be called while holding a read lock on the block. - */ + * Cleanup code run in response to a failed local read. + * Must be called while holding a read lock on the block. + */ private def handleLocalReadFailure(blockId: BlockId): Nothing = { releaseLock(blockId) // Remove the missing block so that its unavailability is reported to the driver @@ -514,8 +515,8 @@ private[spark] class BlockManager( } /** - * Get block from local block manager as an iterator of Java objects. - */ + * Get block from local block manager as an iterator of Java objects. + */ def getLocalValues(blockId: BlockId): Option[BlockResult] = { logDebug(s"Getting local block $blockId") blockInfoManager.lockForReading(blockId) match { @@ -610,7 +611,8 @@ private[spark] class BlockManager( } else { handleLocalReadFailure(blockId) } - } else { // storage level is serialized + } else { + // storage level is serialized if (level.useMemory && memoryStore.contains(blockId)) { new ByteBufferBlockData(memoryStore.getBytes(blockId).get, false) } else if (level.useDisk && diskStore.contains(blockId)) { @@ -625,10 +627,10 @@ private[spark] class BlockManager( } /** - * Get block from remote block managers. - * - * This does not acquire a lock on this block in this JVM. - */ + * Get block from remote block managers. + * + * This does not acquire a lock on this block in this JVM. + */ private def getRemoteValues[T: ClassTag](blockId: BlockId): Option[BlockResult] = { val ct = implicitly[ClassTag[T]] getRemoteBytes(blockId).map { data => @@ -656,8 +658,8 @@ private[spark] class BlockManager( } /** - * Get block from remote block managers as serialized bytes. - */ + * Get block from remote block managers as serialized bytes. + */ def getRemoteBytes(blockId: BlockId): Option[ChunkedByteBuffer] = { logDebug(s"Getting remote block $blockId") require(blockId != null, "BlockId is null") @@ -700,12 +702,12 @@ private[spark] class BlockManager( // or we've refreshed the list of locations from the master, and have still // hit failures after trying locations from the refreshed list. logWarning(s"Failed to fetch block after $totalFailureCount fetch failures. " + - s"Most recent failure cause:", e) + s"Most recent failure cause:", e) return None } logWarning(s"Failed to fetch remote block $blockId " + - s"from $loc (failed attempt $runningFailureCount)", e) + s"from $loc (failed attempt $runningFailureCount)", e) // If there is a large number of executors then locations list can contain a // large number of stale entries causing a large number of retries that may @@ -714,7 +716,7 @@ private[spark] class BlockManager( if (runningFailureCount >= maxFailuresBeforeLocationRefresh) { locationIterator = sortLocations(master.getLocations(blockId)).iterator logDebug(s"Refreshed locations from the driver " + - s"after ${runningFailureCount} fetch failures.") + s"after ${runningFailureCount} fetch failures.") runningFailureCount = 0 } @@ -732,12 +734,12 @@ private[spark] class BlockManager( } /** - * Get a block from the block manager (either local or remote). - * - * This acquires a read lock on the block if the block was stored locally and does not acquire - * any locks if the block was fetched from a remote block manager. The read lock will - * automatically be freed once the result's `data` iterator is fully consumed. - */ + * Get a block from the block manager (either local or remote). + * + * This acquires a read lock on the block if the block was stored locally and does not acquire + * any locks if the block was fetched from a remote block manager. The read lock will + * automatically be freed once the result's `data` iterator is fully consumed. + */ def get[T: ClassTag](blockId: BlockId): Option[BlockResult] = { val local = getLocalValues(blockId) if (local.isDefined) { @@ -753,8 +755,8 @@ private[spark] class BlockManager( } /** - * Downgrades an exclusive write lock to a shared read lock. - */ + * Downgrades an exclusive write lock to a shared read lock. + */ def downgradeLock(blockId: BlockId): Unit = { blockInfoManager.downgradeLock(blockId) } @@ -770,28 +772,28 @@ private[spark] class BlockManager( } /** - * Registers a task with the BlockManager in order to initialize per-task bookkeeping structures. - */ + * Registers a task with the BlockManager in order to initialize per-task bookkeeping structures. + */ def registerTask(taskAttemptId: Long): Unit = { blockInfoManager.registerTask(taskAttemptId) } /** - * Release all locks for the given task. - * - * @return the blocks whose locks were released. - */ + * Release all locks for the given task. + * + * @return the blocks whose locks were released. + */ def releaseAllLocksForTask(taskAttemptId: Long): Seq[BlockId] = { blockInfoManager.releaseAllLocksForTask(taskAttemptId) } /** - * Retrieve the given block if it exists, otherwise call the provided `makeIterator` method - * to compute the block, persist it, and return its values. - * - * @return either a BlockResult if the block was successfully cached, or an iterator if the block - * could not be cached. - */ + * Retrieve the given block if it exists, otherwise call the provided `makeIterator` method + * to compute the block, persist it, and return its values. + * + * @return either a BlockResult if the block was successfully cached, or an iterator if the block + * could not be cached. + */ def getOrElseUpdate[T]( blockId: BlockId, level: StorageLevel, @@ -803,7 +805,7 @@ private[spark] class BlockManager( case Some(block) => return Left(block) case _ => - // Need to compute the block. + // Need to compute the block. } // Initially we hold no locks on this block. doPutIterator(blockId, makeIterator, level, classTag, keepReadLock = true) match { @@ -825,13 +827,13 @@ private[spark] class BlockManager( // The put failed, likely because the data was too large to fit in memory and could not be // dropped to disk. Therefore, we need to pass the input iterator back to the caller so // that they can decide what to do with the values (e.g. process them without caching). - Right(iter) + Right(iter) } } /** - * @return true if the block was stored or false if an error occurred. - */ + * @return true if the block was stored or false if an error occurred. + */ def putIterator[T: ClassTag]( blockId: BlockId, values: Iterator[T], @@ -850,10 +852,10 @@ private[spark] class BlockManager( } /** - * A short circuited method to get a block writer that can write data directly to disk. - * The Block will be appended to the File specified by filename. Callers should handle error - * cases. - */ + * A short circuited method to get a block writer that can write data directly to disk. + * The Block will be appended to the File specified by filename. Callers should handle error + * cases. + */ def getDiskWriter( blockId: BlockId, file: File, @@ -904,7 +906,7 @@ private[spark] class BlockManager( tellMaster: Boolean = true, keepReadLock: Boolean = false): Boolean = { doPut(blockId, level, classTag, tellMaster = tellMaster, keepReadLock = keepReadLock) { info => - val startTimeMs = System.currentTimeMillis + val startTimeMs = if (isDebugEnabled) System.currentTimeMillis else 0L // Since we're storing bytes, initiate the replication before storing them locally. // This is faster as data is already serialized and ready to send. val replicationFuture = if (level.replication > 1) { @@ -983,11 +985,11 @@ private[spark] class BlockManager( } /** - * Helper method used to abstract common code from [[doPutBytes()]] and [[doPutIterator()]]. - * - * @param putBody a function which attempts the actual put() and returns None on success - * or Some on failure. - */ + * Helper method used to abstract common code from [[doPutBytes()]] and [[doPutIterator()]]. + * + * @param putBody a function which attempts the actual put() and returns None on success + * or Some on failure. + */ private def doPut[T]( blockId: BlockId, level: StorageLevel, @@ -1012,7 +1014,7 @@ private[spark] class BlockManager( } } - val startTimeMs = System.currentTimeMillis + val startTimeMs = if (isDebugEnabled) System.currentTimeMillis else 0L var exceptionWasThrown: Boolean = true val result: Option[T] = try { val res = putBody(putBlockInfo) @@ -1053,26 +1055,26 @@ private[spark] class BlockManager( } if (level.replication > 1) { logDebug("Putting block %s with replication took %s" - .format(blockId, Utils.getUsedTimeMs(startTimeMs))) + .format(blockId, Utils.getUsedTimeMs(startTimeMs))) } else { logDebug("Putting block %s without replication took %s" - .format(blockId, Utils.getUsedTimeMs(startTimeMs))) + .format(blockId, Utils.getUsedTimeMs(startTimeMs))) } result } /** - * Put the given block according to the given level in one of the block stores, replicating - * the values if necessary. - * - * If the block already exists, this method will not overwrite it. - * - * @param keepReadLock if true, this method will hold the read lock when it returns (even if the - * block already exists). If false, this method will hold no locks when it - * returns. - * @return None if the block was already present or if the put succeeded, or Some(iterator) - * if the put failed. - */ + * Put the given block according to the given level in one of the block stores, replicating + * the values if necessary. + * + * If the block already exists, this method will not overwrite it. + * + * @param keepReadLock if true, this method will hold the read lock when it returns (even if the + * block already exists). If false, this method will hold no locks when it + * returns. + * @return None if the block was already present or if the put succeeded, or Some(iterator) + * if the put failed. + */ private def doPutIterator[T]( blockId: BlockId, iterator: () => Iterator[T], @@ -1081,7 +1083,7 @@ private[spark] class BlockManager( tellMaster: Boolean = true, keepReadLock: Boolean = false): Option[PartiallyUnrolledIterator[T]] = { doPut(blockId, level, classTag, tellMaster = tellMaster, keepReadLock = keepReadLock) { info => - val startTimeMs = System.currentTimeMillis + val startTimeMs = if (isDebugEnabled) System.currentTimeMillis else 0L var iteratorFromFailedMemoryStorePut: Option[PartiallyUnrolledIterator[T]] = None // Size of the block in bytes var size = 0L @@ -1105,7 +1107,8 @@ private[spark] class BlockManager( iteratorFromFailedMemoryStorePut = Some(iter) } } - } else { // !level.deserialized + } else { + // !level.deserialized memoryStore.putIteratorAsBytes(blockId, iterator(), classTag, level.memoryMode) match { case Right(s) => size = s @@ -1143,7 +1146,7 @@ private[spark] class BlockManager( addUpdatedBlockStatusToTaskMetrics(blockId, putBlockStatus) logDebug("Put block %s locally took %s".format(blockId, Utils.getUsedTimeMs(startTimeMs))) if (level.replication > 1) { - val remoteStartTime = System.currentTimeMillis + val remoteStartTime = if (isDebugEnabled) System.currentTimeMillis else 0L val bytesToReplicate = doGetLocalBytes(blockId, info) // [SPARK-16550] Erase the typed classTag when using default serialization, since // NettyBlockRpcServer crashes when deserializing repl-defined classes. @@ -1159,7 +1162,7 @@ private[spark] class BlockManager( bytesToReplicate.dispose() } logDebug("Put block %s remotely took %s" - .format(blockId, Utils.getUsedTimeMs(remoteStartTime))) + .format(blockId, Utils.getUsedTimeMs(remoteStartTime))) } } assert(blockWasSuccessfullyStored == iteratorFromFailedMemoryStorePut.isEmpty) @@ -1168,14 +1171,14 @@ private[spark] class BlockManager( } /** - * Attempts to cache spilled bytes read from disk into the MemoryStore in order to speed up - * subsequent reads. This method requires the caller to hold a read lock on the block. - * - * @return a copy of the bytes from the memory store if the put succeeded, otherwise None. - * If this returns bytes from the memory store then the original disk store bytes will - * automatically be disposed and the caller should not continue to use them. Otherwise, - * if this returns None then the original disk store bytes will be unaffected. - */ + * Attempts to cache spilled bytes read from disk into the MemoryStore in order to speed up + * subsequent reads. This method requires the caller to hold a read lock on the block. + * + * @return a copy of the bytes from the memory store if the put succeeded, otherwise None. + * If this returns bytes from the memory store then the original disk store bytes will + * automatically be disposed and the caller should not continue to use them. Otherwise, + * if this returns None then the original disk store bytes will be unaffected. + */ private def maybeCacheDiskBytesInMemory( blockInfo: BlockInfo, blockId: BlockId, @@ -1215,12 +1218,12 @@ private[spark] class BlockManager( } /** - * Attempts to cache spilled values read from disk into the MemoryStore in order to speed up - * subsequent reads. This method requires the caller to hold a read lock on the block. - * - * @return a copy of the iterator. The original iterator passed this method should no longer - * be used after this method returns. - */ + * Attempts to cache spilled values read from disk into the MemoryStore in order to speed up + * subsequent reads. This method requires the caller to hold a read lock on the block. + * + * @return a copy of the iterator. The original iterator passed this method should no longer + * be used after this method returns. + */ private def maybeCacheDiskValuesInMemory[T]( blockInfo: BlockInfo, blockId: BlockId, @@ -1252,8 +1255,8 @@ private[spark] class BlockManager( } /** - * Get peer block managers in the system. - */ + * Get peer block managers in the system. + */ private def getPeers(forceFetch: Boolean): Seq[BlockManagerId] = { peerFetchLock.synchronized { val cachedPeersTtl = conf.getInt("spark.storage.cachedPeersTtl", 60 * 1000) // milliseconds @@ -1319,7 +1322,7 @@ private[spark] class BlockManager( replication = 1) val numPeersToReplicateTo = level.replication - 1 - val startTime = System.nanoTime + val startTime = if (isDebugEnabled) System.nanoTime else 0L val peersReplicatedTo = mutable.HashSet.empty ++ existingReplicas val peersFailedToReplicateTo = mutable.HashSet.empty[BlockManagerId] @@ -1337,9 +1340,10 @@ private[spark] class BlockManager( while(numFailures <= maxReplicationFailures && !peersForReplication.isEmpty && peersReplicatedTo.size < numPeersToReplicateTo) { + val peer = peersForReplication.head try { - val onePeerStartTime = System.nanoTime + val onePeerStartTime = if (isDebugEnabled) System.nanoTime else 0L logTrace(s"Trying to replicate $blockId of ${data.size} bytes to $peer") blockTransferService.uploadBlockSync( peer.host, @@ -1350,7 +1354,7 @@ private[spark] class BlockManager( tLevel, classTag) logTrace(s"Replicated $blockId of ${data.size} bytes to $peer" + - s" in ${(System.nanoTime - onePeerStartTime).toDouble / 1e6} ms") + s" in ${(System.nanoTime - onePeerStartTime).toDouble / 1e6} ms") peersForReplication = peersForReplication.tail peersReplicatedTo += peer } catch { @@ -1374,28 +1378,28 @@ private[spark] class BlockManager( } } logDebug(s"Replicating $blockId of ${data.size} bytes to " + - s"${peersReplicatedTo.size} peer(s) took ${(System.nanoTime - startTime) / 1e6} ms") + s"${peersReplicatedTo.size} peer(s) took ${(System.nanoTime - startTime) / 1e6} ms") if (peersReplicatedTo.size < numPeersToReplicateTo) { logWarning(s"Block $blockId replicated to only " + - s"${peersReplicatedTo.size} peer(s) instead of $numPeersToReplicateTo peers") + s"${peersReplicatedTo.size} peer(s) instead of $numPeersToReplicateTo peers") } logDebug(s"block $blockId replicated to ${peersReplicatedTo.mkString(", ")}") } /** - * Read a block consisting of a single object. - */ + * Read a block consisting of a single object. + */ def getSingle[T: ClassTag](blockId: BlockId): Option[T] = { get[T](blockId).map(_.data.next().asInstanceOf[T]) } /** - * Write a block consisting of a single object. - * - * @return true if the block was stored or false if the block was already stored or an - * error occurred. - */ + * Write a block consisting of a single object. + * + * @return true if the block was stored or false if the block was already stored or an + * error occurred. + */ def putSingle[T: ClassTag]( blockId: BlockId, value: T, @@ -1405,16 +1409,16 @@ private[spark] class BlockManager( } /** - * Drop a block from memory, possibly putting it on disk if applicable. Called when the memory - * store reaches its limit and needs to free up space. - * - * If `data` is not put on disk, it won't be created. - * - * The caller of this method must hold a write lock on the block before calling this method. - * This method does not release the write lock. - * - * @return the block's new effective StorageLevel. - */ + * Drop a block from memory, possibly putting it on disk if applicable. Called when the memory + * store reaches its limit and needs to free up space. + * + * If `data` is not put on disk, it won't be created. + * + * The caller of this method must hold a write lock on the block before calling this method. + * This method does not release the write lock. + * + * @return the block's new effective StorageLevel. + */ private[storage] override def dropFromMemory[T: ClassTag]( blockId: BlockId, data: () => Either[Array[T], ChunkedByteBuffer]): StorageLevel = { @@ -1462,10 +1466,10 @@ private[spark] class BlockManager( } /** - * Remove all blocks belonging to the given RDD. - * - * @return The number of blocks removed. - */ + * Remove all blocks belonging to the given RDD. + * + * @return The number of blocks removed. + */ def removeRdd(rddId: Int): Int = { // TODO: Avoid a linear scan by creating another mapping of RDD.id to blocks. logInfo(s"Removing RDD $rddId") @@ -1475,20 +1479,20 @@ private[spark] class BlockManager( } /** - * Remove all blocks belonging to the given broadcast. - */ + * Remove all blocks belonging to the given broadcast. + */ def removeBroadcast(broadcastId: Long, tellMaster: Boolean): Int = { logDebug(s"Removing broadcast $broadcastId") val blocksToRemove = blockInfoManager.entries.map(_._1).collect { - case bid @ BroadcastBlockId(`broadcastId`, _) => bid + case bid@BroadcastBlockId(`broadcastId`, _) => bid } blocksToRemove.foreach { blockId => removeBlock(blockId, tellMaster) } blocksToRemove.size } /** - * Remove a block from both memory and disk. - */ + * Remove a block from both memory and disk. + */ def removeBlock(blockId: BlockId, tellMaster: Boolean = true): Unit = { logDebug(s"Removing block $blockId") blockInfoManager.lockForWriting(blockId) match { @@ -1502,9 +1506,9 @@ private[spark] class BlockManager( } /** - * Internal version of [[removeBlock()]] which assumes that the caller already holds a write - * lock on the block. - */ + * Internal version of [[removeBlock()]] which assumes that the caller already holds a write + * lock on the block. + */ private def removeBlockInternal(blockId: BlockId, tellMaster: Boolean): Unit = { // Removals are idempotent in disk store and memory store. At worst, we get a warning. val removedFromMemory = memoryStore.remove(blockId) diff --git a/core/src/main/scala/org/apache/spark/storage/DiskStore.scala b/core/src/main/scala/org/apache/spark/storage/DiskStore.scala index 39249d411b582..e4c11555bbe09 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskStore.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskStore.scala @@ -60,7 +60,7 @@ private[spark] class DiskStore( throw new IllegalStateException(s"Block $blockId is already present in the disk store") } logDebug(s"Attempting to put block $blockId") - val startTime = System.currentTimeMillis + val startTime = if (isDebugEnabled) System.currentTimeMillis else 0L val file = diskManager.getFile(blockId) val out = new CountingWritableChannel(openForWrite(file)) var threwException: Boolean = true @@ -83,11 +83,10 @@ private[spark] class DiskStore( } } } - val finishTime = System.currentTimeMillis logDebug("Block %s stored as %s file on disk in %d ms".format( file.getName, Utils.bytesToString(file.length()), - finishTime - startTime)) + System.currentTimeMillis - startTime)) } def putBytes(blockId: BlockId, bytes: ChunkedByteBuffer): Unit = { diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala index 98b5a735a4529..5df11ddcaad06 100644 --- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala +++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala @@ -87,7 +87,8 @@ final class ShuffleBlockFetcherIterator( */ private[this] var numBlocksProcessed = 0 - private[this] val startTime = System.currentTimeMillis + private[this] val startTime = + if (isDebugEnabled || isTraceEnabled) System.currentTimeMillis else 0L /** Local blocks to fetch, excluding zero-sized blocks. */ private[this] val localBlocks = new ArrayBuffer[BlockId]() @@ -231,8 +232,12 @@ final class ShuffleBlockFetcherIterator( remainingBlocks -= blockId results.put(new SuccessFetchResult(BlockId(blockId), address, sizeMap(blockId), buf, remainingBlocks.isEmpty)) - logDebug("remainingBlocks: " + remainingBlocks) + if (isDebugEnabled) { + logDebug("remainingBlocks: " + remainingBlocks) + } } + if (isTraceEnabled) logTrace("Got remote block " + blockId + " after " + + Utils.getUsedTimeMs(startTime)) } logTrace("Got remote block " + blockId + " after " + Utils.getUsedTimeMs(startTime)) } @@ -351,11 +356,12 @@ final class ShuffleBlockFetcherIterator( fetchUpToMaxBytes() val numFetches = remoteRequests.size - fetchRequests.size - logInfo("Started " + numFetches + " remote fetches in" + Utils.getUsedTimeMs(startTime)) + if (isDebugEnabled) logDebug("Started " + numFetches + " remote fetches in" + + Utils.getUsedTimeMs(startTime)) // Get Local Blocks fetchLocalBlocks() - logDebug("Got local blocks in " + Utils.getUsedTimeMs(startTime)) + if (isDebugEnabled) logDebug("Got local blocks in " + Utils.getUsedTimeMs(startTime)) } override def hasNext: Boolean = numBlocksProcessed < numBlocksToFetch @@ -382,10 +388,10 @@ final class ShuffleBlockFetcherIterator( // is also corrupt, so the previous stage could be retried. // For local shuffle block, throw FailureFetchResult for the first IOException. while (result == null) { - val startFetchWait = System.currentTimeMillis() + val startFetchWait = System.nanoTime() result = results.take() - val stopFetchWait = System.currentTimeMillis() - shuffleMetrics.incFetchWaitTime(stopFetchWait - startFetchWait) + val stopFetchWait = System.nanoTime() + shuffleMetrics.incFetchWaitTime(math.max(stopFetchWait - startFetchWait, 0L) / 1000000.0) result match { case r @ SuccessFetchResult(blockId, address, size, buf, isNetworkReqDone) => diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala index ff83301d631c4..ea811f45086c2 100644 --- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala +++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala @@ -324,8 +324,9 @@ private[spark] object JsonProtocol { value match { case v: Int => JInt(v) case v: Long => JInt(v) - // We only have 3 kind of internal accumulator types, so if it's not int or long, it must be - // the blocks accumulator, whose type is `java.util.List[(BlockId, BlockStatus)]` + case v: Double => JDouble(v) + // We only have 4 kinds of internal accumulator types, so if it's not int, long or double, + // it must be the blocks accumulator with type `java.util.List[(BlockId, BlockStatus)]` case v => JArray(v.asInstanceOf[java.util.List[(BlockId, BlockStatus)]].asScala.toList.map { case (id, status) => From e25e4b91f8b3e3c044f9ba3e6e28760f31de1947 Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Thu, 2 Feb 2017 07:16:00 +0530 Subject: [PATCH 051/142] Snap 982 (#43) * a) Added a method in SparkContext to manipulate addedJar. This is an workaround for SNAP-1133. b) made repl classloader a variable in Executor.scala * Changed Executor field variable to protected. * Changed build.gradle of launcher and network-yarn to exclude netty dependecies , which was causing some messages to hang. made urlclassLoader in Executor.scala a variable. * Made Utils.doFetchFile method public. * Made Executor.addReplClassLoaderIfNeeded() method as public. --- common/network-yarn/build.gradle | 2 ++ core/src/main/scala/org/apache/spark/SparkContext.scala | 5 +++++ .../src/main/scala/org/apache/spark/executor/Executor.scala | 6 +++--- launcher/build.gradle | 2 ++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/common/network-yarn/build.gradle b/common/network-yarn/build.gradle index bbb6d8c7f81ae..b447d5aecaeaa 100644 --- a/common/network-yarn/build.gradle +++ b/common/network-yarn/build.gradle @@ -41,6 +41,8 @@ dependencies { exclude(group: 'com.sun.jersey') exclude(group: 'com.sun.jersey.jersey-test-framework') exclude(group: 'com.sun.jersey.contribs') + exclude(group: 'io.netty', module: 'netty') + exclude(group: 'io.netty', module: 'netty-all') } /* diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index fcc5fce4bb798..3a0c093587157 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -280,6 +280,11 @@ class SparkContext(config: SparkConf) extends Logging { private[spark] val addedFiles = new ConcurrentHashMap[String, Long]().asScala private[spark] val addedJars = new ConcurrentHashMap[String, Long]().asScala + def removeAddedJar(name : String) { + logInfo(s"Removing jar $name from SparkContext list") + addedJars.remove(name) + } + // Keeps track of all persisted RDDs private[spark] val persistentRdds = { val map: ConcurrentMap[Int, RDD[_]] = new MapMaker().weakValues().makeMap[Int, RDD[_]]() diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 80da4e1a6f1dc..f8cf7a0943584 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -126,8 +126,8 @@ private[spark] class Executor( // Create our ClassLoader // do this after SparkEnv creation so can access the SecurityManager - protected val urlClassLoader = createClassLoader() - private val replClassLoader = addReplClassLoaderIfNeeded(urlClassLoader) + protected var urlClassLoader = createClassLoader() + protected var replClassLoader = addReplClassLoaderIfNeeded(urlClassLoader) // Set the classloader for serializer env.serializer.setDefaultClassLoader(replClassLoader) @@ -726,7 +726,7 @@ private[spark] class Executor( * If the REPL is in use, add another ClassLoader that will read * new classes defined by the REPL as the user types code */ - private def addReplClassLoaderIfNeeded(parent: ClassLoader): ClassLoader = { + def addReplClassLoaderIfNeeded(parent: ClassLoader): ClassLoader = { val classUri = conf.get("spark.repl.class.uri", null) if (classUri != null) { logInfo("Using REPL class URI: " + classUri) diff --git a/launcher/build.gradle b/launcher/build.gradle index 22a32f5227a2a..185fb1bdd875d 100644 --- a/launcher/build.gradle +++ b/launcher/build.gradle @@ -34,6 +34,8 @@ dependencies { exclude(group: 'com.sun.jersey') exclude(group: 'com.sun.jersey.jersey-test-framework') exclude(group: 'com.sun.jersey.contribs') + exclude(group: 'io.netty', module: 'netty') + exclude(group: 'io.netty', module: 'netty-all') } testCompile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion } From 1694567e7386d4de0de979f820f5b6c9bef3487d Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Tue, 8 Aug 2017 16:59:05 +0530 Subject: [PATCH 052/142] Snap 1890 : Snappy Pulse UI suggestions for 1.0 (#69) * SNAP-1890: Snappy Pulse UI suggestions for 1.0 Changes: - SnappyData logo shifted to right most side on navigation tab bar. - Adding SnappyData's own new Pulse logo on left most side on navigation tab bar. - Displaying SnappyData Build details along with product version number on Pulse UI. - Adding CSS,HTML, JS code changes for displaying version details pop up. Conflicts: core/src/main/scala/org/apache/spark/ui/UIUtils.scala --- .../spark/ui/static/snappy-dashboard.css | 2 +- .../snappydata/pulse-snappydata-130X50.png | Bin 0 -> 4650 bytes .../snappydata/pulse-snappydata-152X50.png | Bin 0 -> 9070 bytes .../static/snappydata/snappydata-310X50.png | Bin 0 -> 7750 bytes .../org/apache/spark/ui/static/webui.css | 78 +++++++++++++++++- .../org/apache/spark/ui/static/webui.js | 10 ++- .../scala/org/apache/spark/ui/SparkUI.scala | 9 +- .../scala/org/apache/spark/ui/UIUtils.scala | 63 +++++++++++--- 8 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/pulse-snappydata-130X50.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/pulse-snappydata-152X50.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/snappydata-310X50.png diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css b/core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css index 1ddd1b5263e45..73e740086cb16 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css +++ b/core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css @@ -114,6 +114,6 @@ }*/ .progressValue{ float:right; - width:20%; + /* width:20%; */ text-align:center; } \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/pulse-snappydata-130X50.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/pulse-snappydata-130X50.png new file mode 100644 index 0000000000000000000000000000000000000000..8de1e13897efa4451cd2ba0de0135368f4b325c9 GIT binary patch literal 4650 zcmV+_64mXAP)UW2Nt*xw02y>e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{01=W&L_t(|+U=ZaU{po6$A8tG z1(J{r2w^8|Q5bdwK|n-BKvWFrx}pQ9<0$9|sJH=&qo9rgOvbYVpEvNQRUb{OzxNRokp=oidiU&Az3^&1QLK&z>mNhQ>jCh}0g??V zgDWvhhZ|Dv1daf20#90fIA};#?zMgdc-!72d80KXVg_NZW>-POP(-oXy6Xa*ZnP5_ny zZGq*$5X;*b1kA$|ka`FxG^D(?ycums*$2o13?R<}payUgFctV7Pta<1FH=w z#{u_SJU$J4RmP&+418>5Ng&x&s??{Q6~GKb%4lFKFbr62DmAbsEfxOU$HA5o`~b8v zmCE(g;Zjqnxxg*f^S=!#gD>K28Dm+m2TY}M{d~8?;<1sb)W9mftRdKZcR0vHK6}T+JEkMeSi1gLxlZa1#(}1)IiMuHzs)XBf00<-5Rx zmUrQ;6Kgfo{a3PaswP{6Ce?K3yn&^{i{(Q4q=go)dCriszGanq0?z{X0;2(GD&;-e zR$!(5f$ysM{nIslX=qtZA9<*i9xRiuD}hPMQc#(?RS{6GA!V+q)CS-+L&_I{CxCw$ zQbw6dxh&ThpkYlC=zjw&-c<>}RUVoKSYM{dAuEll0?gj)D(tc9h}PXYh5mXDEE zk!_Ze-BX<*A>}nyRQL~pnU{(p#2Hd9smMR~QG%|(t9Z@`+)>Wf^GfUSHABk4O3dV! zt;gvX5X=pRlnpAfTs})&7b|P4&+BWgRB?fmP(FQq!m=s@YLX&|GM#M>900xm>f_mp zU2Q5gsDc6N1FW;!y4O@{zMpdMSfDKx=xi!gTuzw)JR3;EES%BORO$>y@ZE;aPU>O@ z_E@iO0Y34%yk+r?6+B9_yoN|TVQ25VzXKETL{M7*hy7VCL(1{MKY#*Lsqjld5!U0m zUY2U1?_-v7ya#LoVu0(d^|g~#-C-(~R36CmnN6SK*~iEO-U41XmC7)rj0FCSXP3~+ zLwi%HBHwn(o<}gG^xZJpYw$Xm}M%p#wybc zxZUEdg{4gGOr;Lgq)jKkieM_$$I6>w38|rX=IP2ewxhw3$jhYAPkHnYszjy@hQS#)vePx~)7VuxJ&Ykn$0q_nD?r zBds>?!E;U`4>%4i1Nxdu^|F?)^JcJ5EaH7rspeLIDcPD3$1UEb0(MiWkv>tynoEY1 z%?&BFA?1Jkpyl5r8d#n87vr*PO+S}deMJ~DgsJzh`2WS^@D%i8=Vvc)w5U||i3u$# z{hf2j;Yr#$rkq_)@2=kc+o3XH++Rnw5zz@BSF3(oc}w&}N7vgfMr6@LpMJ{pq5Xoc zmw#T>faBNLbWz3?`rDfyNy^^xf2Ig2!x6wR>CmBwQbCEkkyzfBR5^|+>#(!#vqQHG7kM@++H4yBmfu$&jz+9d z$(^2EmsJW7E)JeKI(Ow?%4=)dd0A=d5n)xd2!Kr& zN4v%8V+)roECdR_0OAv$=rGi_ZT3DL85At27eYbI?7X3US0)u5>>wX`rHf7Ib9ESB$Tc^m0XwQ6ob|PYf2X87yg*_mxf(OxMzm8sQ;!z z_P>(-FS|uRmR-BrwsNnPFZ}!Hx~Af;;XAqNxRmv}N4fXBvR(9=-h!N17j;9TIlTWJ z=T}3t264{N+C?}&W&R+dAF@0XE*BGKMd+fXB|;|vbZ|V-yDdjmh!6+dxSG_}Y0v!A zdH=#b^3KKmbzaFn!kfDYY}QC*>tQn zGb0}#=CFTNPmobUX~R&x_q7Gizs~6;$2>ifLxF3EaWv3--uX%oT|P`KHD@|^bh|up zQs*JIA79$1BkKeT!b2%&FgtJ9)g%=ij&(dcsK3}TOB1}O6jwro{^i^E3Rev%UUH(S z%8Ia&hwp91^V=UzY#7>99R1rQ=k8+yTj^n4eB(#XMFQY*`QWEPqT?vo^1@Cv`>~y# zGYPbDO#0}$fUMhK^J^eZXS(-&zAEYaw*an@rF6QTISC3^@TMW@*BkK*x z&QdEwv*q16Uyj()d*G6j>%-+|!`kuUs+G$*ClFP*G#d~)tIOTl$EXvP=A4snRMJ`>m{nq#qu6G>=md?laOF9KTmGjf?09VOT zId>QCipFy%1Q!8pb-u8mxg7fL$7<yACQjtk8Uj(fN{2lA2~`@24+IUP&(25+ST9h2%3fnflRM!epw zr~TX6b#Zml91bhefnR!!%K%%}_Em!}gH^lvS%1D6okcp4 zpcr@G7w6i4dJNVjbmP5Qm+4(k`d{PR_7XG!w|y8hDB!cXA>wpD8~ekG2~6`!RU^(U zggITKsjd8FQsCCxb&oeDh;8Fqxn6p)dr-#WmA1MiXBX>(^@Xy| zVf-LUe{O%@zWqcubYly=jP%vsaz(mc`iTHt;lzt3{SRIx0NAf%Hvd5LHg4?p)}*DB zy9|J6OxF6dvS^rEY{?==xA;8U*6qoJ#JI3xap1NXoj1Qy7{R&~be$D}Bh&du0NOVn zcyhg(Ib$0!YPsnACpR{1=<~L2QxV}EH=dg2dE{k^b>}y_cwr+!T{T`j_uh!bhKG`Okg1y2so@M)`gE<;mPv@l9*KJnsvPGU~&Vl}Jg!g#3yF|2>ckXN* zeuI3tep`{B(Ak|fA0U_5CM%3~-3@t1y*OzOYDI3INb&@3j?sGIylm9%Qiipc?_b`Ytjn(>ZTE%R@ONONm3_~@ z8tJU1Hf+l?bhGTY9i2j*PUz92s$N4KA@YcrkR_sWBJ7!YSBo7hW~$x)5NI3VdlfAR z2;QAHXWMSNZ~e)3>{JJxwJeMK?5?5eM?m|p_XIuO1)J@1k2TzP}brVIM%!jv6c+1Dl9`9t*S2TzFhfA=G} z0B=7FDP7T$-Dy3SP>AO)MF_y!qPMSqOJ6_Mg~%Vk{P&-6t!*(vxr(eY3GyAy^1hMf z?lo#xK*xZ8(WTax^nPr?+2BUTb*)#W0eiW@oh*r zrBMZ<;Xtjmagvxx>xujcVDoSB#{o}|zUHiJ=ZF(=jwxp~Q~N(mG(Z|fvm6mumJ_&% zxX*FrPJi{9JEmNFkG%TOg#t>-5liUJzdA%wZb0eo5k1fRo;;=(7YZn~A3oQ2jEM6I zLd_d>wp`@!^mFRDleF}wThvOo(z6Z@CBK?|NVkm55n-}4ElmGw#07*qoM6N<$f`f_a$p8QV literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/pulse-snappydata-152X50.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/pulse-snappydata-152X50.png new file mode 100644 index 0000000000000000000000000000000000000000..431452c2ea40be39eb8eb54a0f807e0477503688 GIT binary patch literal 9070 zcmV-!Baz&RP)YM-o02y>e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{03ZNKL_t(|+U=cpm=smk|39}v zr^ymUW&i^Uf)N!=prF8zM3}CT90W72VSROX4XdlWueJ zpnyn}%)rEsRds)VRL{^8Kn31if3KY9>8Go2Ro!sz_nvd^Ij5@OI2{!QigRkdlaG&+ zOX4^Sjv2@lKmvqvx5_|q4x!wwzj@wh$Ki2INKJeMNGDL-Djv$+a-1v20d};KD;pdI zaC{&-0L?&r2(r9dn*G*EInIKkWS=GlQUrirz#t^)F=EHTbzFiS3zFa<5CkNgC%}Wi zgr+K#JlDsJjSb9Ul#Y&i8u36^=W~*aLxE&CtfUC+2Hm4jgg(a zM9?>=O%vJ*Q)T%mLN_w*c!xxm%rvc9LC<34CZi4L~1bbZZ~6^A((v z1Tr9yF$HIvkMO52nG3fvT=HPFBY!ShU5_Vc<^6$gW<3T#+u(zG2eIKo1?O}H*$5;8 zxmzdx^$c_F*8n4pUKE&b9tT)5wK47%?b&@@0PAUiOch8{DIYo0?l!lVy6Pnr>nZ`M zfHxsY;y_5j-jFU!M!s0|{+jEJ-b^tU*Q)ni;{ktbHWuhueE^?=9Z8cvw4Ge^#rEXu zn&th2-Lvkdk1=LcfbcT{=@&>#Cv-!W^L4@0(XOoz^pWt4fUkf%fc1b~7hFAzn|arK z4vZDrKqkJdVf9Mte>FuxhCD%FT{|G?4;JvgFI|ii?n=NWADM2QCZ=YSc`GW(b12+Rhk!(WnU%>Z3JQ-kT1Rr5-(fxEcZaE{91D~iknLjl(KB``t zZY~&8vuhpLm@&TbOq}@`J&kegQwS~4uuu7EJIINoQg-DO^Q}+J!A&t{?>TORxl36+ zODA?h9mb#WQ{buwO(khl#~N3+eWb~@gj*qp?~4A0iX15FtTI7-dqee{E`AuTCxI(r z+V2y@BoZOOBcUU(5rI<8(`s;4d-c~A-Om-qjVESs;EkUKm)lAdfA4#Or#G%H>E+F5 z;>^d$4GzmsqiXrj0T+OS;;SU><=cYYGmX?a$Bq1IN~ff^I0_-_pf---yPipcYI~^h z=PiVays);cu*i}--cK&m^R5QKN3^G`3orQU_OfLT0wED0eH~ zYX&EDY4lzV=hWI0V3$k8Oo@orAAwc!c+t9j0Nst7m{T~BtBkRuQiL@_!O8e#ywKW~ z3AH~jUwRy1myG;f!iVDYILu`g0=W{H@1Sf**kAR?MjcusQ6xA8oxKY3wB&14?A@!B z%SxZgORFi1&w4x22YGoN)f)l1sok52(}Ko8Mgm|?f~O)`Jx1rv2EapPLR~3!g52C` zmN|3CP8zqdP(KACKup2jlg;ja052HXyk}0Ln{n&;SyVkQ4%%5H*<1Ssa*0H|EfM<~ zmWlrE$NgZyvVtcQ6XJB14cJx+*0ZUE|329)u8GI=HAY?MBlfa_Fh@)Nk#={Wiz>x+h$eNYU^aaIaA zk>3p!^Plcn=yI>8F`l=~7g=b&JhPWE@{|mskH}hgXlzqXXbEL&0Zwo#de-0DSpU*x zNxeRt!nX74ZiFP1__eh=SauSWp@BRV#0C%tkWwPhlI*aWpmzmoMMIv~7~TmWhZu(O zqk{cL3OlDvG2dARFvjS?^XA*Y+w?F-pRZwEUB`GQO_p?t$FVO(3Jg`)IFtQf0W7Asc$=7YUb)K^TnF;tK}I}WCMz!1<|S+ z+scvM>qr0ueDy-vXZ3FNV$}nfLVWi#uWl{#m#K;?O~fg{YM>k;z;!#+P$+jRqI`S2FewtS#5 zY^g6S=8l?Cll*2%h%amS7##IjjP7oQvz)sXCG3r_C=b z4?UPYj;w%c;FCj8VT|P6=z`zB{L_ROLoqHgsJ!&mSC$2pRJZ29qNjrF!tJp?l~*?2LMhL zRRfiJ1GK*p^IhZnw3j+`(s1e+$g)IHEBDp4o6Zg1nW5s;i#G*i_!IXwp-MObu#p@{ z08AzUtOl7qm4^X%vuN@=Ap)H(w0S?}qOkCs^|I4*7(g@QfgByF)W3=|8vs^ZBmO^L zVrO~@^?S{n{m;qflAV_YhqCIrrvLW&tT9NFnHwmst-_CWhGyON>b7&CaceUL*|;w6 z(omZNd^@Q2sc(uNgS@$(bSnQY>kTmOPG$vqdL(jo0}QAJdsGo^{P zyA{Y&>P~5U^}Y9yX##;NpsA}GsR9`TEF^&hUuE@9QUSHEF}l5BK(03jL(j@xJ#0p!RnKZT6Ag0o8}cK2m0NqNnD{mz{%XpXwdKWf zITsE0{jbdDVF6ZcgobjrCMLcmq1@W0LiM>LC10IX;_CM5f~~qN%QaH7>h?8^*Z5m7 zQ*Zxz?Opig#)QTLl{pJFoU}UjRi!57yg+d)g>4?649L^#`-f<eBiL8;Riz`W6 zop0n`{YlXi<#Wv$kIgn`o?0POyV!{D8UX(RfFGTc_Z%uYLnB_O6YE!K7M=L2Ip>L6 z%{}qH#>h)EsCd>9%IxjlZZ`w`_pD7d^LVtGwso;^IBbYOagLI-kCjO)N!0m1xQq`P zI^vLVjzgC>(1ws81t7y|VbOb1uK@wuDpKmJ1*s8#OL!{pdG(dIR<*r1FY}DezUw^F z{=I_JrrxGNL>GKTe$6A9{QTt}iHff5W9W1utm2 zUw>T1?pDj&FCC&SH+Gsgc*pmF1DZh47|AFEao|~`1R)**)=QYKKzUW}R(Vzah~Akp zR$QAQGNdbIH@!OZ&FKKA+?Aho_K(Uk4X0f)tJ3u?S529*{9S-+jA2m^mZyH-trY_M z(Ws_xv~9huDjA{RbQj3c02PK&jRqfdttIA;Z#rDL1A9=+ev(CCN}{fnb3M7Ijm9V*h$t%8jzg`Z|DiY ziNtTs{YX6lj6|8xs>2}%zeP2us_#)js1k&5>rK=l-D{;(5s7;wO5#`r6oFJNf4G+@ zPjO}15s<(Rf7@m4l6CFF1NE4q0gAAd^paC{J{=ZkrUazb>6c^!I^OViUtGL-g%tAi z1o{*ZHt-(sDTqj7>>(iC9kvvFp5Y6pR90k^X^Hh$7ae+ck4@5CCRVj{FA;L5WL~f$Ic{bEF3BtSxnFiAZr!0@YJ>3Q`KUK`BTaB$zJ%;fCr^5}}P6 zFL$Do?)U|Y+ecT(c|%@L`u>#uM+`1uSCuv0J8tcn`@iu-RK@MN6dM48Wyz`*4X8@0D^Zz{Noq|d*-oOCo))5xl&x)p zC09bdu*Jd7*Yr`dM61s_r0AB`+MTl+M(v5=*`;Bhbr^oy3igJ8XJ( zSC-pPZ)v){df4ppM$ZnoDh)BOft_mdc0Z~Ud2UAGygQprx?H5~KOk+T*7y~`Nf^!| z31Ddld4sQ{)$YviBHkCzU*0NDkuu{jt|1B1@{6jw3Z@r7;~kQtsLnwm-{>9OP52@g>_-GGS+lhRSQ% zVvDi~_1X55#(1JKMTT-W*D~Y|w*xucdDr!!)bO%yI!m?A?i`%ZB}eRDp4>GLp|@t3 z3wm8+47*>)&IQ>@xQTvIf!r;S-)Y3>%`oTAtYvg2LeDM;S|ucz%a{vq8`L2)WV&6FHy3`vx<+t%Ngck_TU8QCDuPkK1;E`vDU#+0{v7j1 z=bZFWmw$CylR8!r_LV>zfy{I}BujxU3eG2u*!S#2bIJCOcVEY%Ds!?xM^FXyr%cY{wa$+$43+&9S^(W*QVZAI(#&0cQ0$G7G6mll;Ybl>3@ zP)-V90J$J<0ZKt+1OEj69!{_NOkP?N?Q4uk@nSbpuzjwLAtN55rPIy%6#(OmUQABP zI;}ewmlgDK!(+W9*JO-nqGP2Cq$Y4;8g|8l*6Ss0Uw5U@4Z*IwQ1s!?76w0ej(?dw z{41gS{3a9@?xn9W{G$}&m&qjI5TLx*%jV*@j}VM-4p|MB))$sxCr&xnlPAUIN|oG-i2oa5u_|{(D!j1VE>Fs>*MYj$DHEZMgW3;1NC+HS0NEH|wX2zhoXAxOxO{J>wp)B&t?N4k??8?^_uct`Zvh?%w&&rZ zZyUV7@wH#DZ_YtrTs~|xAI*5_P-<5NP67&nslZtvD?s@tQj~@IY_@wE!%shfiWX_k zjt^c)`%H>yFu)i#B!j)HrkiiAYTOO>eelc2d8u(LOU;S&G{$vo#2)#YIe+!h@~(?< zS4s}sdK8*P^Toegf_-Q{OBdsJ&960vZtxQMOdtzm_&y(E44isz-f4Q5;61gjKPP_z z)AL%v-%`ryrG-$Y9fUAs~>Xn^s z{Fm00J^v$7=8ZCXTo2Iu2qokZ0ql(6gQ+L7_hTLF6a{O#gF4AU#B7ufTa;EQpA>y0bDUll&hh*?`$UGxCA&)h*zV_bTkk>7ThG4Moy!70xoTx{@y`SC&ZSmdh5 zKEa_iWebZ6YL553@)$3wlcAktgwZd3y14+u7(lT(jjqP6DJu+=%Z%|C>&SGF`$5i; zh(!{y$HsHcPSJ8Hz$@l_0IoHLwaGW~PL+KTFBH84KsV0=hE6=!NqSnSk1;kgXymsY zYFty>`zxi5Ae4|~84|Vu4(Z4q#+bW3SRNa-jV7gfS#VsspYQ~MR4mk*x<8rq*97lI z`9>IheE?4teZ={BBeBQk^KQ{|0E`Uwb+bW6%LKN)5#S3QYsw4eCm|12j|!ZN1^pT#aywvpE3OGbSlHA zIy=7q^Q=$h0As{M9^!o!9LGa+()3mHUJ~Czt}*8F0FnPua9V(j_+{uX1x0U6%{7Ld zmxA>VkkcgUCZJarDh9_T>U6nnKm1)67Nh_A$l`Od<4S zEBQs|zZb3F*~$1rc561h;iGE)Ub6ON5S=K9zh|1S_h@?U)5?YV%By_DUsaHe(%8{; zDvOq1%Z>A^vo%z#AEt(9cMtkU<@<;}s^GLzaK5}$e%WoI=-Mfj&@&1$OCT$Kg#VP| zY@YSR=u$v50%vj}Cn_Dd*9t_}#Qu9$d{FxHJ5-#e0Im4$%&)oNyi6)zaaXy$y3m}} z5ujD@NzZv=<>MOGr3zAbiJF4bAMdNaNZxA)=x2-?<0E>5K>9sI-U4Ad_#VcMZ8E6%tBQSwK*mz3`0G@&crL&|W7w-cB28m> z<2teCC&|yd{va-$oW_CAUojUw;hI9j@AeYU2dVjG=%s?9H=c4=+csrj>V1=Rtj-Ef zWejDx2diZ&`)-_Keo%HqyGV(?9|(hJTu$p%`9}6ve!1_Cmx>mD?XK`qulk4#kr0;% z)k1q_WXFC>b2_yDL1jm-hSlCd`IkhwBrcRYfT;>vKY?`xh${q+4JdIu7mPIeg$m3! z{^}9@Q=(+5*b;CgiW(x&?eBT5vfjta26Fa@HiKdPGbgcSqJlijL;MLHx&wHKuLIaK zXM*#iG3Jv~h;IZMmEmvFmAfv{uzLyQcM_$!jn;LnzgK@9%TSQ55?L(mqu7>4e5i)q z(nk9ZoQvX!p<6}MIZ)~#q=O?O2u=jbfT~#6-I3mt3P!3pCnQ`TVlUa9QwTqiq+2za zix8Poo@WpLO2=x4%R424_qgze~YU9F$Li zxP#aa(2TvCmN;F;0=A0v`D4!P)B@2T4R+e7do<#EY*eWtcLzko^`E&(JQ3VyB`bhv zn>$7Xd_)Y_CFIPNKouzyJy@9zEOUW`DvFVP>`||Hh_%xR-CAT8ddl!$Wup?sKmHu! zcPTpYTXkY4jlb_J=+6zoQl z__FTC?Yh9}B#14UVa~m;$ej621-?0n2uwvAvFAcJFD_c}syX+Sx#o<|qG)dds4}`| z8_j+IHpTIE&Jq>Rdx-9A?(80NxiLN)WD`Jo*}bBA9I3=_Q;;WniF~w;=N7?NZw)INcRur54IV4hq8c@^6a~o$N|ulnaa=t^LIMgZ%DQbAh`>$6n(gA|Ur= z5pS$OYn_T`x)ft8#5{niW3Rf@xUoG5ze4m|S6WBT1)?h9WhWcsvc1GE)Uc;bHy14d zun_B9%T4a{xC>DAaA5wzqG=nSELvKUF3PVG*o5(I5+ZWGh9$g&9(U)%Qd~fX*C@m< z6iCu3ot&_@(p(nA*B+6PztbGYPTHKRW4&J@PTY3A(er#i@)QL!fvBH~Pc`gIfk+Db z*SMZs>uLcTeX?q=2j>qu_U0@qpRbVWCm`|QEQ_$uuRyknIC)1N$|c6{Psk*E4`BPL znx`QE=h9%u|-;wN&CM!c+z+|b{_-$%tsZAAIQJDiNSC4@9$Ujz7vcLEfR*m4FJ zr1^Z6a$eek^4hU4uo_mMyp6qDxv#Bq*Ri~Aofx9 zatSAVD?us<^&|r6Za7n<-)vmg#s!MZ8sQJziHo-fNTKY5RKgvB z6pi>2mrqVNpu8zN)(5cD6S>W2j5n@K?Ge1Skq`SG4U2Nty{KTHEwGtdvAo7XqGDeu zkn6H2E%g)Wq2f4E0=wE66I6lSUSRX0{n5LM`z!*bG=6HZaor0URQ(-9Oeg-$5$_@; zp{Rr^2i03n*2+xEKUQ$kyhJ9xxc9@=UhEu!UG|*$*(P_e@)Cjb{qsd1ha?=ZP+RXI zYhF_knxSGx1ZlCglbd;VjrC< zA{x=VG~%UA)mn4qZX6*9n!MOh^}q zb9ADc_1RbvltLTjV+Yl*;Peql0n*Nt+rK%YP*IEEqcjsMw?$-nS0C}~RGiid_GSfp z{wwDEkIBDHv88x0jmlNYE+w7#7cK}=dU`0o%S-f&DBk;ZB3V<-H#hb+nvWMaF$?8S z3d;Ekc9n)T=PKqb{>b=qn$EftRpOxt{#!Kcek#^RFVR;E%;K+l8`t+wr=q<@^jZb0 zGL=oY^)HlmLKUn8c_(hwiRkD600PQML_t)J77ES*iCpb>%2!R9^;Q{xCDTkxl(zOD znrT=?7D~34%GrrX(ligz1y7k@{Q$QUGR9n!LTIdq*oRekAJ+)m?O3+HPKHJ~Pzmg0 zT`C~1*Ra-Y7p)^<(srd0z9WUuDq%uvBjej_4&3G=wC1v+l2{ibXKZt}e4ydPG@>U!_w?qQ5>KiC+uXNb}N?Rq!%kxHJCFM~$DGz>$IHLkc6U`((b zJr3~-^ijdSKWm)1#^{49#JHME@+LHR-7uqXwI^Q$igQu}#W~{x#W`yN#W@!RigRif z@JORyt@{?_HKV^VqUQIw;1yhvzI?Q<*hvg(aH*Ekr<8eOOVy^S%c*BHas2NV3So`V~V%R2#_+WV+dQ;se5 zH-_&>qw3w$`11OP%uTV{aj`KMr4oK4o&9%DGvBHgtn6oux+6gNQXf^53e1J?jxnxe zn)&{}eGG{-952ur7=q+sgxm>2HuUt(|L@e+_?>9W{yu)<69h6*r(>-m gf%<{907*qoM6N<$g7(UE!~g&Q literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappydata-310X50.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappydata-310X50.png new file mode 100644 index 0000000000000000000000000000000000000000..39b94b00aba45db1905715ed83a6cd33825be7c0 GIT binary patch literal 7750 zcma)>^;Z<$)5kY}rBhP6yF;YAap^9Rly1pgS|p^qyFqG^T2MieQt6NdSr8Cd8l=8{ zp1qU4U7lCQNUxQRq& z%3iOqr{aidCBI&g-D7vP#>w`APM|<^6qHI-lu*S-<%PBlhu3$SZR!Em~x zjr!Wcy@s2?%ZTZlh@qYSog#wvkK{W1rJB8aAEzE7(TgF;&tG%oq$O+CMim31T_s&#zrUD--T{PECz9e>G7rX4;um;Do3ZF1fH~2o*35 zYIR8t@aG!Ne)h5Q@qcx>Gd^FhpSjmH$MlRTp(9-YxuCIVW5A^1GE%^Cs0U<%Btx*4kZIrD-jsA#1E2|~z+AvOSd#Q&j^{?qaWl_me!ubA(bdlhip`~D zjvRAP^1+utA2i|-D_3+f5ztC{uXL0L$j!!A&WR+KycYb`Xdw4a_SMq z*!!ixQW8yheg>%LkFWAIxdVL!pmNf1Y8A@Bs>YSy|Q@k_~kI}}3_i9&>-IX^Q++ES)*)+44VB)q~|H*@QjgM-vpkRSzxWVXChDsv>1*+6f4kj0)SuASVPA+W#6XIcine)CQ!hldo)!xR_%e~)$_Q%q zb)@h+o_1r0fHyeodB!Oeb!?$zg58rI=3Ge5(VsowBS0$~KbT5mDgonZ-_U+91z#bJ zO!;p9c3^9M+yO~+VFc&FO(+lVh!F7Q#n_;rPvbSP&1}IKV4Gf?3344y4J{Fq^A#T# zBH#ZG!&qcld89b+{MEsi;$VMer2eD%m3jb^NsDN%(GX+Hin_h@NEi1r^`AQApM9SC z40J8vXYmyQ{pz`XW)3>vj$9Ts>6W%h2~($ka8;1ScwEXPNCzH?_#0C0F~u~bvEPsX z;p!Ic2z?^IP_XHSdcOiw03U4^^snV$9lqTpAoCEISUs%hJ~WdP))BO9aNZrBl)RIDhym<3=p`JE(ab(HxQCs;U}2emxB?|ACV9*GzJMlHIqQbDpeN^E?O7f>v+|5(p1I7mctW01@D8_ok{ zHSHwi%+}(q;Cjkw%8hjLT(z=o{7C>Ljw(aG2yfNn@3PSa_xv`9`-W<{V9N3P!jki2 ztQBpP8_GJ1VNFEsG@m4l&3ZW7vDIOo!U`Ve8P!ZXBqg6qH%DAp5#d=o34N*+0Cbm3 zEZ`5_s-C3w%^3;y5TVRiJc9fc$G<8USkzD%csd>=NEF_R_XH0OLb=%Ryd4>aA5Y%X zn`TYDVT2^73WF(We*Y>ZT3yuFeaf&4oR2huAMhRUyb|aZdms>|-|)){l9~{v z=B__Rb7}&)*kjk7vplg6KiV=5Xa|AZm3%JY(_k|Yb*%0qp=X6(a7#^s^lxWCEdW8# zE9?N^_$jUc%aB1(dDP0;O))Wm)l)xF1i^gKmYOKmc-N!C-&E^(%7d!toKYyZ&MgPb ze`g9|BAsgQo$6sJjm2UrMe;OKINA|#20tlb4<4$nfvxQKFh|B~t1^<;Azs<6BjSRP{-Eoc$hI$=js7aSSnZ1p& zWn^&sWC73sw{}fx+{g{VgehGzE(p5nkuHP*F<$A-NWRhc?Bd-7a#xO=3(xWo_Uk3R z^YuT^VTmS-UMg$B^-^fFBCO$!09sFb2IG%CSiRQYoZx+H&xwD^KvNa-94QXKQL3*H zA-dJA`8u^bODugt`GkpP*pXd4ZjQK!J7oAFut@WPCHVOM@8N{^2(d)V0TPjwj7-Pj zWm5?i2Lx;1C{iQazzB#FFazwd$?rL3EXAbseGa7C`Kd!|Lon+bt{nj^k#C#X`VX6q zmWe?5D~%_Tr-b{CMhRWzx|J^_IgrX(g(?xjW|)3>odQuF6~ZCriglLUn;x6OdC1J_VubV<>i@Nj= zMMLGwelCnetX!X~3xa{hICgLdsi*~lDg4aqR7JQLdwQs}hMN|M)vv844E=)mh;fh2 zM7}HfR7He5Etj-}M~!}39z~k+_q@)l8%(7Cwp$+4CMUk??@#=_;p#N3l@XmQVfSvEyJ>S zJW>Fhte8&*h*!FJur1ds(?w0!-M5T6oc8iXN{`Qyh7IR!K3ajv#7t`;+Uj&urE70T zkODM0Ka%~3;r5Gbyx-x)_-*5V|5_%xa{GwHV?^301-;`Y6axnz2OhhOposCy+f4m+A$zy6qq)~17dq(COYOQp=@XU(Rmr0A` zJBz2>N@ozVdwaX4fn%mozAmWrP|?B*w$#bX3y#9hBIDA<}-*IR zUYrPUoLJR_+}iIemcEm@jliAxf#eRyR@d4gx)_y89@@M>J)9J}tsDb1}0zz%@c>UF5vWVS>$iNt35dd>Z{LT^7kn|K`o5 z2j1FEKfuaXzom6mFA7!x8O6Sx!KL;0INwFlpQe=Wmx0NuGou5OnCO z)-%L3gvUCSoiFqwjb>hK^{{{;^)CW#Y%-Vne@z=kZ=@__c^_6 zApVr^VFJxOZ+nW4|I(7|s@6;1Xok6XNycf|1E)z&cEL>d2QzS;*RWGIyNXAs-RsXU zFtNOjFNeb}VGGb4I*y;un*_2yEZh5yx-1@!h);jIj$oNhGqoQ5R5-s7Y7I|8;fC>s zbDRxm&d=EJj8#{zdPzPEy;aB|Ldjcqo{F#HnHv-8;@4?}kfI4uSGYLY*h3O!X-oQa z@|KbHR-73}-f9S(;EDx>&V47HvYzg18`KMwxV^+k2O>G9Hp#V%e4`ZsjavjMx%5`4^{rm7h0Q03a2IVCG+ zM*S*$VMBjECW5XBo}y4lzSjra(kef0Rb{ZE2p%@gav__i12{YGgtF+W<#+J`1!fk6 zz8%kYe_8aGj1Gm2|G?E)bS}F4V;?J2xoYHYuamE^w=+9GXXO7J|2tEjqY#Z-G12P4 z?*ZQcF1pkO0gQ#L<=uE1|)lBeQg8B!Oj#Z?%04&V29&L#s022Xa%{ZtilUH2P zPLl-JvITGoQt38ayTUXb#67D|C@Qc)1jF5ZSSgG3$S_ab=~7(U=Rjz4aXVcyGXHJg z7EzYW=MOkwiclFrr^~j1EXp~rwGOsX3(~XpX!4)zrz%zx zDKfO!!>9o{gSZdgxxKT?H&4HX4;J*rv$GCaN#)tbM#Mvyi=e!toX(@SkdA#n|2=0R z5A^tP)JI3T@f2nmviz|>ahE}e@jrjf@V78cghtTYwK^PDkod`8X1@2-Wr|`O z(Knv@ke!>aaxg+of0d#&V;m{+8}PWwCGQ-g0!dv-Za6!H`7x{YToIb!bUj;S}(yW?d3G!|J@R><9MgAU=yBol3$u9+ExB z#b-Z0Z`|JJnQgbzO?Ak}7M~XM`z16B=_f+hZmUe_yjh-bKy#T=?V)DO|1EB3!)w27 zpKXeg`-O+^(f^@!TnBOh8&n^_U@cc!I367qjKN3bD64Om^EGp^(SL+ zx&j5wFV@GZTzN~u^x(~|{6Gx!i}T3z6*4I6U^^25=7{^eqc{}(rk}XBsS`9-ucO2# z+cnNXD^aj}ohVXkzjZ5KajJLvg%rH$gYk?)t?IXA?jpBhg@6z66ZO|iiQf00th7C5 zodn{uk-FUDN4?l)zE121ER+^ZYyR-w`&b-cJi- zFvWjZMsiJhdpcKt^ooS<2+2OJSi@?$H5t4B9AJ&;r6B`oU@!|w9-k?r?C;3yz zSR|Z8#6F0k=Kwd{&Tgf-J2Hk_PwJ(;hEjbPEPbkp6sB*U-*;-j3txw)3 zmWyUCQPCxD;`D}${CZ>*^|--;6~B~pXNP|!EI4f&xicwN?FaE}{69n^K`<{DWYZb5Pwa}YQ zHNNif?V6M=tYU+DRXUFZ*EO$`1G*exU)>b48$i9;v=l;Wx4C&HM_KOb=g_azk5%gZ zd2RISc;CUJ_ky&iPca|vIi|r19oW--Sm(KMoa0iZ=EpizkJ82t*!QDlA=EZnrGP^I zIC1cLn|z?jIU8_RrqoRz<LJA}t9B8vFwG7pI6yWUi(0F4*6&eZXSeb$!e;_VN372r<9@ss7oa(P0^1Ieh082%8pc6qn`nc?%DhaxL2 zlwz&k3H=Vw3ig0StWKi?Pry0IF3Z;*^OG>#3U>kBT~1D4nfR9kS9=KMbwrw>bl5OM z+;0?W7OQP09!j{mHu15=)!VxI%I{DbvFUrn&UDJ0&K0;R1i8$6)xV`+T-w;~co@+U z*Hy7MJqc&kc|=bbXg-MhR8@=t8)G64Qh6-3!sLW@%L2w1$PQwJ9uv)0>`6Mo%DERr z95)B*^d(&EjgLQwX3=T>Ns!T^RT;zDF#NngHdQ8PjW$NPGCfGfhNASuA2|sUj!;&6 z5c7-&**lWV-OIhHf9yUxQ2mznnKG|H5&6lt-hF8bP76g6!);lT?HozAq6C=%PXT~q zaB=#)n$NZd{p>|&a<*}5s!Wa?q1p3W@u#MwS{vgGYiZvTxdgpO@*4$M$`-Nub;nMW z44xk+Nw;LiUbm_@o`m;|XCy?t9I5rGz9>{V==U0&Q6|Vvt(B`0*yqe=9&v2&81K&g zv%M9t?c@q9_TXM!#Ac(3-=joO)aK|<+EM#w{!}ZA7ud8Po^URzywe!n%wg@<53|%) zur`S{QYMg#)0td0k9NYC#oD3hQFr*qhc9Bk1PM7z^g+-rYRIu_gga7ceZwc#DCqPH zN+E+=ur?dI}qcgS0f{}0Pjj??T zq(LSyd{Bez^B_qEDThCIOPQ;GB7ABxM_#C{RiWd2k>0%4aW1_?Z)xzbp}1=%0Z~P8 zmWj<}6+~ua9?!RLMETojbeZ8rpB`(~+g;r5oS^&a*#c5i9mb1qaaofI^9I|s$qmNz z{p$HnBxfY;TbJ;4YK=E9+ACa8Aa|=3DFMM~di(wD2+D~mdBSw^a}CzB5IgH4yIFN6 zlcYiV5CZg`pLyQP?^izvlSP3zowoMv&JSjb2GeV}I~i6I>~VZN0nBk@2WZ+-UGG&# zP_(2LZJRDCzoN;;?CW8vTdP2>z)k?UAR<`i*3slrb7QZ^;o8%`kb!<&mP?Zc1 zj1d`SxCo<}9f!v;Usec2a=#Y#c<;incWv?vtFvp)AKsxOV__~3^eJZ2W3rj5LN=Ak zWRBV-0!YnCCMXcV%&H>eaK!ovLLS_Gtkzs&6DC-nKFtG7@(q1yH4|i>R}Y<4TcZS@d=5c_EQeY~TFa&WGaSwD7Q$8C zp=-11A66*u$gKO*sR6@gRbog3rx|8UcCbLgQ8TQUIcUuZE-6`XZAWF_=C=AHKFR0J zlP*#hzd>48F+2}7z6Gl=?+tyfFl-tXZJv?N-}LnFEC8<+Z4^|%M z#eQbiK7CQJ#*tli30yF;8pcx3*PWS0T2 zpLHZypsa(YJLfJC%<63wrUIij7$j)6Kzov15>5|pOD#vAuvz2l@=X2cUOp%J#87d| zG){@r-Djx%%}WeJCpfnYC_FX($j9TA?gcPS^i#l4#M*(sN?3XISIWa>2dA$8|QO6*8$CPRLPA*x`N zK1)x|5o26~-@b9CMz;JOgV#GLHOCd;L9Ohxxpb7{us)YWGgeGZ?5BNQSZVl7XKbe$ zvaU4!)RQAiE-Rj8qG>Z1$7WbkMS%i9dbyQf@J5ZfGRb@Ta5r9o z6Bt>;FMLP)s8n{?Lv63a-0sug0#miVxrL;`nSQyOyh)GVz%g`F>jBc#NDh#ve{ZP7 zq*Pik1!<}59?UoYPv9g@CMt8@yhlY51&dx8q`uOHH355Lmg7>YU%3CL+bJYjOXJCC qTwiJF1uRYKyXc^#1^HkioD3 literal 0 HcmV?d00001 diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css index 935d9b1aec615..ac2c8e5102543 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.css +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css @@ -22,7 +22,16 @@ } .navbar .brand { - margin-right: 20px; + float: right; + margin-right: 5px; + margin-bottom: 0; + margin-top: 0; + margin-left: 10px; + padding: 0; +} +.product-brand { + float: left; + margin-right: 10px; margin-bottom: 0; margin-top: 0; margin-left: 10px; @@ -251,4 +260,69 @@ a.expandbutton { .table-cell-width-limited td { max-width: 600px; -} \ No newline at end of file +} + + +/* SnappyData */ +/* Popup container */ +.popup { + position: relative; + display: inline-block; + cursor: pointer; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + line-height: 2.5; + vertical-align: bottom; +} + +/* The actual popup */ +.popup .popuptext { + visibility: hidden; + width: 400px; + background-color: #CCCCCC; + color: #202020; + text-align: left; + text-shadow: none; + border-radius: 5px; + padding: 10px; + position: absolute; + z-index: 1; + right:0%; + font-size: 13px; + line-height: normal; + margin: 5px -10px 0px 0px; + /* bottom: 125%; */ +} + +/* Popup Arrow */ +.popup .popuptext::after { + content: ""; + position: absolute; + bottom: 100%; + left:90%; + border-width: 10px; + border-style: solid; + border-color: transparent transparent #CCCCCC transparent; + /* left: 50%; + margin-left: -5px;*/ +} + +/* Toggle the popup */ +.popup .show { + visibility: visible; + -webkit-animation: fadeIn 1s; + animation: fadeIn 1s; +} + +/* Add animation (fade in the popup) */ +@-webkit-keyframes fadeIn { + from {opacity: 0;} + to {opacity: 1;} +} + +@keyframes fadeIn { + from {opacity: 0;} + to {opacity:1 ;} +} diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js index 0fa1fcf25f8b9..12d8d7d36f6cc 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.js +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js @@ -50,4 +50,12 @@ function collapseTable(thisName, table){ // to remember if it's collapsed on each page reload $(function() { collapseTablePageLoad('collapse-aggregated-metrics','aggregated-metrics'); -}); \ No newline at end of file +}); + +/* SnappyData */ + +// When the user clicks on div, open the popup +function displayVersionDetails() { + var popup = document.getElementById("sdVersionDetails"); + popup.classList.toggle("show"); +} diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index a4f8f2b798b01..3c662e2bf07f3 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -21,6 +21,7 @@ import java.util.{Date, List => JList, ServiceLoader} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer +import scala.collection.mutable.HashMap import org.apache.spark.{JobExecutionStatus, SecurityManager, SparkConf, SparkContext} import org.apache.spark.internal.Logging @@ -160,14 +161,14 @@ private[spark] object SparkUI { val STATIC_RESOURCE_DIR = "org/apache/spark/ui/static" val DEFAULT_POOL_NAME = "default" - var productVersion: String = new String() + var productVersion: HashMap[String, String] = HashMap.empty[String, String] - def getProductVersion: String = { + def getProductVersion: HashMap[String, String] = { productVersion } - def setProductVersion(version: String): Unit = { - productVersion = version + def setProductVersion(versionDetails: HashMap[String, String]): Unit = { + productVersion = versionDetails } def getUIPort(conf: SparkConf): Int = { diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 0ea856466a507..95ff953b5969f 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -230,16 +230,27 @@ private[spark] object UIUtils extends Logging {
    -
    +
    +<<<<<<< HEAD {getProductVersionNode} +||||||| parent of c1b634f... Snap 1890 : Snappy Pulse UI suggestions for 1.0 (#69) + + {getProductUINameNode} + {getProductVersionNode} +======= +
    -

    - {shortAppName} application UI -

    +
    + + +>>>>>>> c1b634f... Snap 1890 : Snappy Pulse UI suggestions for 1.0 (#69) + + {getProductVersionNode} +
    {getProductDocLinkNode()}
      {header}
    @@ -286,16 +297,27 @@ private[spark] object UIUtils extends Logging {
    -
    +
    +<<<<<<< HEAD {getProductVersionNode} +||||||| parent of c1b634f... Snap 1890 : Snappy Pulse UI suggestions for 1.0 (#69) + + {getProductUINameNode} + {getProductVersionNode} +======= +
    -

    - {shortAppName} application UI -

    +
    + + +>>>>>>> c1b634f... Snap 1890 : Snappy Pulse UI suggestions for 1.0 (#69) + + {getProductVersionNode} +
    {getProductDocLinkNode()}
      {header}
    @@ -602,11 +624,28 @@ private[spark] object UIUtils extends Logging { } def getProductVersionNode(): Node = { + val versionDetails = SparkUI.getProductVersion val versionTooltipText = - "SnappyData Ver. " + SparkUI.getProductVersion + " ( Underlying Spark Ver. " + org.apache.spark.SPARK_VERSION + " )" - - {SparkUI.getProductVersion} + "SnappyData Ver. " + versionDetails.getOrElse("productVersion", "") + + " ( Underlying Spark Ver. " + org.apache.spark.SPARK_VERSION + " )" + +
    + { + versionDetails.getOrElse("productVersion", "") + } + +
    + Product Name : {versionDetails.getOrElse("productName", "")}
    + Product Version : {versionDetails.getOrElse("productVersion", "")}
    + Build : { + versionDetails.getOrElse("buildId", "") + " " + + versionDetails.getOrElse("buildDate", "") + }
    + Source Revision : {versionDetails.getOrElse("sourceRevision", "")} +
    +
    } def getProductDocLinkNode(): Node = { From 0c52ebd35f3d426a61887281cb819f19f5c4a65e Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 25 Oct 2016 00:44:58 +0530 Subject: [PATCH 053/142] [SNAP-1067] Optimizations seen in perf analysis related to SnappyData PR#381 (#11) - added hashCode/equals to UnsafeMapData and optimized hashing/equals for Decimal (assuming scale is same for both as in the calls from Spark layer) - optimizations to UTF8String: cached "isAscii" and "hash" - more efficient ByteArrayMethods.arrayEquals (~3ns vs ~9ns for 15 byte array) - reverting aggregate attribute changes (nullability optimization) from Spark layer and instead take care of it on the SnappyData layer; also reverted other changes in HashAggregateExec made earlier for AQP and nullability - copy spark-version-info in generateSources target for IDEA - updating snappy-spark version after the merge Conflicts: build.gradle sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala --- .../spark/sql/execution/aggregate/HashAggregateExec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala index 7e09a40f01cdb..5283d9933c43f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala @@ -323,7 +323,7 @@ case class HashAggregateExec( private val bufferSchema = StructType.fromAttributes(aggregateBufferAttributesForGroup) // The name for Fast HashMap - private var fastHashMapTerm: String = _ +b private var fastHashMapTerm: String = _ private var isFastHashMapEnabled: Boolean = false // whether a vectorized hashmap is used instead From 59d807699d4fa617b610ae6eb6ae7628f5767253 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 26 Oct 2016 22:33:36 +0530 Subject: [PATCH 054/142] [SNAP-1067] Optimizations seen in perf analysis related to SnappyData PR#381 (#11) - added hashCode/equals to UnsafeMapData and optimized hashing/equals for Decimal (assuming scale is same for both as in the calls from Spark layer) - optimizations to UTF8String: cached "isAscii" and "hash" - more efficient ByteArrayMethods.arrayEquals (~3ns vs ~9ns for 15 byte array) - reverting aggregate attribute changes (nullability optimization) from Spark layer and instead take care of it on the SnappyData layer; also reverted other changes in HashAggregateExec made earlier for AQP and nullability - copy spark-version-info in generateSources target for IDEA Conflicts: common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java Conflicts: common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala --- build.gradle | 7 ++++ .../spark/unsafe/array/ByteArrayMethods.java | 37 ++++++++-------- .../apache/spark/unsafe/types/UTF8String.java | 25 +++++++++-- .../expressions/aggregate/Average.scala | 6 --- .../catalyst/expressions/aggregate/Sum.scala | 12 ------ .../expressions/aggregate/interfaces.scala | 8 ---- .../sql/catalyst/optimizer/Optimizer.scala | 14 +++++++ .../org/apache/spark/sql/types/Decimal.scala | 41 ++++++++++++++---- .../sql/execution/aggregate/AggUtils.scala | 2 +- .../aggregate/HashAggregateExec.scala | 42 ++++--------------- .../sql/execution/joins/HashedRelation.scala | 2 +- 11 files changed, 106 insertions(+), 90 deletions(-) diff --git a/build.gradle b/build.gradle index 2ab964fbd6eb4..2f3f420315a64 100644 --- a/build.gradle +++ b/build.gradle @@ -364,6 +364,13 @@ subprojects { task generateSources { dependsOn subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion + ':generateGrammarSource' dependsOn subprojectBase + 'snappy-spark-streaming-flume-sink_' + scalaBinaryVersion + ':generateAvroJava' + // copy extra-resources in normal resource path for IDEA + def coreProject = project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + copy { + from "${coreProject.buildDir}/extra-resources" + include 'spark-version-info.properties' + into "${coreProject.buildDir}/resources/main" + } } if (rootProject.name == 'snappy-spark') { diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java index a6b1f7a16d605..058939c4c0e8a 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java @@ -48,23 +48,22 @@ public static int roundNumberOfBytesToNearestWord(int numBytes) { public static int MAX_ROUNDED_ARRAY_LENGTH = Integer.MAX_VALUE - 15; private static final boolean unaligned = Platform.unaligned(); + /** * Optimized byte array equality check for byte arrays. * @return true if the arrays are equal, false otherwise */ public static boolean arrayEquals( - Object leftBase, long leftOffset, Object rightBase, long rightOffset, final long length) { - int i = 0; - - // check if stars align and we can get both offsets to be aligned - if ((leftOffset % 8) == (rightOffset % 8)) { - while ((leftOffset + i) % 8 != 0 && i < length) { - if (Platform.getByte(leftBase, leftOffset + i) != - Platform.getByte(rightBase, rightOffset + i)) { - return false; - } - i += 1; + final Object leftBase, long leftOffset, final Object rightBase, + long rightOffset, final long length) { + long endOffset = leftOffset + length - 8; + while (leftOffset <= endOffset) { + if (Platform.getLong(leftBase, leftOffset) != + Platform.getLong(rightBase, rightOffset)) { + return false; } + leftOffset += 8; + rightOffset += 8; } // for architectures that support unaligned accesses, chew it up 8 bytes at a time if (unaligned || (((leftOffset + i) % 8 == 0) && ((rightOffset + i) % 8 == 0))) { @@ -75,15 +74,17 @@ public static boolean arrayEquals( } i += 8; } + leftOffset += 4; + rightOffset += 4; } - // this will finish off the unaligned comparisons, or do the entire aligned - // comparison whichever is needed. - while (i < length) { - if (Platform.getByte(leftBase, leftOffset + i) != - Platform.getByte(rightBase, rightOffset + i)) { - return false; + endOffset += 4; + while (leftOffset < endOffset) { + if (Platform.getByte(leftBase, leftOffset) != + Platform.getByte(rightBase, rightOffset)) { + return false; } - i += 1; + leftOffset++; + rightOffset++; } return true; } diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index 712129391cd80..b45022fc728bd 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -54,6 +54,9 @@ public final class UTF8String implements Comparable, Externalizable, private long offset; private int numBytes; + private transient int hash; + private transient boolean isAscii; + public Object getBaseObject() { return base; } public long getBaseOffset() { return offset; } @@ -187,6 +190,7 @@ public void writeTo(OutputStream out) throws IOException { * @param b The first byte of a code point */ private static int numBytesForFirstByte(final byte b) { + if (b >= 0) return 1; final int offset = (b & 0xFF) - 192; return (offset >= 0) ? bytesOfCodePointInUTF8[offset] : 1; } @@ -202,10 +206,14 @@ public int numBytes() { * Returns the number of code points in it. */ public int numChars() { + if (isAscii) return numBytes; + final long endOffset = offset + numBytes; int len = 0; - for (int i = 0; i < numBytes; i += numBytesForFirstByte(getByte(i))) { - len += 1; + for (long offset = this.offset; offset < endOffset; + offset += numBytesForFirstByte(Platform.getByte(base, offset))) { + len++; } + if (len == numBytes) isAscii = true; return len; } @@ -332,7 +340,7 @@ public boolean contains(final UTF8String substring) { /** * Returns the byte at position `i`. */ - private byte getByte(int i) { + public byte getByte(int i) { return Platform.getByte(base, offset + i); } @@ -1256,6 +1264,12 @@ public boolean equals(final Object other) { } } + public boolean equals(final UTF8String o) { + final int numBytes = this.numBytes; + return o != null && numBytes == o.numBytes && ByteArrayMethods.arrayEquals( + base, offset, o.base, o.offset, numBytes); + } + /** * Levenshtein distance is a metric for measuring the distance of two strings. The distance is * defined by the minimum number of single-character edits (i.e. insertions, deletions or @@ -1322,7 +1336,10 @@ public int levenshteinDistance(UTF8String other) { @Override public int hashCode() { - return Murmur3_x86_32.hashUnsafeBytes(base, offset, numBytes, 42); + final int h = this.hash; + if (h != 0) return h; + return (this.hash = Murmur3_x86_32.hashUnsafeBytes( + base, offset, numBytes, 42)); } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala index 70331bb183cd2..7a296590c37e9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala @@ -57,12 +57,6 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit override lazy val aggBufferAttributes = sum :: count :: Nil - override lazy val aggBufferAttributesForGroup: Seq[AttributeReference] = { - if (child.nullable) aggBufferAttributes - else sum.copy(nullable = false)(sum.exprId, sum.qualifier, - sum.isGenerated) :: count :: Nil - } - override lazy val initialValues = Seq( /* sum = */ Cast(Literal(0), sumDataType), /* count = */ Literal(0L) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala index 6dacb7148072c..86e40a9713b36 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala @@ -53,22 +53,10 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast override lazy val aggBufferAttributes = sum :: Nil - override lazy val aggBufferAttributesForGroup: Seq[AttributeReference] = { - if (child.nullable) aggBufferAttributes - else sum.copy(nullable = false)(sum.exprId, sum.qualifier, - sum.isGenerated) :: Nil - } - override lazy val initialValues: Seq[Expression] = Seq( /* sum = */ Literal.create(null, sumDataType) ) - override lazy val initialValuesForGroup: Seq[Expression] = Seq( - /* sum = */ - if (child.nullable) Literal.create(null, sumDataType) - else Cast(Literal(0), sumDataType) - ) - override lazy val updateExpressions: Seq[Expression] = { if (child.nullable) { Seq( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala index d060fdc2b3c3c..d6d394f5b8ae9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala @@ -194,9 +194,6 @@ abstract class AggregateFunction extends Expression { /** Attributes of fields in aggBufferSchema. */ def aggBufferAttributes: Seq[AttributeReference] - /** Attributes of fields in aggBufferSchema used for group by. */ - def aggBufferAttributesForGroup: Seq[AttributeReference] = aggBufferAttributes - /** * Attributes of fields in input aggregation buffers (immutable aggregation buffers that are * merged with mutable aggregation buffers in the merge() function or merge expressions). @@ -378,11 +375,6 @@ abstract class DeclarativeAggregate */ val initialValues: Seq[Expression] - /** - * Expressions for initializing empty aggregation buffers for group by. - */ - def initialValuesForGroup: Seq[Expression] = initialValues - /** * Expressions for updating the mutable aggregation buffer based on an input row. */ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index a28b6a0feb8f9..da69ea621b59c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -1158,6 +1158,14 @@ object DecimalAggregates extends Rule[LogicalPlan] { Divide(newAggExpr, Literal.create(math.pow(10.0, scale), DoubleType)), DecimalType(prec + 4, scale + 4), Option(SQLConf.get.sessionLocalTimeZone)) + case Max(e @ DecimalType.Expression(prec, scale)) if prec <= MAX_LONG_DIGITS => + MakeDecimal(we.copy(windowFunction = ae.copy( + aggregateFunction = Max(UnscaledValue(e)))), prec, scale) + + case Min(e @ DecimalType.Expression(prec, scale)) if prec <= MAX_LONG_DIGITS => + MakeDecimal(we.copy(windowFunction = ae.copy( + aggregateFunction = Min(UnscaledValue(e)))), prec, scale) + case _ => we } case ae @ AggregateExpression(af, _, _, _) => af match { @@ -1170,6 +1178,12 @@ object DecimalAggregates extends Rule[LogicalPlan] { Divide(newAggExpr, Literal.create(math.pow(10.0, scale), DoubleType)), DecimalType(prec + 4, scale + 4), Option(SQLConf.get.sessionLocalTimeZone)) + case Max(e @ DecimalType.Expression(prec, scale)) if prec <= MAX_LONG_DIGITS => + MakeDecimal(ae.copy(aggregateFunction = Max(UnscaledValue(e))), prec, scale) + + case Min(e @ DecimalType.Expression(prec, scale)) if prec <= MAX_LONG_DIGITS => + MakeDecimal(ae.copy(aggregateFunction = Min(UnscaledValue(e))), prec, scale) + case _ => ae } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala index 6da4f28b12962..98f612e8c24ba 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala @@ -190,10 +190,10 @@ final class Decimal extends Ordered[Decimal] with Serializable { def toJavaBigInteger: java.math.BigInteger = java.math.BigInteger.valueOf(toLong) def toUnscaledLong: Long = { - if (decimalVal.ne(null)) { - decimalVal.underlying().unscaledValue().longValueExact() - } else { + if (decimalVal eq null) { longVal + } else { + decimalVal.underlying().unscaledValue().longValueExact() } } @@ -339,14 +339,41 @@ final class Decimal extends Ordered[Decimal] with Serializable { } override def equals(other: Any): Boolean = other match { - case d: Decimal => - compare(d) == 0 - case _ => - false + case d: Decimal => equals(d) + case _ => false } override def hashCode(): Int = toBigDecimal.hashCode() + def equals(other: Decimal): Boolean = { + if (other != null) { + val decimalVal = this.decimalVal + val otherDecimalVal = other.decimalVal + if (decimalVal eq null) { + if (otherDecimalVal eq null) { + if (_scale == other._scale) longVal == other.longVal + else toJavaBigDecimal.equals(other.toJavaBigDecimal) + } else { + toJavaBigDecimal.equals(otherDecimalVal.bigDecimal) + } + } else if (otherDecimalVal ne null) { + decimalVal.bigDecimal.equals(otherDecimalVal.bigDecimal) + } else { + decimalVal.bigDecimal.equals(other.toJavaBigDecimal) + } + } else false + } + + def fastHashCode(): Int = { + val decimalVal = this.decimalVal + if (decimalVal != null) { + decimalVal.bigDecimal.hashCode() + } else { + val longVal = this.longVal + (longVal ^ (longVal >>> 32)).toInt + } + } + def isZero: Boolean = if (decimalVal.ne(null)) decimalVal == BIG_DEC_ZERO else longVal == 0 def + (that: Decimal): Decimal = { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala index 0c3161662e717..60d3e70dbb2d4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala @@ -82,7 +82,7 @@ object AggUtils { aggregateExpressions = aggregateExpressions, aggregateAttributes = aggregateAttributes, initialInputBufferOffset = initialInputBufferOffset, - __resultExpressions = resultExpressions, + resultExpressions = resultExpressions, child = child) } else { val objectHashEnabled = child.sqlContext.conf.useObjectHashAggregation diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala index 5283d9933c43f..23dfc8a4310a3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala @@ -14,24 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ package org.apache.spark.sql.execution.aggregate @@ -60,20 +42,14 @@ case class HashAggregateExec( aggregateExpressions: Seq[AggregateExpression], aggregateAttributes: Seq[Attribute], initialInputBufferOffset: Int, - __resultExpressions: Seq[NamedExpression], + resultExpressions: Seq[NamedExpression], child: SparkPlan) extends UnaryExecNode with CodegenSupport { - @transient lazy val resultExpressions = __resultExpressions - - @transient lazy private[this] val aggregateBufferAttributes = { + private[this] val aggregateBufferAttributes = { aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes) } - @transient lazy private[this] val aggregateBufferAttributesForGroup = { - aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributesForGroup) - } - require(HashAggregateExec.supportsAggregate(aggregateBufferAttributes)) override lazy val allAttributes: AttributeSeq = @@ -320,7 +296,7 @@ case class HashAggregateExec( private val declFunctions = aggregateExpressions.map(_.aggregateFunction) .filter(_.isInstanceOf[DeclarativeAggregate]) .map(_.asInstanceOf[DeclarativeAggregate]) - private val bufferSchema = StructType.fromAttributes(aggregateBufferAttributesForGroup) + private val bufferSchema = StructType.fromAttributes(aggregateBufferAttributes) // The name for Fast HashMap b private var fastHashMapTerm: String = _ @@ -340,7 +316,7 @@ b private var fastHashMapTerm: String = _ */ def createHashMap(): UnsafeFixedWidthAggregationMap = { // create initialized aggregate buffer - val initExpr = declFunctions.flatMap(_.initialValuesForGroup) + val initExpr = declFunctions.flatMap(f => f.initialValues) val initialBuffer = UnsafeProjection.create(initExpr)(EmptyRow) // create hashMap @@ -409,7 +385,7 @@ b private var fastHashMapTerm: String = _ val mergeExpr = declFunctions.flatMap(_.mergeExpressions) val mergeProjection = newMutableProjection( mergeExpr, - aggregateBufferAttributesForGroup ++ declFunctions.flatMap(_.inputAggBufferAttributes), + aggregateBufferAttributes ++ declFunctions.flatMap(_.inputAggBufferAttributes), subexpressionEliminationEnabled) val joinedRow = new JoinedRow() @@ -477,14 +453,14 @@ b private var fastHashMapTerm: String = _ } val evaluateKeyVars = evaluateVariables(keyVars) ctx.INPUT_ROW = bufferTerm - val bufferVars = aggregateBufferAttributesForGroup.zipWithIndex.map { case (e, i) => + val bufferVars = aggregateBufferAttributes.zipWithIndex.map { case (e, i) => BoundReference(i, e.dataType, e.nullable).genCode(ctx) } val evaluateBufferVars = evaluateVariables(bufferVars) // evaluate the aggregation result ctx.currentVars = bufferVars val aggResults = declFunctions.map(_.evaluateExpression).map { e => - BindReferences.bindReference(e, aggregateBufferAttributesForGroup).genCode(ctx) + BindReferences.bindReference(e, aggregateBufferAttributes).genCode(ctx) } val evaluateAggResults = evaluateVariables(aggResults) // generate the final result @@ -779,8 +755,8 @@ b private var fastHashMapTerm: String = _ val hashExpr = Murmur3Hash(groupingExpressions, 42) val hashEval = BindReferences.bindReference(hashExpr, child.output).genCode(ctx) - val inputAttr = aggregateBufferAttributesForGroup ++ child.output - ctx.currentVars = new Array[ExprCode](aggregateBufferAttributesForGroup.length) ++ input + val inputAttr = aggregateBufferAttributes ++ child.output + ctx.currentVars = new Array[ExprCode](aggregateBufferAttributes.length) ++ input val (checkFallbackForGeneratedHashMap, checkFallbackForBytesToBytesMap, resetCounter, incCounter) = if (testFallbackStartsAt.isDefined) { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 678262ada8bee..04d9a6d03f30c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -171,7 +171,7 @@ private[joins] class UnsafeHashedRelation( def getValue(key: InternalRow): InternalRow = { val unsafeKey = key.asInstanceOf[UnsafeRow] val map = binaryMap // avoid the compiler error - val loc = new map.Location // this could be allocated in stack + val loc = mapLoc binaryMap.safeLookup(unsafeKey.getBaseObject, unsafeKey.getBaseOffset, unsafeKey.getSizeInBytes, loc, unsafeKey.hashCode()) if (loc.isDefined) { From b500538bbb0c7c76eb13d007246f7ed3c02f6d47 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Mon, 28 Nov 2016 16:21:36 +0530 Subject: [PATCH 055/142] [SNAP-1136] Kryo closure serialtization support and optimizations (#27) - added back configurable closure serializer in Spark which was removed in SPARK-12414; some minor changes taken from closed Spark PR https://github.com/apache/spark/pull/6361 - added optimized Kryo serialization for multiple classes; currently registration and string sharing fix for kryo (https://github.com/EsotericSoftware/kryo/issues/128) is only in the SnappyData layer PooledKryoSerializer implementation; classes providing maximum benefit have added KryoSerializable notably Accumulators and *Metrics - use closureSerializer for Netty messaging too instead of fixed JavaSerializer - updated kryo to 4.0.0 to get the fix for kryo#342 - actually fixing scalastyle errors introduced by d80ef1b4 - set ordering field with kryo serialization in GenerateOrdering - removed warning if non-closure passed for cleaning Conflicts: core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala core/src/main/scala/org/apache/spark/scheduler/Task.scala core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala core/src/main/scala/org/apache/spark/storage/BlockId.scala core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala --- build.gradle | 3 +- common/unsafe/build.gradle | 5 +- core/build.gradle | 9 +- .../scala/org/apache/spark/SparkEnv.scala | 58 ++++++--- .../apache/spark/executor/InputMetrics.scala | 20 +++- .../apache/spark/executor/OutputMetrics.scala | 20 +++- .../spark/executor/ShuffleReadMetrics.scala | 28 ++++- .../spark/executor/ShuffleWriteMetrics.scala | 22 +++- .../apache/spark/executor/TaskMetrics.scala | 39 ++++++- .../netty/NettyBlockTransferService.scala | 7 +- .../main/scala/org/apache/spark/rdd/RDD.scala | 5 +- .../spark/rdd/ZippedPartitionsRDD.scala | 30 ++++- .../org/apache/spark/rpc/RpcEndpointRef.scala | 10 +- .../scala/org/apache/spark/rpc/RpcEnv.scala | 4 + .../apache/spark/rpc/netty/NettyRpcEnv.scala | 58 +++++++-- .../apache/spark/scheduler/ResultTask.scala | 27 ++++- .../spark/scheduler/ShuffleMapTask.scala | 19 ++- .../org/apache/spark/scheduler/Task.scala | 53 +++++++-- .../spark/scheduler/TaskDescription.scala | 65 +++++++---- .../spark/scheduler/TaskSetManager.scala | 10 +- .../cluster/CoarseGrainedClusterMessage.scala | 28 ++++- .../org/apache/spark/storage/BlockId.scala | 19 ++- .../spark/storage/BlockManagerMessages.scala | 55 ++++++++- .../org/apache/spark/util/AccumulatorV2.scala | 110 +++++++++++++++++- .../apache/spark/util/ClosureCleaner.scala | 2 +- .../spark/util/SerializableBuffer.scala | 23 +++- .../apache/spark/util/collection/BitSet.scala | 32 ++++- .../codegen/GenerateOrdering.scala | 5 +- .../sql/execution/metric/SQLMetrics.scala | 21 +++- 29 files changed, 666 insertions(+), 121 deletions(-) diff --git a/build.gradle b/build.gradle index 2f3f420315a64..01d46b079a504 100644 --- a/build.gradle +++ b/build.gradle @@ -61,7 +61,8 @@ allprojects { javaxServletVersion = '3.1.0' guavaVersion = '14.0.1' hiveVersion = '1.2.1.spark2' - chillVersion = '0.8.0' + chillVersion = '0.8.1' + kryoVersion = '4.0.0' nettyVersion = '3.8.0.Final' nettyAllVersion = '4.0.29.Final' derbyVersion = '10.12.1.1' diff --git a/common/unsafe/build.gradle b/common/unsafe/build.gradle index ee2347c9eb872..b14fed1ab31d7 100644 --- a/common/unsafe/build.gradle +++ b/common/unsafe/build.gradle @@ -20,7 +20,10 @@ description = 'Spark Project Unsafe' dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile group: 'com.twitter', name: 'chill_' + scalaBinaryVersion, version: chillVersion + compile group: 'com.esotericsoftware', name: 'kryo-shaded', version: kryoVersion + compile(group: 'com.twitter', name: 'chill_' + scalaBinaryVersion, version: chillVersion) { + exclude(group: 'com.esotericsoftware', module: 'kryo-shaded') + } compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version compile group: 'com.google.guava', name: 'guava', version: guavaVersion diff --git a/core/build.gradle b/core/build.gradle index 1caee72201e40..ebeff567df64d 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -40,8 +40,13 @@ dependencies { exclude(group: 'org.apache.avro', module: 'avro-ipc') } compile group: 'com.google.guava', name: 'guava', version: guavaVersion - compile group: 'com.twitter', name: 'chill_' + scalaBinaryVersion, version: chillVersion - compile group: 'com.twitter', name: 'chill-java', version: chillVersion + compile group: 'com.esotericsoftware', name: 'kryo-shaded', version: kryoVersion + compile(group: 'com.twitter', name: 'chill_' + scalaBinaryVersion, version: chillVersion) { + exclude(group: 'com.esotericsoftware', module: 'kryo-shaded') + } + compile(group: 'com.twitter', name: 'chill-java', version: chillVersion) { + exclude(group: 'com.esotericsoftware', module: 'kryo-shaded') + } compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' // explicitly include netty from akka-remote to not let zookeeper override it compile group: 'io.netty', name: 'netty', version: nettyVersion diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index 2ff379cced1a8..8a9d8a9292b34 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -170,6 +170,43 @@ object SparkEnv extends Logging { env } + // Create an instance of the class with the given name, possibly initializing it with our conf + def instantiateClass[T](className: String, conf: SparkConf, + isDriver: Boolean): T = { + val cls = Utils.classForName(className) + // Look for a constructor taking a SparkConf and a boolean isDriver, then one taking just + // SparkConf, then one taking no arguments + try { + cls.getConstructor(classOf[SparkConf], java.lang.Boolean.TYPE) + .newInstance(conf, new java.lang.Boolean(isDriver)) + .asInstanceOf[T] + } catch { + case _: NoSuchMethodException => + try { + cls.getConstructor(classOf[SparkConf]).newInstance(conf).asInstanceOf[T] + } catch { + case _: NoSuchMethodException => + cls.getConstructor().newInstance().asInstanceOf[T] + } + } + } + + def getClosureSerializer(conf: SparkConf, doLog: Boolean = false): Serializer = { + val defaultClosureSerializerClass = classOf[JavaSerializer].getName + val closureSerializerClass = conf.get("spark.closure.serializer", + defaultClosureSerializerClass) + val closureSerializer = instantiateClass[Serializer]( + closureSerializerClass, conf, isDriver = false) + if (doLog) { + if (closureSerializerClass != defaultClosureSerializerClass) { + logInfo(s"Using non-default closure serializer: $closureSerializerClass") + } else { + logDebug(s"Using closure serializer: $closureSerializerClass") + } + } + closureSerializer + } + /** * Create a SparkEnv for the driver. */ @@ -272,26 +309,9 @@ object SparkEnv extends Logging { conf.set("spark.driver.port", rpcEnv.address.port.toString) } - // Create an instance of the class with the given name, possibly initializing it with our conf def instantiateClass[T](className: String): T = { - val cls = Utils.classForName(className) - // Look for a constructor taking a SparkConf and a boolean isDriver, then one taking just - // SparkConf, then one taking no arguments - try { - cls.getConstructor(classOf[SparkConf], java.lang.Boolean.TYPE) - .newInstance(conf, new java.lang.Boolean(isDriver)) - .asInstanceOf[T] - } catch { - case _: NoSuchMethodException => - try { - cls.getConstructor(classOf[SparkConf]).newInstance(conf).asInstanceOf[T] - } catch { - case _: NoSuchMethodException => - cls.getConstructor().newInstance().asInstanceOf[T] - } - } + SparkEnv.instantiateClass(className, conf, isDriver) } - // Create an instance of the class named by the given SparkConf property, or defaultClassName // if the property is not set, possibly initializing it with our conf def instantiateClassFromConf[T](propertyName: String, defaultClassName: String): T = { @@ -304,7 +324,7 @@ object SparkEnv extends Logging { val serializerManager = new SerializerManager(serializer, conf, ioEncryptionKey) - val closureSerializer = new JavaSerializer(conf) + val closureSerializer = getClosureSerializer(conf, doLog = true) def registerOrLookupEndpoint( name: String, endpointCreator: => RpcEndpoint): diff --git a/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala b/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala index 3d15f3a0396e1..1647b06ce0481 100644 --- a/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala @@ -17,6 +17,10 @@ package org.apache.spark.executor +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + +import org.apache.spark.TaskContext import org.apache.spark.annotation.DeveloperApi import org.apache.spark.util.LongAccumulator @@ -39,7 +43,7 @@ object DataReadMethod extends Enumeration with Serializable { * A collection of accumulators that represents metrics about reading data from external systems. */ @DeveloperApi -class InputMetrics private[spark] () extends Serializable { +class InputMetrics private[spark] () extends Serializable with KryoSerializable { private[executor] val _bytesRead = new LongAccumulator private[executor] val _recordsRead = new LongAccumulator @@ -56,4 +60,18 @@ class InputMetrics private[spark] () extends Serializable { private[spark] def incBytesRead(v: Long): Unit = _bytesRead.add(v) private[spark] def incRecordsRead(v: Long): Unit = _recordsRead.add(v) private[spark] def setBytesRead(v: Long): Unit = _bytesRead.setValue(v) + + override def write(kryo: Kryo, output: Output): Unit = { + _bytesRead.write(kryo, output) + _recordsRead.write(kryo, output) + } + + override final def read(kryo: Kryo, input: Input): Unit = { + read(kryo, input, context = null) + } + + def read(kryo: Kryo, input: Input, context: TaskContext): Unit = { + _bytesRead.read(kryo, input, context) + _recordsRead.read(kryo, input, context) + } } diff --git a/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala b/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala index dada9697c1cf9..418a831c7555f 100644 --- a/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala @@ -17,6 +17,10 @@ package org.apache.spark.executor +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + +import org.apache.spark.TaskContext import org.apache.spark.annotation.DeveloperApi import org.apache.spark.util.LongAccumulator @@ -38,7 +42,7 @@ object DataWriteMethod extends Enumeration with Serializable { * A collection of accumulators that represents metrics about writing data to external systems. */ @DeveloperApi -class OutputMetrics private[spark] () extends Serializable { +class OutputMetrics private[spark] () extends Serializable with KryoSerializable { private[executor] val _bytesWritten = new LongAccumulator private[executor] val _recordsWritten = new LongAccumulator @@ -54,4 +58,18 @@ class OutputMetrics private[spark] () extends Serializable { private[spark] def setBytesWritten(v: Long): Unit = _bytesWritten.setValue(v) private[spark] def setRecordsWritten(v: Long): Unit = _recordsWritten.setValue(v) + + override def write(kryo: Kryo, output: Output): Unit = { + _bytesWritten.write(kryo, output) + _recordsWritten.write(kryo, output) + } + + override final def read(kryo: Kryo, input: Input): Unit = { + read(kryo, input, context = null) + } + + def read(kryo: Kryo, input: Input, context: TaskContext): Unit = { + _bytesWritten.read(kryo, input, context) + _recordsWritten.read(kryo, input, context) + } } diff --git a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala index 5a435f2f9a1ba..6b49c8cbeb629 100644 --- a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala @@ -17,6 +17,10 @@ package org.apache.spark.executor +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + +import org.apache.spark.TaskContext import org.apache.spark.annotation.DeveloperApi import org.apache.spark.util.{DoubleAccumulator, LongAccumulator} @@ -27,7 +31,7 @@ import org.apache.spark.util.{DoubleAccumulator, LongAccumulator} * Operations are not thread-safe. */ @DeveloperApi -class ShuffleReadMetrics private[spark] () extends Serializable { +class ShuffleReadMetrics private[spark] () extends Serializable with KryoSerializable { private[executor] val _remoteBlocksFetched = new LongAccumulator private[executor] val _localBlocksFetched = new LongAccumulator private[executor] val _remoteBytesRead = new LongAccumulator @@ -121,6 +125,28 @@ class ShuffleReadMetrics private[spark] () extends Serializable { _recordsRead.add(metric.recordsRead) } } + + override def write(kryo: Kryo, output: Output): Unit = { + _remoteBlocksFetched.write(kryo, output) + _localBlocksFetched.write(kryo, output) + _remoteBytesRead.write(kryo, output) + _localBytesRead.write(kryo, output) + _fetchWaitTime.write(kryo, output) + _recordsRead.write(kryo, output) + } + + override final def read(kryo: Kryo, input: Input): Unit = { + read(kryo, input, context = null) + } + + def read(kryo: Kryo, input: Input, context: TaskContext): Unit = { + _remoteBlocksFetched.read(kryo, input, context) + _localBlocksFetched.read(kryo, input, context) + _remoteBytesRead.read(kryo, input, context) + _localBytesRead.read(kryo, input, context) + _fetchWaitTime.read(kryo, input, context) + _recordsRead.read(kryo, input, context) + } } /** diff --git a/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala b/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala index ada2e1bc08593..f6aaf90d93b9c 100644 --- a/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala @@ -17,6 +17,10 @@ package org.apache.spark.executor +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + +import org.apache.spark.TaskContext import org.apache.spark.annotation.DeveloperApi import org.apache.spark.util.LongAccumulator @@ -27,7 +31,7 @@ import org.apache.spark.util.LongAccumulator * Operations are not thread-safe. */ @DeveloperApi -class ShuffleWriteMetrics private[spark] () extends Serializable { +class ShuffleWriteMetrics private[spark] () extends Serializable with KryoSerializable { private[executor] val _bytesWritten = new LongAccumulator private[executor] val _recordsWritten = new LongAccumulator private[executor] val _writeTime = new LongAccumulator @@ -57,6 +61,22 @@ class ShuffleWriteMetrics private[spark] () extends Serializable { _recordsWritten.setValue(recordsWritten - v) } + override def write(kryo: Kryo, output: Output): Unit = { + _bytesWritten.write(kryo, output) + _recordsWritten.write(kryo, output) + _writeTime.write(kryo, output) + } + + override def read(kryo: Kryo, input: Input): Unit = { + read(kryo, input, context = null) + } + + def read(kryo: Kryo, input: Input, context: TaskContext): Unit = { + _bytesWritten.read(kryo, input, context) + _recordsWritten.read(kryo, input, context) + _writeTime.read(kryo, input, context) + } + // Legacy methods for backward compatibility. // TODO: remove these once we make this class private. @deprecated("use bytesWritten instead", "2.0.0") diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala index dc6dc6878f567..3facda3df95bc 100644 --- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala @@ -20,6 +20,9 @@ package org.apache.spark.executor import scala.collection.JavaConverters._ import scala.collection.mutable.{ArrayBuffer, LinkedHashMap} +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark._ import org.apache.spark.annotation.DeveloperApi import org.apache.spark.internal.Logging @@ -42,7 +45,7 @@ import org.apache.spark.util._ * be sent to the driver. */ @DeveloperApi -class TaskMetrics private[spark] () extends Serializable { +class TaskMetrics private[spark] () extends Serializable with KryoSerializable { // Each metric is internally represented as an accumulator private val _executorDeserializeTime = new DoubleAccumulator private val _executorDeserializeCpuTime = new LongAccumulator @@ -263,6 +266,40 @@ class TaskMetrics private[spark] () extends Serializable { // value will be updated at driver side. internalAccums.filter(a => !a.isZero || a == _resultSize) } + + override def write(kryo: Kryo, output: Output): Unit = { + _executorDeserializeTime.write(kryo, output) + _executorRunTime.write(kryo, output) + _resultSize.write(kryo, output) + _jvmGCTime.write(kryo, output) + _resultSerializationTime.write(kryo, output) + _memoryBytesSpilled.write(kryo, output) + _diskBytesSpilled.write(kryo, output) + _peakExecutionMemory.write(kryo, output) + _updatedBlockStatuses.write(kryo, output) + inputMetrics.write(kryo, output) + outputMetrics.write(kryo, output) + shuffleReadMetrics.write(kryo, output) + shuffleWriteMetrics.write(kryo, output) + } + + override def read(kryo: Kryo, input: Input): Unit = { + // read the TaskContext thread-local once + val taskContext = TaskContext.get() + _executorDeserializeTime.read(kryo, input, taskContext) + _executorRunTime.read(kryo, input, taskContext) + _resultSize.read(kryo, input, taskContext) + _jvmGCTime.read(kryo, input, taskContext) + _resultSerializationTime.read(kryo, input, taskContext) + _memoryBytesSpilled.read(kryo, input, taskContext) + _diskBytesSpilled.read(kryo, input, taskContext) + _peakExecutionMemory.read(kryo, input, taskContext) + _updatedBlockStatuses.read(kryo, input, taskContext) + inputMetrics.read(kryo, input, taskContext) + outputMetrics.read(kryo, input, taskContext) + shuffleReadMetrics.read(kryo, input, taskContext) + shuffleWriteMetrics.read(kryo, input, taskContext) + } } diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala index b7d8c35032763..e718336af82ca 100644 --- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala +++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala @@ -24,9 +24,7 @@ import scala.collection.JavaConverters._ import scala.concurrent.{Future, Promise} import scala.reflect.ClassTag -import com.codahale.metrics.{Metric, MetricSet} - -import org.apache.spark.{SecurityManager, SparkConf} +import org.apache.spark.{SecurityManager, SparkConf, SparkEnv} import org.apache.spark.network._ import org.apache.spark.network.buffer.ManagedBuffer import org.apache.spark.network.client.{RpcResponseCallback, TransportClientBootstrap, TransportClientFactory} @@ -35,7 +33,6 @@ import org.apache.spark.network.server._ import org.apache.spark.network.shuffle.{BlockFetchingListener, OneForOneBlockFetcher, RetryingBlockFetcher, TempFileManager} import org.apache.spark.network.shuffle.protocol.UploadBlock import org.apache.spark.network.util.JavaUtils -import org.apache.spark.serializer.JavaSerializer import org.apache.spark.storage.{BlockId, StorageLevel} import org.apache.spark.util.Utils @@ -52,7 +49,7 @@ private[spark] class NettyBlockTransferService( extends BlockTransferService { // TODO: Don't use Java serialization, use a more cross-version compatible serialization format. - private val serializer = new JavaSerializer(conf) + private val serializer = SparkEnv.getClosureSerializer(conf) private val authEnabled = securityManager.isAuthenticationEnabled() private val transportConf = SparkTransportConf.fromSparkConf(conf, "shuffle", numCores) diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index 0574abdca32ac..74871f836191e 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -147,7 +147,8 @@ abstract class RDD[T: ClassTag]( def sparkContext: SparkContext = sc /** A unique ID for this RDD (within its SparkContext). */ - val id: Int = sc.newRddId() + protected var _id: Int = sc.newRddId() + def id: Int = _id /** A friendly name for this RDD */ @transient var name: String = _ @@ -1651,7 +1652,7 @@ abstract class RDD[T: ClassTag]( // Other internal methods and fields // ======================================================================= - private var storageLevel: StorageLevel = StorageLevel.NONE + protected var storageLevel: StorageLevel = StorageLevel.NONE /** User code that created this RDD (e.g. `textFile`, `parallelize`). */ @transient private[spark] val creationSite = sc.getCallSite() diff --git a/core/src/main/scala/org/apache/spark/rdd/ZippedPartitionsRDD.scala b/core/src/main/scala/org/apache/spark/rdd/ZippedPartitionsRDD.scala index 3cb1231bd3477..7d4e5595fe860 100644 --- a/core/src/main/scala/org/apache/spark/rdd/ZippedPartitionsRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/ZippedPartitionsRDD.scala @@ -21,16 +21,19 @@ import java.io.{IOException, ObjectOutputStream} import scala.reflect.ClassTag +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark.{OneToOneDependency, Partition, SparkContext, TaskContext} import org.apache.spark.util.Utils private[spark] class ZippedPartitionsPartition( - idx: Int, + private var idx: Int, @transient private val rdds: Seq[RDD[_]], @transient val preferredLocations: Seq[String]) - extends Partition { + extends Partition with KryoSerializable { - override val index: Int = idx + override def index: Int = idx var partitionValues = rdds.map(rdd => rdd.partitions(idx)) def partitions: Seq[Partition] = partitionValues @@ -40,6 +43,27 @@ private[spark] class ZippedPartitionsPartition( partitionValues = rdds.map(rdd => rdd.partitions(idx)) oos.defaultWriteObject() } + + override def write(kryo: Kryo, output: Output): Unit = { + // Update the reference to parent split at the time of task serialization + partitionValues = rdds.map(rdd => rdd.partitions(idx)) + output.writeVarInt(idx, true) + output.writeVarInt(partitionValues.length, true) + for (p <- partitionValues) { + kryo.writeClassAndObject(output, p) + } + } + + override def read(kryo: Kryo, input: Input): Unit = { + idx = input.readVarInt(true) + var numPartitions = input.readVarInt(true) + val partitionBuilder = Seq.newBuilder[Partition] + while (numPartitions > 0) { + partitionBuilder += kryo.readClassAndObject(input).asInstanceOf[Partition] + numPartitions -= 1 + } + partitionValues = partitionBuilder.result() + } } private[spark] abstract class ZippedPartitionsBaseRDD[V: ClassTag]( diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala index 4d39f144dd198..4788230bbd000 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala @@ -27,12 +27,12 @@ import org.apache.spark.util.RpcUtils /** * A reference for a remote [[RpcEndpoint]]. [[RpcEndpointRef]] is thread-safe. */ -private[spark] abstract class RpcEndpointRef(conf: SparkConf) - extends Serializable with Logging { +private[spark] abstract class RpcEndpointRef(conf: SparkConf, + _env: RpcEnv) extends Serializable with Logging { - private[this] val maxRetries = RpcUtils.numRetries(conf) - private[this] val retryWaitMs = RpcUtils.retryWaitMs(conf) - private[this] val defaultAskTimeout = RpcUtils.askRpcTimeout(conf) + @transient protected var maxRetries = _env.maxRetries + @transient protected var retryWaitMs = _env.retryWaitMs + @transient protected var defaultAskTimeout = _env.defaultAskTimeout /** * return the address for the [[RpcEndpointRef]] diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala index de2cc56bc6b16..59fce6a4e731b 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala @@ -72,6 +72,10 @@ private[spark] abstract class RpcEnv(conf: SparkConf) { private[spark] val defaultLookupTimeout = RpcUtils.lookupRpcTimeout(conf) + private[spark] val maxRetries = RpcUtils.numRetries(conf) + private[spark] val retryWaitMs = RpcUtils.retryWaitMs(conf) + private[spark] val defaultAskTimeout = RpcUtils.askRpcTimeout(conf) + /** * Return RpcEndpointRef of the registered [[RpcEndpoint]]. Will be used to implement * [[RpcEndpoint.self]]. Return `null` if the corresponding [[RpcEndpointRef]] does not exist. diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala index a2936d6ad539c..79639f2f38745 100644 --- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala +++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala @@ -29,7 +29,10 @@ import scala.reflect.ClassTag import scala.util.{DynamicVariable, Failure, Success, Try} import scala.util.control.NonFatal -import org.apache.spark.{SecurityManager, SparkConf} +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + +import org.apache.spark.{SecurityManager, SparkConf, SparkEnv} import org.apache.spark.internal.Logging import org.apache.spark.network.TransportContext import org.apache.spark.network.client._ @@ -37,12 +40,12 @@ import org.apache.spark.network.crypto.{AuthClientBootstrap, AuthServerBootstrap import org.apache.spark.network.netty.SparkTransportConf import org.apache.spark.network.server._ import org.apache.spark.rpc._ -import org.apache.spark.serializer.{JavaSerializer, JavaSerializerInstance, SerializationStream} -import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, ThreadUtils, Utils} +import org.apache.spark.serializer.{JavaSerializer, JavaSerializerInstance, Serializer, SerializerInstance, SerializationStream} +import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, RpcUtils, ThreadUtils, Utils} private[netty] class NettyRpcEnv( val conf: SparkConf, - javaSerializerInstance: JavaSerializerInstance, + serializer: Serializer, host: String, securityManager: SecurityManager, numUsableCores: Int) extends RpcEnv(conf) with Logging { @@ -52,6 +55,10 @@ private[netty] class NettyRpcEnv( "rpc", conf.getInt("spark.rpc.io.threads", 0)) + private val serializerInstance = new ThreadLocal[SerializerInstance] { + override def initialValue(): SerializerInstance = serializer.newInstance() + } + private val dispatcher: Dispatcher = new Dispatcher(this, numUsableCores) private val streamManager = new NettyStreamManager(this) @@ -255,7 +262,7 @@ private[netty] class NettyRpcEnv( } private[netty] def serialize(content: Any): ByteBuffer = { - javaSerializerInstance.serialize(content) + serializerInstance.get().serialize(content) } /** @@ -268,7 +275,7 @@ private[netty] class NettyRpcEnv( private[netty] def deserialize[T: ClassTag](client: TransportClient, bytes: ByteBuffer): T = { NettyRpcEnv.currentClient.withValue(client) { deserialize { () => - javaSerializerInstance.deserialize[T](bytes) + serializerInstance.get().deserialize[T](bytes) } } } @@ -453,10 +460,7 @@ private[rpc] class NettyRpcEnvFactory extends RpcEnvFactory with Logging { def create(config: RpcEnvConfig): RpcEnv = { val sparkConf = config.conf - // Use JavaSerializerInstance in multiple threads is safe. However, if we plan to support - // KryoSerializer in future, we have to use ThreadLocal to store SerializerInstance - val javaSerializerInstance = - new JavaSerializer(sparkConf).newInstance().asInstanceOf[JavaSerializerInstance] + val serializer = SparkEnv.getClosureSerializer(sparkConf) val nettyEnv = new NettyRpcEnv(sparkConf, javaSerializerInstance, config.advertiseAddress, config.securityManager, config.numUsableCores) @@ -500,7 +504,8 @@ private[rpc] class NettyRpcEnvFactory extends RpcEnvFactory with Logging { private[netty] class NettyRpcEndpointRef( @transient private val conf: SparkConf, private val endpointAddress: RpcEndpointAddress, - @transient @volatile private var nettyEnv: NettyRpcEnv) extends RpcEndpointRef(conf) { + @transient @volatile private var nettyEnv: NettyRpcEnv) + extends RpcEndpointRef(conf, nettyEnv) with Serializable with KryoSerializable with Logging { @transient @volatile var client: TransportClient = _ @@ -511,6 +516,10 @@ private[netty] class NettyRpcEndpointRef( in.defaultReadObject() nettyEnv = NettyRpcEnv.currentEnv.value client = NettyRpcEnv.currentClient.value + + maxRetries = nettyEnv.maxRetries + retryWaitMs = nettyEnv.retryWaitMs + defaultAskTimeout = nettyEnv.defaultAskTimeout } private def writeObject(out: ObjectOutputStream): Unit = { @@ -519,6 +528,33 @@ private[netty] class NettyRpcEndpointRef( override def name: String = endpointAddress.name + override def write(kryo: Kryo, output: Output): Unit = { + val addr = address + output.writeString(_name) + if (addr != null && addr.host != null) { + output.writeString(addr.host) + output.writeInt(addr.port) + } else { + output.writeString(null) + } + } + + override def read(kryo: Kryo, input: Input): Unit = { + _name = input.readString() + _address = null + val host = input.readString() + if (host != null) { + val port = input.readInt() + _address = RpcEndpointAddress(host, port, _name) + } + nettyEnv = NettyRpcEnv.currentEnv.value + client = NettyRpcEnv.currentClient.value + + maxRetries = nettyEnv.maxRetries + retryWaitMs = nettyEnv.retryWaitMs + defaultAskTimeout = nettyEnv.defaultAskTimeout + } + override def ask[T: ClassTag](message: Any, timeout: RpcTimeout): Future[T] = { nettyEnv.ask(new RequestMessage(nettyEnv.address, this, message), timeout) } diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index 2acac37dcff94..86f188dbb6022 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -22,6 +22,9 @@ import java.lang.management.ManagementFactory import java.nio.ByteBuffer import java.util.Properties +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark._ import org.apache.spark.broadcast.Broadcast import org.apache.spark.rdd.RDD @@ -38,7 +41,7 @@ import org.apache.spark.rdd.RDD * (RDD[T], (TaskContext, Iterator[T]) => U). * @param partition partition of the RDD this task is associated with * @param locs preferred task execution locations for locality scheduling - * @param outputId index of the task in this job (a job can launch tasks on only a subset of the + * @param _outputId index of the task in this job (a job can launch tasks on only a subset of the * input RDD's partitions). * @param localProperties copy of thread-local properties set by the user on the driver side. * @param serializedTaskMetrics a `TaskMetrics` that is created and serialized on the driver side @@ -52,10 +55,10 @@ import org.apache.spark.rdd.RDD private[spark] class ResultTask[T, U]( stageId: Int, stageAttemptId: Int, - taskBinary: Broadcast[Array[Byte]], - partition: Partition, + private var taskBinary: Broadcast[Array[Byte]], + private var partition: Partition, locs: Seq[TaskLocation], - val outputId: Int, + private var _outputId: Int, localProperties: Properties, serializedTaskMetrics: Array[Byte], jobId: Option[Int] = None, @@ -65,6 +68,8 @@ private[spark] class ResultTask[T, U]( jobId, appId, appAttemptId) with Serializable { + final def outputId: Int = _outputId + @transient private[this] val preferredLocs: Seq[TaskLocation] = { if (locs == null) Nil else locs.toSet.toSeq } @@ -91,4 +96,18 @@ private[spark] class ResultTask[T, U]( override def preferredLocations: Seq[TaskLocation] = preferredLocs override def toString: String = "ResultTask(" + stageId + ", " + partitionId + ")" + + override def write(kryo: Kryo, output: Output): Unit = { + super.write(kryo, output) + kryo.writeClassAndObject(output, taskBinary) + kryo.writeClassAndObject(output, partition) + output.writeInt(_outputId) + } + + override def read(kryo: Kryo, input: Input): Unit = { + super.read(kryo, input) + taskBinary = kryo.readClassAndObject(input).asInstanceOf[Broadcast[Array[Byte]]] + partition = kryo.readClassAndObject(input).asInstanceOf[Partition] + _outputId = input.readInt() + } } diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index ef2b3e8764b71..c5e757024ed45 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -23,6 +23,9 @@ import java.util.Properties import scala.language.existentials +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark._ import org.apache.spark.broadcast.Broadcast import org.apache.spark.internal.Logging @@ -53,8 +56,8 @@ import org.apache.spark.shuffle.ShuffleWriter private[spark] class ShuffleMapTask( stageId: Int, stageAttemptId: Int, - taskBinary: Broadcast[Array[Byte]], - partition: Partition, + private var taskBinary: Broadcast[Array[Byte]], + private var partition: Partition, @transient private var locs: Seq[TaskLocation], localProperties: Properties, serializedTaskMetrics: Array[Byte], @@ -112,4 +115,16 @@ private[spark] class ShuffleMapTask( override def preferredLocations: Seq[TaskLocation] = preferredLocs override def toString: String = "ShuffleMapTask(%d, %d)".format(stageId, partitionId) + + override def write(kryo: Kryo, output: Output): Unit = { + super.write(kryo, output) + kryo.writeClassAndObject(output, taskBinary) + kryo.writeClassAndObject(output, partition) + } + + override def read(kryo: Kryo, input: Input): Unit = { + super.read(kryo, input) + taskBinary = kryo.readClassAndObject(input).asInstanceOf[Broadcast[Array[Byte]]] + partition = kryo.readClassAndObject(input).asInstanceOf[Partition] + } } diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index 2a9b0628c1131..867bca3b84de2 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -20,6 +20,12 @@ package org.apache.spark.scheduler import java.nio.ByteBuffer import java.util.Properties +import scala.collection.mutable +import scala.collection.mutable.HashMap + +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark._ import org.apache.spark.executor.TaskMetrics import org.apache.spark.internal.config.APP_CALLER_CONTEXT @@ -38,9 +44,10 @@ import org.apache.spark.util._ * and sends the task output back to the driver application. A ShuffleMapTask executes the task * and divides the task output to multiple buckets (based on the task's partitioner). * - * @param stageId id of the stage this task belongs to - * @param stageAttemptId attempt id of the stage this task belongs to - * @param partitionId index of the number in the RDD + * @param _stageId id of the stage this task belongs to + * @param _stageAttemptId attempt id of the stage this task belongs to + * @param _partitionId index of the number in the RDD + * @param _metrics a [[TaskMetrics]] that is created at driver side and sent to executor side. * @param localProperties copy of thread-local properties set by the user on the driver side. * @param serializedTaskMetrics a `TaskMetrics` that is created and serialized on the driver side * and sent to executor side. @@ -51,16 +58,27 @@ import org.apache.spark.util._ * @param appAttemptId attempt id of the app this task belongs to */ private[spark] abstract class Task[T]( - val stageId: Int, - val stageAttemptId: Int, - val partitionId: Int, + private var _stageId: Int, + private var _stageAttemptId: Int, + private var _partitionId: Int, + // The default value is only used in tests. + private var _metrics: TaskMetrics = TaskMetrics.registered, @transient var localProperties: Properties = new Properties, // The default value is only used in tests. serializedTaskMetrics: Array[Byte] = SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array(), val jobId: Option[Int] = None, val appId: Option[String] = None, - val appAttemptId: Option[String] = None) extends Serializable { + val appAttemptId: Option[String] = None) extends Serializable + with KryoSerializable { + + final def stageId: Int = _stageId + + final def stageAttemptId: Int = _stageAttemptId + + final def partitionId: Int = _partitionId + + final def metrics: TaskMetrics = _metrics @transient lazy val metrics: TaskMetrics = SparkEnv.get.closureSerializer.newInstance().deserialize(ByteBuffer.wrap(serializedTaskMetrics)) @@ -147,7 +165,7 @@ private[spark] abstract class Task[T]( } } - private var taskMemoryManager: TaskMemoryManager = _ + @transient private var taskMemoryManager: TaskMemoryManager = _ def setTaskMemoryManager(taskMemoryManager: TaskMemoryManager): Unit = { this.taskMemoryManager = taskMemoryManager @@ -216,4 +234,23 @@ private[spark] abstract class Task[T]( taskThread.interrupt() } } + + override def write(kryo: Kryo, output: Output): Unit = { + output.writeInt(_stageId) + output.writeVarInt(_stageAttemptId, true) + output.writeVarInt(_partitionId, true) + output.writeLong(epoch) + output.writeLong(_executorDeserializeTime) + _metrics.write(kryo, output) + } + + override def read(kryo: Kryo, input: Input): Unit = { + _stageId = input.readInt() + _stageAttemptId = input.readVarInt(true) + _partitionId = input.readVarInt(true) + epoch = input.readLong() + _executorDeserializeTime = input.readLong() + _metrics = new TaskMetrics + _metrics.read(kryo, input) + } } diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala index c98b87148e404..583898e12145f 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala @@ -25,7 +25,9 @@ import java.util.Properties import scala.collection.JavaConverters._ import scala.collection.mutable.{HashMap, Map} -import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, Utils} +import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, SerializableBuffer, Utils} +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} /** * Description of a task that gets passed onto executors to be executed, usually created by @@ -45,31 +47,46 @@ import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, Uti * (which can introduce significant overhead when the maps are small). */ private[spark] class TaskDescription( - val taskId: Long, - val attemptNumber: Int, - val executorId: String, - val name: String, - val index: Int, // Index within this task's TaskSet - val addedFiles: Map[String, Long], - val addedJars: Map[String, Long], - val properties: Properties, - val serializedTask: ByteBuffer) { - - override def toString: String = "TaskDescription(TID=%d, index=%d)".format(taskId, index) -} - -private[spark] object TaskDescription { - private def serializeStringLongMap(map: Map[String, Long], dataOut: DataOutputStream): Unit = { - dataOut.writeInt(map.size) - for ((key, value) <- map) { - dataOut.writeUTF(key) - dataOut.writeLong(value) - } + private var _taskId: Long, + private var _attemptNumber: Int, + private var _executorId: String, + private var _name: String, + private var _index: Int, // Index within this task's TaskSet + @transient private var _serializedTask: ByteBuffer) + extends Serializable with KryoSerializable { + + def taskId: Long = _taskId + def attemptNumber: Int = _attemptNumber + def executorId: String = _executorId + def name: String = _name + def index: Int = _index + + // Because ByteBuffers are not serializable, wrap the task in a SerializableBuffer + private val buffer = + if (_serializedTask ne null) new SerializableBuffer(_serializedTask) else null + + def serializedTask: ByteBuffer = + if (_serializedTask ne null) _serializedTask else buffer.value + + override def write(kryo: Kryo, output: Output): Unit = { + output.writeLong(_taskId) + output.writeVarInt(_attemptNumber, true) + output.writeString(_executorId) + output.writeString(_name) + output.writeInt(_index) + output.writeInt(_serializedTask.remaining()) + Utils.writeByteBuffer(_serializedTask, output) } - def encode(taskDescription: TaskDescription): ByteBuffer = { - val bytesOut = new ByteBufferOutputStream(4096) - val dataOut = new DataOutputStream(bytesOut) + override def read(kryo: Kryo, input: Input): Unit = { + _taskId = input.readLong() + _attemptNumber = input.readVarInt(true) + _executorId = input.readString() + _name = input.readString() + _index = input.readInt() + val len = input.readInt() + _serializedTask = ByteBuffer.wrap(input.readBytes(len)) + } dataOut.writeLong(taskDescription.taskId) dataOut.writeInt(taskDescription.attemptNumber) diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 3da2116f10d97..3519669e96fe6 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -427,7 +427,7 @@ private[spark] class TaskSetManager( } if (TaskLocality.isAllowed(maxLocality, TaskLocality.NODE_LOCAL)) { - for (index <- dequeueTaskFromList(execId, getPendingTasksForHost(host)) + for (index <- dequeueTaskFromList(execId, host, getPendingTasksForHost(host)) // don't return executor-local tasks that are still alive if canRunOnExecutor(execId, index)) { return Some((index, TaskLocality.NODE_LOCAL, false)) @@ -444,7 +444,7 @@ private[spark] class TaskSetManager( if (TaskLocality.isAllowed(maxLocality, TaskLocality.RACK_LOCAL)) { for { rack <- sched.getRackForHost(host) - index <- dequeueTaskFromList(execId, getPendingTasksForRack(rack)) + index <- dequeueTaskFromList(execId, host, getPendingTasksForRack(rack)) // don't return executor-local tasks that are still alive if canRunOnExecutor(execId, index) } { @@ -453,7 +453,7 @@ private[spark] class TaskSetManager( } if (TaskLocality.isAllowed(maxLocality, TaskLocality.ANY)) { - for (index <- dequeueTaskFromList(execId, allPendingTasks) + for (index <- dequeueTaskFromList(execId, host, allPendingTasks) // don't return executor-local tasks that are still alive if canRunOnExecutor(execId, index)) { return Some((index, TaskLocality.ANY, false)) @@ -547,8 +547,8 @@ private[spark] class TaskSetManager( sched.dagScheduler.taskStarted(task, info) new TaskDescription( - taskId, - attemptNum, + _taskId = taskId, + _attemptNumber = attemptNum, execId, taskName, index, diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala index e8b7fc0ef100a..fc5115c57fb03 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala @@ -19,10 +19,13 @@ package org.apache.spark.scheduler.cluster import java.nio.ByteBuffer +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark.TaskState.TaskState import org.apache.spark.rpc.RpcEndpointRef import org.apache.spark.scheduler.ExecutorLossReason -import org.apache.spark.util.SerializableBuffer +import org.apache.spark.util.{SerializableBuffer, Utils} private[spark] sealed trait CoarseGrainedClusterMessage extends Serializable @@ -66,8 +69,27 @@ private[spark] object CoarseGrainedClusterMessages { logUrls: Map[String, String]) extends CoarseGrainedClusterMessage - case class StatusUpdate(executorId: String, taskId: Long, state: TaskState, - data: SerializableBuffer) extends CoarseGrainedClusterMessage + case class StatusUpdate(var executorId: String, var taskId: Long, + var state: TaskState, var data: SerializableBuffer) + extends CoarseGrainedClusterMessage with KryoSerializable { + + override def write(kryo: Kryo, output: Output): Unit = { + output.writeString(executorId) + output.writeLong(taskId) + output.writeVarInt(state.id, true) + val buffer = data.buffer + output.writeInt(buffer.remaining()) + Utils.writeByteBuffer(buffer, output) + } + + override def read(kryo: Kryo, input: Input): Unit = { + executorId = input.readString() + taskId = input.readLong() + state = org.apache.spark.TaskState(input.readVarInt(true)) + val len = input.readInt() + data = new SerializableBuffer(ByteBuffer.wrap(input.readBytes(len))) + } + } object StatusUpdate { /** Alternate factory method that takes a ByteBuffer directly for the data field */ diff --git a/core/src/main/scala/org/apache/spark/storage/BlockId.scala b/core/src/main/scala/org/apache/spark/storage/BlockId.scala index 7ac2c71c18eb3..d0bfaafe8088f 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockId.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockId.scala @@ -20,6 +20,10 @@ package org.apache.spark.storage import java.util.UUID import org.apache.spark.SparkException + +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark.annotation.DeveloperApi /** @@ -45,8 +49,19 @@ sealed abstract class BlockId { } @DeveloperApi -case class RDDBlockId(rddId: Int, splitIndex: Int) extends BlockId { - override def name: String = "rdd_" + rddId + "_" + splitIndex +case class RDDBlockId(var rddId: Int, var splitIndex: Int) + extends BlockId with KryoSerializable { + @transient override lazy val name: String = "rdd_" + rddId + "_" + splitIndex + + override def write(kryo: Kryo, output: Output): Unit = { + output.writeInt(rddId) + output.writeVarInt(splitIndex, true) + } + + override def read(kryo: Kryo, input: Input): Unit = { + rddId = input.readInt() + splitIndex = input.readVarInt(true) + } } // Format of the shuffle block ids (including data and index) should be kept in sync with diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala index 1bbe7a5b39509..b6f45c4f894c6 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala @@ -19,6 +19,9 @@ package org.apache.spark.storage import java.io.{Externalizable, ObjectInput, ObjectOutput} +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark.rpc.RpcEndpointRef import org.apache.spark.util.Utils @@ -30,21 +33,63 @@ private[spark] object BlockManagerMessages { // Remove a block from the slaves that have it. This can only be used to remove // blocks that the master knows about. - case class RemoveBlock(blockId: BlockId) extends ToBlockManagerSlave + case class RemoveBlock(private var blockId: BlockId) extends ToBlockManagerSlave + with KryoSerializable { + + override def write(kryo: Kryo, output: Output): Unit = { + output.writeString(blockId.name) + } + + override def read(kryo: Kryo, input: Input): Unit = { + blockId = BlockId(input.readString()) + } + } // Replicate blocks that were lost due to executor failure case class ReplicateBlock(blockId: BlockId, replicas: Seq[BlockManagerId], maxReplicas: Int) extends ToBlockManagerSlave // Remove all blocks belonging to a specific RDD. - case class RemoveRdd(rddId: Int) extends ToBlockManagerSlave + case class RemoveRdd(private var rddId: Int) extends ToBlockManagerSlave + with KryoSerializable { + + override def write(kryo: Kryo, output: Output): Unit = { + output.writeInt(rddId) + } + + override def read(kryo: Kryo, input: Input): Unit = { + rddId = input.readInt() + } + } // Remove all blocks belonging to a specific shuffle. - case class RemoveShuffle(shuffleId: Int) extends ToBlockManagerSlave + case class RemoveShuffle(private var shuffleId: Int) extends ToBlockManagerSlave + with KryoSerializable { + + override def write(kryo: Kryo, output: Output): Unit = { + output.writeInt(shuffleId) + } + + override def read(kryo: Kryo, input: Input): Unit = { + shuffleId = input.readInt() + } + } // Remove all blocks belonging to a specific broadcast. - case class RemoveBroadcast(broadcastId: Long, removeFromDriver: Boolean = true) - extends ToBlockManagerSlave + case class RemoveBroadcast(private var broadcastId: Long, + private var removeFromDriver: Boolean = true) + extends ToBlockManagerSlave with KryoSerializable { + + override def write(kryo: Kryo, output: Output): Unit = { + output.writeLong(broadcastId) + output.writeBoolean(removeFromDriver) + } + + override def read(kryo: Kryo, input: Input): Unit = { + broadcastId = input.readLong() + removeFromDriver = input.readBoolean() + } + } /** * Driver to Executor message to trigger a thread dump. diff --git a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala index f4a736d6d439a..42010a261c553 100644 --- a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala +++ b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala @@ -23,6 +23,11 @@ import java.util.{ArrayList, Collections} import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong +import scala.collection.JavaConverters._ + +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark.{InternalAccumulator, SparkContext, TaskContext} import org.apache.spark.scheduler.AccumulableInfo @@ -41,7 +46,7 @@ private[spark] case class AccumulatorMetadata( */ abstract class AccumulatorV2[IN, OUT] extends Serializable { private[spark] var metadata: AccumulatorMetadata = _ - private[this] var atDriverSide = true + private[spark] var atDriverSide = true private[spark] def register( sc: SparkContext, @@ -207,6 +212,63 @@ abstract class AccumulatorV2[IN, OUT] extends Serializable { } } +abstract class AccumulatorV2Kryo[IN, OUT] + extends AccumulatorV2[IN, OUT] with KryoSerializable { + + /** + * Child classes cannot override this and must instead implement + * writeKryo/readKryo for consistent writeReplace() behavior. + */ + override final def write(kryo: Kryo, output: Output): Unit = { + var instance = this + if (atDriverSide) { + instance = copyAndReset().asInstanceOf[AccumulatorV2Kryo[IN, OUT]] + assert(instance.isZero, "copyAndReset must return a zero value copy") + instance.metadata = this.metadata + } + val metadata = instance.metadata + output.writeLong(metadata.id) + metadata.name match { + case None => output.writeString(null) + case Some(name) => output.writeString(name) + } + output.writeBoolean(metadata.countFailedValues) + output.writeBoolean(instance.atDriverSide) + + instance.writeKryo(kryo, output) + } + + /** + * Child classes must implement readKryo() and cannot override this. + */ + override final def read(kryo: Kryo, input: Input): Unit = { + read(kryo, input, context = null) + } + + final def read(kryo: Kryo, input: Input, context: TaskContext): Unit = { + val id = input.readLong() + val name = input.readString() + metadata = AccumulatorMetadata(id, Option(name), input.readBoolean()) + atDriverSide = input.readBoolean() + if (atDriverSide) { + atDriverSide = false + // Automatically register the accumulator when it is deserialized with the task closure. + // This is for external accumulators and internal ones that do not represent task level + // metrics, e.g. internal SQL metrics, which are per-operator. + val taskContext = if (context != null) context else TaskContext.get() + if (taskContext != null) { + taskContext.registerAccumulator(this) + } + } else { + atDriverSide = true + } + + readKryo(kryo, input) + } + + def writeKryo(kryo: Kryo, output: Output): Unit + def readKryo(kryo: Kryo, input: Input): Unit +} /** * An internal class used to track accumulators by Spark itself. @@ -285,7 +347,8 @@ private[spark] object AccumulatorContext { * * @since 2.0.0 */ -class LongAccumulator extends AccumulatorV2[jl.Long, jl.Long] { +class LongAccumulator extends AccumulatorV2Kryo[jl.Long, jl.Long] + with KryoSerializable { private var _sum = 0L private var _count = 0L @@ -355,6 +418,16 @@ class LongAccumulator extends AccumulatorV2[jl.Long, jl.Long] { private[spark] def setValue(newValue: Long): Unit = _sum = newValue override def value: jl.Long = _sum + + override def writeKryo(kryo: Kryo, output: Output): Unit = { + output.writeLong(_sum) + output.writeLong(_count) + } + + override def readKryo(kryo: Kryo, input: Input): Unit = { + _sum = input.readLong() + _count = input.readLong() + } } @@ -364,7 +437,8 @@ class LongAccumulator extends AccumulatorV2[jl.Long, jl.Long] { * * @since 2.0.0 */ -class DoubleAccumulator extends AccumulatorV2[jl.Double, jl.Double] { +class DoubleAccumulator extends AccumulatorV2Kryo[jl.Double, jl.Double] + with KryoSerializable { private var _sum = 0.0 private var _count = 0L @@ -430,6 +504,16 @@ class DoubleAccumulator extends AccumulatorV2[jl.Double, jl.Double] { private[spark] def setValue(newValue: Double): Unit = _sum = newValue override def value: jl.Double = _sum + + override def writeKryo(kryo: Kryo, output: Output): Unit = { + output.writeDouble(_sum) + output.writeVarLong(_count, true) + } + + override def readKryo(kryo: Kryo, input: Input): Unit = { + _sum = input.readDouble() + _count = input.readVarLong(true) + } } @@ -438,7 +522,8 @@ class DoubleAccumulator extends AccumulatorV2[jl.Double, jl.Double] { * * @since 2.0.0 */ -class CollectionAccumulator[T] extends AccumulatorV2[T, java.util.List[T]] { +class CollectionAccumulator[T] extends AccumulatorV2Kryo[T, java.util.List[T]] + with KryoSerializable { private val _list: java.util.List[T] = Collections.synchronizedList(new ArrayList[T]()) override def isZero: Boolean = _list.isEmpty @@ -471,6 +556,23 @@ class CollectionAccumulator[T] extends AccumulatorV2[T, java.util.List[T]] { _list.clear() _list.addAll(newValue) } + + override def writeKryo(kryo: Kryo, output: Output): Unit = { + output.writeVarInt(_list.size(), true) + val iter = _list.iterator() + while (iter.hasNext) { + kryo.writeClassAndObject(output, iter.next()) + } + } + + override def readKryo(kryo: Kryo, input: Input): Unit = { + var len = input.readVarInt(true) + if (!_list.isEmpty) _list.clear() + while (len > 0) { + _list.add(kryo.readClassAndObject(input).asInstanceOf[T]) + len -= 1 + } + } } diff --git a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala index 40616421b5bca..01c0588707594 100644 --- a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala +++ b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala @@ -207,7 +207,7 @@ private[spark] object ClosureCleaner extends Logging { accessedFields: Map[Class[_], Set[String]]): Unit = { if (!isClosure(func.getClass)) { - logWarning("Expected a closure; got " + func.getClass.getName) + // logWarning("Expected a closure; got " + func.getClass.getName) return } diff --git a/core/src/main/scala/org/apache/spark/util/SerializableBuffer.scala b/core/src/main/scala/org/apache/spark/util/SerializableBuffer.scala index a06b6f84ef11b..5b27fe5cdc6eb 100644 --- a/core/src/main/scala/org/apache/spark/util/SerializableBuffer.scala +++ b/core/src/main/scala/org/apache/spark/util/SerializableBuffer.scala @@ -21,12 +21,17 @@ import java.io.{EOFException, IOException, ObjectInputStream, ObjectOutputStream import java.nio.ByteBuffer import java.nio.channels.Channels +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + /** * A wrapper around a java.nio.ByteBuffer that is serializable through Java serialization, to make * it easier to pass ByteBuffers in case class messages. */ private[spark] -class SerializableBuffer(@transient var buffer: ByteBuffer) extends Serializable { +class SerializableBuffer(@transient var buffer: ByteBuffer) + extends Serializable with KryoSerializable { + def value: ByteBuffer = buffer private def readObject(in: ObjectInputStream): Unit = Utils.tryOrIOException { @@ -51,4 +56,20 @@ class SerializableBuffer(@transient var buffer: ByteBuffer) extends Serializable } buffer.rewind() // Allow us to write it again later } + + override def write(kryo: Kryo, output: Output) { + if (buffer.position() != 0) { + throw new IOException(s"Unexpected buffer position ${buffer.position()}") + } + output.writeInt(buffer.limit()) + output.writeBytes(buffer.array(), buffer.arrayOffset(), buffer.limit()) + } + + override def read(kryo: Kryo, input: Input) { + val length = input.readInt() + val b = new Array[Byte](length) + input.readBytes(b) + buffer = ByteBuffer.wrap(b) + buffer.rewind() // Allow us to read it later + } } diff --git a/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala b/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala index e63e0e3e1f68f..953699fe37b7a 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala @@ -19,14 +19,17 @@ package org.apache.spark.util.collection import java.util.Arrays +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + /** * A simple, fixed-size bit set implementation. This implementation is fast because it avoids * safety/bound checking. */ -class BitSet(numBits: Int) extends Serializable { +class BitSet(numBits: Int) extends Serializable with KryoSerializable { - private val words = new Array[Long](bit2words(numBits)) - private val numWords = words.length + private var words = new Array[Long](bit2words(numBits)) + private var numWords = words.length /** * Compute the capacity (number of bits) that can be represented @@ -238,4 +241,27 @@ class BitSet(numBits: Int) extends Serializable { /** Return the number of longs it would take to hold numBits. */ private def bit2words(numBits: Int) = ((numBits - 1) >> 6) + 1 + + override def write(kryo: Kryo, output: Output): Unit = { + val words = this.words + val numWords = this.numWords + output.writeVarInt(numWords, true) + var i = 0 + while (i < numWords) { + output.writeLong(words(i)) + i += 1 + } + } + + override def read(kryo: Kryo, input: Input): Unit = { + val numWords = input.readVarInt(true) + val words = new Array[Long](numWords) + var i = 0 + while (i < numWords) { + words(i) = input.readLong() + i += 1 + } + this.words = words + this.numWords = numWords + } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala index 4a459571ed634..c49a8026af15c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala @@ -197,7 +197,7 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR /** * A lazily generated row ordering comparator. */ -class LazilyGeneratedOrdering(val ordering: Seq[SortOrder]) +class LazilyGeneratedOrdering(private var ordering: Seq[SortOrder]) extends Ordering[InternalRow] with KryoSerializable { def this(ordering: Seq[SortOrder], inputSchema: Seq[Attribute]) = @@ -220,7 +220,8 @@ class LazilyGeneratedOrdering(val ordering: Seq[SortOrder]) } override def read(kryo: Kryo, in: Input): Unit = Utils.tryOrIOException { - generatedOrdering = GenerateOrdering.generate(kryo.readObject(in, classOf[Array[SortOrder]])) + ordering = kryo.readObject(in, classOf[Array[SortOrder]]) + generatedOrdering = GenerateOrdering.generate(ordering) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala index 67bb66da5f38b..6713464f336b7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala @@ -20,10 +20,13 @@ package org.apache.spark.sql.execution.metric import java.text.NumberFormat import java.util.Locale +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark.SparkContext import org.apache.spark.scheduler.AccumulableInfo import org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates -import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, Utils} +import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, AccumulatorV2Kryo, Utils} /** @@ -32,12 +35,12 @@ import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, Utils} * on the driver side must be explicitly posted using [[SQLMetrics.postDriverMetricUpdates()]]. */ final class SQLMetric(val metricType: String, initValue: Long = 0L) - extends AccumulatorV2[Long, Long] { + extends AccumulatorV2Kryo[Long, Long] with KryoSerializable { // This is a workaround for SPARK-11013. // We may use -1 as initial value of the accumulator, if the accumulator is valid, we will // update it at the end of task and the value will be at least 0. Then we can filter out the -1 // values before calculate max, min, etc. - private[this] var _value = initValue + private var _value = initValue private var _zeroValue = initValue override def copy(): SQLMetric = { @@ -78,6 +81,18 @@ final class SQLMetric(val metricType: String, initValue: Long = 0L) new AccumulableInfo( id, name, update, value, true, true, Some(AccumulatorContext.SQL_ACCUM_IDENTIFIER)) } + + override def writeKryo(kryo: Kryo, output: Output): Unit = { + output.writeString(metricType) + output.writeLong(_value) + output.writeLong(_zeroValue) + } + + override def readKryo(kryo: Kryo, input: Input): Unit = { + metricType = input.readString() + _value = input.readLong() + _zeroValue = input.readLong() + } } object SQLMetrics { From c874a10c6cf6f9fe07cee2fb76a87dd172c974e7 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 3 Dec 2016 17:36:38 +0530 Subject: [PATCH 056/142] [SNAP-1190] Reduce partition message overhead from driver to executor (#31) - DAGScheduler: - For small enough common task data (RDD + closure) send inline with the Task instead of a broadcast - Transiently store task binary data in Stage to re-use if possible - Compress the common task bytes to save on network cost - Task: New TaskData class to encapsulate task compressed bytes from above, the uncompressed length and reference index if TaskData is being read from a separate list (see next comments) - CoarseGrainedClusterMessage: Added new LaunchTasks message to encapsulate multiple Task messages to same executor - CoarseGrainedSchedulerBackend: - Create LaunchTasks by grouping messages in ExecutorTaskGroup per executor - Actual TaskData is sent as part of TaskDescription and not the Task to easily separate out the common portions in a separate list - Send the common TaskData as a separate ArrayBuffer of data with the index into this list set in the original task's TaskData - CoarseGrainedExecutorBackend: Handle LaunchTasks by splitting into individual jobs - CompressionCodec: added bytes compress/decompress methods for more efficient byte array compression - Executor: - Set the common decompressed task data back into the Task object. - Avoid additional serialization of TaskResult just to determine the serialization time. Instead now calculate the time inline during serialization write/writeExternal methods - TaskMetrics: more generic handling for DoubleAccumulator case - Task: Handling of TaskData during serialization to send a flag to indicate whether data is inlined or will be received via broadcast - ResultTask, ShuffleMapTask: delegate handling of TaskData to parent Task class - SparkEnv: encapsulate codec creation as a zero-arg function to avoid repeated conf lookups - SparkContext.clean: avoid checking serializability in case non-default closure serializer is being used - Test updates for above Conflicts: core/src/main/scala/org/apache/spark/SparkEnv.scala core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala core/src/main/scala/org/apache/spark/scheduler/Task.scala core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala Conflicts: core/src/main/scala/org/apache/spark/SparkContext.scala core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala core/src/main/scala/org/apache/spark/executor/Executor.scala core/src/main/scala/org/apache/spark/io/CompressionCodec.scala core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala core/src/main/scala/org/apache/spark/scheduler/Task.scala core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosTaskLaunchData.scala resource-managers/mesos/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackend.scala resource-managers/mesos/src/test/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcherSuite.scala --- .../scala/org/apache/spark/SparkContext.scala | 6 +- .../scala/org/apache/spark/SparkEnv.scala | 6 + .../CoarseGrainedExecutorBackend.scala | 27 +++-- .../org/apache/spark/executor/Executor.scala | 18 ++- .../apache/spark/executor/TaskMetrics.scala | 8 +- .../apache/spark/io/CompressionCodec.scala | 74 ++++++++++-- .../apache/spark/scheduler/DAGScheduler.scala | 30 ++++- .../apache/spark/scheduler/ResultTask.scala | 23 ++-- .../spark/scheduler/ShuffleMapTask.scala | 26 +++-- .../org/apache/spark/scheduler/Stage.scala | 4 +- .../org/apache/spark/scheduler/Task.scala | 33 +++++- .../spark/scheduler/TaskDescription.scala | 5 +- .../apache/spark/scheduler/TaskResult.scala | 101 +++++++++++------ .../spark/scheduler/TaskSetManager.scala | 2 +- .../cluster/CoarseGrainedClusterMessage.scala | 60 +++++++++- .../CoarseGrainedSchedulerBackend.scala | 106 ++++++++++++++++-- .../local/LocalSchedulerBackend.scala | 3 +- .../spark/scheduler/TaskSetManagerSuite.scala | 3 +- .../spark/executor/MesosExecutorBackend.scala | 3 +- .../spark/sql/execution/SparkPlan.scala | 4 +- 20 files changed, 423 insertions(+), 119 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 3a0c093587157..004bfe9756c4c 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -72,6 +72,7 @@ import org.apache.spark.scheduler._ import org.apache.spark.scheduler.cluster.{CoarseGrainedSchedulerBackend, StandaloneSchedulerBackend} import org.apache.spark.scheduler.local.LocalSchedulerBackend import org.apache.spark.status.AppStatusStore +import org.apache.spark.serializer.JavaSerializer import org.apache.spark.storage._ import org.apache.spark.storage.BlockManagerMessages.TriggerThreadDump import org.apache.spark.ui.{ConsoleProgressBar, SparkUI} @@ -230,6 +231,7 @@ class SparkContext(config: SparkConf) extends Logging { private var _files: Seq[String] = _ private var _shutdownHookRef: AnyRef = _ private var _statusStore: AppStatusStore = _ + private var _isDefaultClosureSerializer: Boolean = true /* ------------------------------------------------------------------------------------- * | Accessors and public fields. These provide access to the internal state of the | @@ -446,6 +448,8 @@ class SparkContext(config: SparkConf) extends Logging { _env = createSparkEnv(_conf, isLocal, listenerBus) SparkEnv.set(_env) + _isDefaultClosureSerializer = _env.closureSerializer.isInstanceOf[JavaSerializer] + // If running the REPL, register the repl's output dir with the file server. _conf.getOption("spark.repl.class.outputDir").foreach { path => val replUri = _env.rpcEnv.fileServer.addDirectory("/classes", new File(path)) @@ -2312,7 +2316,7 @@ class SparkContext(config: SparkConf) extends Logging { * @return the cleaned closure */ private[spark] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = { - ClosureCleaner.clean(f, checkSerializable) + ClosureCleaner.clean(f, checkSerializable && _isDefaultClosureSerializer) f } diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index 8a9d8a9292b34..97c0f5bcc2c7f 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -49,6 +49,7 @@ import org.apache.spark.api.python.PythonWorkerFactory import org.apache.spark.broadcast.BroadcastManager import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ +import org.apache.spark.io.CompressionCodec import org.apache.spark.memory.{MemoryManager, StaticMemoryManager, UnifiedMemoryManager} import org.apache.spark.metrics.MetricsSystem import org.apache.spark.network.netty.NettyBlockTransferService @@ -97,6 +98,11 @@ class SparkEnv ( private[spark] var driverTmpDir: Option[String] = None + private val codecCreator = CompressionCodec.codecCreator(conf, + CompressionCodec.getCodecName(conf)) + + def createCompressionCodec: CompressionCodec = codecCreator() + private[spark] def stop() { if (!isStopped) { diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala index fab79c8904c07..f0fdbcefb80cc 100644 --- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala +++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala @@ -34,7 +34,6 @@ import org.apache.spark.internal.Logging import org.apache.spark.rpc._ import org.apache.spark.scheduler.{ExecutorLossReason, TaskDescription} import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._ -import org.apache.spark.serializer.SerializerInstance import org.apache.spark.util.{ThreadUtils, Utils} private[spark] class CoarseGrainedExecutorBackend( @@ -51,10 +50,6 @@ private[spark] class CoarseGrainedExecutorBackend( var executor: Executor = null @volatile var driver: Option[RpcEndpointRef] = None - // If this CoarseGrainedExecutorBackend is changed to support multiple threads, then this may need - // to be changed so that we don't share the serializer instance across threads - private[this] val ser: SerializerInstance = env.closureSerializer.newInstance() - override def onStart() { logInfo("Connecting to driver: " + driverUrl) rpcEnv.asyncSetupEndpointRefByURI(driverUrl).flatMap { ref => @@ -94,14 +89,30 @@ private[spark] class CoarseGrainedExecutorBackend( logError("Slave registration failed: " + message) exitExecutor() - case LaunchTask(data) => + case LaunchTask(taskDesc) => if (executor == null) { logError("Received LaunchTask command but executor was null") exitExecutor() } else { - val taskDesc = TaskDescription.decode(data.value) + val taskDesc = ser.deserialize[TaskDescription](data.value) logInfo("Got assigned task " + taskDesc.taskId) - executor.launchTask(this, taskDesc) + executor.launchTask(this, taskId = taskDesc.taskId, attemptNumber = taskDesc.attemptNumber, + taskDesc.name, taskDesc.serializedTask, taskDesc.taskData.decompress(env)) + } + + case LaunchTasks(tasks, taskDataList) => + if (executor ne null) { + logDebug("Got assigned tasks " + tasks.map(_.taskId).mkString(",")) + for (task <- tasks) { + logInfo("Got assigned task " + task.taskId) + val ref = task.taskData.reference + val taskData = if (ref >= 0) taskDataList(ref) else task.taskData + executor.launchTask(this, taskId = task.taskId, + attemptNumber = task.attemptNumber, task.name, task.serializedTask, + taskData.decompress(env)) + } + } else { + exitExecutor(1, "Received LaunchTasks command but executor was null") } case KillTask(taskId, _, interruptThread, reason) => diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index f8cf7a0943584..e724f824d15c4 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -246,7 +246,12 @@ private[spark] class Executor( class TaskRunner( execBackend: ExecutorBackend, - private val taskDescription: TaskDescription) + private val taskDescription: TaskDescription, + val taskId: Long, + val attemptNumber: Int, + taskName: String, + serializedTask: ByteBuffer, + taskDataBytes: Array[Byte]) extends Runnable { val taskId = taskDescription.taskId @@ -404,11 +409,6 @@ private[spark] class Executor( // If the task has been killed, let's fail it. task.context.killTaskIfInterrupted() - val resultSer = env.serializer.newInstance() - val beforeSerialization = System.nanoTime() - val valueBytes = resultSer.serialize(value) - val afterSerialization = System.nanoTime() - // Deserialization happens in two parts: first, we deserialize a Task object, which // includes the Partition. Second, Task.run() deserializes the RDD and function to be run. task.metrics.setExecutorDeserializeTime(math.max( @@ -421,8 +421,6 @@ private[spark] class Executor( task.metrics.setExecutorCpuTime( (taskFinishCpu - taskStartCpu) - task.executorDeserializeCpuTime) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) - task.metrics.setResultSerializationTime(math.max( - afterSerialization - beforeSerialization, 0L) / 1000000.0) // Expose task metrics using the Dropwizard metrics system. // Update task metrics counters @@ -467,8 +465,8 @@ private[spark] class Executor( // Note: accumulator updates must be collected after TaskMetrics is updated val accumUpdates = task.collectAccumulatorUpdates() - // TODO: do not serialize value twice - val directResult = new DirectTaskResult(valueBytes, accumUpdates) + val directResult = new DirectTaskResult(value, accumUpdates, + Some(task.metrics.resultSerializationTimeMetric)) val serializedDirectResult = ser.serialize(directResult) val resultSize = serializedDirectResult.limit() diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala index 3facda3df95bc..445ca611118cf 100644 --- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala @@ -139,6 +139,7 @@ class TaskMetrics private[spark] () extends Serializable with KryoSerializable { private[spark] def setJvmGCTime(v: Long): Unit = _jvmGCTime.setValue(v) private[spark] def setResultSerializationTime(v: Double): Unit = _resultSerializationTime.setValue(v) + private[spark] def resultSerializationTimeMetric = _resultSerializationTime private[spark] def incMemoryBytesSpilled(v: Long): Unit = _memoryBytesSpilled.add(v) private[spark] def incDiskBytesSpilled(v: Long): Unit = _diskBytesSpilled.add(v) private[spark] def incPeakExecutionMemory(v: Long): Unit = _peakExecutionMemory.add(v) @@ -338,7 +339,12 @@ private[spark] object TaskMetrics extends Logging { } else { tm.nameToAccums.get(name).foreach { case l: LongAccumulator => l.setValue(value.asInstanceOf[Long]) - case d => d.asInstanceOf[DoubleAccumulator].setValue(value.asInstanceOf[Double]) + case d: DoubleAccumulator => value match { + case v: Double => d.setValue(v) + case _ => d.setValue(value.asInstanceOf[Long]) + } + case o => throw new UnsupportedOperationException( + s"Unexpected accumulator $o for TaskMetrics") } } } diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala index 7722db56ee297..22c663686d6ef 100644 --- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala +++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala @@ -21,8 +21,8 @@ import java.io._ import java.util.Locale import com.github.luben.zstd.{ZstdInputStream, ZstdOutputStream} -import com.ning.compress.lzf.{LZFInputStream, LZFOutputStream} -import net.jpountz.lz4.{LZ4BlockInputStream, LZ4BlockOutputStream} +import com.ning.compress.lzf.{LZFDecoder, LZFEncoder, LZFInputStream, LZFOutputStream} +import net.jpountz.lz4.{LZ4BlockInputStream, LZ4BlockOutputStream, LZ4Factory} import org.xerial.snappy.{Snappy, SnappyInputStream, SnappyOutputStream} import org.apache.spark.SparkConf @@ -43,6 +43,11 @@ trait CompressionCodec { def compressedOutputStream(s: OutputStream): OutputStream def compressedInputStream(s: InputStream): InputStream + + def compress(input: Array[Byte], inputLen: Int): Array[Byte] + + def decompress(input: Array[Byte], inputOffset: Int, inputLen: Int, + outputLen: Int): Array[Byte] } private[spark] object CompressionCodec { @@ -69,16 +74,32 @@ private[spark] object CompressionCodec { } def createCodec(conf: SparkConf, codecName: String): CompressionCodec = { - val codecClass = - shortCompressionCodecNames.getOrElse(codecName.toLowerCase(Locale.ROOT), codecName) - val codec = try { + codecCreator(conf, codecName)() + } + + def codecCreator(conf: SparkConf, codecName: String): () => CompressionCodec = { + if (codecName == DEFAULT_COMPRESSION_CODEC) { + return () => new LZ4CompressionCodec(conf) + } + val codecClass = shortCompressionCodecNames.getOrElse(codecName.toLowerCase, codecName) + try { val ctor = Utils.classForName(codecClass).getConstructor(classOf[SparkConf]) - Some(ctor.newInstance(conf).asInstanceOf[CompressionCodec]) + () => { + try { + ctor.newInstance(conf).asInstanceOf[CompressionCodec] + } catch { + case e: IllegalArgumentException => throw fail(codecName) + } + } } catch { - case _: ClassNotFoundException | _: IllegalArgumentException => None + case e: ClassNotFoundException => throw fail(codecName) + case e: NoSuchMethodException => throw fail(codecName) } - codec.getOrElse(throw new IllegalArgumentException(s"Codec [$codecName] is not available. " + - s"Consider setting $configKey=$FALLBACK_COMPRESSION_CODEC")) + } + + private def fail(codecName: String): IllegalArgumentException = { + new IllegalArgumentException(s"Codec [$codecName] is not available. " + + s"Consider setting $configKey=$FALLBACK_COMPRESSION_CODEC") } /** @@ -117,9 +138,16 @@ class LZ4CompressionCodec(conf: SparkConf) extends CompressionCodec { new LZ4BlockOutputStream(s, blockSize) } - override def compressedInputStream(s: InputStream): InputStream = { - val disableConcatenationOfByteStream = false - new LZ4BlockInputStream(s, disableConcatenationOfByteStream) + override def compressedInputStream(s: InputStream): InputStream = new LZ4BlockInputStream(s) + + override def compress(input: Array[Byte], inputLen: Int): Array[Byte] = { + LZ4Factory.fastestInstance().fastCompressor().compress(input, 0, inputLen) + } + + override def decompress(input: Array[Byte], inputOffset: Int, inputLen: Int, + outputLen: Int): Array[Byte] = { + LZ4Factory.fastestInstance().fastDecompressor().decompress(input, + inputOffset, outputLen) } } @@ -140,6 +168,17 @@ class LZFCompressionCodec(conf: SparkConf) extends CompressionCodec { } override def compressedInputStream(s: InputStream): InputStream = new LZFInputStream(s) + + override def compress(input: Array[Byte], inputLen: Int): Array[Byte] = { + LZFEncoder.encode(input, 0, inputLen) + } + + override def decompress(input: Array[Byte], inputOffset: Int, inputLen: Int, + outputLen: Int): Array[Byte] = { + val output = new Array[Byte](outputLen) + LZFDecoder.decode(input, inputOffset, inputLen, output) + output + } } @@ -162,6 +201,17 @@ class SnappyCompressionCodec(conf: SparkConf) extends CompressionCodec { } override def compressedInputStream(s: InputStream): InputStream = new SnappyInputStream(s) + + override def compress(input: Array[Byte], inputLen: Int): Array[Byte] = { + Snappy.rawCompress(input, inputLen) + } + + override def decompress(input: Array[Byte], inputOffset: Int, + inputLen: Int, outputLen: Int): Array[Byte] = { + val output = new Array[Byte](outputLen) + Snappy.uncompress(input, inputOffset, inputLen, output, 0) + output + } } /** diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 8c46a84323392..396dbd46946a4 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -1020,6 +1020,7 @@ class DAGScheduler( try { // For ShuffleMapTask, serialize and broadcast (rdd, shuffleDep). // For ResultTask, serialize and broadcast (rdd, func). +<<<<<<< HEAD var taskBinaryBytes: Array[Byte] = null // taskBinaryBytes and partitions are both effected by the checkpoint status. We need // this synchronization in case another concurrent job is checkpointing this RDD, so we get a @@ -1035,8 +1036,23 @@ class DAGScheduler( partitions = stage.rdd.partitions } - - taskBinary = sc.broadcast(taskBinaryBytes) + if (bytes == null) stage.taskBinaryBytes = taskBinaryBytes + + // use direct byte shipping for small size or if number of partitions is small + val taskBytesLen = taskBinaryBytes.length + if (taskBytesLen <= DAGScheduler.TASK_INLINE_LIMIT || + partitionsToCompute.length <= DAGScheduler.TASK_INLINE_PARTITION_LIMIT) { + if (stage.taskData.uncompressedLen > 0) { + taskData = stage.taskData + } else { + // compress inline task data (broadcast compresses as per conf) + taskData = new TaskData(env.createCompressionCodec.compress( + taskBinaryBytes, taskBytesLen), taskBytesLen) + stage.taskData = taskData + } + } else { + taskBinary = Some(sc.broadcast(taskBinaryBytes)) + } } catch { // In the case of a failure during serialization, abort the stage. case e: NotSerializableException => @@ -1545,7 +1561,7 @@ class DAGScheduler( * Marks a stage as finished and removes it from the list of running stages. */ private def markStageAsFinished(stage: Stage, errorMessage: Option[String] = None): Unit = { - val serviceTime = stage.latestInfo.submissionTime match { + val serviceTime = if (!log.isInfoEnabled) 0L else stage.latestInfo.submissionTime match { case Some(t) => "%.03f".format((clock.getTimeMillis() - t) / 1000.0) case _ => "Unknown" } @@ -1847,4 +1863,12 @@ private[spark] object DAGScheduler { // Number of consecutive stage attempts allowed before a stage is aborted val DEFAULT_MAX_CONSECUTIVE_STAGE_ATTEMPTS = 4 + + // The maximum size of uncompressed common task bytes (rdd, closure) + // that will be shipped with the task else will be broadcast separately. + val TASK_INLINE_LIMIT = 100 * 1024 + + // The maximum number of partitions below which common task bytes will be + // shipped with the task else will be broadcast separately. + val TASK_INLINE_PARTITION_LIMIT = 8 } diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index 86f188dbb6022..f94aad960e41a 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -22,7 +22,7 @@ import java.lang.management.ManagementFactory import java.nio.ByteBuffer import java.util.Properties -import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.spark._ @@ -36,7 +36,9 @@ import org.apache.spark.rdd.RDD * * @param stageId id of the stage this task belongs to * @param stageAttemptId attempt id of the stage this task belongs to - * @param taskBinary broadcasted version of the serialized RDD and the function to apply on each + * @param _taskData if serialized RDD and function are small, then it is compressed + * and sent with its original decompressed size + * @param _taskBinary broadcasted version of the serialized RDD and the function to apply on each * partition of the given RDD. Once deserialized, the type should be * (RDD[T], (TaskContext, Iterator[T]) => U). * @param partition partition of the RDD this task is associated with @@ -55,7 +57,8 @@ import org.apache.spark.rdd.RDD private[spark] class ResultTask[T, U]( stageId: Int, stageAttemptId: Int, - private var taskBinary: Broadcast[Array[Byte]], + _taskData: TaskData, + _taskBinary: Option[Broadcast[Array[Byte]]], private var partition: Partition, locs: Seq[TaskLocation], private var _outputId: Int, @@ -64,9 +67,9 @@ private[spark] class ResultTask[T, U]( jobId: Option[Int] = None, appId: Option[String] = None, appAttemptId: Option[String] = None) - extends Task[U](stageId, stageAttemptId, partition.index, localProperties, serializedTaskMetrics, - jobId, appId, appAttemptId) - with Serializable { + extends Task[U](stageId, stageAttemptId, partition.index, _taskData, + _taskBinary, metrics, localProperties, jobId, appId, appAttemptId) + with Serializable with KryoSerializable { final def outputId: Int = _outputId @@ -83,7 +86,7 @@ private[spark] class ResultTask[T, U]( } else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, func) = ser.deserialize[(RDD[T], (TaskContext, Iterator[T]) => U)]( - ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) + ByteBuffer.wrap(getTaskBytes), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = math.max(System.nanoTime() - deserializeStartTime, 0L) _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime @@ -98,15 +101,13 @@ private[spark] class ResultTask[T, U]( override def toString: String = "ResultTask(" + stageId + ", " + partitionId + ")" override def write(kryo: Kryo, output: Output): Unit = { - super.write(kryo, output) - kryo.writeClassAndObject(output, taskBinary) + super.writeKryo(kryo, output) kryo.writeClassAndObject(output, partition) output.writeInt(_outputId) } override def read(kryo: Kryo, input: Input): Unit = { - super.read(kryo, input) - taskBinary = kryo.readClassAndObject(input).asInstanceOf[Broadcast[Array[Byte]]] + super.readKryo(kryo, input) partition = kryo.readClassAndObject(input).asInstanceOf[Partition] _outputId = input.readInt() } diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index c5e757024ed45..46e03c033f36c 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -23,7 +23,7 @@ import java.util.Properties import scala.language.existentials -import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.spark._ @@ -40,7 +40,9 @@ import org.apache.spark.shuffle.ShuffleWriter * * @param stageId id of the stage this task belongs to * @param stageAttemptId attempt id of the stage this task belongs to - * @param taskBinary broadcast version of the RDD and the ShuffleDependency. Once deserialized, + * @param _taskData if serialized RDD and function are small, then it is compressed + * and sent with its original decompressed size + * @param _taskBinary broadcast version of the RDD and the ShuffleDependency. Once deserialized, * the type should be (RDD[_], ShuffleDependency[_, _, _]). * @param partition partition of the RDD this task is associated with * @param locs preferred task execution locations for locality scheduling @@ -56,7 +58,8 @@ import org.apache.spark.shuffle.ShuffleWriter private[spark] class ShuffleMapTask( stageId: Int, stageAttemptId: Int, - private var taskBinary: Broadcast[Array[Byte]], + _taskData: TaskData, + _taskBinary: Option[Broadcast[Array[Byte]]], private var partition: Partition, @transient private var locs: Seq[TaskLocation], localProperties: Properties, @@ -64,13 +67,14 @@ private[spark] class ShuffleMapTask( jobId: Option[Int] = None, appId: Option[String] = None, appAttemptId: Option[String] = None) - extends Task[MapStatus](stageId, stageAttemptId, partition.index, localProperties, - serializedTaskMetrics, jobId, appId, appAttemptId) - with Logging { + extends Task[MapStatus](stageId, stageAttemptId, partition.index, _taskData, + _taskBinary, metrics, localProperties, jobId, appId, appAttemptId) +with KryoSerializable with Logging { /** A constructor used only in test suites. This does not require passing in an RDD. */ def this(partitionId: Int) { - this(0, 0, null, new Partition { override def index: Int = 0 }, null, new Properties, null) + this(0, 0, TaskData.EMPTY, null, new Partition { override def index: Int = 0 }, + null, null, new Properties) } @transient private val preferredLocs: Seq[TaskLocation] = { @@ -86,7 +90,7 @@ private[spark] class ShuffleMapTask( } else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, dep) = ser.deserialize[(RDD[_], ShuffleDependency[_, _, _])]( - ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) + ByteBuffer.wrap(getTaskBytes), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = math.max(System.nanoTime() - deserializeStartTime, 0L) _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime @@ -117,14 +121,12 @@ private[spark] class ShuffleMapTask( override def toString: String = "ShuffleMapTask(%d, %d)".format(stageId, partitionId) override def write(kryo: Kryo, output: Output): Unit = { - super.write(kryo, output) - kryo.writeClassAndObject(output, taskBinary) + super.writeKryo(kryo, output) kryo.writeClassAndObject(output, partition) } override def read(kryo: Kryo, input: Input): Unit = { - super.read(kryo, input) - taskBinary = kryo.readClassAndObject(input).asInstanceOf[Broadcast[Array[Byte]]] + super.readKryo(kryo, input) partition = kryo.readClassAndObject(input).asInstanceOf[Partition] } } diff --git a/core/src/main/scala/org/apache/spark/scheduler/Stage.scala b/core/src/main/scala/org/apache/spark/scheduler/Stage.scala index 290fd073caf27..65af6bdd50334 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Stage.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Stage.scala @@ -59,7 +59,9 @@ private[scheduler] abstract class Stage( val numTasks: Int, val parents: List[Stage], val firstJobId: Int, - val callSite: CallSite) + val callSite: CallSite, + @transient private[scheduler] var taskBinaryBytes: Array[Byte] = null, + @transient private[scheduler] var taskData: TaskData = TaskData.EMPTY) extends Logging { val numPartitions = rdd.partitions.length diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index 867bca3b84de2..67153e58096cf 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -23,10 +23,11 @@ import java.util.Properties import scala.collection.mutable import scala.collection.mutable.HashMap -import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.{KryoSerializable, Kryo} import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.spark._ +import org.apache.spark.broadcast.Broadcast import org.apache.spark.executor.TaskMetrics import org.apache.spark.internal.config.APP_CALLER_CONTEXT import org.apache.spark.memory.{MemoryMode, TaskMemoryManager} @@ -61,7 +62,9 @@ private[spark] abstract class Task[T]( private var _stageId: Int, private var _stageAttemptId: Int, private var _partitionId: Int, + @transient private[spark] var taskData: TaskData = TaskData.EMPTY, // The default value is only used in tests. + protected var taskBinary: Option[Broadcast[Array[Byte]]] = None, private var _metrics: TaskMetrics = TaskMetrics.registered, @transient var localProperties: Properties = new Properties, // The default value is only used in tests. @@ -78,11 +81,16 @@ private[spark] abstract class Task[T]( final def partitionId: Int = _partitionId - final def metrics: TaskMetrics = _metrics - @transient lazy val metrics: TaskMetrics = SparkEnv.get.closureSerializer.newInstance().deserialize(ByteBuffer.wrap(serializedTaskMetrics)) + @transient private[spark] var taskDataBytes: Array[Byte] = _ + + protected final def getTaskBytes: Array[Byte] = { + val bytes = taskDataBytes + if ((bytes ne null) && bytes.length > 0) bytes else taskBinary.get.value + } + /** * Called by [[org.apache.spark.executor.Executor]] to run this task. * @@ -235,21 +243,36 @@ private[spark] abstract class Task[T]( } } - override def write(kryo: Kryo, output: Output): Unit = { + protected def writeKryo(kryo: Kryo, output: Output): Unit = { output.writeInt(_stageId) output.writeVarInt(_stageAttemptId, true) output.writeVarInt(_partitionId, true) output.writeLong(epoch) output.writeLong(_executorDeserializeTime) + if ((taskData ne null) && taskData.uncompressedLen > 0) { + // actual bytes will be shipped in TaskDescription + output.writeBoolean(true) + } else { + output.writeBoolean(false) + kryo.writeClassAndObject(output, taskBinary.get) + } _metrics.write(kryo, output) } - override def read(kryo: Kryo, input: Input): Unit = { + def readKryo(kryo: Kryo, input: Input): Unit = { _stageId = input.readInt() _stageAttemptId = input.readVarInt(true) _partitionId = input.readVarInt(true) epoch = input.readLong() _executorDeserializeTime = input.readLong() + // actual bytes are shipped in TaskDescription + taskData = TaskData.EMPTY + if (input.readBoolean()) { + taskBinary = None + } else { + taskBinary = Some(kryo.readClassAndObject(input) + .asInstanceOf[Broadcast[Array[Byte]]]) + } _metrics = new TaskMetrics _metrics.read(kryo, input) } diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala index 583898e12145f..fb86409cc5a89 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala @@ -52,7 +52,8 @@ private[spark] class TaskDescription( private var _executorId: String, private var _name: String, private var _index: Int, // Index within this task's TaskSet - @transient private var _serializedTask: ByteBuffer) + @transient private var _serializedTask: ByteBuffer, + private[spark] var taskData: TaskData = TaskData.EMPTY) extends Serializable with KryoSerializable { def taskId: Long = _taskId @@ -76,6 +77,7 @@ private[spark] class TaskDescription( output.writeInt(_index) output.writeInt(_serializedTask.remaining()) Utils.writeByteBuffer(_serializedTask, output) + TaskData.write(taskData, output) } override def read(kryo: Kryo, input: Input): Unit = { @@ -86,6 +88,7 @@ private[spark] class TaskDescription( _index = input.readInt() val len = input.readInt() _serializedTask = ByteBuffer.wrap(input.readBytes(len)) + taskData = TaskData.read(input) } dataOut.writeLong(taskDescription.taskId) diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala index 836769e1723d5..5b564366439a8 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala @@ -18,14 +18,14 @@ package org.apache.spark.scheduler import java.io._ -import java.nio.ByteBuffer import scala.collection.mutable.ArrayBuffer -import org.apache.spark.SparkEnv -import org.apache.spark.serializer.SerializerInstance +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark.storage.BlockId -import org.apache.spark.util.{AccumulatorV2, Utils} +import org.apache.spark.util.{AccumulatorV2, DoubleAccumulator, Utils} // Task result. Also contains updates to accumulator variables. private[spark] sealed trait TaskResult[T] @@ -36,27 +36,32 @@ private[spark] case class IndirectTaskResult[T](blockId: BlockId, size: Int) /** A TaskResult that contains the task's return value and accumulator updates. */ private[spark] class DirectTaskResult[T]( - var valueBytes: ByteBuffer, - var accumUpdates: Seq[AccumulatorV2[_, _]]) - extends TaskResult[T] with Externalizable { - - private var valueObjectDeserialized = false - private var valueObject: T = _ + private var _value: Any, + var accumUpdates: Seq[AccumulatorV2[_, _]], + private val serializationTimeMetric: Option[DoubleAccumulator] = None) + extends TaskResult[T] with Externalizable with KryoSerializable { - def this() = this(null.asInstanceOf[ByteBuffer], null) + def this() = this(null, null) override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException { - out.writeInt(valueBytes.remaining) - Utils.writeByteBuffer(valueBytes, out) - out.writeInt(accumUpdates.size) - accumUpdates.foreach(out.writeObject) + serializationTimeMetric match { + case Some(timeMetric) => + val start = System.nanoTime() + out.writeObject(_value) + out.writeInt(accumUpdates.size + 1) + accumUpdates.foreach(out.writeObject) + val end = System.nanoTime() + timeMetric.setValue(math.max(end - start, 0L) / 1000000.0) + out.writeObject(timeMetric) + case None => + out.writeObject(_value) + out.writeInt(accumUpdates.size) + accumUpdates.foreach(out.writeObject) + } } override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException { - val blen = in.readInt() - val byteVal = new Array[Byte](blen) - in.readFully(byteVal) - valueBytes = ByteBuffer.wrap(byteVal) + _value = in.readObject() val numUpdates = in.readInt if (numUpdates == 0) { @@ -68,26 +73,50 @@ private[spark] class DirectTaskResult[T]( } accumUpdates = _accumUpdates } - valueObjectDeserialized = false } - /** - * When `value()` is called at the first time, it needs to deserialize `valueObject` from - * `valueBytes`. It may cost dozens of seconds for a large instance. So when calling `value` at - * the first time, the caller should avoid to block other threads. - * - * After the first time, `value()` is trivial and just returns the deserialized `valueObject`. - */ - def value(resultSer: SerializerInstance = null): T = { - if (valueObjectDeserialized) { - valueObject + override def write(kryo: Kryo, output: Output): Unit = Utils.tryOrIOException { + serializationTimeMetric match { + case Some(timeMetric) => + val start = System.nanoTime() + kryo.writeClassAndObject(output, _value) + output.writeVarInt(accumUpdates.size, true) + output.writeBoolean(true) // indicates additional timeMetric + accumUpdates.foreach(kryo.writeClassAndObject(output, _)) + val end = System.nanoTime() + timeMetric.setValue(math.max(end - start, 0L) / 1000000.0) + timeMetric.write(kryo, output) + case None => + kryo.writeClassAndObject(output, _value) + output.writeVarInt(accumUpdates.size, true) + output.writeBoolean(false) // indicates no timeMetric + accumUpdates.foreach(kryo.writeClassAndObject(output, _)) + } + } + + override def read(kryo: Kryo, input: Input): Unit = Utils.tryOrIOException { + _value = kryo.readClassAndObject(input) + + var numUpdates = input.readVarInt(true) + val hasTimeMetric = input.readBoolean() + if (numUpdates == 0 && !hasTimeMetric) { + accumUpdates = null } else { - // This should not run when holding a lock because it may cost dozens of seconds for a large - // value - val ser = if (resultSer == null) SparkEnv.get.serializer.newInstance() else resultSer - valueObject = ser.deserialize(valueBytes) - valueObjectDeserialized = true - valueObject + val _accumUpdates = new ArrayBuffer[AccumulatorV2[_, _]]( + if (hasTimeMetric) numUpdates + 1 else numUpdates) + while (numUpdates > 0) { + _accumUpdates += kryo.readClassAndObject(input) + .asInstanceOf[AccumulatorV2[_, _]] + numUpdates -= 1 + } + if (hasTimeMetric) { + val timeMetric = new DoubleAccumulator + timeMetric.read(kryo, input) + _accumUpdates += timeMetric + } + accumUpdates = _accumUpdates } } + + def value(): T = _value.asInstanceOf[T] } diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 3519669e96fe6..25029ec9f48b5 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -1085,5 +1085,5 @@ private[spark] class TaskSetManager( private[spark] object TaskSetManager { // The user will be warned if any stages contain a task that has a serialized size greater than // this. - val TASK_SIZE_TO_WARN_KB = 100 + val TASK_SIZE_TO_WARN_KB = 128 } diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala index fc5115c57fb03..a22593c66ab52 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala @@ -19,12 +19,14 @@ package org.apache.spark.scheduler.cluster import java.nio.ByteBuffer +import scala.collection.mutable + import com.esotericsoftware.kryo.{Kryo, KryoSerializable} import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.spark.TaskState.TaskState import org.apache.spark.rpc.RpcEndpointRef -import org.apache.spark.scheduler.ExecutorLossReason +import org.apache.spark.scheduler.{ExecutorLossReason, TaskData, TaskDescription} import org.apache.spark.util.{SerializableBuffer, Utils} private[spark] sealed trait CoarseGrainedClusterMessage extends Serializable @@ -42,7 +44,61 @@ private[spark] object CoarseGrainedClusterMessages { case object RetrieveLastAllocatedExecutorId extends CoarseGrainedClusterMessage // Driver to executors - case class LaunchTask(data: SerializableBuffer) extends CoarseGrainedClusterMessage + case class LaunchTask(private var task: TaskDescription) + extends CoarseGrainedClusterMessage with KryoSerializable { + + override def write(kryo: Kryo, output: Output): Unit = { + task.write(kryo, output) + } + + override def read(kryo: Kryo, input: Input): Unit = { + task = new TaskDescription(0L, 0, null, null, 0, null) + task.read(kryo, input) + } + } + + case class LaunchTasks(private var tasks: mutable.ArrayBuffer[TaskDescription], + private var taskDataList: mutable.ArrayBuffer[TaskData]) + extends CoarseGrainedClusterMessage with KryoSerializable { + + override def write(kryo: Kryo, output: Output): Unit = Utils.tryOrIOException { + val tasks = this.tasks + val numTasks = tasks.length + output.writeVarInt(numTasks, true) + var i = 0 + while (i < numTasks) { + tasks(i).write(kryo, output) + i += 1 + } + val taskDataList = this.taskDataList + val numData = taskDataList.length + output.writeVarInt(numData, true) + i = 0 + while (i < numData) { + TaskData.write(taskDataList(i), output) + i += 1 + } + } + + override def read(kryo: Kryo, input: Input): Unit = Utils.tryOrIOException { + var numTasks = input.readVarInt(true) + val tasks = new mutable.ArrayBuffer[TaskDescription](numTasks) + while (numTasks > 0) { + val task = new TaskDescription(0, 0, null, null, 0, null) + task.read(kryo, input) + tasks += task + numTasks -= 1 + } + var numData = input.readVarInt(true) + val taskDataList = new mutable.ArrayBuffer[TaskData](numData) + while (numData > 0) { + taskDataList += TaskData.read(input) + numData -= 1 + } + this.tasks = tasks + this.taskDataList = taskDataList + } + } case class KillTask(taskId: Long, executor: String, interruptThread: Boolean, reason: String) extends CoarseGrainedClusterMessage diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index 7ffae9b68cb0e..76877fb691334 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -42,13 +42,14 @@ import javax.annotation.concurrent.GuardedBy import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet} import scala.concurrent.Future -import org.apache.spark.{ExecutorAllocationClient, SparkEnv, SparkException, TaskState} +import org.apache.spark.{SparkEnv, ExecutorAllocationClient, SparkException, TaskState} import org.apache.spark.internal.Logging import org.apache.spark.rpc._ import org.apache.spark.scheduler._ import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._ import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.ENDPOINT_NAME -import org.apache.spark.util.{RpcUtils, SerializableBuffer, ThreadUtils, Utils} +import org.apache.spark.util.{RpcUtils, ThreadUtils, Utils} +import org.apache.spark.util.collection.OpenHashMap /** * A scheduler backend that waits for coarse-grained executors to connect. @@ -324,16 +325,54 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp } } } - else { - val executorData = executorDataMap(task.executorId) - executorData.freeCores -= scheduler.CPUS_PER_TASK - - logDebug(s"Launching task ${task.taskId} on executor id: ${task.executorId} hostname: " + - s"${executorData.executorHost}.") + false + } else true + } - executorData.executorEndpoint.send(LaunchTask(new SerializableBuffer(serializedTask))) + // Launch tasks returned by a set of resource offers + protected def launchTasks(tasks: Seq[Seq[TaskDescription]]): Unit = { + val executorTaskGroupMap = new OpenHashMap[String, ExecutorTaskGroup](8) + for (taskSet <- tasks) { + for (task <- taskSet) { + val taskLimit = task.serializedTask.limit + val taskSize = taskLimit + task.taskData.compressedBytes.length + if (checkTaskSizeLimit(task, taskSize)) { + // group tasks per executor as long as message limit is not breached + executorTaskGroupMap.changeValue(task.executorId, { + val executorData = executorDataMap(task.executorId) + val executorTaskGroup = new ExecutorTaskGroup(executorData, taskSize) + executorTaskGroup.taskGroup += task + executorTaskGroup.taskDataList += task.taskData + // add reference to first index in taskDataList + task.taskData = TaskData(0) + executorTaskGroup + }, { executorTaskGroup => + // group into existing if size fits in the max allowed + if (!executorTaskGroup.addTask(task, taskLimit, maxRpcMessageSize)) { + // send this task separately + val executorData = executorTaskGroup.executorData + executorData.freeCores -= scheduler.CPUS_PER_TASK + logInfo(s"Launching task ${task.taskId} on executor id: " + + s"${task.executorId} hostname: ${executorData.executorHost}.") + + executorData.executorEndpoint.send(LaunchTask(task)) + } + executorTaskGroup + }) + } } } + // send the accumulated task groups per executor + executorTaskGroupMap.foreach { case (executorId, executorTaskGroup) => + val taskGroup = executorTaskGroup.taskGroup + val executorData = executorTaskGroup.executorData + + executorData.freeCores -= (scheduler.CPUS_PER_TASK * taskGroup.length) + logDebug(s"Launching tasks ${taskGroup.map(_.taskId).mkString(",")} on " + + s"executor id: $executorId hostname: ${executorData.executorHost}.") + executorData.executorEndpoint.send(LaunchTasks(taskGroup, + executorTaskGroup.taskDataList)) + } } // Remove a disconnected slave from the cluster @@ -705,3 +744,52 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp private[spark] object CoarseGrainedSchedulerBackend { val ENDPOINT_NAME = "CoarseGrainedScheduler" } + +private[spark] final class ExecutorTaskGroup( + private[cluster] var executorData: ExecutorData, + private var groupSize: Int = 0) { + + private[cluster] val taskGroup = new ArrayBuffer[TaskDescription](2) + // field to carry around common task data + private[cluster] val taskDataList = new ArrayBuffer[TaskData](2) + + def addTask(task: TaskDescription, taskLimit: Int, limit: Int): Boolean = { + val newGroupSize = groupSize + taskLimit + if (newGroupSize > limit) return false + + groupSize = newGroupSize + // linear search is best since there cannot be many different + // tasks in a single taskSet + if (task.taskData.uncompressedLen == 0 || + findOrAddTaskData(task, taskDataList, limit)) { + taskGroup += task + true + } else { + // task rejected from group + groupSize -= taskLimit + false + } + } + + private def findOrAddTaskData(task: TaskDescription, + taskDataList: ArrayBuffer[TaskData], limit: Int): Boolean = { + val data = task.taskData + val numData = taskDataList.length + var i = 0 + while (i < numData) { + if (taskDataList(i) eq data) { + // add reference to index `i` in taskDataList + task.taskData = TaskData(i) + return true + } + i += 1 + } + val newGroupSize = groupSize + data.compressedBytes.length + if (newGroupSize <= limit) { + groupSize = newGroupSize + taskDataList += data + task.taskData = TaskData(numData) + true + } else false + } +} diff --git a/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala index 4c614c5c0f602..83198c0d7c10d 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala @@ -84,7 +84,8 @@ private[spark] class LocalEndpoint( val offers = IndexedSeq(new WorkerOffer(localExecutorId, localExecutorHostname, freeCores)) for (task <- scheduler.resourceOffers(offers).flatten) { freeCores -= scheduler.CPUS_PER_TASK - executor.launchTask(executorBackend, task) + executor.launchTask(executorBackend, taskId = task.taskId, attemptNumber = task.attemptNumber, + task.name, task.serializedTask, task.taskData.decompress()) } } } diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala index 2ce81ae27daf6..92bc89722b872 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala @@ -1365,7 +1365,6 @@ class TaskSetManagerSuite extends SparkFunSuite with LocalSparkContext with Logg private def createTaskResult( id: Int, accumUpdates: Seq[AccumulatorV2[_, _]] = Seq.empty): DirectTaskResult[Int] = { - val valueSer = SparkEnv.get.serializer.newInstance() - new DirectTaskResult[Int](valueSer.serialize(id), accumUpdates) + new DirectTaskResult[Int](id, accumUpdates) } } diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala index 61bfa27a84fd8..6c61540fe92b6 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala @@ -89,7 +89,8 @@ private[spark] class MesosExecutorBackend logError("Received launchTask but executor was null") } else { SparkHadoopUtil.get.runAsSparkUser { () => - executor.launchTask(this, taskDescription) + executor.launchTask(this, taskId = taskId, attemptNumber = taskData.attemptNumber, + taskInfo.getName, taskData.serializedTask, taskData.taskData.decompress()) } } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala index 398758a3331b4..b237960deddc6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala @@ -247,7 +247,7 @@ abstract class SparkPlan extends QueryPlan[SparkPlan] with Logging with Serializ execute().mapPartitionsInternal { iter => var count = 0 val buffer = new Array[Byte](4 << 10) // 4K - val codec = CompressionCodec.createCodec(SparkEnv.get.conf) + val codec = SparkEnv.get.createCompressionCodec val bos = new ByteArrayOutputStream() val out = new DataOutputStream(codec.compressedOutputStream(bos)) while (iter.hasNext && (n < 0 || count < n)) { @@ -269,7 +269,7 @@ abstract class SparkPlan extends QueryPlan[SparkPlan] with Logging with Serializ private def decodeUnsafeRows(bytes: Array[Byte]): Iterator[InternalRow] = { val nFields = schema.length - val codec = CompressionCodec.createCodec(SparkEnv.get.conf) + val codec = SparkEnv.get.createCompressionCodec val bis = new ByteArrayInputStream(bytes) val ins = new DataInputStream(codec.compressedInputStream(bis)) From 47955537e35095c8331062bb5079e64cc40d8620 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 3 Dec 2016 17:37:31 +0530 Subject: [PATCH 057/142] [SNAP-1202] Reduce serialization overheads of biggest contributors in queries (#34) - Properties serialization in Task now walks through the properties and writes to same buffer instead of using java serialization writeObject on a separate buffer - Cloning of properties uses SerializationUtils which is inefficient. Instead added Utils.cloneProperties that will clone by walking all its entries (including defaults if requested) - Separate out WholeStageCodegenExec closure invocation into its own WholeStageCodegenRDD for optimal serialization of its components including base RDD and CodeAndComment. This RDD also removes the limitation of having a max of only 2 RDDs in inputRDDs(). Conflicts: core/src/main/scala/org/apache/spark/scheduler/Task.scala sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala --- .../scala/org/apache/spark/SparkContext.scala | 3 +- .../apache/spark/scheduler/DAGScheduler.scala | 8 +- .../org/apache/spark/scheduler/Task.scala | 154 +++++++++++++++++ .../CoarseGrainedSchedulerBackend.scala | 2 +- .../scala/org/apache/spark/util/Utils.scala | 34 ++++ .../expressions/codegen/CodeGenerator.scala | 13 +- .../sql/execution/WholeStageCodegenExec.scala | 158 ++++++++++++++---- .../spark/streaming/StreamingContext.scala | 3 +- .../streaming/scheduler/JobScheduler.scala | 6 +- 9 files changed, 330 insertions(+), 51 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 004bfe9756c4c..ebd6610b81742 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -50,7 +50,6 @@ import scala.reflect.{classTag, ClassTag} import scala.util.control.NonFatal import com.google.common.collect.MapMaker -import org.apache.commons.lang3.SerializationUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.io.{ArrayWritable, BooleanWritable, BytesWritable, DoubleWritable, FloatWritable, IntWritable, LongWritable, NullWritable, Text, Writable} @@ -354,7 +353,7 @@ class SparkContext(config: SparkConf) extends Logging { override protected def childValue(parent: Properties): Properties = { // Note: make a clone such that changes in the parent properties aren't reflected in // the those of the children threads, which has confusing semantics (SPARK-10563). - SerializationUtils.clone(parent) + Utils.cloneProperties(parent) } override protected def initialValue(): Properties = new Properties() } diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 396dbd46946a4..7e206626f35ef 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -30,8 +30,6 @@ import scala.language.existentials import scala.language.postfixOps import scala.util.control.NonFatal -import org.apache.commons.lang3.SerializationUtils - import org.apache.spark._ import org.apache.spark.broadcast.Broadcast import org.apache.spark.executor.TaskMetrics @@ -603,7 +601,7 @@ class DAGScheduler( val waiter = new JobWaiter(this, jobId, partitions.size, resultHandler) eventProcessLoop.post(JobSubmitted( jobId, rdd, func2, partitions.toArray, callSite, waiter, - SerializationUtils.clone(properties))) + Utils.cloneProperties(properties))) waiter } @@ -668,7 +666,7 @@ class DAGScheduler( val partitions = (0 until rdd.partitions.length).toArray val jobId = nextJobId.getAndIncrement() eventProcessLoop.post(JobSubmitted( - jobId, rdd, func2, partitions, callSite, listener, SerializationUtils.clone(properties))) + jobId, rdd, func2, partitions, callSite, listener, Utils.cloneProperties(properties))) listener.awaitResult() // Will throw an exception if the job fails } @@ -703,7 +701,7 @@ class DAGScheduler( // the map output tracker and some node failures had caused the output statistics to be lost. val waiter = new JobWaiter(this, jobId, 1, (i: Int, r: MapOutputStatistics) => callback(r)) eventProcessLoop.post(MapStageSubmitted( - jobId, dependency, callSite, waiter, SerializationUtils.clone(properties))) + jobId, dependency, callSite, waiter, Utils.cloneProperties(properties))) waiter } diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index 67153e58096cf..ee5a65aa79219 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -277,3 +277,157 @@ private[spark] abstract class Task[T]( _metrics.read(kryo, input) } } + +/** + * Handles transmission of tasks and their dependencies, because this can be slightly tricky. We + * need to send the list of JARs and files added to the SparkContext with each task to ensure that + * worker nodes find out about it, but we can't make it part of the Task because the user's code in + * the task might depend on one of the JARs. Thus we serialize each task as multiple objects, by + * first writing out its dependencies. + */ +private[spark] object Task { + /** + * Serialize a task and the current app dependencies (files and JARs added to the SparkContext) + */ + def serializeWithDependencies( + task: Task[_], + currentFiles: mutable.Map[String, Long], + currentJars: mutable.Map[String, Long], + serializer: SerializerInstance) + : ByteBuffer = { + + val out = new ByteBufferOutputStream(4096) + val dataOut = new DataOutputStream(out) + + // Write currentFiles + dataOut.writeInt(currentFiles.size) + for ((name, timestamp) <- currentFiles) { + dataOut.writeUTF(name) + dataOut.writeLong(timestamp) + } + + // Write currentJars + dataOut.writeInt(currentJars.size) + for ((name, timestamp) <- currentJars) { + dataOut.writeUTF(name) + dataOut.writeLong(timestamp) + } + + // Write the task properties separately so it is available before full task deserialization. + val propBytes = Utils.serialize(task.localProperties) + dataOut.writeInt(propBytes.length) + dataOut.write(propBytes) + + // Write the task itself and finish + dataOut.flush() + val taskBytes = serializer.serialize(task) + Utils.writeByteBuffer(taskBytes, out) + out.close() + out.toByteBuffer + } + + /** + * Deserialize the list of dependencies in a task serialized with serializeWithDependencies, + * and return the task itself as a serialized ByteBuffer. The caller can then update its + * ClassLoaders and deserialize the task. + * + * @return (taskFiles, taskJars, taskProps, taskBytes) + */ + def deserializeWithDependencies(serializedTask: ByteBuffer) + : (HashMap[String, Long], HashMap[String, Long], Properties, ByteBuffer) = { + + val in = new ByteBufferInputStream(serializedTask) + val dataIn = new DataInputStream(in) + + // Read task's files + val taskFiles = new HashMap[String, Long]() + val numFiles = dataIn.readInt() + for (i <- 0 until numFiles) { + taskFiles(dataIn.readUTF()) = dataIn.readLong() + } + + // Read task's JARs + val taskJars = new HashMap[String, Long]() + val numJars = dataIn.readInt() + for (i <- 0 until numJars) { + taskJars(dataIn.readUTF()) = dataIn.readLong() + } + + val propLength = dataIn.readInt() + val propBytes = new Array[Byte](propLength) + dataIn.readFully(propBytes, 0, propLength) + val taskProps = Utils.deserialize[Properties](propBytes) + + // Create a sub-buffer for the rest of the data, which is the serialized Task object + val subBuffer = serializedTask.slice() // ByteBufferInputStream will have read just up to task + (taskFiles, taskJars, taskProps, subBuffer) + } +} + +private[spark] final class TaskData private(var compressedBytes: Array[Byte], + var uncompressedLen: Int, var reference: Int) extends Serializable { + + def this(compressedBytes: Array[Byte], uncompressedLen: Int) = + this(compressedBytes, uncompressedLen, TaskData.NO_REF) + + @transient private var decompressed: Array[Byte] = _ + + /** decompress the common task data if present */ + def decompress(env: SparkEnv = SparkEnv.get): Array[Byte] = { + if (uncompressedLen > 0) { + if (decompressed eq null) { + decompressed = env.createCompressionCodec.decompress(compressedBytes, + 0, compressedBytes.length, uncompressedLen) + } + decompressed + } else TaskData.EMPTY_BYTES + } + + override def hashCode(): Int = java.util.Arrays.hashCode(compressedBytes) + + override def equals(obj: Any): Boolean = obj match { + case d: TaskData => + uncompressedLen == d.uncompressedLen && + reference == d.reference && + java.util.Arrays.equals(compressedBytes, d.compressedBytes) + case _ => false + } +} + +private[spark] object TaskData { + + private val NO_REF: Int = -1 + private val EMPTY_BYTES: Array[Byte] = Array.empty[Byte] + private val FIRST: TaskData = new TaskData(EMPTY_BYTES, 0, 0) + val EMPTY: TaskData = new TaskData(EMPTY_BYTES, 0, -2) + + def apply(reference: Int): TaskData = { + if (reference == 0) FIRST + else if (reference > 0) new TaskData(EMPTY_BYTES, 0, reference) + else EMPTY + } + + def write(data: TaskData, output: Output): Unit = Utils.tryOrIOException { + if (data.reference != NO_REF) { + output.writeVarInt(data.reference, false) + } else { + val bytes = data.compressedBytes + assert(bytes != null) + output.writeVarInt(NO_REF, false) + output.writeVarInt(data.uncompressedLen, true) + output.writeVarInt(bytes.length, true) + output.writeBytes(bytes) + } + } + + def read(input: Input): TaskData = Utils.tryOrIOException { + val reference = input.readVarInt(false) + if (reference != NO_REF) { + TaskData(reference) + } else { + val uncompressedLen = input.readVarInt(true) + val bytesLen = input.readVarInt(true) + new TaskData(input.readBytes(bytesLen), uncompressedLen) + } + } +} diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index 76877fb691334..12929002b4a8f 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -644,7 +644,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp protected def doRequestTotalExecutors(requestedTotal: Int): Future[Boolean] = Future.successful(false) - /** + /**W * Request that the cluster manager kill the specified executors. * * When asking the executor to be replaced, the executor loss is considered a failure, and diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 5706a7ba35ff3..32574f8147f54 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -157,6 +157,40 @@ private[spark] object Utils extends Logging { /** Shorthand for calling truncatedString() without start or end strings. */ def truncatedString[T](seq: Seq[T], sep: String): String = truncatedString(seq, "", sep, "") + def cloneProperties(properties: Properties, + withDefaults: Boolean = false): Properties = { + val newProperties = new Properties() + // first put the keys other than the ones only in defaults + var numStringKeys = 0 + if (!properties.isEmpty) { + val entries = properties.entrySet().iterator() + while (entries.hasNext) { + val entry = entries.next + val key = entry.getKey + if (withDefaults && key.isInstanceOf[String]) { + numStringKeys += 1 + } + newProperties.put(key, entry.getValue) + } + } + if (withDefaults) { + // list the string properties if there are any that are only in defaults + val stringKeys = properties.stringPropertyNames() + // check if any extra keys in defaults exist (only String keys are useful + // since those are the only ones that can be queried from defaults) + if (stringKeys.size() > numStringKeys) { + val iterator = stringKeys.iterator() + while (iterator.hasNext) { + val key = iterator.next() + if (!newProperties.contains(key)) { + newProperties.setProperty(key, properties.getProperty(key)) + } + } + } + } + newProperties + } + /** Serialize an object using Java serialization */ def serialize[T](o: T): Array[Byte] = { val bos = new ByteArrayOutputStream() diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 4dcbb702893da..6308701d7e4ea 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1285,12 +1285,23 @@ abstract class GeneratedClass { */ class CodeAndComment(val body: String, val comment: collection.Map[String, String]) extends Serializable { + + private[sql] var hash: Int = 0 + override def equals(that: Any): Boolean = that match { case t: CodeAndComment if t.body == body => true case _ => false } - override def hashCode(): Int = body.hashCode + // noinspection HashCodeUsesVar + override def hashCode(): Int = { + val h = hash + if (h != 0) h + else { + hash = body.hashCode + hash + } + } } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala index 0e525b1e22eb9..2f3f5dc71cb53 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala @@ -24,6 +24,11 @@ import scala.collection.mutable import org.apache.spark.broadcast import org.apache.spark.rdd.RDD +import com.esotericsoftware.kryo.io.{Input, Output} +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} + +import org.apache.spark.{Partition, SparkContext, TaskContext, broadcast} +import org.apache.spark.rdd.{RDD, ZippedPartitionsBaseRDD, ZippedPartitionsPartition} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen._ @@ -31,9 +36,10 @@ import org.apache.spark.sql.catalyst.plans.physical.Partitioning import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.aggregate.HashAggregateExec import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, SortMergeJoinExec} -import org.apache.spark.sql.execution.metric.SQLMetrics +import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ +import org.apache.spark.storage.StorageLevel import org.apache.spark.util.Utils /** @@ -603,41 +609,8 @@ case class WholeStageCodegenExec(child: SparkPlan)(val codegenStageId: Int) val durationMs = longMetric("pipelineTime") val rdds = child.asInstanceOf[CodegenSupport].inputRDDs() - assert(rdds.size <= 2, "Up to two input RDDs can be supported") - if (rdds.length == 1) { - rdds.head.mapPartitionsWithIndex { (index, iter) => - val (clazz, _) = CodeGenerator.compile(cleanedSource) - val buffer = clazz.generate(references).asInstanceOf[BufferedRowIterator] - buffer.init(index, Array(iter)) - new Iterator[InternalRow] { - override def hasNext: Boolean = { - val v = buffer.hasNext - if (!v) durationMs += buffer.durationMs() - v - } - override def next: InternalRow = buffer.next() - } - } - } else { - // Right now, we support up to two input RDDs. - rdds.head.zipPartitions(rdds(1)) { (leftIter, rightIter) => - Iterator((leftIter, rightIter)) - // a small hack to obtain the correct partition index - }.mapPartitionsWithIndex { (index, zippedIter) => - val (leftIter, rightIter) = zippedIter.next() - val (clazz, _) = CodeGenerator.compile(cleanedSource) - val buffer = clazz.generate(references).asInstanceOf[BufferedRowIterator] - buffer.init(index, Array(leftIter, rightIter)) - new Iterator[InternalRow] { - override def hasNext: Boolean = { - val v = buffer.hasNext - if (!v) durationMs += buffer.durationMs() - v - } - override def next: InternalRow = buffer.next() - } - } - } + new WholeStageCodegenRDD(sqlContext.sparkContext, cleanedSource, + references, durationMs, rdds) } override def inputRDDs(): Seq[RDD[InternalRow]] = { @@ -737,3 +710,116 @@ case class CollapseCodegenStages(conf: SQLConf) extends Rule[SparkPlan] { } } } + +class WholeStageCodegenRDD(sc: SparkContext, var source: CodeAndComment, + var references: Array[Any], var durationMs: SQLMetric, + inputRDDs: Seq[RDD[InternalRow]]) + extends ZippedPartitionsBaseRDD[InternalRow](sc, inputRDDs) + with Serializable with KryoSerializable { + + override def getPartitions: Array[Partition] = { + if (rdds.length == 1) rdds.head.partitions + else super.getPartitions + } + + override def getPreferredLocations(s: Partition): Seq[String] = { + if (rdds.length == 1) rdds.head.preferredLocations(s) + else s.asInstanceOf[ZippedPartitionsPartition].preferredLocations + } + + override def compute(split: Partition, + context: TaskContext): Iterator[InternalRow] = { + val clazz = CodeGenerator.compile(source) + val buffer = clazz.generate(references).asInstanceOf[BufferedRowIterator] + if (rdds.length == 1) { + buffer.init(split.index, Array(rdds.head.iterator(split, context) + .asInstanceOf[Iterator[InternalRow]])) + } else { + val zippedPartition = split.asInstanceOf[ZippedPartitionsPartition] + val partitions = zippedPartition.partitions + val iterators = new Array[Iterator[InternalRow]](partitions.length) + for (i <- partitions.indices) { + iterators(i) = rdds(i).iterator(partitions(i), context) + .asInstanceOf[Iterator[InternalRow]] + } + buffer.init(zippedPartition.index, iterators) + } + new Iterator[InternalRow] { + override def hasNext: Boolean = { + val v = buffer.hasNext + if (!v) durationMs += buffer.durationMs() + v + } + override def next: InternalRow = buffer.next() + } + } + + override def write(kryo: Kryo, output: Output): Unit = { + output.writeInt(_id) + + // write CodeAndComment + output.writeInt(source.hashCode()) + output.writeString(source.body) + val comment = source.comment + output.writeVarInt(comment.size, true) + for ((k, v) <- comment) { + output.writeString(k) + output.writeString(v) + } + + val refsLen = if (references != null) references.length else 0 + output.writeVarInt(refsLen, true) + var i = 0 + while (i < refsLen) { + kryo.writeClassAndObject(output, references(i)) + i += 1 + } + durationMs.write(kryo, output) + + output.writeVarInt(rdds.length, true) + for (rdd <- rdds) { + kryo.writeClassAndObject(output, rdd) + } + } + + override def read(kryo: Kryo, input: Input): Unit = { + _id = input.readInt() + storageLevel = StorageLevel.NONE + checkpointData = None + + val hash = input.readInt() + val body = input.readString() + var commentSize = input.readVarInt(true) + val comment = new scala.collection.mutable.HashMap[String, String]() + while (commentSize > 0) { + val k = input.readString() + val v = input.readString() + comment.put(k, v) + commentSize -= 1 + } + source = new CodeAndComment(body, comment) + source.hash = hash + + val refsLen = input.readVarInt(true) + if (refsLen > 0) { + references = new Array[Any](refsLen) + var i = 0 + while (i < refsLen) { + references(i) = kryo.readClassAndObject(input) + i += 1 + } + } else { + references = null + } + durationMs = new SQLMetric(null) + durationMs.read(kryo, input) + + val rddsBuilder = IndexedSeq.newBuilder[RDD[InternalRow]] + var rddsLen = input.readVarInt(true) + while (rddsLen > 0) { + rddsBuilder += kryo.readClassAndObject(input).asInstanceOf[RDD[InternalRow]] + rddsLen -= 1 + } + rdds = rddsBuilder.result() + } +} diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala index 027403816f538..c7ab48059e788 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala @@ -26,7 +26,6 @@ import scala.collection.mutable.Queue import scala.reflect.ClassTag import scala.util.control.NonFatal -import org.apache.commons.lang3.SerializationUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.io.{BytesWritable, LongWritable, Text} @@ -579,7 +578,7 @@ class StreamingContext private[streaming] ( sparkContext.setCallSite(startSite.get) sparkContext.clearJobGroup() sparkContext.setLocalProperty(SparkContext.SPARK_JOB_INTERRUPT_ON_CANCEL, "false") - savedProperties.set(SerializationUtils.clone(sparkContext.localProperties.get())) + savedProperties.set(Utils.cloneProperties(sparkContext.localProperties.get())) scheduler.start() } state = StreamingContextState.ACTIVE diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala index 2fa3bf7d5230b..2db2fe7f9712c 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala @@ -22,8 +22,6 @@ import java.util.concurrent.{ConcurrentHashMap, TimeUnit} import scala.collection.JavaConverters._ import scala.util.Failure -import org.apache.commons.lang3.SerializationUtils - import org.apache.spark.ExecutorAllocationClient import org.apache.spark.internal.Logging import org.apache.spark.internal.io.SparkHadoopWriterUtils @@ -31,7 +29,7 @@ import org.apache.spark.rdd.RDD import org.apache.spark.streaming._ import org.apache.spark.streaming.api.python.PythonDStream import org.apache.spark.streaming.ui.UIUtils -import org.apache.spark.util.{EventLoop, ThreadUtils} +import org.apache.spark.util.{EventLoop, ThreadUtils, Utils} private[scheduler] sealed trait JobSchedulerEvent @@ -230,7 +228,7 @@ class JobScheduler(val ssc: StreamingContext) extends Logging { def run() { val oldProps = ssc.sparkContext.getLocalProperties try { - ssc.sparkContext.setLocalProperties(SerializationUtils.clone(ssc.savedProperties.get())) + ssc.sparkContext.setLocalProperties(Utils.cloneProperties(ssc.savedProperties.get())) val formattedTime = UIUtils.formatBatchTime( job.time.milliseconds, ssc.graph.batchDuration.milliseconds, showYYYYMMSS = false) val batchUrl = s"/streaming/batch/?id=${job.time.milliseconds}" From 7a6836caaa37960ad99cb33f4073a15940ef0c8b Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Thu, 15 Dec 2016 16:52:29 +0530 Subject: [PATCH 058/142] [SNAPPYDATA] Code changes for displaying product version details. (#38) Conflicts: core/src/main/scala/org/apache/spark/ui/SparkUI.scala core/src/main/scala/org/apache/spark/ui/UIUtils.scala --- .../scala/org/apache/spark/ui/UIUtils.scala | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 95ff953b5969f..7813d44738c42 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -232,22 +232,14 @@ private[spark] object UIUtils extends Logging {
    -<<<<<<< HEAD - + {getProductVersionNode} -||||||| parent of c1b634f... Snap 1890 : Snappy Pulse UI suggestions for 1.0 (#69) - - {getProductUINameNode} - {getProductVersionNode} -======= -
    ->>>>>>> c1b634f... Snap 1890 : Snappy Pulse UI suggestions for 1.0 (#69) {getProductVersionNode}
    @@ -299,22 +291,17 @@ private[spark] object UIUtils extends Logging {
    -<<<<<<< HEAD - - - {getProductVersionNode} -||||||| parent of c1b634f... Snap 1890 : Snappy Pulse UI suggestions for 1.0 (#69) {getProductUINameNode} {getProductVersionNode} -=======
    ->>>>>>> c1b634f... Snap 1890 : Snappy Pulse UI suggestions for 1.0 (#69) + + {getProductVersionNode} {getProductVersionNode}
    From b0d91184e1778d79694d9a4b98d0b8932b3e822f Mon Sep 17 00:00:00 2001 From: Amogh Shetkar Date: Wed, 25 Jan 2017 14:21:46 +0530 Subject: [PATCH 059/142] [SNAPPYDATA] Update to gradle-scalatest version 0.13.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 01d46b079a504..00424bcf2287c 100644 --- a/build.gradle +++ b/build.gradle @@ -25,7 +25,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'io.snappydata:gradle-scalatest:0.13-1' + classpath 'io.snappydata:gradle-scalatest:0.13.1' classpath 'org.github.ngbinh.scalastyle:gradle-scalastyle-plugin_2.11:0.8.2' } } From b0fe78d695ad902479a47e7f67df12de1a027c10 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 12 Jan 2017 18:00:15 +0530 Subject: [PATCH 060/142] [SNAPPYDATA] Skip cast if non-nullable type is being inserted in nullable target Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala --- .../sql/catalyst/optimizer/Optimizer.scala | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index da69ea621b59c..b0c13ac931329 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -1071,6 +1071,49 @@ object PushPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper { } } +/** + * Removes [[Cast Casts]] that are unnecessary because the input is already the correct type. + */ +object SimplifyCasts extends Rule[LogicalPlan] { + def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { + case Cast(e, dataType) if e.dataType == dataType || + (e.dataType.getClass == dataType.getClass && + e.dataType.asNullable == dataType) => e + } +} + +/** + * Removes nodes that are not necessary. + */ +object RemoveDispensableExpressions extends Rule[LogicalPlan] { + def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { + case UnaryPositive(child) => child + case PromotePrecision(child) => child + } +} + +/** +======= + * Removes [[Cast Casts]] that are unnecessary because the input is already the correct type. + */ +object SimplifyCasts extends Rule[LogicalPlan] { + def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { + case Cast(e, dataType) if e.dataType == dataType || + (e.dataType.getClass == dataType.getClass && + e.dataType.asNullable == dataType) => e + } +} + +/** + * Removes nodes that are not necessary. + */ +object RemoveDispensableExpressions extends Rule[LogicalPlan] { + def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { + case UnaryPositive(child) => child + case PromotePrecision(child) => child + } +} + /** * Combines two adjacent [[Limit]] operators into one, merging the * expressions into one single expression. From db700b678805975e02b806a7a8c832c3bcbc4c71 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 12 Jan 2017 18:55:20 +0530 Subject: [PATCH 061/142] [SNAPPYDATA] optimized versions for a couple of string functions --- .../sql/catalyst/expressions/stringExpressions.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index d7612e30b4c57..dd96ee11b9a97 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -1808,9 +1808,9 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp override def inputTypes: Seq[DataType] = Seq(StringType) protected override def nullSafeEval(string: Any): Any = { - val bytes = string.asInstanceOf[UTF8String].getBytes - if (bytes.length > 0) { - bytes(0).asInstanceOf[Int] + val str = string.asInstanceOf[UTF8String] + if (str.numBytes() > 0) { + str.getByte(0).asInstanceOf[Int] } else { 0 } @@ -1818,11 +1818,9 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { nullSafeCodeGen(ctx, ev, (child) => { - val bytes = ctx.freshName("bytes") s""" - byte[] $bytes = $child.getBytes(); - if ($bytes.length > 0) { - ${ev.value} = (int) $bytes[0]; + if ($child.numBytes() > 0) { + ${ev.value} = (int)$child.getByte(0); } else { ${ev.value} = 0; } From 573dbd479205003b0cd0e42a711666b00570c11f Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 9 Feb 2017 17:36:52 +0530 Subject: [PATCH 062/142] [SNAPPYDATA] Increasing the code generation cache eviction size to 300 from 100 --- .../spark/sql/catalyst/expressions/codegen/CodeGenerator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 6308701d7e4ea..38a410774b271 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1500,7 +1500,7 @@ object CodeGenerator extends Logging { * weak keys/values and thus does not respond to memory pressure. */ private val cache = CacheBuilder.newBuilder() - .maximumSize(100) + .maximumSize(300) .build( new CacheLoader[CodeAndComment, (GeneratedClass, Int)]() { override def load(code: CodeAndComment): (GeneratedClass, Int) = { From 216c3e5fb7bc076bea7e0123be6aaef58be6e8a2 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Fri, 10 Mar 2017 13:20:16 +0530 Subject: [PATCH 063/142] [SNAP-1398] Update janino version to latest 3.0.x This works around some of the limitations of older janino versions causing SNAP-1398 --- sql/catalyst/build.gradle | 2 +- sql/hive/build.gradle | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/build.gradle b/sql/catalyst/build.gradle index cc0e9bbf2822e..aef0611699b4d 100644 --- a/sql/catalyst/build.gradle +++ b/sql/catalyst/build.gradle @@ -26,7 +26,7 @@ dependencies { compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion compile group: 'org.scala-lang.modules', name: 'scala-parser-combinators_' + scalaBinaryVersion, version: '1.0.4' - compile group: 'org.codehaus.janino', name: 'janino', version: '2.7.8' + compile group: 'org.codehaus.janino', name: 'janino', version: '3.0.6' compile group: 'org.antlr', name: 'antlr4-runtime', version: antlrVersion compile group: 'commons-codec', name: 'commons-codec', version: commonsCodecVersion antlr group: 'org.antlr', name: 'antlr4', version: antlrVersion diff --git a/sql/hive/build.gradle b/sql/hive/build.gradle index 25f6d76d4d7eb..64ba9c361ea65 100644 --- a/sql/hive/build.gradle +++ b/sql/hive/build.gradle @@ -89,6 +89,7 @@ dependencies { exclude(group: 'com.google.guava', module: 'guava') exclude(group: 'com.google.code.findbugs', module: 'jsr305') exclude(group: 'org.codehaus.janino', module: 'janino') + exclude(group: 'org.codehaus.janino', module: 'commons-compiler') exclude(group: 'org.hsqldb', module: 'hsqldb') exclude(group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm') } From 535ebb77608a16e8378f1e5244c2133492a576f6 Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Thu, 30 Mar 2017 11:24:06 +0530 Subject: [PATCH 064/142] [SNAPPYDATA] made some methods protected to be used by SnappyUnifiedManager (#47) --- .../scala/org/apache/spark/memory/UnifiedMemoryManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala index 78edd2c4d7faa..994d86e522005 100644 --- a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala +++ b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala @@ -55,7 +55,7 @@ private[spark] class UnifiedMemoryManager private[memory] ( onHeapStorageRegionSize, maxHeapMemory - onHeapStorageRegionSize) { - private def assertInvariants(): Unit = { + protected def assertInvariants(): Unit = { assert(onHeapExecutionMemoryPool.poolSize + onHeapStorageMemoryPool.poolSize == maxHeapMemory) assert( offHeapExecutionMemoryPool.poolSize + offHeapStorageMemoryPool.poolSize == maxOffHeapMemory) From bf9784f5951dd6d9ba881780065074c075ad1d40 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 9 May 2017 12:58:36 -0700 Subject: [PATCH 065/142] [SNAPPYDATA] Reducing file read/write buffer sizes Reduced buffer sizes from 1M to 64K to reduce unaccounted memory overhead. Disk read/write buffers beyond 32K don't help in performance in any case. Conflicts: core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java --- .../org/apache/spark/shuffle/sort/ShuffleExternalSorter.java | 2 +- .../java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java | 2 +- .../util/collection/unsafe/sort/UnsafeSorterSpillReader.java | 2 +- .../util/collection/unsafe/sort/UnsafeSorterSpillWriter.java | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java index c3a07b2abf896..79b667b83b30f 100644 --- a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java +++ b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java @@ -68,7 +68,7 @@ final class ShuffleExternalSorter extends MemoryConsumer { private static final Logger logger = LoggerFactory.getLogger(ShuffleExternalSorter.class); @VisibleForTesting - static final int DISK_WRITE_BUFFER_SIZE = 1024 * 1024; + static final int DISK_WRITE_BUFFER_SIZE = 64 * 1024; private final int numPartitions; private final TaskMemoryManager taskMemoryManager; diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java index 4839d04522f10..005037fd57f9e 100644 --- a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java +++ b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java @@ -217,7 +217,7 @@ private void open() { partitioner.numPartitions(), sparkConf, writeMetrics); - serBuffer = new MyByteArrayOutputStream(DEFAULT_INITIAL_SER_BUFFER_SIZE); + serBuffer = new MyByteArrayOutputStream(64 * 1024); // DEFAULT_INITIAL_SER_BUFFER_SIZE serOutputStream = serializer.serializeStream(serBuffer); } diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java index 2c53c8d809d2e..414de7af848e7 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java @@ -49,7 +49,7 @@ public final class UnsafeSorterSpillReader extends UnsafeSorterIterator implemen private int numRecords; private int numRecordsRemaining; - private byte[] arr = new byte[1024 * 1024]; + private byte[] arr = new byte[64 * 1024]; private Object baseObject = arr; private final long baseOffset = Platform.BYTE_ARRAY_OFFSET; private final TaskContext taskContext = TaskContext.get(); diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java index 9399024f01783..700902136f43f 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java @@ -45,6 +45,7 @@ public final class UnsafeSorterSpillWriter { /** The buffer size to use when writing the sorted records to an on-disk file */ private final int diskWriteBufferSize = (int) (long) conf.get(package$.MODULE$.SHUFFLE_DISK_WRITE_BUFFER_SIZE()); + static final int DISK_WRITE_BUFFER_SIZE = 64 * 1024; // Small writes to DiskBlockObjectWriter will be fairly inefficient. Since there doesn't seem to // be an API to directly transfer bytes from managed memory to the disk writer, we buffer From ec242aa7b2042804690570485e689b43f1dbb1c4 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Mon, 29 May 2017 12:02:02 +0530 Subject: [PATCH 066/142] [SNAP-1486] make QueryPlan.cleanArgs a transient lazy val (#51) cleanArgs can end up holding transient fields of the class which can be recalculated on the other side if required in any case. Also added full exception stack for cases of task listener failures. Conflicts: core/src/main/scala/org/apache/spark/TaskContextImpl.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala --- .../main/scala/org/apache/spark/TaskContextImpl.scala | 2 +- .../apache/spark/sql/catalyst/plans/QueryPlan.scala | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/TaskContextImpl.scala b/core/src/main/scala/org/apache/spark/TaskContextImpl.scala index cccd3ea457ba4..c9fc158ca7284 100644 --- a/core/src/main/scala/org/apache/spark/TaskContextImpl.scala +++ b/core/src/main/scala/org/apache/spark/TaskContextImpl.scala @@ -131,7 +131,7 @@ private[spark] class TaskContextImpl( callback(listener) } catch { case e: Throwable => - errorMsgs += e.getMessage + errorMsgs += Utils.exceptionString(e) logError(s"Error in $name", e) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala index d3dbb7851b828..a06f53b8ab2b3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala @@ -319,4 +319,14 @@ object QueryPlan extends PredicateHelper { Nil } } + + /** Args that have cleaned such that differences in expression id should not affect equality */ + @transient protected lazy val cleanArgs: Seq[Any] = { + def cleanArg(arg: Any): Any = arg match { + // Children are checked using sameResult above. + case tn: TreeNode[_] if containsChild(tn) => null + case e: Expression => cleanExpression(e).canonicalized + case other => other + } + } } From 7bf9d900e5626afd9534ea8cf80d343779f9a324 Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Fri, 2 Jun 2017 18:27:42 +0530 Subject: [PATCH 067/142] [SPARK-19500] [SQL] Fix off-by-one bug in BytesToBytesMap (#53) Merging Spark fix. Radix sort require that half of array as free (as temporary space), so we use 0.5 as the scale factor to make sure that BytesToBytesMap will not have more items than 1/2 of capacity. Turned out this is not true, the current implementation of append() could leave 1 more item than the threshold (1/2 of capacity) in the array, which break the requirement of radix sort (fail the assert in 2.2, or fail to insert into InMemorySorter in 2.1). This PR fix the off-by-one bug in BytesToBytesMap. This PR also fix a bug that the array will never grow if it fail to grow once (stay as initial capacity), introduced by #15722 . Conflicts: core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java Conflicts: sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala From 366007ef95ac36e91e87daf6b74a9830ffad68bc Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Sat, 3 Jun 2017 00:28:55 +0530 Subject: [PATCH 068/142] SNAP-1545: Snappy Dashboard UI Revamping (#52) Changes: - Adding new methods simpleSparkPageWithTabs_2 and commonHeaderNodes_2 for custom snappy UI changes - Adding javascript librarires d3.js, liquidFillGauge.js and snappy-dashboard.js for snappy UI new widgets and styling changes. - Updating snappy-dashboard.css for new widgets and UI content stylings - Relocating snappy-dashboard.css into ui/static/snappydata directory. --- .../apache/spark/ui/static/snappydata/d3.js | 5 + .../ui/static/snappydata/liquidFillGauge.js | 268 ++++++++++++++++++ .../{ => snappydata}/snappy-dashboard.css | 24 +- .../ui/static/snappydata/snappy-dashboard.js | 49 ++++ .../scala/org/apache/spark/ui/UIUtils.scala | 71 ++++- 5 files changed, 415 insertions(+), 2 deletions(-) create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/d3.js create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/liquidFillGauge.js rename core/src/main/resources/org/apache/spark/ui/static/{ => snappydata}/snappy-dashboard.css (87%) create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.js diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/d3.js b/core/src/main/resources/org/apache/spark/ui/static/snappydata/d3.js new file mode 100644 index 0000000000000..166487309a774 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/snappydata/d3.js @@ -0,0 +1,5 @@ +!function(){function n(n){return n&&(n.ownerDocument||n.document||n).documentElement}function t(n){return n&&(n.ownerDocument&&n.ownerDocument.defaultView||n.document&&n||n.defaultView)}function e(n,t){return t>n?-1:n>t?1:n>=t?0:NaN}function r(n){return null===n?NaN:+n}function i(n){return!isNaN(n)}function u(n){return{left:function(t,e,r,i){for(arguments.length<3&&(r=0),arguments.length<4&&(i=t.length);i>r;){var u=r+i>>>1;n(t[u],e)<0?r=u+1:i=u}return r},right:function(t,e,r,i){for(arguments.length<3&&(r=0),arguments.length<4&&(i=t.length);i>r;){var u=r+i>>>1;n(t[u],e)>0?i=u:r=u+1}return r}}}function o(n){return n.length}function a(n){for(var t=1;n*t%1;)t*=10;return t}function l(n,t){for(var e in t)Object.defineProperty(n.prototype,e,{value:t[e],enumerable:!1})}function c(){this._=Object.create(null)}function f(n){return(n+="")===bo||n[0]===_o?_o+n:n}function s(n){return(n+="")[0]===_o?n.slice(1):n}function h(n){return f(n)in this._}function p(n){return(n=f(n))in this._&&delete this._[n]}function g(){var n=[];for(var t in this._)n.push(s(t));return n}function v(){var n=0;for(var t in this._)++n;return n}function d(){for(var n in this._)return!1;return!0}function y(){this._=Object.create(null)}function m(n){return n}function M(n,t,e){return function(){var r=e.apply(t,arguments);return r===t?n:r}}function x(n,t){if(t in n)return t;t=t.charAt(0).toUpperCase()+t.slice(1);for(var e=0,r=wo.length;r>e;++e){var i=wo[e]+t;if(i in n)return i}}function b(){}function _(){}function w(n){function t(){for(var t,r=e,i=-1,u=r.length;++ie;e++)for(var i,u=n[e],o=0,a=u.length;a>o;o++)(i=u[o])&&t(i,o,e);return n}function Z(n){return ko(n,qo),n}function V(n){var t,e;return function(r,i,u){var o,a=n[u].update,l=a.length;for(u!=e&&(e=u,t=0),i>=t&&(t=i+1);!(o=a[t])&&++t0&&(n=n.slice(0,a));var c=To.get(n);return c&&(n=c,l=B),a?t?i:r:t?b:u}function $(n,t){return function(e){var r=ao.event;ao.event=e,t[0]=this.__data__;try{n.apply(this,t)}finally{ao.event=r}}}function B(n,t){var e=$(n,t);return function(n){var t=this,r=n.relatedTarget;r&&(r===t||8&r.compareDocumentPosition(t))||e.call(t,n)}}function W(e){var r=".dragsuppress-"+ ++Do,i="click"+r,u=ao.select(t(e)).on("touchmove"+r,S).on("dragstart"+r,S).on("selectstart"+r,S);if(null==Ro&&(Ro="onselectstart"in e?!1:x(e.style,"userSelect")),Ro){var o=n(e).style,a=o[Ro];o[Ro]="none"}return function(n){if(u.on(r,null),Ro&&(o[Ro]=a),n){var t=function(){u.on(i,null)};u.on(i,function(){S(),t()},!0),setTimeout(t,0)}}}function J(n,e){e.changedTouches&&(e=e.changedTouches[0]);var r=n.ownerSVGElement||n;if(r.createSVGPoint){var i=r.createSVGPoint();if(0>Po){var u=t(n);if(u.scrollX||u.scrollY){r=ao.select("body").append("svg").style({position:"absolute",top:0,left:0,margin:0,padding:0,border:"none"},"important");var o=r[0][0].getScreenCTM();Po=!(o.f||o.e),r.remove()}}return Po?(i.x=e.pageX,i.y=e.pageY):(i.x=e.clientX,i.y=e.clientY),i=i.matrixTransform(n.getScreenCTM().inverse()),[i.x,i.y]}var a=n.getBoundingClientRect();return[e.clientX-a.left-n.clientLeft,e.clientY-a.top-n.clientTop]}function G(){return ao.event.changedTouches[0].identifier}function K(n){return n>0?1:0>n?-1:0}function Q(n,t,e){return(t[0]-n[0])*(e[1]-n[1])-(t[1]-n[1])*(e[0]-n[0])}function nn(n){return n>1?0:-1>n?Fo:Math.acos(n)}function tn(n){return n>1?Io:-1>n?-Io:Math.asin(n)}function en(n){return((n=Math.exp(n))-1/n)/2}function rn(n){return((n=Math.exp(n))+1/n)/2}function un(n){return((n=Math.exp(2*n))-1)/(n+1)}function on(n){return(n=Math.sin(n/2))*n}function an(){}function ln(n,t,e){return this instanceof ln?(this.h=+n,this.s=+t,void(this.l=+e)):arguments.length<2?n instanceof ln?new ln(n.h,n.s,n.l):_n(""+n,wn,ln):new ln(n,t,e)}function cn(n,t,e){function r(n){return n>360?n-=360:0>n&&(n+=360),60>n?u+(o-u)*n/60:180>n?o:240>n?u+(o-u)*(240-n)/60:u}function i(n){return Math.round(255*r(n))}var u,o;return n=isNaN(n)?0:(n%=360)<0?n+360:n,t=isNaN(t)?0:0>t?0:t>1?1:t,e=0>e?0:e>1?1:e,o=.5>=e?e*(1+t):e+t-e*t,u=2*e-o,new mn(i(n+120),i(n),i(n-120))}function fn(n,t,e){return this instanceof fn?(this.h=+n,this.c=+t,void(this.l=+e)):arguments.length<2?n instanceof fn?new fn(n.h,n.c,n.l):n instanceof hn?gn(n.l,n.a,n.b):gn((n=Sn((n=ao.rgb(n)).r,n.g,n.b)).l,n.a,n.b):new fn(n,t,e)}function sn(n,t,e){return isNaN(n)&&(n=0),isNaN(t)&&(t=0),new hn(e,Math.cos(n*=Yo)*t,Math.sin(n)*t)}function hn(n,t,e){return this instanceof hn?(this.l=+n,this.a=+t,void(this.b=+e)):arguments.length<2?n instanceof hn?new hn(n.l,n.a,n.b):n instanceof fn?sn(n.h,n.c,n.l):Sn((n=mn(n)).r,n.g,n.b):new hn(n,t,e)}function pn(n,t,e){var r=(n+16)/116,i=r+t/500,u=r-e/200;return i=vn(i)*na,r=vn(r)*ta,u=vn(u)*ea,new mn(yn(3.2404542*i-1.5371385*r-.4985314*u),yn(-.969266*i+1.8760108*r+.041556*u),yn(.0556434*i-.2040259*r+1.0572252*u))}function gn(n,t,e){return n>0?new fn(Math.atan2(e,t)*Zo,Math.sqrt(t*t+e*e),n):new fn(NaN,NaN,n)}function vn(n){return n>.206893034?n*n*n:(n-4/29)/7.787037}function dn(n){return n>.008856?Math.pow(n,1/3):7.787037*n+4/29}function yn(n){return Math.round(255*(.00304>=n?12.92*n:1.055*Math.pow(n,1/2.4)-.055))}function mn(n,t,e){return this instanceof mn?(this.r=~~n,this.g=~~t,void(this.b=~~e)):arguments.length<2?n instanceof mn?new mn(n.r,n.g,n.b):_n(""+n,mn,cn):new mn(n,t,e)}function Mn(n){return new mn(n>>16,n>>8&255,255&n)}function xn(n){return Mn(n)+""}function bn(n){return 16>n?"0"+Math.max(0,n).toString(16):Math.min(255,n).toString(16)}function _n(n,t,e){var r,i,u,o=0,a=0,l=0;if(r=/([a-z]+)\((.*)\)/.exec(n=n.toLowerCase()))switch(i=r[2].split(","),r[1]){case"hsl":return e(parseFloat(i[0]),parseFloat(i[1])/100,parseFloat(i[2])/100);case"rgb":return t(Nn(i[0]),Nn(i[1]),Nn(i[2]))}return(u=ua.get(n))?t(u.r,u.g,u.b):(null==n||"#"!==n.charAt(0)||isNaN(u=parseInt(n.slice(1),16))||(4===n.length?(o=(3840&u)>>4,o=o>>4|o,a=240&u,a=a>>4|a,l=15&u,l=l<<4|l):7===n.length&&(o=(16711680&u)>>16,a=(65280&u)>>8,l=255&u)),t(o,a,l))}function wn(n,t,e){var r,i,u=Math.min(n/=255,t/=255,e/=255),o=Math.max(n,t,e),a=o-u,l=(o+u)/2;return a?(i=.5>l?a/(o+u):a/(2-o-u),r=n==o?(t-e)/a+(e>t?6:0):t==o?(e-n)/a+2:(n-t)/a+4,r*=60):(r=NaN,i=l>0&&1>l?0:r),new ln(r,i,l)}function Sn(n,t,e){n=kn(n),t=kn(t),e=kn(e);var r=dn((.4124564*n+.3575761*t+.1804375*e)/na),i=dn((.2126729*n+.7151522*t+.072175*e)/ta),u=dn((.0193339*n+.119192*t+.9503041*e)/ea);return hn(116*i-16,500*(r-i),200*(i-u))}function kn(n){return(n/=255)<=.04045?n/12.92:Math.pow((n+.055)/1.055,2.4)}function Nn(n){var t=parseFloat(n);return"%"===n.charAt(n.length-1)?Math.round(2.55*t):t}function En(n){return"function"==typeof n?n:function(){return n}}function An(n){return function(t,e,r){return 2===arguments.length&&"function"==typeof e&&(r=e,e=null),Cn(t,e,n,r)}}function Cn(n,t,e,r){function i(){var n,t=l.status;if(!t&&Ln(l)||t>=200&&300>t||304===t){try{n=e.call(u,l)}catch(r){return void o.error.call(u,r)}o.load.call(u,n)}else o.error.call(u,l)}var u={},o=ao.dispatch("beforesend","progress","load","error"),a={},l=new XMLHttpRequest,c=null;return!this.XDomainRequest||"withCredentials"in l||!/^(http(s)?:)?\/\//.test(n)||(l=new XDomainRequest),"onload"in l?l.onload=l.onerror=i:l.onreadystatechange=function(){l.readyState>3&&i()},l.onprogress=function(n){var t=ao.event;ao.event=n;try{o.progress.call(u,l)}finally{ao.event=t}},u.header=function(n,t){return n=(n+"").toLowerCase(),arguments.length<2?a[n]:(null==t?delete a[n]:a[n]=t+"",u)},u.mimeType=function(n){return arguments.length?(t=null==n?null:n+"",u):t},u.responseType=function(n){return arguments.length?(c=n,u):c},u.response=function(n){return e=n,u},["get","post"].forEach(function(n){u[n]=function(){return u.send.apply(u,[n].concat(co(arguments)))}}),u.send=function(e,r,i){if(2===arguments.length&&"function"==typeof r&&(i=r,r=null),l.open(e,n,!0),null==t||"accept"in a||(a.accept=t+",*/*"),l.setRequestHeader)for(var f in a)l.setRequestHeader(f,a[f]);return null!=t&&l.overrideMimeType&&l.overrideMimeType(t),null!=c&&(l.responseType=c),null!=i&&u.on("error",i).on("load",function(n){i(null,n)}),o.beforesend.call(u,l),l.send(null==r?null:r),u},u.abort=function(){return l.abort(),u},ao.rebind(u,o,"on"),null==r?u:u.get(zn(r))}function zn(n){return 1===n.length?function(t,e){n(null==t?e:null)}:n}function Ln(n){var t=n.responseType;return t&&"text"!==t?n.response:n.responseText}function qn(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now());var i=e+t,u={c:n,t:i,n:null};return aa?aa.n=u:oa=u,aa=u,la||(ca=clearTimeout(ca),la=1,fa(Tn)),u}function Tn(){var n=Rn(),t=Dn()-n;t>24?(isFinite(t)&&(clearTimeout(ca),ca=setTimeout(Tn,t)),la=0):(la=1,fa(Tn))}function Rn(){for(var n=Date.now(),t=oa;t;)n>=t.t&&t.c(n-t.t)&&(t.c=null),t=t.n;return n}function Dn(){for(var n,t=oa,e=1/0;t;)t.c?(t.t8?function(n){return n/e}:function(n){return n*e},symbol:n}}function jn(n){var t=n.decimal,e=n.thousands,r=n.grouping,i=n.currency,u=r&&e?function(n,t){for(var i=n.length,u=[],o=0,a=r[0],l=0;i>0&&a>0&&(l+a+1>t&&(a=Math.max(1,t-l)),u.push(n.substring(i-=a,i+a)),!((l+=a+1)>t));)a=r[o=(o+1)%r.length];return u.reverse().join(e)}:m;return function(n){var e=ha.exec(n),r=e[1]||" ",o=e[2]||">",a=e[3]||"-",l=e[4]||"",c=e[5],f=+e[6],s=e[7],h=e[8],p=e[9],g=1,v="",d="",y=!1,m=!0;switch(h&&(h=+h.substring(1)),(c||"0"===r&&"="===o)&&(c=r="0",o="="),p){case"n":s=!0,p="g";break;case"%":g=100,d="%",p="f";break;case"p":g=100,d="%",p="r";break;case"b":case"o":case"x":case"X":"#"===l&&(v="0"+p.toLowerCase());case"c":m=!1;case"d":y=!0,h=0;break;case"s":g=-1,p="r"}"$"===l&&(v=i[0],d=i[1]),"r"!=p||h||(p="g"),null!=h&&("g"==p?h=Math.max(1,Math.min(21,h)):"e"!=p&&"f"!=p||(h=Math.max(0,Math.min(20,h)))),p=pa.get(p)||Fn;var M=c&&s;return function(n){var e=d;if(y&&n%1)return"";var i=0>n||0===n&&0>1/n?(n=-n,"-"):"-"===a?"":a;if(0>g){var l=ao.formatPrefix(n,h);n=l.scale(n),e=l.symbol+d}else n*=g;n=p(n,h);var x,b,_=n.lastIndexOf(".");if(0>_){var w=m?n.lastIndexOf("e"):-1;0>w?(x=n,b=""):(x=n.substring(0,w),b=n.substring(w))}else x=n.substring(0,_),b=t+n.substring(_+1);!c&&s&&(x=u(x,1/0));var S=v.length+x.length+b.length+(M?0:i.length),k=f>S?new Array(S=f-S+1).join(r):"";return M&&(x=u(k+x,k.length?f-b.length:1/0)),i+=v,n=x+b,("<"===o?i+n+k:">"===o?k+i+n:"^"===o?k.substring(0,S>>=1)+i+n+k.substring(S):i+(M?n:k+n))+e}}}function Fn(n){return n+""}function Hn(){this._=new Date(arguments.length>1?Date.UTC.apply(this,arguments):arguments[0])}function On(n,t,e){function r(t){var e=n(t),r=u(e,1);return r-t>t-e?e:r}function i(e){return t(e=n(new va(e-1)),1),e}function u(n,e){return t(n=new va(+n),e),n}function o(n,r,u){var o=i(n),a=[];if(u>1)for(;r>o;)e(o)%u||a.push(new Date(+o)),t(o,1);else for(;r>o;)a.push(new Date(+o)),t(o,1);return a}function a(n,t,e){try{va=Hn;var r=new Hn;return r._=n,o(r,t,e)}finally{va=Date}}n.floor=n,n.round=r,n.ceil=i,n.offset=u,n.range=o;var l=n.utc=In(n);return l.floor=l,l.round=In(r),l.ceil=In(i),l.offset=In(u),l.range=a,n}function In(n){return function(t,e){try{va=Hn;var r=new Hn;return r._=t,n(r,e)._}finally{va=Date}}}function Yn(n){function t(n){function t(t){for(var e,i,u,o=[],a=-1,l=0;++aa;){if(r>=c)return-1;if(i=t.charCodeAt(a++),37===i){if(o=t.charAt(a++),u=C[o in ya?t.charAt(a++):o],!u||(r=u(n,e,r))<0)return-1}else if(i!=e.charCodeAt(r++))return-1}return r}function r(n,t,e){_.lastIndex=0;var r=_.exec(t.slice(e));return r?(n.w=w.get(r[0].toLowerCase()),e+r[0].length):-1}function i(n,t,e){x.lastIndex=0;var r=x.exec(t.slice(e));return r?(n.w=b.get(r[0].toLowerCase()),e+r[0].length):-1}function u(n,t,e){N.lastIndex=0;var r=N.exec(t.slice(e));return r?(n.m=E.get(r[0].toLowerCase()),e+r[0].length):-1}function o(n,t,e){S.lastIndex=0;var r=S.exec(t.slice(e));return r?(n.m=k.get(r[0].toLowerCase()),e+r[0].length):-1}function a(n,t,r){return e(n,A.c.toString(),t,r)}function l(n,t,r){return e(n,A.x.toString(),t,r)}function c(n,t,r){return e(n,A.X.toString(),t,r)}function f(n,t,e){var r=M.get(t.slice(e,e+=2).toLowerCase());return null==r?-1:(n.p=r,e)}var s=n.dateTime,h=n.date,p=n.time,g=n.periods,v=n.days,d=n.shortDays,y=n.months,m=n.shortMonths;t.utc=function(n){function e(n){try{va=Hn;var t=new va;return t._=n,r(t)}finally{va=Date}}var r=t(n);return e.parse=function(n){try{va=Hn;var t=r.parse(n);return t&&t._}finally{va=Date}},e.toString=r.toString,e},t.multi=t.utc.multi=ct;var M=ao.map(),x=Vn(v),b=Xn(v),_=Vn(d),w=Xn(d),S=Vn(y),k=Xn(y),N=Vn(m),E=Xn(m);g.forEach(function(n,t){M.set(n.toLowerCase(),t)});var A={a:function(n){return d[n.getDay()]},A:function(n){return v[n.getDay()]},b:function(n){return m[n.getMonth()]},B:function(n){return y[n.getMonth()]},c:t(s),d:function(n,t){return Zn(n.getDate(),t,2)},e:function(n,t){return Zn(n.getDate(),t,2)},H:function(n,t){return Zn(n.getHours(),t,2)},I:function(n,t){return Zn(n.getHours()%12||12,t,2)},j:function(n,t){return Zn(1+ga.dayOfYear(n),t,3)},L:function(n,t){return Zn(n.getMilliseconds(),t,3)},m:function(n,t){return Zn(n.getMonth()+1,t,2)},M:function(n,t){return Zn(n.getMinutes(),t,2)},p:function(n){return g[+(n.getHours()>=12)]},S:function(n,t){return Zn(n.getSeconds(),t,2)},U:function(n,t){return Zn(ga.sundayOfYear(n),t,2)},w:function(n){return n.getDay()},W:function(n,t){return Zn(ga.mondayOfYear(n),t,2)},x:t(h),X:t(p),y:function(n,t){return Zn(n.getFullYear()%100,t,2)},Y:function(n,t){return Zn(n.getFullYear()%1e4,t,4)},Z:at,"%":function(){return"%"}},C={a:r,A:i,b:u,B:o,c:a,d:tt,e:tt,H:rt,I:rt,j:et,L:ot,m:nt,M:it,p:f,S:ut,U:Bn,w:$n,W:Wn,x:l,X:c,y:Gn,Y:Jn,Z:Kn,"%":lt};return t}function Zn(n,t,e){var r=0>n?"-":"",i=(r?-n:n)+"",u=i.length;return r+(e>u?new Array(e-u+1).join(t)+i:i)}function Vn(n){return new RegExp("^(?:"+n.map(ao.requote).join("|")+")","i")}function Xn(n){for(var t=new c,e=-1,r=n.length;++e68?1900:2e3)}function nt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.m=r[0]-1,e+r[0].length):-1}function tt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.d=+r[0],e+r[0].length):-1}function et(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+3));return r?(n.j=+r[0],e+r[0].length):-1}function rt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.H=+r[0],e+r[0].length):-1}function it(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.M=+r[0],e+r[0].length):-1}function ut(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.S=+r[0],e+r[0].length):-1}function ot(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+3));return r?(n.L=+r[0],e+r[0].length):-1}function at(n){var t=n.getTimezoneOffset(),e=t>0?"-":"+",r=xo(t)/60|0,i=xo(t)%60;return e+Zn(r,"0",2)+Zn(i,"0",2)}function lt(n,t,e){Ma.lastIndex=0;var r=Ma.exec(t.slice(e,e+1));return r?e+r[0].length:-1}function ct(n){for(var t=n.length,e=-1;++e=0?1:-1,a=o*e,l=Math.cos(t),c=Math.sin(t),f=u*c,s=i*l+f*Math.cos(a),h=f*o*Math.sin(a);ka.add(Math.atan2(h,s)),r=n,i=l,u=c}var t,e,r,i,u;Na.point=function(o,a){Na.point=n,r=(t=o)*Yo,i=Math.cos(a=(e=a)*Yo/2+Fo/4),u=Math.sin(a)},Na.lineEnd=function(){n(t,e)}}function dt(n){var t=n[0],e=n[1],r=Math.cos(e);return[r*Math.cos(t),r*Math.sin(t),Math.sin(e)]}function yt(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]}function mt(n,t){return[n[1]*t[2]-n[2]*t[1],n[2]*t[0]-n[0]*t[2],n[0]*t[1]-n[1]*t[0]]}function Mt(n,t){n[0]+=t[0],n[1]+=t[1],n[2]+=t[2]}function xt(n,t){return[n[0]*t,n[1]*t,n[2]*t]}function bt(n){var t=Math.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2]);n[0]/=t,n[1]/=t,n[2]/=t}function _t(n){return[Math.atan2(n[1],n[0]),tn(n[2])]}function wt(n,t){return xo(n[0]-t[0])a;++a)i.point((e=n[a])[0],e[1]);return void i.lineEnd()}var l=new Tt(e,n,null,!0),c=new Tt(e,null,l,!1);l.o=c,u.push(l),o.push(c),l=new Tt(r,n,null,!1),c=new Tt(r,null,l,!0),l.o=c,u.push(l),o.push(c)}}),o.sort(t),qt(u),qt(o),u.length){for(var a=0,l=e,c=o.length;c>a;++a)o[a].e=l=!l;for(var f,s,h=u[0];;){for(var p=h,g=!0;p.v;)if((p=p.n)===h)return;f=p.z,i.lineStart();do{if(p.v=p.o.v=!0,p.e){if(g)for(var a=0,c=f.length;c>a;++a)i.point((s=f[a])[0],s[1]);else r(p.x,p.n.x,1,i);p=p.n}else{if(g){f=p.p.z;for(var a=f.length-1;a>=0;--a)i.point((s=f[a])[0],s[1])}else r(p.x,p.p.x,-1,i);p=p.p}p=p.o,f=p.z,g=!g}while(!p.v);i.lineEnd()}}}function qt(n){if(t=n.length){for(var t,e,r=0,i=n[0];++r0){for(b||(u.polygonStart(),b=!0),u.lineStart();++o1&&2&t&&e.push(e.pop().concat(e.shift())),p.push(e.filter(Dt))}var p,g,v,d=t(u),y=i.invert(r[0],r[1]),m={point:o,lineStart:l,lineEnd:c,polygonStart:function(){m.point=f,m.lineStart=s,m.lineEnd=h,p=[],g=[]},polygonEnd:function(){m.point=o,m.lineStart=l,m.lineEnd=c,p=ao.merge(p);var n=Ot(y,g);p.length?(b||(u.polygonStart(),b=!0),Lt(p,Ut,n,e,u)):n&&(b||(u.polygonStart(),b=!0),u.lineStart(),e(null,null,1,u),u.lineEnd()),b&&(u.polygonEnd(),b=!1),p=g=null},sphere:function(){u.polygonStart(),u.lineStart(),e(null,null,1,u),u.lineEnd(),u.polygonEnd()}},M=Pt(),x=t(M),b=!1;return m}}function Dt(n){return n.length>1}function Pt(){var n,t=[];return{lineStart:function(){t.push(n=[])},point:function(t,e){n.push([t,e])},lineEnd:b,buffer:function(){var e=t;return t=[],n=null,e},rejoin:function(){t.length>1&&t.push(t.pop().concat(t.shift()))}}}function Ut(n,t){return((n=n.x)[0]<0?n[1]-Io-Uo:Io-n[1])-((t=t.x)[0]<0?t[1]-Io-Uo:Io-t[1])}function jt(n){var t,e=NaN,r=NaN,i=NaN;return{lineStart:function(){n.lineStart(),t=1},point:function(u,o){var a=u>0?Fo:-Fo,l=xo(u-e);xo(l-Fo)0?Io:-Io),n.point(i,r),n.lineEnd(),n.lineStart(),n.point(a,r),n.point(u,r),t=0):i!==a&&l>=Fo&&(xo(e-i)Uo?Math.atan((Math.sin(t)*(u=Math.cos(r))*Math.sin(e)-Math.sin(r)*(i=Math.cos(t))*Math.sin(n))/(i*u*o)):(t+r)/2}function Ht(n,t,e,r){var i;if(null==n)i=e*Io,r.point(-Fo,i),r.point(0,i),r.point(Fo,i),r.point(Fo,0),r.point(Fo,-i),r.point(0,-i),r.point(-Fo,-i),r.point(-Fo,0),r.point(-Fo,i);else if(xo(n[0]-t[0])>Uo){var u=n[0]a;++a){var c=t[a],f=c.length;if(f)for(var s=c[0],h=s[0],p=s[1]/2+Fo/4,g=Math.sin(p),v=Math.cos(p),d=1;;){d===f&&(d=0),n=c[d];var y=n[0],m=n[1]/2+Fo/4,M=Math.sin(m),x=Math.cos(m),b=y-h,_=b>=0?1:-1,w=_*b,S=w>Fo,k=g*M;if(ka.add(Math.atan2(k*_*Math.sin(w),v*x+k*Math.cos(w))),u+=S?b+_*Ho:b,S^h>=e^y>=e){var N=mt(dt(s),dt(n));bt(N);var E=mt(i,N);bt(E);var A=(S^b>=0?-1:1)*tn(E[2]);(r>A||r===A&&(N[0]||N[1]))&&(o+=S^b>=0?1:-1)}if(!d++)break;h=y,g=M,v=x,s=n}}return(-Uo>u||Uo>u&&-Uo>ka)^1&o}function It(n){function t(n,t){return Math.cos(n)*Math.cos(t)>u}function e(n){var e,u,l,c,f;return{lineStart:function(){c=l=!1,f=1},point:function(s,h){var p,g=[s,h],v=t(s,h),d=o?v?0:i(s,h):v?i(s+(0>s?Fo:-Fo),h):0;if(!e&&(c=l=v)&&n.lineStart(),v!==l&&(p=r(e,g),(wt(e,p)||wt(g,p))&&(g[0]+=Uo,g[1]+=Uo,v=t(g[0],g[1]))),v!==l)f=0,v?(n.lineStart(),p=r(g,e),n.point(p[0],p[1])):(p=r(e,g),n.point(p[0],p[1]),n.lineEnd()),e=p;else if(a&&e&&o^v){var y;d&u||!(y=r(g,e,!0))||(f=0,o?(n.lineStart(),n.point(y[0][0],y[0][1]),n.point(y[1][0],y[1][1]),n.lineEnd()):(n.point(y[1][0],y[1][1]),n.lineEnd(),n.lineStart(),n.point(y[0][0],y[0][1])))}!v||e&&wt(e,g)||n.point(g[0],g[1]),e=g,l=v,u=d},lineEnd:function(){l&&n.lineEnd(),e=null},clean:function(){return f|(c&&l)<<1}}}function r(n,t,e){var r=dt(n),i=dt(t),o=[1,0,0],a=mt(r,i),l=yt(a,a),c=a[0],f=l-c*c;if(!f)return!e&&n;var s=u*l/f,h=-u*c/f,p=mt(o,a),g=xt(o,s),v=xt(a,h);Mt(g,v);var d=p,y=yt(g,d),m=yt(d,d),M=y*y-m*(yt(g,g)-1);if(!(0>M)){var x=Math.sqrt(M),b=xt(d,(-y-x)/m);if(Mt(b,g),b=_t(b),!e)return b;var _,w=n[0],S=t[0],k=n[1],N=t[1];w>S&&(_=w,w=S,S=_);var E=S-w,A=xo(E-Fo)E;if(!A&&k>N&&(_=k,k=N,N=_),C?A?k+N>0^b[1]<(xo(b[0]-w)Fo^(w<=b[0]&&b[0]<=S)){var z=xt(d,(-y+x)/m);return Mt(z,g),[b,_t(z)]}}}function i(t,e){var r=o?n:Fo-n,i=0;return-r>t?i|=1:t>r&&(i|=2),-r>e?i|=4:e>r&&(i|=8),i}var u=Math.cos(n),o=u>0,a=xo(u)>Uo,l=ve(n,6*Yo);return Rt(t,e,l,o?[0,-n]:[-Fo,n-Fo])}function Yt(n,t,e,r){return function(i){var u,o=i.a,a=i.b,l=o.x,c=o.y,f=a.x,s=a.y,h=0,p=1,g=f-l,v=s-c;if(u=n-l,g||!(u>0)){if(u/=g,0>g){if(h>u)return;p>u&&(p=u)}else if(g>0){if(u>p)return;u>h&&(h=u)}if(u=e-l,g||!(0>u)){if(u/=g,0>g){if(u>p)return;u>h&&(h=u)}else if(g>0){if(h>u)return;p>u&&(p=u)}if(u=t-c,v||!(u>0)){if(u/=v,0>v){if(h>u)return;p>u&&(p=u)}else if(v>0){if(u>p)return;u>h&&(h=u)}if(u=r-c,v||!(0>u)){if(u/=v,0>v){if(u>p)return;u>h&&(h=u)}else if(v>0){if(h>u)return;p>u&&(p=u)}return h>0&&(i.a={x:l+h*g,y:c+h*v}),1>p&&(i.b={x:l+p*g,y:c+p*v}),i}}}}}}function Zt(n,t,e,r){function i(r,i){return xo(r[0]-n)0?0:3:xo(r[0]-e)0?2:1:xo(r[1]-t)0?1:0:i>0?3:2}function u(n,t){return o(n.x,t.x)}function o(n,t){var e=i(n,1),r=i(t,1);return e!==r?e-r:0===e?t[1]-n[1]:1===e?n[0]-t[0]:2===e?n[1]-t[1]:t[0]-n[0]}return function(a){function l(n){for(var t=0,e=d.length,r=n[1],i=0;e>i;++i)for(var u,o=1,a=d[i],l=a.length,c=a[0];l>o;++o)u=a[o],c[1]<=r?u[1]>r&&Q(c,u,n)>0&&++t:u[1]<=r&&Q(c,u,n)<0&&--t,c=u;return 0!==t}function c(u,a,l,c){var f=0,s=0;if(null==u||(f=i(u,l))!==(s=i(a,l))||o(u,a)<0^l>0){do c.point(0===f||3===f?n:e,f>1?r:t);while((f=(f+l+4)%4)!==s)}else c.point(a[0],a[1])}function f(i,u){return i>=n&&e>=i&&u>=t&&r>=u}function s(n,t){f(n,t)&&a.point(n,t)}function h(){C.point=g,d&&d.push(y=[]),S=!0,w=!1,b=_=NaN}function p(){v&&(g(m,M),x&&w&&E.rejoin(),v.push(E.buffer())),C.point=s,w&&a.lineEnd()}function g(n,t){n=Math.max(-Ha,Math.min(Ha,n)),t=Math.max(-Ha,Math.min(Ha,t));var e=f(n,t);if(d&&y.push([n,t]),S)m=n,M=t,x=e,S=!1,e&&(a.lineStart(),a.point(n,t));else if(e&&w)a.point(n,t);else{var r={a:{x:b,y:_},b:{x:n,y:t}};A(r)?(w||(a.lineStart(),a.point(r.a.x,r.a.y)),a.point(r.b.x,r.b.y),e||a.lineEnd(),k=!1):e&&(a.lineStart(),a.point(n,t),k=!1)}b=n,_=t,w=e}var v,d,y,m,M,x,b,_,w,S,k,N=a,E=Pt(),A=Yt(n,t,e,r),C={point:s,lineStart:h,lineEnd:p,polygonStart:function(){a=E,v=[],d=[],k=!0},polygonEnd:function(){a=N,v=ao.merge(v);var t=l([n,r]),e=k&&t,i=v.length;(e||i)&&(a.polygonStart(),e&&(a.lineStart(),c(null,null,1,a),a.lineEnd()),i&&Lt(v,u,t,c,a),a.polygonEnd()),v=d=y=null}};return C}}function Vt(n){var t=0,e=Fo/3,r=ae(n),i=r(t,e);return i.parallels=function(n){return arguments.length?r(t=n[0]*Fo/180,e=n[1]*Fo/180):[t/Fo*180,e/Fo*180]},i}function Xt(n,t){function e(n,t){var e=Math.sqrt(u-2*i*Math.sin(t))/i;return[e*Math.sin(n*=i),o-e*Math.cos(n)]}var r=Math.sin(n),i=(r+Math.sin(t))/2,u=1+r*(2*i-r),o=Math.sqrt(u)/i;return e.invert=function(n,t){var e=o-t;return[Math.atan2(n,e)/i,tn((u-(n*n+e*e)*i*i)/(2*i))]},e}function $t(){function n(n,t){Ia+=i*n-r*t,r=n,i=t}var t,e,r,i;$a.point=function(u,o){$a.point=n,t=r=u,e=i=o},$a.lineEnd=function(){n(t,e)}}function Bt(n,t){Ya>n&&(Ya=n),n>Va&&(Va=n),Za>t&&(Za=t),t>Xa&&(Xa=t)}function Wt(){function n(n,t){o.push("M",n,",",t,u)}function t(n,t){o.push("M",n,",",t),a.point=e}function e(n,t){o.push("L",n,",",t)}function r(){a.point=n}function i(){o.push("Z")}var u=Jt(4.5),o=[],a={point:n,lineStart:function(){a.point=t},lineEnd:r,polygonStart:function(){a.lineEnd=i},polygonEnd:function(){a.lineEnd=r,a.point=n},pointRadius:function(n){return u=Jt(n),a},result:function(){if(o.length){var n=o.join("");return o=[],n}}};return a}function Jt(n){return"m0,"+n+"a"+n+","+n+" 0 1,1 0,"+-2*n+"a"+n+","+n+" 0 1,1 0,"+2*n+"z"}function Gt(n,t){Ca+=n,za+=t,++La}function Kt(){function n(n,r){var i=n-t,u=r-e,o=Math.sqrt(i*i+u*u);qa+=o*(t+n)/2,Ta+=o*(e+r)/2,Ra+=o,Gt(t=n,e=r)}var t,e;Wa.point=function(r,i){Wa.point=n,Gt(t=r,e=i)}}function Qt(){Wa.point=Gt}function ne(){function n(n,t){var e=n-r,u=t-i,o=Math.sqrt(e*e+u*u);qa+=o*(r+n)/2,Ta+=o*(i+t)/2,Ra+=o,o=i*n-r*t,Da+=o*(r+n),Pa+=o*(i+t),Ua+=3*o,Gt(r=n,i=t)}var t,e,r,i;Wa.point=function(u,o){Wa.point=n,Gt(t=r=u,e=i=o)},Wa.lineEnd=function(){n(t,e)}}function te(n){function t(t,e){n.moveTo(t+o,e),n.arc(t,e,o,0,Ho)}function e(t,e){n.moveTo(t,e),a.point=r}function r(t,e){n.lineTo(t,e)}function i(){a.point=t}function u(){n.closePath()}var o=4.5,a={point:t,lineStart:function(){a.point=e},lineEnd:i,polygonStart:function(){a.lineEnd=u},polygonEnd:function(){a.lineEnd=i,a.point=t},pointRadius:function(n){return o=n,a},result:b};return a}function ee(n){function t(n){return(a?r:e)(n)}function e(t){return ue(t,function(e,r){e=n(e,r),t.point(e[0],e[1])})}function r(t){function e(e,r){e=n(e,r),t.point(e[0],e[1])}function r(){M=NaN,S.point=u,t.lineStart()}function u(e,r){var u=dt([e,r]),o=n(e,r);i(M,x,m,b,_,w,M=o[0],x=o[1],m=e,b=u[0],_=u[1],w=u[2],a,t),t.point(M,x)}function o(){S.point=e,t.lineEnd()}function l(){ +r(),S.point=c,S.lineEnd=f}function c(n,t){u(s=n,h=t),p=M,g=x,v=b,d=_,y=w,S.point=u}function f(){i(M,x,m,b,_,w,p,g,s,v,d,y,a,t),S.lineEnd=o,o()}var s,h,p,g,v,d,y,m,M,x,b,_,w,S={point:e,lineStart:r,lineEnd:o,polygonStart:function(){t.polygonStart(),S.lineStart=l},polygonEnd:function(){t.polygonEnd(),S.lineStart=r}};return S}function i(t,e,r,a,l,c,f,s,h,p,g,v,d,y){var m=f-t,M=s-e,x=m*m+M*M;if(x>4*u&&d--){var b=a+p,_=l+g,w=c+v,S=Math.sqrt(b*b+_*_+w*w),k=Math.asin(w/=S),N=xo(xo(w)-1)u||xo((m*z+M*L)/x-.5)>.3||o>a*p+l*g+c*v)&&(i(t,e,r,a,l,c,A,C,N,b/=S,_/=S,w,d,y),y.point(A,C),i(A,C,N,b,_,w,f,s,h,p,g,v,d,y))}}var u=.5,o=Math.cos(30*Yo),a=16;return t.precision=function(n){return arguments.length?(a=(u=n*n)>0&&16,t):Math.sqrt(u)},t}function re(n){var t=ee(function(t,e){return n([t*Zo,e*Zo])});return function(n){return le(t(n))}}function ie(n){this.stream=n}function ue(n,t){return{point:t,sphere:function(){n.sphere()},lineStart:function(){n.lineStart()},lineEnd:function(){n.lineEnd()},polygonStart:function(){n.polygonStart()},polygonEnd:function(){n.polygonEnd()}}}function oe(n){return ae(function(){return n})()}function ae(n){function t(n){return n=a(n[0]*Yo,n[1]*Yo),[n[0]*h+l,c-n[1]*h]}function e(n){return n=a.invert((n[0]-l)/h,(c-n[1])/h),n&&[n[0]*Zo,n[1]*Zo]}function r(){a=Ct(o=se(y,M,x),u);var n=u(v,d);return l=p-n[0]*h,c=g+n[1]*h,i()}function i(){return f&&(f.valid=!1,f=null),t}var u,o,a,l,c,f,s=ee(function(n,t){return n=u(n,t),[n[0]*h+l,c-n[1]*h]}),h=150,p=480,g=250,v=0,d=0,y=0,M=0,x=0,b=Fa,_=m,w=null,S=null;return t.stream=function(n){return f&&(f.valid=!1),f=le(b(o,s(_(n)))),f.valid=!0,f},t.clipAngle=function(n){return arguments.length?(b=null==n?(w=n,Fa):It((w=+n)*Yo),i()):w},t.clipExtent=function(n){return arguments.length?(S=n,_=n?Zt(n[0][0],n[0][1],n[1][0],n[1][1]):m,i()):S},t.scale=function(n){return arguments.length?(h=+n,r()):h},t.translate=function(n){return arguments.length?(p=+n[0],g=+n[1],r()):[p,g]},t.center=function(n){return arguments.length?(v=n[0]%360*Yo,d=n[1]%360*Yo,r()):[v*Zo,d*Zo]},t.rotate=function(n){return arguments.length?(y=n[0]%360*Yo,M=n[1]%360*Yo,x=n.length>2?n[2]%360*Yo:0,r()):[y*Zo,M*Zo,x*Zo]},ao.rebind(t,s,"precision"),function(){return u=n.apply(this,arguments),t.invert=u.invert&&e,r()}}function le(n){return ue(n,function(t,e){n.point(t*Yo,e*Yo)})}function ce(n,t){return[n,t]}function fe(n,t){return[n>Fo?n-Ho:-Fo>n?n+Ho:n,t]}function se(n,t,e){return n?t||e?Ct(pe(n),ge(t,e)):pe(n):t||e?ge(t,e):fe}function he(n){return function(t,e){return t+=n,[t>Fo?t-Ho:-Fo>t?t+Ho:t,e]}}function pe(n){var t=he(n);return t.invert=he(-n),t}function ge(n,t){function e(n,t){var e=Math.cos(t),a=Math.cos(n)*e,l=Math.sin(n)*e,c=Math.sin(t),f=c*r+a*i;return[Math.atan2(l*u-f*o,a*r-c*i),tn(f*u+l*o)]}var r=Math.cos(n),i=Math.sin(n),u=Math.cos(t),o=Math.sin(t);return e.invert=function(n,t){var e=Math.cos(t),a=Math.cos(n)*e,l=Math.sin(n)*e,c=Math.sin(t),f=c*u-l*o;return[Math.atan2(l*u+c*o,a*r+f*i),tn(f*r-a*i)]},e}function ve(n,t){var e=Math.cos(n),r=Math.sin(n);return function(i,u,o,a){var l=o*t;null!=i?(i=de(e,i),u=de(e,u),(o>0?u>i:i>u)&&(i+=o*Ho)):(i=n+o*Ho,u=n-.5*l);for(var c,f=i;o>0?f>u:u>f;f-=l)a.point((c=_t([e,-r*Math.cos(f),-r*Math.sin(f)]))[0],c[1])}}function de(n,t){var e=dt(t);e[0]-=n,bt(e);var r=nn(-e[1]);return((-e[2]<0?-r:r)+2*Math.PI-Uo)%(2*Math.PI)}function ye(n,t,e){var r=ao.range(n,t-Uo,e).concat(t);return function(n){return r.map(function(t){return[n,t]})}}function me(n,t,e){var r=ao.range(n,t-Uo,e).concat(t);return function(n){return r.map(function(t){return[t,n]})}}function Me(n){return n.source}function xe(n){return n.target}function be(n,t,e,r){var i=Math.cos(t),u=Math.sin(t),o=Math.cos(r),a=Math.sin(r),l=i*Math.cos(n),c=i*Math.sin(n),f=o*Math.cos(e),s=o*Math.sin(e),h=2*Math.asin(Math.sqrt(on(r-t)+i*o*on(e-n))),p=1/Math.sin(h),g=h?function(n){var t=Math.sin(n*=h)*p,e=Math.sin(h-n)*p,r=e*l+t*f,i=e*c+t*s,o=e*u+t*a;return[Math.atan2(i,r)*Zo,Math.atan2(o,Math.sqrt(r*r+i*i))*Zo]}:function(){return[n*Zo,t*Zo]};return g.distance=h,g}function _e(){function n(n,i){var u=Math.sin(i*=Yo),o=Math.cos(i),a=xo((n*=Yo)-t),l=Math.cos(a);Ja+=Math.atan2(Math.sqrt((a=o*Math.sin(a))*a+(a=r*u-e*o*l)*a),e*u+r*o*l),t=n,e=u,r=o}var t,e,r;Ga.point=function(i,u){t=i*Yo,e=Math.sin(u*=Yo),r=Math.cos(u),Ga.point=n},Ga.lineEnd=function(){Ga.point=Ga.lineEnd=b}}function we(n,t){function e(t,e){var r=Math.cos(t),i=Math.cos(e),u=n(r*i);return[u*i*Math.sin(t),u*Math.sin(e)]}return e.invert=function(n,e){var r=Math.sqrt(n*n+e*e),i=t(r),u=Math.sin(i),o=Math.cos(i);return[Math.atan2(n*u,r*o),Math.asin(r&&e*u/r)]},e}function Se(n,t){function e(n,t){o>0?-Io+Uo>t&&(t=-Io+Uo):t>Io-Uo&&(t=Io-Uo);var e=o/Math.pow(i(t),u);return[e*Math.sin(u*n),o-e*Math.cos(u*n)]}var r=Math.cos(n),i=function(n){return Math.tan(Fo/4+n/2)},u=n===t?Math.sin(n):Math.log(r/Math.cos(t))/Math.log(i(t)/i(n)),o=r*Math.pow(i(n),u)/u;return u?(e.invert=function(n,t){var e=o-t,r=K(u)*Math.sqrt(n*n+e*e);return[Math.atan2(n,e)/u,2*Math.atan(Math.pow(o/r,1/u))-Io]},e):Ne}function ke(n,t){function e(n,t){var e=u-t;return[e*Math.sin(i*n),u-e*Math.cos(i*n)]}var r=Math.cos(n),i=n===t?Math.sin(n):(r-Math.cos(t))/(t-n),u=r/i+n;return xo(i)i;i++){for(;r>1&&Q(n[e[r-2]],n[e[r-1]],n[i])<=0;)--r;e[r++]=i}return e.slice(0,r)}function qe(n,t){return n[0]-t[0]||n[1]-t[1]}function Te(n,t,e){return(e[0]-t[0])*(n[1]-t[1])<(e[1]-t[1])*(n[0]-t[0])}function Re(n,t,e,r){var i=n[0],u=e[0],o=t[0]-i,a=r[0]-u,l=n[1],c=e[1],f=t[1]-l,s=r[1]-c,h=(a*(l-c)-s*(i-u))/(s*o-a*f);return[i+h*o,l+h*f]}function De(n){var t=n[0],e=n[n.length-1];return!(t[0]-e[0]||t[1]-e[1])}function Pe(){rr(this),this.edge=this.site=this.circle=null}function Ue(n){var t=cl.pop()||new Pe;return t.site=n,t}function je(n){Be(n),ol.remove(n),cl.push(n),rr(n)}function Fe(n){var t=n.circle,e=t.x,r=t.cy,i={x:e,y:r},u=n.P,o=n.N,a=[n];je(n);for(var l=u;l.circle&&xo(e-l.circle.x)f;++f)c=a[f],l=a[f-1],nr(c.edge,l.site,c.site,i);l=a[0],c=a[s-1],c.edge=Ke(l.site,c.site,null,i),$e(l),$e(c)}function He(n){for(var t,e,r,i,u=n.x,o=n.y,a=ol._;a;)if(r=Oe(a,o)-u,r>Uo)a=a.L;else{if(i=u-Ie(a,o),!(i>Uo)){r>-Uo?(t=a.P,e=a):i>-Uo?(t=a,e=a.N):t=e=a;break}if(!a.R){t=a;break}a=a.R}var l=Ue(n);if(ol.insert(t,l),t||e){if(t===e)return Be(t),e=Ue(t.site),ol.insert(l,e),l.edge=e.edge=Ke(t.site,l.site),$e(t),void $e(e);if(!e)return void(l.edge=Ke(t.site,l.site));Be(t),Be(e);var c=t.site,f=c.x,s=c.y,h=n.x-f,p=n.y-s,g=e.site,v=g.x-f,d=g.y-s,y=2*(h*d-p*v),m=h*h+p*p,M=v*v+d*d,x={x:(d*m-p*M)/y+f,y:(h*M-v*m)/y+s};nr(e.edge,c,g,x),l.edge=Ke(c,n,null,x),e.edge=Ke(n,g,null,x),$e(t),$e(e)}}function Oe(n,t){var e=n.site,r=e.x,i=e.y,u=i-t;if(!u)return r;var o=n.P;if(!o)return-(1/0);e=o.site;var a=e.x,l=e.y,c=l-t;if(!c)return a;var f=a-r,s=1/u-1/c,h=f/c;return s?(-h+Math.sqrt(h*h-2*s*(f*f/(-2*c)-l+c/2+i-u/2)))/s+r:(r+a)/2}function Ie(n,t){var e=n.N;if(e)return Oe(e,t);var r=n.site;return r.y===t?r.x:1/0}function Ye(n){this.site=n,this.edges=[]}function Ze(n){for(var t,e,r,i,u,o,a,l,c,f,s=n[0][0],h=n[1][0],p=n[0][1],g=n[1][1],v=ul,d=v.length;d--;)if(u=v[d],u&&u.prepare())for(a=u.edges,l=a.length,o=0;l>o;)f=a[o].end(),r=f.x,i=f.y,c=a[++o%l].start(),t=c.x,e=c.y,(xo(r-t)>Uo||xo(i-e)>Uo)&&(a.splice(o,0,new tr(Qe(u.site,f,xo(r-s)Uo?{x:s,y:xo(t-s)Uo?{x:xo(e-g)Uo?{x:h,y:xo(t-h)Uo?{x:xo(e-p)=-jo)){var p=l*l+c*c,g=f*f+s*s,v=(s*p-c*g)/h,d=(l*g-f*p)/h,s=d+a,y=fl.pop()||new Xe;y.arc=n,y.site=i,y.x=v+o,y.y=s+Math.sqrt(v*v+d*d),y.cy=s,n.circle=y;for(var m=null,M=ll._;M;)if(y.yd||d>=a)return;if(h>g){if(u){if(u.y>=c)return}else u={x:d,y:l};e={x:d,y:c}}else{if(u){if(u.yr||r>1)if(h>g){if(u){if(u.y>=c)return}else u={x:(l-i)/r,y:l};e={x:(c-i)/r,y:c}}else{if(u){if(u.yp){if(u){if(u.x>=a)return}else u={x:o,y:r*o+i};e={x:a,y:r*a+i}}else{if(u){if(u.xu||s>o||r>h||i>p)){if(g=n.point){var g,v=t-n.x,d=e-n.y,y=v*v+d*d;if(l>y){var m=Math.sqrt(l=y);r=t-m,i=e-m,u=t+m,o=e+m,a=g}}for(var M=n.nodes,x=.5*(f+h),b=.5*(s+p),_=t>=x,w=e>=b,S=w<<1|_,k=S+4;k>S;++S)if(n=M[3&S])switch(3&S){case 0:c(n,f,s,x,b);break;case 1:c(n,x,s,h,b);break;case 2:c(n,f,b,x,p);break;case 3:c(n,x,b,h,p)}}}(n,r,i,u,o),a}function vr(n,t){n=ao.rgb(n),t=ao.rgb(t);var e=n.r,r=n.g,i=n.b,u=t.r-e,o=t.g-r,a=t.b-i;return function(n){return"#"+bn(Math.round(e+u*n))+bn(Math.round(r+o*n))+bn(Math.round(i+a*n))}}function dr(n,t){var e,r={},i={};for(e in n)e in t?r[e]=Mr(n[e],t[e]):i[e]=n[e];for(e in t)e in n||(i[e]=t[e]);return function(n){for(e in r)i[e]=r[e](n);return i}}function yr(n,t){return n=+n,t=+t,function(e){return n*(1-e)+t*e}}function mr(n,t){var e,r,i,u=hl.lastIndex=pl.lastIndex=0,o=-1,a=[],l=[];for(n+="",t+="";(e=hl.exec(n))&&(r=pl.exec(t));)(i=r.index)>u&&(i=t.slice(u,i),a[o]?a[o]+=i:a[++o]=i),(e=e[0])===(r=r[0])?a[o]?a[o]+=r:a[++o]=r:(a[++o]=null,l.push({i:o,x:yr(e,r)})),u=pl.lastIndex;return ur;++r)a[(e=l[r]).i]=e.x(n);return a.join("")})}function Mr(n,t){for(var e,r=ao.interpolators.length;--r>=0&&!(e=ao.interpolators[r](n,t)););return e}function xr(n,t){var e,r=[],i=[],u=n.length,o=t.length,a=Math.min(n.length,t.length);for(e=0;a>e;++e)r.push(Mr(n[e],t[e]));for(;u>e;++e)i[e]=n[e];for(;o>e;++e)i[e]=t[e];return function(n){for(e=0;a>e;++e)i[e]=r[e](n);return i}}function br(n){return function(t){return 0>=t?0:t>=1?1:n(t)}}function _r(n){return function(t){return 1-n(1-t)}}function wr(n){return function(t){return.5*(.5>t?n(2*t):2-n(2-2*t))}}function Sr(n){return n*n}function kr(n){return n*n*n}function Nr(n){if(0>=n)return 0;if(n>=1)return 1;var t=n*n,e=t*n;return 4*(.5>n?e:3*(n-t)+e-.75)}function Er(n){return function(t){return Math.pow(t,n)}}function Ar(n){return 1-Math.cos(n*Io)}function Cr(n){return Math.pow(2,10*(n-1))}function zr(n){return 1-Math.sqrt(1-n*n)}function Lr(n,t){var e;return arguments.length<2&&(t=.45),arguments.length?e=t/Ho*Math.asin(1/n):(n=1,e=t/4),function(r){return 1+n*Math.pow(2,-10*r)*Math.sin((r-e)*Ho/t)}}function qr(n){return n||(n=1.70158),function(t){return t*t*((n+1)*t-n)}}function Tr(n){return 1/2.75>n?7.5625*n*n:2/2.75>n?7.5625*(n-=1.5/2.75)*n+.75:2.5/2.75>n?7.5625*(n-=2.25/2.75)*n+.9375:7.5625*(n-=2.625/2.75)*n+.984375}function Rr(n,t){n=ao.hcl(n),t=ao.hcl(t);var e=n.h,r=n.c,i=n.l,u=t.h-e,o=t.c-r,a=t.l-i;return isNaN(o)&&(o=0,r=isNaN(r)?t.c:r),isNaN(u)?(u=0,e=isNaN(e)?t.h:e):u>180?u-=360:-180>u&&(u+=360),function(n){return sn(e+u*n,r+o*n,i+a*n)+""}}function Dr(n,t){n=ao.hsl(n),t=ao.hsl(t);var e=n.h,r=n.s,i=n.l,u=t.h-e,o=t.s-r,a=t.l-i;return isNaN(o)&&(o=0,r=isNaN(r)?t.s:r),isNaN(u)?(u=0,e=isNaN(e)?t.h:e):u>180?u-=360:-180>u&&(u+=360),function(n){return cn(e+u*n,r+o*n,i+a*n)+""}}function Pr(n,t){n=ao.lab(n),t=ao.lab(t);var e=n.l,r=n.a,i=n.b,u=t.l-e,o=t.a-r,a=t.b-i;return function(n){return pn(e+u*n,r+o*n,i+a*n)+""}}function Ur(n,t){return t-=n,function(e){return Math.round(n+t*e)}}function jr(n){var t=[n.a,n.b],e=[n.c,n.d],r=Hr(t),i=Fr(t,e),u=Hr(Or(e,t,-i))||0;t[0]*e[1]180?t+=360:t-n>180&&(n+=360),r.push({i:e.push(Ir(e)+"rotate(",null,")")-2,x:yr(n,t)})):t&&e.push(Ir(e)+"rotate("+t+")")}function Vr(n,t,e,r){n!==t?r.push({i:e.push(Ir(e)+"skewX(",null,")")-2,x:yr(n,t)}):t&&e.push(Ir(e)+"skewX("+t+")")}function Xr(n,t,e,r){if(n[0]!==t[0]||n[1]!==t[1]){var i=e.push(Ir(e)+"scale(",null,",",null,")");r.push({i:i-4,x:yr(n[0],t[0])},{i:i-2,x:yr(n[1],t[1])})}else 1===t[0]&&1===t[1]||e.push(Ir(e)+"scale("+t+")")}function $r(n,t){var e=[],r=[];return n=ao.transform(n),t=ao.transform(t),Yr(n.translate,t.translate,e,r),Zr(n.rotate,t.rotate,e,r),Vr(n.skew,t.skew,e,r),Xr(n.scale,t.scale,e,r),n=t=null,function(n){for(var t,i=-1,u=r.length;++i=0;)e.push(i[r])}function oi(n,t){for(var e=[n],r=[];null!=(n=e.pop());)if(r.push(n),(u=n.children)&&(i=u.length))for(var i,u,o=-1;++oe;++e)(t=n[e][1])>i&&(r=e,i=t);return r}function yi(n){return n.reduce(mi,0)}function mi(n,t){return n+t[1]}function Mi(n,t){return xi(n,Math.ceil(Math.log(t.length)/Math.LN2+1))}function xi(n,t){for(var e=-1,r=+n[0],i=(n[1]-r)/t,u=[];++e<=t;)u[e]=i*e+r;return u}function bi(n){return[ao.min(n),ao.max(n)]}function _i(n,t){return n.value-t.value}function wi(n,t){var e=n._pack_next;n._pack_next=t,t._pack_prev=n,t._pack_next=e,e._pack_prev=t}function Si(n,t){n._pack_next=t,t._pack_prev=n}function ki(n,t){var e=t.x-n.x,r=t.y-n.y,i=n.r+t.r;return.999*i*i>e*e+r*r}function Ni(n){function t(n){f=Math.min(n.x-n.r,f),s=Math.max(n.x+n.r,s),h=Math.min(n.y-n.r,h),p=Math.max(n.y+n.r,p)}if((e=n.children)&&(c=e.length)){var e,r,i,u,o,a,l,c,f=1/0,s=-(1/0),h=1/0,p=-(1/0);if(e.forEach(Ei),r=e[0],r.x=-r.r,r.y=0,t(r),c>1&&(i=e[1],i.x=i.r,i.y=0,t(i),c>2))for(u=e[2],zi(r,i,u),t(u),wi(r,u),r._pack_prev=u,wi(u,i),i=r._pack_next,o=3;c>o;o++){zi(r,i,u=e[o]);var g=0,v=1,d=1;for(a=i._pack_next;a!==i;a=a._pack_next,v++)if(ki(a,u)){g=1;break}if(1==g)for(l=r._pack_prev;l!==a._pack_prev&&!ki(l,u);l=l._pack_prev,d++);g?(d>v||v==d&&i.ro;o++)u=e[o],u.x-=y,u.y-=m,M=Math.max(M,u.r+Math.sqrt(u.x*u.x+u.y*u.y));n.r=M,e.forEach(Ai)}}function Ei(n){n._pack_next=n._pack_prev=n}function Ai(n){delete n._pack_next,delete n._pack_prev}function Ci(n,t,e,r){var i=n.children;if(n.x=t+=r*n.x,n.y=e+=r*n.y,n.r*=r,i)for(var u=-1,o=i.length;++u=0;)t=i[u],t.z+=e,t.m+=e,e+=t.s+(r+=t.c)}function Pi(n,t,e){return n.a.parent===t.parent?n.a:e}function Ui(n){return 1+ao.max(n,function(n){return n.y})}function ji(n){return n.reduce(function(n,t){return n+t.x},0)/n.length}function Fi(n){var t=n.children;return t&&t.length?Fi(t[0]):n}function Hi(n){var t,e=n.children;return e&&(t=e.length)?Hi(e[t-1]):n}function Oi(n){return{x:n.x,y:n.y,dx:n.dx,dy:n.dy}}function Ii(n,t){var e=n.x+t[3],r=n.y+t[0],i=n.dx-t[1]-t[3],u=n.dy-t[0]-t[2];return 0>i&&(e+=i/2,i=0),0>u&&(r+=u/2,u=0),{x:e,y:r,dx:i,dy:u}}function Yi(n){var t=n[0],e=n[n.length-1];return e>t?[t,e]:[e,t]}function Zi(n){return n.rangeExtent?n.rangeExtent():Yi(n.range())}function Vi(n,t,e,r){var i=e(n[0],n[1]),u=r(t[0],t[1]);return function(n){return u(i(n))}}function Xi(n,t){var e,r=0,i=n.length-1,u=n[r],o=n[i];return u>o&&(e=r,r=i,i=e,e=u,u=o,o=e),n[r]=t.floor(u),n[i]=t.ceil(o),n}function $i(n){return n?{floor:function(t){return Math.floor(t/n)*n},ceil:function(t){return Math.ceil(t/n)*n}}:Sl}function Bi(n,t,e,r){var i=[],u=[],o=0,a=Math.min(n.length,t.length)-1;for(n[a]2?Bi:Vi,l=r?Wr:Br;return o=i(n,t,l,e),a=i(t,n,l,Mr),u}function u(n){return o(n)}var o,a;return u.invert=function(n){return a(n)},u.domain=function(t){return arguments.length?(n=t.map(Number),i()):n},u.range=function(n){return arguments.length?(t=n,i()):t},u.rangeRound=function(n){return u.range(n).interpolate(Ur)},u.clamp=function(n){return arguments.length?(r=n,i()):r},u.interpolate=function(n){return arguments.length?(e=n,i()):e},u.ticks=function(t){return Qi(n,t)},u.tickFormat=function(t,e){return nu(n,t,e)},u.nice=function(t){return Gi(n,t),i()},u.copy=function(){return Wi(n,t,e,r)},i()}function Ji(n,t){return ao.rebind(n,t,"range","rangeRound","interpolate","clamp")}function Gi(n,t){return Xi(n,$i(Ki(n,t)[2])),Xi(n,$i(Ki(n,t)[2])),n}function Ki(n,t){null==t&&(t=10);var e=Yi(n),r=e[1]-e[0],i=Math.pow(10,Math.floor(Math.log(r/t)/Math.LN10)),u=t/r*i;return.15>=u?i*=10:.35>=u?i*=5:.75>=u&&(i*=2),e[0]=Math.ceil(e[0]/i)*i,e[1]=Math.floor(e[1]/i)*i+.5*i,e[2]=i,e}function Qi(n,t){return ao.range.apply(ao,Ki(n,t))}function nu(n,t,e){var r=Ki(n,t);if(e){var i=ha.exec(e);if(i.shift(),"s"===i[8]){var u=ao.formatPrefix(Math.max(xo(r[0]),xo(r[1])));return i[7]||(i[7]="."+tu(u.scale(r[2]))),i[8]="f",e=ao.format(i.join("")),function(n){return e(u.scale(n))+u.symbol}}i[7]||(i[7]="."+eu(i[8],r)),e=i.join("")}else e=",."+tu(r[2])+"f";return ao.format(e)}function tu(n){return-Math.floor(Math.log(n)/Math.LN10+.01)}function eu(n,t){var e=tu(t[2]);return n in kl?Math.abs(e-tu(Math.max(xo(t[0]),xo(t[1]))))+ +("e"!==n):e-2*("%"===n)}function ru(n,t,e,r){function i(n){return(e?Math.log(0>n?0:n):-Math.log(n>0?0:-n))/Math.log(t)}function u(n){return e?Math.pow(t,n):-Math.pow(t,-n)}function o(t){return n(i(t))}return o.invert=function(t){return u(n.invert(t))},o.domain=function(t){return arguments.length?(e=t[0]>=0,n.domain((r=t.map(Number)).map(i)),o):r},o.base=function(e){return arguments.length?(t=+e,n.domain(r.map(i)),o):t},o.nice=function(){var t=Xi(r.map(i),e?Math:El);return n.domain(t),r=t.map(u),o},o.ticks=function(){var n=Yi(r),o=[],a=n[0],l=n[1],c=Math.floor(i(a)),f=Math.ceil(i(l)),s=t%1?2:t;if(isFinite(f-c)){if(e){for(;f>c;c++)for(var h=1;s>h;h++)o.push(u(c)*h);o.push(u(c))}else for(o.push(u(c));c++0;h--)o.push(u(c)*h);for(c=0;o[c]l;f--);o=o.slice(c,f)}return o},o.tickFormat=function(n,e){if(!arguments.length)return Nl;arguments.length<2?e=Nl:"function"!=typeof e&&(e=ao.format(e));var r=Math.max(1,t*n/o.ticks().length);return function(n){var o=n/u(Math.round(i(n)));return t-.5>o*t&&(o*=t),r>=o?e(n):""}},o.copy=function(){return ru(n.copy(),t,e,r)},Ji(o,n)}function iu(n,t,e){function r(t){return n(i(t))}var i=uu(t),u=uu(1/t);return r.invert=function(t){return u(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain((e=t.map(Number)).map(i)),r):e},r.ticks=function(n){return Qi(e,n)},r.tickFormat=function(n,t){return nu(e,n,t)},r.nice=function(n){return r.domain(Gi(e,n))},r.exponent=function(o){return arguments.length?(i=uu(t=o),u=uu(1/t),n.domain(e.map(i)),r):t},r.copy=function(){return iu(n.copy(),t,e)},Ji(r,n)}function uu(n){return function(t){return 0>t?-Math.pow(-t,n):Math.pow(t,n)}}function ou(n,t){function e(e){return u[((i.get(e)||("range"===t.t?i.set(e,n.push(e)):NaN))-1)%u.length]}function r(t,e){return ao.range(n.length).map(function(n){return t+e*n})}var i,u,o;return e.domain=function(r){if(!arguments.length)return n;n=[],i=new c;for(var u,o=-1,a=r.length;++oe?[NaN,NaN]:[e>0?a[e-1]:n[0],et?NaN:t/u+n,[t,t+1/u]},r.copy=function(){return lu(n,t,e)},i()}function cu(n,t){function e(e){return e>=e?t[ao.bisect(n,e)]:void 0}return e.domain=function(t){return arguments.length?(n=t,e):n},e.range=function(n){return arguments.length?(t=n,e):t},e.invertExtent=function(e){return e=t.indexOf(e),[n[e-1],n[e]]},e.copy=function(){return cu(n,t)},e}function fu(n){function t(n){return+n}return t.invert=t,t.domain=t.range=function(e){return arguments.length?(n=e.map(t),t):n},t.ticks=function(t){return Qi(n,t)},t.tickFormat=function(t,e){return nu(n,t,e)},t.copy=function(){return fu(n)},t}function su(){return 0}function hu(n){return n.innerRadius}function pu(n){return n.outerRadius}function gu(n){return n.startAngle}function vu(n){return n.endAngle}function du(n){return n&&n.padAngle}function yu(n,t,e,r){return(n-e)*t-(t-r)*n>0?0:1}function mu(n,t,e,r,i){var u=n[0]-t[0],o=n[1]-t[1],a=(i?r:-r)/Math.sqrt(u*u+o*o),l=a*o,c=-a*u,f=n[0]+l,s=n[1]+c,h=t[0]+l,p=t[1]+c,g=(f+h)/2,v=(s+p)/2,d=h-f,y=p-s,m=d*d+y*y,M=e-r,x=f*p-h*s,b=(0>y?-1:1)*Math.sqrt(Math.max(0,M*M*m-x*x)),_=(x*y-d*b)/m,w=(-x*d-y*b)/m,S=(x*y+d*b)/m,k=(-x*d+y*b)/m,N=_-g,E=w-v,A=S-g,C=k-v;return N*N+E*E>A*A+C*C&&(_=S,w=k),[[_-l,w-c],[_*e/M,w*e/M]]}function Mu(n){function t(t){function o(){c.push("M",u(n(f),a))}for(var l,c=[],f=[],s=-1,h=t.length,p=En(e),g=En(r);++s1?n.join("L"):n+"Z"}function bu(n){return n.join("L")+"Z"}function _u(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t1&&i.push("H",r[0]),i.join("")}function wu(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t1){a=t[1],u=n[l],l++,r+="C"+(i[0]+o[0])+","+(i[1]+o[1])+","+(u[0]-a[0])+","+(u[1]-a[1])+","+u[0]+","+u[1];for(var c=2;c9&&(i=3*t/Math.sqrt(i),o[a]=i*e,o[a+1]=i*r));for(a=-1;++a<=l;)i=(n[Math.min(l,a+1)][0]-n[Math.max(0,a-1)][0])/(6*(1+o[a]*o[a])),u.push([i||0,o[a]*i||0]);return u}function Fu(n){return n.length<3?xu(n):n[0]+Au(n,ju(n))}function Hu(n){for(var t,e,r,i=-1,u=n.length;++i=t?o(n-t):void(f.c=o)}function o(e){var i=g.active,u=g[i];u&&(u.timer.c=null,u.timer.t=NaN,--g.count,delete g[i],u.event&&u.event.interrupt.call(n,n.__data__,u.index));for(var o in g)if(r>+o){var c=g[o];c.timer.c=null,c.timer.t=NaN,--g.count,delete g[o]}f.c=a,qn(function(){return f.c&&a(e||1)&&(f.c=null,f.t=NaN),1},0,l),g.active=r,v.event&&v.event.start.call(n,n.__data__,t),p=[],v.tween.forEach(function(e,r){(r=r.call(n,n.__data__,t))&&p.push(r)}),h=v.ease,s=v.duration}function a(i){for(var u=i/s,o=h(u),a=p.length;a>0;)p[--a].call(n,o);return u>=1?(v.event&&v.event.end.call(n,n.__data__,t),--g.count?delete g[r]:delete n[e],1):void 0}var l,f,s,h,p,g=n[e]||(n[e]={active:0,count:0}),v=g[r];v||(l=i.time,f=qn(u,0,l),v=g[r]={tween:new c,time:l,timer:f,delay:i.delay,duration:i.duration,ease:i.ease,index:t},i=null,++g.count)}function no(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate("+(isFinite(r)?r:e(n))+",0)"})}function to(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate(0,"+(isFinite(r)?r:e(n))+")"})}function eo(n){return n.toISOString()}function ro(n,t,e){function r(t){return n(t)}function i(n,e){var r=n[1]-n[0],i=r/e,u=ao.bisect(Kl,i);return u==Kl.length?[t.year,Ki(n.map(function(n){return n/31536e6}),e)[2]]:u?t[i/Kl[u-1]1?{floor:function(t){for(;e(t=n.floor(t));)t=io(t-1);return t},ceil:function(t){for(;e(t=n.ceil(t));)t=io(+t+1);return t}}:n))},r.ticks=function(n,t){var e=Yi(r.domain()),u=null==n?i(e,10):"number"==typeof n?i(e,n):!n.range&&[{range:n},t];return u&&(n=u[0],t=u[1]),n.range(e[0],io(+e[1]+1),1>t?1:t)},r.tickFormat=function(){return e},r.copy=function(){return ro(n.copy(),t,e)},Ji(r,n)}function io(n){return new Date(n)}function uo(n){return JSON.parse(n.responseText)}function oo(n){var t=fo.createRange();return t.selectNode(fo.body),t.createContextualFragment(n.responseText)}var ao={version:"3.5.17"},lo=[].slice,co=function(n){return lo.call(n)},fo=this.document;if(fo)try{co(fo.documentElement.childNodes)[0].nodeType}catch(so){co=function(n){for(var t=n.length,e=new Array(t);t--;)e[t]=n[t];return e}}if(Date.now||(Date.now=function(){return+new Date}),fo)try{fo.createElement("DIV").style.setProperty("opacity",0,"")}catch(ho){var po=this.Element.prototype,go=po.setAttribute,vo=po.setAttributeNS,yo=this.CSSStyleDeclaration.prototype,mo=yo.setProperty;po.setAttribute=function(n,t){go.call(this,n,t+"")},po.setAttributeNS=function(n,t,e){vo.call(this,n,t,e+"")},yo.setProperty=function(n,t,e){mo.call(this,n,t+"",e)}}ao.ascending=e,ao.descending=function(n,t){return n>t?-1:t>n?1:t>=n?0:NaN},ao.min=function(n,t){var e,r,i=-1,u=n.length;if(1===arguments.length){for(;++i=r){e=r;break}for(;++ir&&(e=r)}else{for(;++i=r){e=r;break}for(;++ir&&(e=r)}return e},ao.max=function(n,t){var e,r,i=-1,u=n.length;if(1===arguments.length){for(;++i=r){e=r;break}for(;++ie&&(e=r)}else{for(;++i=r){e=r;break}for(;++ie&&(e=r)}return e},ao.extent=function(n,t){var e,r,i,u=-1,o=n.length;if(1===arguments.length){for(;++u=r){e=i=r;break}for(;++ur&&(e=r),r>i&&(i=r))}else{for(;++u=r){e=i=r;break}for(;++ur&&(e=r),r>i&&(i=r))}return[e,i]},ao.sum=function(n,t){var e,r=0,u=n.length,o=-1;if(1===arguments.length)for(;++o1?l/(f-1):void 0},ao.deviation=function(){var n=ao.variance.apply(this,arguments);return n?Math.sqrt(n):n};var Mo=u(e);ao.bisectLeft=Mo.left,ao.bisect=ao.bisectRight=Mo.right,ao.bisector=function(n){return u(1===n.length?function(t,r){return e(n(t),r)}:n)},ao.shuffle=function(n,t,e){(u=arguments.length)<3&&(e=n.length,2>u&&(t=0));for(var r,i,u=e-t;u;)i=Math.random()*u--|0,r=n[u+t],n[u+t]=n[i+t],n[i+t]=r;return n},ao.permute=function(n,t){for(var e=t.length,r=new Array(e);e--;)r[e]=n[t[e]];return r},ao.pairs=function(n){for(var t,e=0,r=n.length-1,i=n[0],u=new Array(0>r?0:r);r>e;)u[e]=[t=i,i=n[++e]];return u},ao.transpose=function(n){if(!(i=n.length))return[];for(var t=-1,e=ao.min(n,o),r=new Array(e);++t=0;)for(r=n[i],t=r.length;--t>=0;)e[--o]=r[t];return e};var xo=Math.abs;ao.range=function(n,t,e){if(arguments.length<3&&(e=1,arguments.length<2&&(t=n,n=0)),(t-n)/e===1/0)throw new Error("infinite range");var r,i=[],u=a(xo(e)),o=-1;if(n*=u,t*=u,e*=u,0>e)for(;(r=n+e*++o)>t;)i.push(r/u);else for(;(r=n+e*++o)=u.length)return r?r.call(i,o):e?o.sort(e):o;for(var l,f,s,h,p=-1,g=o.length,v=u[a++],d=new c;++p=u.length)return n;var r=[],i=o[e++];return n.forEach(function(n,i){r.push({key:n,values:t(i,e)})}),i?r.sort(function(n,t){return i(n.key,t.key)}):r}var e,r,i={},u=[],o=[];return i.map=function(t,e){return n(e,t,0)},i.entries=function(e){return t(n(ao.map,e,0),0)},i.key=function(n){return u.push(n),i},i.sortKeys=function(n){return o[u.length-1]=n,i},i.sortValues=function(n){return e=n,i},i.rollup=function(n){return r=n,i},i},ao.set=function(n){var t=new y;if(n)for(var e=0,r=n.length;r>e;++e)t.add(n[e]);return t},l(y,{has:h,add:function(n){return this._[f(n+="")]=!0,n},remove:p,values:g,size:v,empty:d,forEach:function(n){for(var t in this._)n.call(this,s(t))}}),ao.behavior={},ao.rebind=function(n,t){for(var e,r=1,i=arguments.length;++r=0&&(r=n.slice(e+1),n=n.slice(0,e)),n)return arguments.length<2?this[n].on(r):this[n].on(r,t);if(2===arguments.length){if(null==t)for(n in this)this.hasOwnProperty(n)&&this[n].on(r,null);return this}},ao.event=null,ao.requote=function(n){return n.replace(So,"\\$&")};var So=/[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g,ko={}.__proto__?function(n,t){n.__proto__=t}:function(n,t){for(var e in t)n[e]=t[e]},No=function(n,t){return t.querySelector(n)},Eo=function(n,t){return t.querySelectorAll(n)},Ao=function(n,t){var e=n.matches||n[x(n,"matchesSelector")];return(Ao=function(n,t){return e.call(n,t)})(n,t)};"function"==typeof Sizzle&&(No=function(n,t){return Sizzle(n,t)[0]||null},Eo=Sizzle,Ao=Sizzle.matchesSelector),ao.selection=function(){return ao.select(fo.documentElement)};var Co=ao.selection.prototype=[];Co.select=function(n){var t,e,r,i,u=[];n=A(n);for(var o=-1,a=this.length;++o=0&&"xmlns"!==(e=n.slice(0,t))&&(n=n.slice(t+1)),Lo.hasOwnProperty(e)?{space:Lo[e],local:n}:n}},Co.attr=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node();return n=ao.ns.qualify(n),n.local?e.getAttributeNS(n.space,n.local):e.getAttribute(n)}for(t in n)this.each(z(t,n[t]));return this}return this.each(z(n,t))},Co.classed=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node(),r=(n=T(n)).length,i=-1;if(t=e.classList){for(;++ii){if("string"!=typeof n){2>i&&(e="");for(r in n)this.each(P(r,n[r],e));return this}if(2>i){var u=this.node();return t(u).getComputedStyle(u,null).getPropertyValue(n)}r=""}return this.each(P(n,e,r))},Co.property=function(n,t){if(arguments.length<2){if("string"==typeof n)return this.node()[n];for(t in n)this.each(U(t,n[t]));return this}return this.each(U(n,t))},Co.text=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.textContent=null==t?"":t}:null==n?function(){this.textContent=""}:function(){this.textContent=n}):this.node().textContent},Co.html=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.innerHTML=null==t?"":t}:null==n?function(){this.innerHTML=""}:function(){this.innerHTML=n}):this.node().innerHTML},Co.append=function(n){return n=j(n),this.select(function(){return this.appendChild(n.apply(this,arguments))})},Co.insert=function(n,t){return n=j(n),t=A(t),this.select(function(){return this.insertBefore(n.apply(this,arguments),t.apply(this,arguments)||null)})},Co.remove=function(){return this.each(F)},Co.data=function(n,t){function e(n,e){var r,i,u,o=n.length,s=e.length,h=Math.min(o,s),p=new Array(s),g=new Array(s),v=new Array(o);if(t){var d,y=new c,m=new Array(o);for(r=-1;++rr;++r)g[r]=H(e[r]);for(;o>r;++r)v[r]=n[r]}g.update=p,g.parentNode=p.parentNode=v.parentNode=n.parentNode,a.push(g),l.push(p),f.push(v)}var r,i,u=-1,o=this.length;if(!arguments.length){for(n=new Array(o=(r=this[0]).length);++uu;u++){i.push(t=[]),t.parentNode=(e=this[u]).parentNode;for(var a=0,l=e.length;l>a;a++)(r=e[a])&&n.call(r,r.__data__,a,u)&&t.push(r)}return E(i)},Co.order=function(){for(var n=-1,t=this.length;++n=0;)(e=r[i])&&(u&&u!==e.nextSibling&&u.parentNode.insertBefore(e,u),u=e);return this},Co.sort=function(n){n=I.apply(this,arguments);for(var t=-1,e=this.length;++tn;n++)for(var e=this[n],r=0,i=e.length;i>r;r++){var u=e[r];if(u)return u}return null},Co.size=function(){var n=0;return Y(this,function(){++n}),n};var qo=[];ao.selection.enter=Z,ao.selection.enter.prototype=qo,qo.append=Co.append,qo.empty=Co.empty,qo.node=Co.node,qo.call=Co.call,qo.size=Co.size,qo.select=function(n){for(var t,e,r,i,u,o=[],a=-1,l=this.length;++ar){if("string"!=typeof n){2>r&&(t=!1);for(e in n)this.each(X(e,n[e],t));return this}if(2>r)return(r=this.node()["__on"+n])&&r._;e=!1}return this.each(X(n,t,e))};var To=ao.map({mouseenter:"mouseover",mouseleave:"mouseout"});fo&&To.forEach(function(n){"on"+n in fo&&To.remove(n)});var Ro,Do=0;ao.mouse=function(n){return J(n,k())};var Po=this.navigator&&/WebKit/.test(this.navigator.userAgent)?-1:0;ao.touch=function(n,t,e){if(arguments.length<3&&(e=t,t=k().changedTouches),t)for(var r,i=0,u=t.length;u>i;++i)if((r=t[i]).identifier===e)return J(n,r)},ao.behavior.drag=function(){function n(){this.on("mousedown.drag",u).on("touchstart.drag",o)}function e(n,t,e,u,o){return function(){function a(){var n,e,r=t(h,v);r&&(n=r[0]-M[0],e=r[1]-M[1],g|=n|e,M=r,p({type:"drag",x:r[0]+c[0],y:r[1]+c[1],dx:n,dy:e}))}function l(){t(h,v)&&(y.on(u+d,null).on(o+d,null),m(g),p({type:"dragend"}))}var c,f=this,s=ao.event.target.correspondingElement||ao.event.target,h=f.parentNode,p=r.of(f,arguments),g=0,v=n(),d=".drag"+(null==v?"":"-"+v),y=ao.select(e(s)).on(u+d,a).on(o+d,l),m=W(s),M=t(h,v);i?(c=i.apply(f,arguments),c=[c.x-M[0],c.y-M[1]]):c=[0,0],p({type:"dragstart"})}}var r=N(n,"drag","dragstart","dragend"),i=null,u=e(b,ao.mouse,t,"mousemove","mouseup"),o=e(G,ao.touch,m,"touchmove","touchend");return n.origin=function(t){return arguments.length?(i=t,n):i},ao.rebind(n,r,"on")},ao.touches=function(n,t){return arguments.length<2&&(t=k().touches),t?co(t).map(function(t){var e=J(n,t);return e.identifier=t.identifier,e}):[]};var Uo=1e-6,jo=Uo*Uo,Fo=Math.PI,Ho=2*Fo,Oo=Ho-Uo,Io=Fo/2,Yo=Fo/180,Zo=180/Fo,Vo=Math.SQRT2,Xo=2,$o=4;ao.interpolateZoom=function(n,t){var e,r,i=n[0],u=n[1],o=n[2],a=t[0],l=t[1],c=t[2],f=a-i,s=l-u,h=f*f+s*s;if(jo>h)r=Math.log(c/o)/Vo,e=function(n){return[i+n*f,u+n*s,o*Math.exp(Vo*n*r)]};else{var p=Math.sqrt(h),g=(c*c-o*o+$o*h)/(2*o*Xo*p),v=(c*c-o*o-$o*h)/(2*c*Xo*p),d=Math.log(Math.sqrt(g*g+1)-g),y=Math.log(Math.sqrt(v*v+1)-v);r=(y-d)/Vo,e=function(n){var t=n*r,e=rn(d),a=o/(Xo*p)*(e*un(Vo*t+d)-en(d));return[i+a*f,u+a*s,o*e/rn(Vo*t+d)]}}return e.duration=1e3*r,e},ao.behavior.zoom=function(){function n(n){n.on(L,s).on(Wo+".zoom",p).on("dblclick.zoom",g).on(R,h)}function e(n){return[(n[0]-k.x)/k.k,(n[1]-k.y)/k.k]}function r(n){return[n[0]*k.k+k.x,n[1]*k.k+k.y]}function i(n){k.k=Math.max(A[0],Math.min(A[1],n))}function u(n,t){t=r(t),k.x+=n[0]-t[0],k.y+=n[1]-t[1]}function o(t,e,r,o){t.__chart__={x:k.x,y:k.y,k:k.k},i(Math.pow(2,o)),u(d=e,r),t=ao.select(t),C>0&&(t=t.transition().duration(C)),t.call(n.event)}function a(){b&&b.domain(x.range().map(function(n){return(n-k.x)/k.k}).map(x.invert)),w&&w.domain(_.range().map(function(n){return(n-k.y)/k.k}).map(_.invert))}function l(n){z++||n({type:"zoomstart"})}function c(n){a(),n({type:"zoom",scale:k.k,translate:[k.x,k.y]})}function f(n){--z||(n({type:"zoomend"}),d=null)}function s(){function n(){a=1,u(ao.mouse(i),h),c(o)}function r(){s.on(q,null).on(T,null),p(a),f(o)}var i=this,o=D.of(i,arguments),a=0,s=ao.select(t(i)).on(q,n).on(T,r),h=e(ao.mouse(i)),p=W(i);Il.call(i),l(o)}function h(){function n(){var n=ao.touches(g);return p=k.k,n.forEach(function(n){n.identifier in d&&(d[n.identifier]=e(n))}),n}function t(){var t=ao.event.target;ao.select(t).on(x,r).on(b,a),_.push(t);for(var e=ao.event.changedTouches,i=0,u=e.length;u>i;++i)d[e[i].identifier]=null;var l=n(),c=Date.now();if(1===l.length){if(500>c-M){var f=l[0];o(g,f,d[f.identifier],Math.floor(Math.log(k.k)/Math.LN2)+1),S()}M=c}else if(l.length>1){var f=l[0],s=l[1],h=f[0]-s[0],p=f[1]-s[1];y=h*h+p*p}}function r(){var n,t,e,r,o=ao.touches(g);Il.call(g);for(var a=0,l=o.length;l>a;++a,r=null)if(e=o[a],r=d[e.identifier]){if(t)break;n=e,t=r}if(r){var f=(f=e[0]-n[0])*f+(f=e[1]-n[1])*f,s=y&&Math.sqrt(f/y);n=[(n[0]+e[0])/2,(n[1]+e[1])/2],t=[(t[0]+r[0])/2,(t[1]+r[1])/2],i(s*p)}M=null,u(n,t),c(v)}function a(){if(ao.event.touches.length){for(var t=ao.event.changedTouches,e=0,r=t.length;r>e;++e)delete d[t[e].identifier];for(var i in d)return void n()}ao.selectAll(_).on(m,null),w.on(L,s).on(R,h),N(),f(v)}var p,g=this,v=D.of(g,arguments),d={},y=0,m=".zoom-"+ao.event.changedTouches[0].identifier,x="touchmove"+m,b="touchend"+m,_=[],w=ao.select(g),N=W(g);t(),l(v),w.on(L,null).on(R,t)}function p(){var n=D.of(this,arguments);m?clearTimeout(m):(Il.call(this),v=e(d=y||ao.mouse(this)),l(n)),m=setTimeout(function(){m=null,f(n)},50),S(),i(Math.pow(2,.002*Bo())*k.k),u(d,v),c(n)}function g(){var n=ao.mouse(this),t=Math.log(k.k)/Math.LN2;o(this,n,e(n),ao.event.shiftKey?Math.ceil(t)-1:Math.floor(t)+1)}var v,d,y,m,M,x,b,_,w,k={x:0,y:0,k:1},E=[960,500],A=Jo,C=250,z=0,L="mousedown.zoom",q="mousemove.zoom",T="mouseup.zoom",R="touchstart.zoom",D=N(n,"zoomstart","zoom","zoomend");return Wo||(Wo="onwheel"in fo?(Bo=function(){return-ao.event.deltaY*(ao.event.deltaMode?120:1)},"wheel"):"onmousewheel"in fo?(Bo=function(){return ao.event.wheelDelta},"mousewheel"):(Bo=function(){return-ao.event.detail},"MozMousePixelScroll")),n.event=function(n){n.each(function(){var n=D.of(this,arguments),t=k;Hl?ao.select(this).transition().each("start.zoom",function(){k=this.__chart__||{x:0,y:0,k:1},l(n)}).tween("zoom:zoom",function(){var e=E[0],r=E[1],i=d?d[0]:e/2,u=d?d[1]:r/2,o=ao.interpolateZoom([(i-k.x)/k.k,(u-k.y)/k.k,e/k.k],[(i-t.x)/t.k,(u-t.y)/t.k,e/t.k]);return function(t){var r=o(t),a=e/r[2];this.__chart__=k={x:i-r[0]*a,y:u-r[1]*a,k:a},c(n)}}).each("interrupt.zoom",function(){f(n)}).each("end.zoom",function(){f(n)}):(this.__chart__=k,l(n),c(n),f(n))})},n.translate=function(t){return arguments.length?(k={x:+t[0],y:+t[1],k:k.k},a(),n):[k.x,k.y]},n.scale=function(t){return arguments.length?(k={x:k.x,y:k.y,k:null},i(+t),a(),n):k.k},n.scaleExtent=function(t){return arguments.length?(A=null==t?Jo:[+t[0],+t[1]],n):A},n.center=function(t){return arguments.length?(y=t&&[+t[0],+t[1]],n):y},n.size=function(t){return arguments.length?(E=t&&[+t[0],+t[1]],n):E},n.duration=function(t){return arguments.length?(C=+t,n):C},n.x=function(t){return arguments.length?(b=t,x=t.copy(),k={x:0,y:0,k:1},n):b},n.y=function(t){return arguments.length?(w=t,_=t.copy(),k={x:0,y:0,k:1},n):w},ao.rebind(n,D,"on")};var Bo,Wo,Jo=[0,1/0];ao.color=an,an.prototype.toString=function(){return this.rgb()+""},ao.hsl=ln;var Go=ln.prototype=new an;Go.brighter=function(n){return n=Math.pow(.7,arguments.length?n:1),new ln(this.h,this.s,this.l/n)},Go.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new ln(this.h,this.s,n*this.l)},Go.rgb=function(){return cn(this.h,this.s,this.l)},ao.hcl=fn;var Ko=fn.prototype=new an;Ko.brighter=function(n){return new fn(this.h,this.c,Math.min(100,this.l+Qo*(arguments.length?n:1)))},Ko.darker=function(n){return new fn(this.h,this.c,Math.max(0,this.l-Qo*(arguments.length?n:1)))},Ko.rgb=function(){return sn(this.h,this.c,this.l).rgb()},ao.lab=hn;var Qo=18,na=.95047,ta=1,ea=1.08883,ra=hn.prototype=new an;ra.brighter=function(n){return new hn(Math.min(100,this.l+Qo*(arguments.length?n:1)),this.a,this.b)},ra.darker=function(n){return new hn(Math.max(0,this.l-Qo*(arguments.length?n:1)),this.a,this.b)},ra.rgb=function(){return pn(this.l,this.a,this.b)},ao.rgb=mn;var ia=mn.prototype=new an;ia.brighter=function(n){n=Math.pow(.7,arguments.length?n:1);var t=this.r,e=this.g,r=this.b,i=30;return t||e||r?(t&&i>t&&(t=i),e&&i>e&&(e=i),r&&i>r&&(r=i),new mn(Math.min(255,t/n),Math.min(255,e/n),Math.min(255,r/n))):new mn(i,i,i)},ia.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new mn(n*this.r,n*this.g,n*this.b)},ia.hsl=function(){return wn(this.r,this.g,this.b)},ia.toString=function(){return"#"+bn(this.r)+bn(this.g)+bn(this.b)};var ua=ao.map({aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,rebeccapurple:6697881,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074});ua.forEach(function(n,t){ua.set(n,Mn(t))}),ao.functor=En,ao.xhr=An(m),ao.dsv=function(n,t){function e(n,e,u){arguments.length<3&&(u=e,e=null);var o=Cn(n,t,null==e?r:i(e),u);return o.row=function(n){return arguments.length?o.response(null==(e=n)?r:i(n)):e},o}function r(n){return e.parse(n.responseText)}function i(n){return function(t){return e.parse(t.responseText,n)}}function u(t){return t.map(o).join(n)}function o(n){return a.test(n)?'"'+n.replace(/\"/g,'""')+'"':n}var a=new RegExp('["'+n+"\n]"),l=n.charCodeAt(0);return e.parse=function(n,t){var r;return e.parseRows(n,function(n,e){if(r)return r(n,e-1);var i=new Function("d","return {"+n.map(function(n,t){return JSON.stringify(n)+": d["+t+"]"}).join(",")+"}");r=t?function(n,e){return t(i(n),e)}:i})},e.parseRows=function(n,t){function e(){if(f>=c)return o;if(i)return i=!1,u;var t=f;if(34===n.charCodeAt(t)){for(var e=t;e++f;){var r=n.charCodeAt(f++),a=1;if(10===r)i=!0;else if(13===r)i=!0,10===n.charCodeAt(f)&&(++f,++a);else if(r!==l)continue;return n.slice(t,f-a)}return n.slice(t)}for(var r,i,u={},o={},a=[],c=n.length,f=0,s=0;(r=e())!==o;){for(var h=[];r!==u&&r!==o;)h.push(r),r=e();t&&null==(h=t(h,s++))||a.push(h)}return a},e.format=function(t){if(Array.isArray(t[0]))return e.formatRows(t);var r=new y,i=[];return t.forEach(function(n){for(var t in n)r.has(t)||i.push(r.add(t))}),[i.map(o).join(n)].concat(t.map(function(t){return i.map(function(n){return o(t[n])}).join(n)})).join("\n")},e.formatRows=function(n){return n.map(u).join("\n")},e},ao.csv=ao.dsv(",","text/csv"),ao.tsv=ao.dsv(" ","text/tab-separated-values");var oa,aa,la,ca,fa=this[x(this,"requestAnimationFrame")]||function(n){setTimeout(n,17)};ao.timer=function(){qn.apply(this,arguments)},ao.timer.flush=function(){Rn(),Dn()},ao.round=function(n,t){return t?Math.round(n*(t=Math.pow(10,t)))/t:Math.round(n)};var sa=["y","z","a","f","p","n","\xb5","m","","k","M","G","T","P","E","Z","Y"].map(Un);ao.formatPrefix=function(n,t){var e=0;return(n=+n)&&(0>n&&(n*=-1),t&&(n=ao.round(n,Pn(n,t))),e=1+Math.floor(1e-12+Math.log(n)/Math.LN10),e=Math.max(-24,Math.min(24,3*Math.floor((e-1)/3)))),sa[8+e/3]};var ha=/(?:([^{])?([<>=^]))?([+\- ])?([$#])?(0)?(\d+)?(,)?(\.-?\d+)?([a-z%])?/i,pa=ao.map({b:function(n){return n.toString(2)},c:function(n){return String.fromCharCode(n)},o:function(n){return n.toString(8)},x:function(n){return n.toString(16)},X:function(n){return n.toString(16).toUpperCase()},g:function(n,t){return n.toPrecision(t)},e:function(n,t){return n.toExponential(t)},f:function(n,t){return n.toFixed(t)},r:function(n,t){return(n=ao.round(n,Pn(n,t))).toFixed(Math.max(0,Math.min(20,Pn(n*(1+1e-15),t))))}}),ga=ao.time={},va=Date;Hn.prototype={getDate:function(){return this._.getUTCDate()},getDay:function(){return this._.getUTCDay()},getFullYear:function(){return this._.getUTCFullYear()},getHours:function(){return this._.getUTCHours()},getMilliseconds:function(){return this._.getUTCMilliseconds()},getMinutes:function(){return this._.getUTCMinutes()},getMonth:function(){return this._.getUTCMonth()},getSeconds:function(){return this._.getUTCSeconds()},getTime:function(){return this._.getTime()},getTimezoneOffset:function(){return 0},valueOf:function(){return this._.valueOf()},setDate:function(){da.setUTCDate.apply(this._,arguments)},setDay:function(){da.setUTCDay.apply(this._,arguments)},setFullYear:function(){da.setUTCFullYear.apply(this._,arguments)},setHours:function(){da.setUTCHours.apply(this._,arguments)},setMilliseconds:function(){da.setUTCMilliseconds.apply(this._,arguments)},setMinutes:function(){da.setUTCMinutes.apply(this._,arguments)},setMonth:function(){da.setUTCMonth.apply(this._,arguments)},setSeconds:function(){da.setUTCSeconds.apply(this._,arguments)},setTime:function(){da.setTime.apply(this._,arguments)}};var da=Date.prototype;ga.year=On(function(n){return n=ga.day(n),n.setMonth(0,1),n},function(n,t){n.setFullYear(n.getFullYear()+t)},function(n){return n.getFullYear()}),ga.years=ga.year.range,ga.years.utc=ga.year.utc.range,ga.day=On(function(n){var t=new va(2e3,0);return t.setFullYear(n.getFullYear(),n.getMonth(),n.getDate()),t},function(n,t){n.setDate(n.getDate()+t)},function(n){return n.getDate()-1}),ga.days=ga.day.range,ga.days.utc=ga.day.utc.range,ga.dayOfYear=function(n){var t=ga.year(n);return Math.floor((n-t-6e4*(n.getTimezoneOffset()-t.getTimezoneOffset()))/864e5)},["sunday","monday","tuesday","wednesday","thursday","friday","saturday"].forEach(function(n,t){t=7-t;var e=ga[n]=On(function(n){return(n=ga.day(n)).setDate(n.getDate()-(n.getDay()+t)%7),n},function(n,t){n.setDate(n.getDate()+7*Math.floor(t))},function(n){var e=ga.year(n).getDay();return Math.floor((ga.dayOfYear(n)+(e+t)%7)/7)-(e!==t)});ga[n+"s"]=e.range,ga[n+"s"].utc=e.utc.range,ga[n+"OfYear"]=function(n){var e=ga.year(n).getDay();return Math.floor((ga.dayOfYear(n)+(e+t)%7)/7)}}),ga.week=ga.sunday,ga.weeks=ga.sunday.range,ga.weeks.utc=ga.sunday.utc.range,ga.weekOfYear=ga.sundayOfYear;var ya={"-":"",_:" ",0:"0"},ma=/^\s*\d+/,Ma=/^%/;ao.locale=function(n){return{numberFormat:jn(n),timeFormat:Yn(n)}};var xa=ao.locale({decimal:".",thousands:",",grouping:[3],currency:["$",""],dateTime:"%a %b %e %X %Y",date:"%m/%d/%Y",time:"%H:%M:%S",periods:["AM","PM"],days:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"], +shortDays:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],months:["January","February","March","April","May","June","July","August","September","October","November","December"],shortMonths:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]});ao.format=xa.numberFormat,ao.geo={},ft.prototype={s:0,t:0,add:function(n){st(n,this.t,ba),st(ba.s,this.s,this),this.s?this.t+=ba.t:this.s=ba.t},reset:function(){this.s=this.t=0},valueOf:function(){return this.s}};var ba=new ft;ao.geo.stream=function(n,t){n&&_a.hasOwnProperty(n.type)?_a[n.type](n,t):ht(n,t)};var _a={Feature:function(n,t){ht(n.geometry,t)},FeatureCollection:function(n,t){for(var e=n.features,r=-1,i=e.length;++rn?4*Fo+n:n,Na.lineStart=Na.lineEnd=Na.point=b}};ao.geo.bounds=function(){function n(n,t){M.push(x=[f=n,h=n]),s>t&&(s=t),t>p&&(p=t)}function t(t,e){var r=dt([t*Yo,e*Yo]);if(y){var i=mt(y,r),u=[i[1],-i[0],0],o=mt(u,i);bt(o),o=_t(o);var l=t-g,c=l>0?1:-1,v=o[0]*Zo*c,d=xo(l)>180;if(d^(v>c*g&&c*t>v)){var m=o[1]*Zo;m>p&&(p=m)}else if(v=(v+360)%360-180,d^(v>c*g&&c*t>v)){var m=-o[1]*Zo;s>m&&(s=m)}else s>e&&(s=e),e>p&&(p=e);d?g>t?a(f,t)>a(f,h)&&(h=t):a(t,h)>a(f,h)&&(f=t):h>=f?(f>t&&(f=t),t>h&&(h=t)):t>g?a(f,t)>a(f,h)&&(h=t):a(t,h)>a(f,h)&&(f=t)}else n(t,e);y=r,g=t}function e(){b.point=t}function r(){x[0]=f,x[1]=h,b.point=n,y=null}function i(n,e){if(y){var r=n-g;m+=xo(r)>180?r+(r>0?360:-360):r}else v=n,d=e;Na.point(n,e),t(n,e)}function u(){Na.lineStart()}function o(){i(v,d),Na.lineEnd(),xo(m)>Uo&&(f=-(h=180)),x[0]=f,x[1]=h,y=null}function a(n,t){return(t-=n)<0?t+360:t}function l(n,t){return n[0]-t[0]}function c(n,t){return t[0]<=t[1]?t[0]<=n&&n<=t[1]:nka?(f=-(h=180),s=-(p=90)):m>Uo?p=90:-Uo>m&&(s=-90),x[0]=f,x[1]=h}};return function(n){p=h=-(f=s=1/0),M=[],ao.geo.stream(n,b);var t=M.length;if(t){M.sort(l);for(var e,r=1,i=M[0],u=[i];t>r;++r)e=M[r],c(e[0],i)||c(e[1],i)?(a(i[0],e[1])>a(i[0],i[1])&&(i[1]=e[1]),a(e[0],i[1])>a(i[0],i[1])&&(i[0]=e[0])):u.push(i=e);for(var o,e,g=-(1/0),t=u.length-1,r=0,i=u[t];t>=r;i=e,++r)e=u[r],(o=a(i[1],e[0]))>g&&(g=o,f=e[0],h=i[1])}return M=x=null,f===1/0||s===1/0?[[NaN,NaN],[NaN,NaN]]:[[f,s],[h,p]]}}(),ao.geo.centroid=function(n){Ea=Aa=Ca=za=La=qa=Ta=Ra=Da=Pa=Ua=0,ao.geo.stream(n,ja);var t=Da,e=Pa,r=Ua,i=t*t+e*e+r*r;return jo>i&&(t=qa,e=Ta,r=Ra,Uo>Aa&&(t=Ca,e=za,r=La),i=t*t+e*e+r*r,jo>i)?[NaN,NaN]:[Math.atan2(e,t)*Zo,tn(r/Math.sqrt(i))*Zo]};var Ea,Aa,Ca,za,La,qa,Ta,Ra,Da,Pa,Ua,ja={sphere:b,point:St,lineStart:Nt,lineEnd:Et,polygonStart:function(){ja.lineStart=At},polygonEnd:function(){ja.lineStart=Nt}},Fa=Rt(zt,jt,Ht,[-Fo,-Fo/2]),Ha=1e9;ao.geo.clipExtent=function(){var n,t,e,r,i,u,o={stream:function(n){return i&&(i.valid=!1),i=u(n),i.valid=!0,i},extent:function(a){return arguments.length?(u=Zt(n=+a[0][0],t=+a[0][1],e=+a[1][0],r=+a[1][1]),i&&(i.valid=!1,i=null),o):[[n,t],[e,r]]}};return o.extent([[0,0],[960,500]])},(ao.geo.conicEqualArea=function(){return Vt(Xt)}).raw=Xt,ao.geo.albers=function(){return ao.geo.conicEqualArea().rotate([96,0]).center([-.6,38.7]).parallels([29.5,45.5]).scale(1070)},ao.geo.albersUsa=function(){function n(n){var u=n[0],o=n[1];return t=null,e(u,o),t||(r(u,o),t)||i(u,o),t}var t,e,r,i,u=ao.geo.albers(),o=ao.geo.conicEqualArea().rotate([154,0]).center([-2,58.5]).parallels([55,65]),a=ao.geo.conicEqualArea().rotate([157,0]).center([-3,19.9]).parallels([8,18]),l={point:function(n,e){t=[n,e]}};return n.invert=function(n){var t=u.scale(),e=u.translate(),r=(n[0]-e[0])/t,i=(n[1]-e[1])/t;return(i>=.12&&.234>i&&r>=-.425&&-.214>r?o:i>=.166&&.234>i&&r>=-.214&&-.115>r?a:u).invert(n)},n.stream=function(n){var t=u.stream(n),e=o.stream(n),r=a.stream(n);return{point:function(n,i){t.point(n,i),e.point(n,i),r.point(n,i)},sphere:function(){t.sphere(),e.sphere(),r.sphere()},lineStart:function(){t.lineStart(),e.lineStart(),r.lineStart()},lineEnd:function(){t.lineEnd(),e.lineEnd(),r.lineEnd()},polygonStart:function(){t.polygonStart(),e.polygonStart(),r.polygonStart()},polygonEnd:function(){t.polygonEnd(),e.polygonEnd(),r.polygonEnd()}}},n.precision=function(t){return arguments.length?(u.precision(t),o.precision(t),a.precision(t),n):u.precision()},n.scale=function(t){return arguments.length?(u.scale(t),o.scale(.35*t),a.scale(t),n.translate(u.translate())):u.scale()},n.translate=function(t){if(!arguments.length)return u.translate();var c=u.scale(),f=+t[0],s=+t[1];return e=u.translate(t).clipExtent([[f-.455*c,s-.238*c],[f+.455*c,s+.238*c]]).stream(l).point,r=o.translate([f-.307*c,s+.201*c]).clipExtent([[f-.425*c+Uo,s+.12*c+Uo],[f-.214*c-Uo,s+.234*c-Uo]]).stream(l).point,i=a.translate([f-.205*c,s+.212*c]).clipExtent([[f-.214*c+Uo,s+.166*c+Uo],[f-.115*c-Uo,s+.234*c-Uo]]).stream(l).point,n},n.scale(1070)};var Oa,Ia,Ya,Za,Va,Xa,$a={point:b,lineStart:b,lineEnd:b,polygonStart:function(){Ia=0,$a.lineStart=$t},polygonEnd:function(){$a.lineStart=$a.lineEnd=$a.point=b,Oa+=xo(Ia/2)}},Ba={point:Bt,lineStart:b,lineEnd:b,polygonStart:b,polygonEnd:b},Wa={point:Gt,lineStart:Kt,lineEnd:Qt,polygonStart:function(){Wa.lineStart=ne},polygonEnd:function(){Wa.point=Gt,Wa.lineStart=Kt,Wa.lineEnd=Qt}};ao.geo.path=function(){function n(n){return n&&("function"==typeof a&&u.pointRadius(+a.apply(this,arguments)),o&&o.valid||(o=i(u)),ao.geo.stream(n,o)),u.result()}function t(){return o=null,n}var e,r,i,u,o,a=4.5;return n.area=function(n){return Oa=0,ao.geo.stream(n,i($a)),Oa},n.centroid=function(n){return Ca=za=La=qa=Ta=Ra=Da=Pa=Ua=0,ao.geo.stream(n,i(Wa)),Ua?[Da/Ua,Pa/Ua]:Ra?[qa/Ra,Ta/Ra]:La?[Ca/La,za/La]:[NaN,NaN]},n.bounds=function(n){return Va=Xa=-(Ya=Za=1/0),ao.geo.stream(n,i(Ba)),[[Ya,Za],[Va,Xa]]},n.projection=function(n){return arguments.length?(i=(e=n)?n.stream||re(n):m,t()):e},n.context=function(n){return arguments.length?(u=null==(r=n)?new Wt:new te(n),"function"!=typeof a&&u.pointRadius(a),t()):r},n.pointRadius=function(t){return arguments.length?(a="function"==typeof t?t:(u.pointRadius(+t),+t),n):a},n.projection(ao.geo.albersUsa()).context(null)},ao.geo.transform=function(n){return{stream:function(t){var e=new ie(t);for(var r in n)e[r]=n[r];return e}}},ie.prototype={point:function(n,t){this.stream.point(n,t)},sphere:function(){this.stream.sphere()},lineStart:function(){this.stream.lineStart()},lineEnd:function(){this.stream.lineEnd()},polygonStart:function(){this.stream.polygonStart()},polygonEnd:function(){this.stream.polygonEnd()}},ao.geo.projection=oe,ao.geo.projectionMutator=ae,(ao.geo.equirectangular=function(){return oe(ce)}).raw=ce.invert=ce,ao.geo.rotation=function(n){function t(t){return t=n(t[0]*Yo,t[1]*Yo),t[0]*=Zo,t[1]*=Zo,t}return n=se(n[0]%360*Yo,n[1]*Yo,n.length>2?n[2]*Yo:0),t.invert=function(t){return t=n.invert(t[0]*Yo,t[1]*Yo),t[0]*=Zo,t[1]*=Zo,t},t},fe.invert=ce,ao.geo.circle=function(){function n(){var n="function"==typeof r?r.apply(this,arguments):r,t=se(-n[0]*Yo,-n[1]*Yo,0).invert,i=[];return e(null,null,1,{point:function(n,e){i.push(n=t(n,e)),n[0]*=Zo,n[1]*=Zo}}),{type:"Polygon",coordinates:[i]}}var t,e,r=[0,0],i=6;return n.origin=function(t){return arguments.length?(r=t,n):r},n.angle=function(r){return arguments.length?(e=ve((t=+r)*Yo,i*Yo),n):t},n.precision=function(r){return arguments.length?(e=ve(t*Yo,(i=+r)*Yo),n):i},n.angle(90)},ao.geo.distance=function(n,t){var e,r=(t[0]-n[0])*Yo,i=n[1]*Yo,u=t[1]*Yo,o=Math.sin(r),a=Math.cos(r),l=Math.sin(i),c=Math.cos(i),f=Math.sin(u),s=Math.cos(u);return Math.atan2(Math.sqrt((e=s*o)*e+(e=c*f-l*s*a)*e),l*f+c*s*a)},ao.geo.graticule=function(){function n(){return{type:"MultiLineString",coordinates:t()}}function t(){return ao.range(Math.ceil(u/d)*d,i,d).map(h).concat(ao.range(Math.ceil(c/y)*y,l,y).map(p)).concat(ao.range(Math.ceil(r/g)*g,e,g).filter(function(n){return xo(n%d)>Uo}).map(f)).concat(ao.range(Math.ceil(a/v)*v,o,v).filter(function(n){return xo(n%y)>Uo}).map(s))}var e,r,i,u,o,a,l,c,f,s,h,p,g=10,v=g,d=90,y=360,m=2.5;return n.lines=function(){return t().map(function(n){return{type:"LineString",coordinates:n}})},n.outline=function(){return{type:"Polygon",coordinates:[h(u).concat(p(l).slice(1),h(i).reverse().slice(1),p(c).reverse().slice(1))]}},n.extent=function(t){return arguments.length?n.majorExtent(t).minorExtent(t):n.minorExtent()},n.majorExtent=function(t){return arguments.length?(u=+t[0][0],i=+t[1][0],c=+t[0][1],l=+t[1][1],u>i&&(t=u,u=i,i=t),c>l&&(t=c,c=l,l=t),n.precision(m)):[[u,c],[i,l]]},n.minorExtent=function(t){return arguments.length?(r=+t[0][0],e=+t[1][0],a=+t[0][1],o=+t[1][1],r>e&&(t=r,r=e,e=t),a>o&&(t=a,a=o,o=t),n.precision(m)):[[r,a],[e,o]]},n.step=function(t){return arguments.length?n.majorStep(t).minorStep(t):n.minorStep()},n.majorStep=function(t){return arguments.length?(d=+t[0],y=+t[1],n):[d,y]},n.minorStep=function(t){return arguments.length?(g=+t[0],v=+t[1],n):[g,v]},n.precision=function(t){return arguments.length?(m=+t,f=ye(a,o,90),s=me(r,e,m),h=ye(c,l,90),p=me(u,i,m),n):m},n.majorExtent([[-180,-90+Uo],[180,90-Uo]]).minorExtent([[-180,-80-Uo],[180,80+Uo]])},ao.geo.greatArc=function(){function n(){return{type:"LineString",coordinates:[t||r.apply(this,arguments),e||i.apply(this,arguments)]}}var t,e,r=Me,i=xe;return n.distance=function(){return ao.geo.distance(t||r.apply(this,arguments),e||i.apply(this,arguments))},n.source=function(e){return arguments.length?(r=e,t="function"==typeof e?null:e,n):r},n.target=function(t){return arguments.length?(i=t,e="function"==typeof t?null:t,n):i},n.precision=function(){return arguments.length?n:0},n},ao.geo.interpolate=function(n,t){return be(n[0]*Yo,n[1]*Yo,t[0]*Yo,t[1]*Yo)},ao.geo.length=function(n){return Ja=0,ao.geo.stream(n,Ga),Ja};var Ja,Ga={sphere:b,point:b,lineStart:_e,lineEnd:b,polygonStart:b,polygonEnd:b},Ka=we(function(n){return Math.sqrt(2/(1+n))},function(n){return 2*Math.asin(n/2)});(ao.geo.azimuthalEqualArea=function(){return oe(Ka)}).raw=Ka;var Qa=we(function(n){var t=Math.acos(n);return t&&t/Math.sin(t)},m);(ao.geo.azimuthalEquidistant=function(){return oe(Qa)}).raw=Qa,(ao.geo.conicConformal=function(){return Vt(Se)}).raw=Se,(ao.geo.conicEquidistant=function(){return Vt(ke)}).raw=ke;var nl=we(function(n){return 1/n},Math.atan);(ao.geo.gnomonic=function(){return oe(nl)}).raw=nl,Ne.invert=function(n,t){return[n,2*Math.atan(Math.exp(t))-Io]},(ao.geo.mercator=function(){return Ee(Ne)}).raw=Ne;var tl=we(function(){return 1},Math.asin);(ao.geo.orthographic=function(){return oe(tl)}).raw=tl;var el=we(function(n){return 1/(1+n)},function(n){return 2*Math.atan(n)});(ao.geo.stereographic=function(){return oe(el)}).raw=el,Ae.invert=function(n,t){return[-t,2*Math.atan(Math.exp(n))-Io]},(ao.geo.transverseMercator=function(){var n=Ee(Ae),t=n.center,e=n.rotate;return n.center=function(n){return n?t([-n[1],n[0]]):(n=t(),[n[1],-n[0]])},n.rotate=function(n){return n?e([n[0],n[1],n.length>2?n[2]+90:90]):(n=e(),[n[0],n[1],n[2]-90])},e([0,0,90])}).raw=Ae,ao.geom={},ao.geom.hull=function(n){function t(n){if(n.length<3)return[];var t,i=En(e),u=En(r),o=n.length,a=[],l=[];for(t=0;o>t;t++)a.push([+i.call(this,n[t],t),+u.call(this,n[t],t),t]);for(a.sort(qe),t=0;o>t;t++)l.push([a[t][0],-a[t][1]]);var c=Le(a),f=Le(l),s=f[0]===c[0],h=f[f.length-1]===c[c.length-1],p=[];for(t=c.length-1;t>=0;--t)p.push(n[a[c[t]][2]]);for(t=+s;t=r&&c.x<=u&&c.y>=i&&c.y<=o?[[r,o],[u,o],[u,i],[r,i]]:[];f.point=n[a]}),t}function e(n){return n.map(function(n,t){return{x:Math.round(u(n,t)/Uo)*Uo,y:Math.round(o(n,t)/Uo)*Uo,i:t}})}var r=Ce,i=ze,u=r,o=i,a=sl;return n?t(n):(t.links=function(n){return ar(e(n)).edges.filter(function(n){return n.l&&n.r}).map(function(t){return{source:n[t.l.i],target:n[t.r.i]}})},t.triangles=function(n){var t=[];return ar(e(n)).cells.forEach(function(e,r){for(var i,u,o=e.site,a=e.edges.sort(Ve),l=-1,c=a.length,f=a[c-1].edge,s=f.l===o?f.r:f.l;++l=c,h=r>=f,p=h<<1|s;n.leaf=!1,n=n.nodes[p]||(n.nodes[p]=hr()),s?i=c:a=c,h?o=f:l=f,u(n,t,e,r,i,o,a,l)}var f,s,h,p,g,v,d,y,m,M=En(a),x=En(l);if(null!=t)v=t,d=e,y=r,m=i;else if(y=m=-(v=d=1/0),s=[],h=[],g=n.length,o)for(p=0;g>p;++p)f=n[p],f.xy&&(y=f.x),f.y>m&&(m=f.y),s.push(f.x),h.push(f.y);else for(p=0;g>p;++p){var b=+M(f=n[p],p),_=+x(f,p);v>b&&(v=b),d>_&&(d=_),b>y&&(y=b),_>m&&(m=_),s.push(b),h.push(_)}var w=y-v,S=m-d;w>S?m=d+w:y=v+S;var k=hr();if(k.add=function(n){u(k,n,+M(n,++p),+x(n,p),v,d,y,m)},k.visit=function(n){pr(n,k,v,d,y,m)},k.find=function(n){return gr(k,n[0],n[1],v,d,y,m)},p=-1,null==t){for(;++p=0?n.slice(0,t):n,r=t>=0?n.slice(t+1):"in";return e=vl.get(e)||gl,r=dl.get(r)||m,br(r(e.apply(null,lo.call(arguments,1))))},ao.interpolateHcl=Rr,ao.interpolateHsl=Dr,ao.interpolateLab=Pr,ao.interpolateRound=Ur,ao.transform=function(n){var t=fo.createElementNS(ao.ns.prefix.svg,"g");return(ao.transform=function(n){if(null!=n){t.setAttribute("transform",n);var e=t.transform.baseVal.consolidate()}return new jr(e?e.matrix:yl)})(n)},jr.prototype.toString=function(){return"translate("+this.translate+")rotate("+this.rotate+")skewX("+this.skew+")scale("+this.scale+")"};var yl={a:1,b:0,c:0,d:1,e:0,f:0};ao.interpolateTransform=$r,ao.layout={},ao.layout.bundle=function(){return function(n){for(var t=[],e=-1,r=n.length;++ea*a/y){if(v>l){var c=t.charge/l;n.px-=u*c,n.py-=o*c}return!0}if(t.point&&l&&v>l){var c=t.pointCharge/l;n.px-=u*c,n.py-=o*c}}return!t.charge}}function t(n){n.px=ao.event.x,n.py=ao.event.y,l.resume()}var e,r,i,u,o,a,l={},c=ao.dispatch("start","tick","end"),f=[1,1],s=.9,h=ml,p=Ml,g=-30,v=xl,d=.1,y=.64,M=[],x=[];return l.tick=function(){if((i*=.99)<.005)return e=null,c.end({type:"end",alpha:i=0}),!0;var t,r,l,h,p,v,y,m,b,_=M.length,w=x.length;for(r=0;w>r;++r)l=x[r],h=l.source,p=l.target,m=p.x-h.x,b=p.y-h.y,(v=m*m+b*b)&&(v=i*o[r]*((v=Math.sqrt(v))-u[r])/v,m*=v,b*=v,p.x-=m*(y=h.weight+p.weight?h.weight/(h.weight+p.weight):.5),p.y-=b*y,h.x+=m*(y=1-y),h.y+=b*y);if((y=i*d)&&(m=f[0]/2,b=f[1]/2,r=-1,y))for(;++r<_;)l=M[r],l.x+=(m-l.x)*y,l.y+=(b-l.y)*y;if(g)for(ri(t=ao.geom.quadtree(M),i,a),r=-1;++r<_;)(l=M[r]).fixed||t.visit(n(l));for(r=-1;++r<_;)l=M[r],l.fixed?(l.x=l.px,l.y=l.py):(l.x-=(l.px-(l.px=l.x))*s,l.y-=(l.py-(l.py=l.y))*s);c.tick({type:"tick",alpha:i})},l.nodes=function(n){return arguments.length?(M=n,l):M},l.links=function(n){return arguments.length?(x=n,l):x},l.size=function(n){return arguments.length?(f=n,l):f},l.linkDistance=function(n){return arguments.length?(h="function"==typeof n?n:+n,l):h},l.distance=l.linkDistance,l.linkStrength=function(n){return arguments.length?(p="function"==typeof n?n:+n,l):p},l.friction=function(n){return arguments.length?(s=+n,l):s},l.charge=function(n){return arguments.length?(g="function"==typeof n?n:+n,l):g},l.chargeDistance=function(n){return arguments.length?(v=n*n,l):Math.sqrt(v)},l.gravity=function(n){return arguments.length?(d=+n,l):d},l.theta=function(n){return arguments.length?(y=n*n,l):Math.sqrt(y)},l.alpha=function(n){return arguments.length?(n=+n,i?n>0?i=n:(e.c=null,e.t=NaN,e=null,c.end({type:"end",alpha:i=0})):n>0&&(c.start({type:"start",alpha:i=n}),e=qn(l.tick)),l):i},l.start=function(){function n(n,r){if(!e){for(e=new Array(i),l=0;i>l;++l)e[l]=[];for(l=0;c>l;++l){var u=x[l];e[u.source.index].push(u.target),e[u.target.index].push(u.source)}}for(var o,a=e[t],l=-1,f=a.length;++lt;++t)(r=M[t]).index=t,r.weight=0;for(t=0;c>t;++t)r=x[t],"number"==typeof r.source&&(r.source=M[r.source]),"number"==typeof r.target&&(r.target=M[r.target]),++r.source.weight,++r.target.weight;for(t=0;i>t;++t)r=M[t],isNaN(r.x)&&(r.x=n("x",s)),isNaN(r.y)&&(r.y=n("y",v)),isNaN(r.px)&&(r.px=r.x),isNaN(r.py)&&(r.py=r.y);if(u=[],"function"==typeof h)for(t=0;c>t;++t)u[t]=+h.call(this,x[t],t);else for(t=0;c>t;++t)u[t]=h;if(o=[],"function"==typeof p)for(t=0;c>t;++t)o[t]=+p.call(this,x[t],t);else for(t=0;c>t;++t)o[t]=p;if(a=[],"function"==typeof g)for(t=0;i>t;++t)a[t]=+g.call(this,M[t],t);else for(t=0;i>t;++t)a[t]=g;return l.resume()},l.resume=function(){return l.alpha(.1)},l.stop=function(){return l.alpha(0)},l.drag=function(){return r||(r=ao.behavior.drag().origin(m).on("dragstart.force",Qr).on("drag.force",t).on("dragend.force",ni)),arguments.length?void this.on("mouseover.force",ti).on("mouseout.force",ei).call(r):r},ao.rebind(l,c,"on")};var ml=20,Ml=1,xl=1/0;ao.layout.hierarchy=function(){function n(i){var u,o=[i],a=[];for(i.depth=0;null!=(u=o.pop());)if(a.push(u),(c=e.call(n,u,u.depth))&&(l=c.length)){for(var l,c,f;--l>=0;)o.push(f=c[l]),f.parent=u,f.depth=u.depth+1;r&&(u.value=0),u.children=c}else r&&(u.value=+r.call(n,u,u.depth)||0),delete u.children;return oi(i,function(n){var e,i;t&&(e=n.children)&&e.sort(t),r&&(i=n.parent)&&(i.value+=n.value)}),a}var t=ci,e=ai,r=li;return n.sort=function(e){return arguments.length?(t=e,n):t},n.children=function(t){return arguments.length?(e=t,n):e},n.value=function(t){return arguments.length?(r=t,n):r},n.revalue=function(t){return r&&(ui(t,function(n){n.children&&(n.value=0)}),oi(t,function(t){var e;t.children||(t.value=+r.call(n,t,t.depth)||0),(e=t.parent)&&(e.value+=t.value)})),t},n},ao.layout.partition=function(){function n(t,e,r,i){var u=t.children;if(t.x=e,t.y=t.depth*i,t.dx=r,t.dy=i,u&&(o=u.length)){var o,a,l,c=-1;for(r=t.value?r/t.value:0;++cs?-1:1),g=ao.sum(c),v=g?(s-l*p)/g:0,d=ao.range(l),y=[];return null!=e&&d.sort(e===bl?function(n,t){return c[t]-c[n]}:function(n,t){return e(o[n],o[t])}),d.forEach(function(n){y[n]={data:o[n],value:a=c[n],startAngle:f,endAngle:f+=a*v+p,padAngle:h}}),y}var t=Number,e=bl,r=0,i=Ho,u=0;return n.value=function(e){return arguments.length?(t=e,n):t},n.sort=function(t){return arguments.length?(e=t,n):e},n.startAngle=function(t){return arguments.length?(r=t,n):r},n.endAngle=function(t){return arguments.length?(i=t,n):i},n.padAngle=function(t){return arguments.length?(u=t,n):u},n};var bl={};ao.layout.stack=function(){function n(a,l){if(!(h=a.length))return a;var c=a.map(function(e,r){return t.call(n,e,r)}),f=c.map(function(t){return t.map(function(t,e){return[u.call(n,t,e),o.call(n,t,e)]})}),s=e.call(n,f,l);c=ao.permute(c,s),f=ao.permute(f,s);var h,p,g,v,d=r.call(n,f,l),y=c[0].length;for(g=0;y>g;++g)for(i.call(n,c[0][g],v=d[g],f[0][g][1]),p=1;h>p;++p)i.call(n,c[p][g],v+=f[p-1][g][1],f[p][g][1]);return a}var t=m,e=gi,r=vi,i=pi,u=si,o=hi;return n.values=function(e){return arguments.length?(t=e,n):t},n.order=function(t){return arguments.length?(e="function"==typeof t?t:_l.get(t)||gi,n):e},n.offset=function(t){return arguments.length?(r="function"==typeof t?t:wl.get(t)||vi,n):r},n.x=function(t){return arguments.length?(u=t,n):u},n.y=function(t){return arguments.length?(o=t,n):o},n.out=function(t){return arguments.length?(i=t,n):i},n};var _l=ao.map({"inside-out":function(n){var t,e,r=n.length,i=n.map(di),u=n.map(yi),o=ao.range(r).sort(function(n,t){return i[n]-i[t]}),a=0,l=0,c=[],f=[];for(t=0;r>t;++t)e=o[t],l>a?(a+=u[e],c.push(e)):(l+=u[e],f.push(e));return f.reverse().concat(c)},reverse:function(n){return ao.range(n.length).reverse()},"default":gi}),wl=ao.map({silhouette:function(n){var t,e,r,i=n.length,u=n[0].length,o=[],a=0,l=[];for(e=0;u>e;++e){for(t=0,r=0;i>t;t++)r+=n[t][e][1];r>a&&(a=r),o.push(r)}for(e=0;u>e;++e)l[e]=(a-o[e])/2;return l},wiggle:function(n){var t,e,r,i,u,o,a,l,c,f=n.length,s=n[0],h=s.length,p=[];for(p[0]=l=c=0,e=1;h>e;++e){for(t=0,i=0;f>t;++t)i+=n[t][e][1];for(t=0,u=0,a=s[e][0]-s[e-1][0];f>t;++t){for(r=0,o=(n[t][e][1]-n[t][e-1][1])/(2*a);t>r;++r)o+=(n[r][e][1]-n[r][e-1][1])/a;u+=o*n[t][e][1]}p[e]=l-=i?u/i*a:0,c>l&&(c=l)}for(e=0;h>e;++e)p[e]-=c;return p},expand:function(n){var t,e,r,i=n.length,u=n[0].length,o=1/i,a=[];for(e=0;u>e;++e){for(t=0,r=0;i>t;t++)r+=n[t][e][1];if(r)for(t=0;i>t;t++)n[t][e][1]/=r;else for(t=0;i>t;t++)n[t][e][1]=o}for(e=0;u>e;++e)a[e]=0;return a},zero:vi});ao.layout.histogram=function(){function n(n,u){for(var o,a,l=[],c=n.map(e,this),f=r.call(this,c,u),s=i.call(this,f,c,u),u=-1,h=c.length,p=s.length-1,g=t?1:1/h;++u0)for(u=-1;++u=f[0]&&a<=f[1]&&(o=l[ao.bisect(s,a,1,p)-1],o.y+=g,o.push(n[u]));return l}var t=!0,e=Number,r=bi,i=Mi;return n.value=function(t){return arguments.length?(e=t,n):e},n.range=function(t){return arguments.length?(r=En(t),n):r},n.bins=function(t){return arguments.length?(i="number"==typeof t?function(n){return xi(n,t)}:En(t),n):i},n.frequency=function(e){return arguments.length?(t=!!e,n):t},n},ao.layout.pack=function(){function n(n,u){var o=e.call(this,n,u),a=o[0],l=i[0],c=i[1],f=null==t?Math.sqrt:"function"==typeof t?t:function(){return t};if(a.x=a.y=0,oi(a,function(n){n.r=+f(n.value)}),oi(a,Ni),r){var s=r*(t?1:Math.max(2*a.r/l,2*a.r/c))/2;oi(a,function(n){n.r+=s}),oi(a,Ni),oi(a,function(n){n.r-=s})}return Ci(a,l/2,c/2,t?1:1/Math.max(2*a.r/l,2*a.r/c)),o}var t,e=ao.layout.hierarchy().sort(_i),r=0,i=[1,1];return n.size=function(t){return arguments.length?(i=t,n):i},n.radius=function(e){return arguments.length?(t=null==e||"function"==typeof e?e:+e,n):t},n.padding=function(t){return arguments.length?(r=+t,n):r},ii(n,e)},ao.layout.tree=function(){function n(n,i){var f=o.call(this,n,i),s=f[0],h=t(s);if(oi(h,e),h.parent.m=-h.z,ui(h,r),c)ui(s,u);else{var p=s,g=s,v=s;ui(s,function(n){n.xg.x&&(g=n),n.depth>v.depth&&(v=n)});var d=a(p,g)/2-p.x,y=l[0]/(g.x+a(g,p)/2+d),m=l[1]/(v.depth||1);ui(s,function(n){n.x=(n.x+d)*y,n.y=n.depth*m})}return f}function t(n){for(var t,e={A:null,children:[n]},r=[e];null!=(t=r.pop());)for(var i,u=t.children,o=0,a=u.length;a>o;++o)r.push((u[o]=i={_:u[o],parent:t,children:(i=u[o].children)&&i.slice()||[],A:null,a:null,z:0,m:0,c:0,s:0,t:null,i:o}).a=i);return e.children[0]}function e(n){var t=n.children,e=n.parent.children,r=n.i?e[n.i-1]:null;if(t.length){Di(n);var u=(t[0].z+t[t.length-1].z)/2;r?(n.z=r.z+a(n._,r._),n.m=n.z-u):n.z=u}else r&&(n.z=r.z+a(n._,r._));n.parent.A=i(n,r,n.parent.A||e[0])}function r(n){n._.x=n.z+n.parent.m,n.m+=n.parent.m}function i(n,t,e){if(t){for(var r,i=n,u=n,o=t,l=i.parent.children[0],c=i.m,f=u.m,s=o.m,h=l.m;o=Ti(o),i=qi(i),o&&i;)l=qi(l),u=Ti(u),u.a=n,r=o.z+s-i.z-c+a(o._,i._),r>0&&(Ri(Pi(o,n,e),n,r),c+=r,f+=r),s+=o.m,c+=i.m,h+=l.m,f+=u.m;o&&!Ti(u)&&(u.t=o,u.m+=s-f),i&&!qi(l)&&(l.t=i,l.m+=c-h,e=n)}return e}function u(n){n.x*=l[0],n.y=n.depth*l[1]}var o=ao.layout.hierarchy().sort(null).value(null),a=Li,l=[1,1],c=null;return n.separation=function(t){return arguments.length?(a=t,n):a},n.size=function(t){return arguments.length?(c=null==(l=t)?u:null,n):c?null:l},n.nodeSize=function(t){return arguments.length?(c=null==(l=t)?null:u,n):c?l:null},ii(n,o)},ao.layout.cluster=function(){function n(n,u){var o,a=t.call(this,n,u),l=a[0],c=0;oi(l,function(n){var t=n.children;t&&t.length?(n.x=ji(t),n.y=Ui(t)):(n.x=o?c+=e(n,o):0,n.y=0,o=n)});var f=Fi(l),s=Hi(l),h=f.x-e(f,s)/2,p=s.x+e(s,f)/2;return oi(l,i?function(n){n.x=(n.x-l.x)*r[0],n.y=(l.y-n.y)*r[1]}:function(n){n.x=(n.x-h)/(p-h)*r[0],n.y=(1-(l.y?n.y/l.y:1))*r[1]}),a}var t=ao.layout.hierarchy().sort(null).value(null),e=Li,r=[1,1],i=!1;return n.separation=function(t){return arguments.length?(e=t,n):e},n.size=function(t){return arguments.length?(i=null==(r=t),n):i?null:r},n.nodeSize=function(t){return arguments.length?(i=null!=(r=t),n):i?r:null},ii(n,t)},ao.layout.treemap=function(){function n(n,t){for(var e,r,i=-1,u=n.length;++it?0:t),e.area=isNaN(r)||0>=r?0:r}function t(e){var u=e.children;if(u&&u.length){var o,a,l,c=s(e),f=[],h=u.slice(),g=1/0,v="slice"===p?c.dx:"dice"===p?c.dy:"slice-dice"===p?1&e.depth?c.dy:c.dx:Math.min(c.dx,c.dy);for(n(h,c.dx*c.dy/e.value),f.area=0;(l=h.length)>0;)f.push(o=h[l-1]),f.area+=o.area,"squarify"!==p||(a=r(f,v))<=g?(h.pop(),g=a):(f.area-=f.pop().area,i(f,v,c,!1),v=Math.min(c.dx,c.dy),f.length=f.area=0,g=1/0);f.length&&(i(f,v,c,!0),f.length=f.area=0),u.forEach(t)}}function e(t){var r=t.children;if(r&&r.length){var u,o=s(t),a=r.slice(),l=[];for(n(a,o.dx*o.dy/t.value),l.area=0;u=a.pop();)l.push(u),l.area+=u.area,null!=u.z&&(i(l,u.z?o.dx:o.dy,o,!a.length),l.length=l.area=0);r.forEach(e)}}function r(n,t){for(var e,r=n.area,i=0,u=1/0,o=-1,a=n.length;++oe&&(u=e),e>i&&(i=e));return r*=r,t*=t,r?Math.max(t*i*g/r,r/(t*u*g)):1/0}function i(n,t,e,r){var i,u=-1,o=n.length,a=e.x,c=e.y,f=t?l(n.area/t):0; +if(t==e.dx){for((r||f>e.dy)&&(f=e.dy);++ue.dx)&&(f=e.dx);++ue&&(t=1),1>e&&(n=0),function(){var e,r,i;do e=2*Math.random()-1,r=2*Math.random()-1,i=e*e+r*r;while(!i||i>1);return n+t*e*Math.sqrt(-2*Math.log(i)/i)}},logNormal:function(){var n=ao.random.normal.apply(ao,arguments);return function(){return Math.exp(n())}},bates:function(n){var t=ao.random.irwinHall(n);return function(){return t()/n}},irwinHall:function(n){return function(){for(var t=0,e=0;n>e;e++)t+=Math.random();return t}}},ao.scale={};var Sl={floor:m,ceil:m};ao.scale.linear=function(){return Wi([0,1],[0,1],Mr,!1)};var kl={s:1,g:1,p:1,r:1,e:1};ao.scale.log=function(){return ru(ao.scale.linear().domain([0,1]),10,!0,[1,10])};var Nl=ao.format(".0e"),El={floor:function(n){return-Math.ceil(-n)},ceil:function(n){return-Math.floor(-n)}};ao.scale.pow=function(){return iu(ao.scale.linear(),1,[0,1])},ao.scale.sqrt=function(){return ao.scale.pow().exponent(.5)},ao.scale.ordinal=function(){return ou([],{t:"range",a:[[]]})},ao.scale.category10=function(){return ao.scale.ordinal().range(Al)},ao.scale.category20=function(){return ao.scale.ordinal().range(Cl)},ao.scale.category20b=function(){return ao.scale.ordinal().range(zl)},ao.scale.category20c=function(){return ao.scale.ordinal().range(Ll)};var Al=[2062260,16744206,2924588,14034728,9725885,9197131,14907330,8355711,12369186,1556175].map(xn),Cl=[2062260,11454440,16744206,16759672,2924588,10018698,14034728,16750742,9725885,12955861,9197131,12885140,14907330,16234194,8355711,13092807,12369186,14408589,1556175,10410725].map(xn),zl=[3750777,5395619,7040719,10264286,6519097,9216594,11915115,13556636,9202993,12426809,15186514,15190932,8666169,11356490,14049643,15177372,8077683,10834324,13528509,14589654].map(xn),Ll=[3244733,7057110,10406625,13032431,15095053,16616764,16625259,16634018,3253076,7652470,10607003,13101504,7695281,10394312,12369372,14342891,6513507,9868950,12434877,14277081].map(xn);ao.scale.quantile=function(){return au([],[])},ao.scale.quantize=function(){return lu(0,1,[0,1])},ao.scale.threshold=function(){return cu([.5],[0,1])},ao.scale.identity=function(){return fu([0,1])},ao.svg={},ao.svg.arc=function(){function n(){var n=Math.max(0,+e.apply(this,arguments)),c=Math.max(0,+r.apply(this,arguments)),f=o.apply(this,arguments)-Io,s=a.apply(this,arguments)-Io,h=Math.abs(s-f),p=f>s?0:1;if(n>c&&(g=c,c=n,n=g),h>=Oo)return t(c,p)+(n?t(n,1-p):"")+"Z";var g,v,d,y,m,M,x,b,_,w,S,k,N=0,E=0,A=[];if((y=(+l.apply(this,arguments)||0)/2)&&(d=u===ql?Math.sqrt(n*n+c*c):+u.apply(this,arguments),p||(E*=-1),c&&(E=tn(d/c*Math.sin(y))),n&&(N=tn(d/n*Math.sin(y)))),c){m=c*Math.cos(f+E),M=c*Math.sin(f+E),x=c*Math.cos(s-E),b=c*Math.sin(s-E);var C=Math.abs(s-f-2*E)<=Fo?0:1;if(E&&yu(m,M,x,b)===p^C){var z=(f+s)/2;m=c*Math.cos(z),M=c*Math.sin(z),x=b=null}}else m=M=0;if(n){_=n*Math.cos(s-N),w=n*Math.sin(s-N),S=n*Math.cos(f+N),k=n*Math.sin(f+N);var L=Math.abs(f-s+2*N)<=Fo?0:1;if(N&&yu(_,w,S,k)===1-p^L){var q=(f+s)/2;_=n*Math.cos(q),w=n*Math.sin(q),S=k=null}}else _=w=0;if(h>Uo&&(g=Math.min(Math.abs(c-n)/2,+i.apply(this,arguments)))>.001){v=c>n^p?0:1;var T=g,R=g;if(Fo>h){var D=null==S?[_,w]:null==x?[m,M]:Re([m,M],[S,k],[x,b],[_,w]),P=m-D[0],U=M-D[1],j=x-D[0],F=b-D[1],H=1/Math.sin(Math.acos((P*j+U*F)/(Math.sqrt(P*P+U*U)*Math.sqrt(j*j+F*F)))/2),O=Math.sqrt(D[0]*D[0]+D[1]*D[1]);R=Math.min(g,(n-O)/(H-1)),T=Math.min(g,(c-O)/(H+1))}if(null!=x){var I=mu(null==S?[_,w]:[S,k],[m,M],c,T,p),Y=mu([x,b],[_,w],c,T,p);g===T?A.push("M",I[0],"A",T,",",T," 0 0,",v," ",I[1],"A",c,",",c," 0 ",1-p^yu(I[1][0],I[1][1],Y[1][0],Y[1][1]),",",p," ",Y[1],"A",T,",",T," 0 0,",v," ",Y[0]):A.push("M",I[0],"A",T,",",T," 0 1,",v," ",Y[0])}else A.push("M",m,",",M);if(null!=S){var Z=mu([m,M],[S,k],n,-R,p),V=mu([_,w],null==x?[m,M]:[x,b],n,-R,p);g===R?A.push("L",V[0],"A",R,",",R," 0 0,",v," ",V[1],"A",n,",",n," 0 ",p^yu(V[1][0],V[1][1],Z[1][0],Z[1][1]),",",1-p," ",Z[1],"A",R,",",R," 0 0,",v," ",Z[0]):A.push("L",V[0],"A",R,",",R," 0 0,",v," ",Z[0])}else A.push("L",_,",",w)}else A.push("M",m,",",M),null!=x&&A.push("A",c,",",c," 0 ",C,",",p," ",x,",",b),A.push("L",_,",",w),null!=S&&A.push("A",n,",",n," 0 ",L,",",1-p," ",S,",",k);return A.push("Z"),A.join("")}function t(n,t){return"M0,"+n+"A"+n+","+n+" 0 1,"+t+" 0,"+-n+"A"+n+","+n+" 0 1,"+t+" 0,"+n}var e=hu,r=pu,i=su,u=ql,o=gu,a=vu,l=du;return n.innerRadius=function(t){return arguments.length?(e=En(t),n):e},n.outerRadius=function(t){return arguments.length?(r=En(t),n):r},n.cornerRadius=function(t){return arguments.length?(i=En(t),n):i},n.padRadius=function(t){return arguments.length?(u=t==ql?ql:En(t),n):u},n.startAngle=function(t){return arguments.length?(o=En(t),n):o},n.endAngle=function(t){return arguments.length?(a=En(t),n):a},n.padAngle=function(t){return arguments.length?(l=En(t),n):l},n.centroid=function(){var n=(+e.apply(this,arguments)+ +r.apply(this,arguments))/2,t=(+o.apply(this,arguments)+ +a.apply(this,arguments))/2-Io;return[Math.cos(t)*n,Math.sin(t)*n]},n};var ql="auto";ao.svg.line=function(){return Mu(m)};var Tl=ao.map({linear:xu,"linear-closed":bu,step:_u,"step-before":wu,"step-after":Su,basis:zu,"basis-open":Lu,"basis-closed":qu,bundle:Tu,cardinal:Eu,"cardinal-open":ku,"cardinal-closed":Nu,monotone:Fu});Tl.forEach(function(n,t){t.key=n,t.closed=/-closed$/.test(n)});var Rl=[0,2/3,1/3,0],Dl=[0,1/3,2/3,0],Pl=[0,1/6,2/3,1/6];ao.svg.line.radial=function(){var n=Mu(Hu);return n.radius=n.x,delete n.x,n.angle=n.y,delete n.y,n},wu.reverse=Su,Su.reverse=wu,ao.svg.area=function(){return Ou(m)},ao.svg.area.radial=function(){var n=Ou(Hu);return n.radius=n.x,delete n.x,n.innerRadius=n.x0,delete n.x0,n.outerRadius=n.x1,delete n.x1,n.angle=n.y,delete n.y,n.startAngle=n.y0,delete n.y0,n.endAngle=n.y1,delete n.y1,n},ao.svg.chord=function(){function n(n,a){var l=t(this,u,n,a),c=t(this,o,n,a);return"M"+l.p0+r(l.r,l.p1,l.a1-l.a0)+(e(l,c)?i(l.r,l.p1,l.r,l.p0):i(l.r,l.p1,c.r,c.p0)+r(c.r,c.p1,c.a1-c.a0)+i(c.r,c.p1,l.r,l.p0))+"Z"}function t(n,t,e,r){var i=t.call(n,e,r),u=a.call(n,i,r),o=l.call(n,i,r)-Io,f=c.call(n,i,r)-Io;return{r:u,a0:o,a1:f,p0:[u*Math.cos(o),u*Math.sin(o)],p1:[u*Math.cos(f),u*Math.sin(f)]}}function e(n,t){return n.a0==t.a0&&n.a1==t.a1}function r(n,t,e){return"A"+n+","+n+" 0 "+ +(e>Fo)+",1 "+t}function i(n,t,e,r){return"Q 0,0 "+r}var u=Me,o=xe,a=Iu,l=gu,c=vu;return n.radius=function(t){return arguments.length?(a=En(t),n):a},n.source=function(t){return arguments.length?(u=En(t),n):u},n.target=function(t){return arguments.length?(o=En(t),n):o},n.startAngle=function(t){return arguments.length?(l=En(t),n):l},n.endAngle=function(t){return arguments.length?(c=En(t),n):c},n},ao.svg.diagonal=function(){function n(n,i){var u=t.call(this,n,i),o=e.call(this,n,i),a=(u.y+o.y)/2,l=[u,{x:u.x,y:a},{x:o.x,y:a},o];return l=l.map(r),"M"+l[0]+"C"+l[1]+" "+l[2]+" "+l[3]}var t=Me,e=xe,r=Yu;return n.source=function(e){return arguments.length?(t=En(e),n):t},n.target=function(t){return arguments.length?(e=En(t),n):e},n.projection=function(t){return arguments.length?(r=t,n):r},n},ao.svg.diagonal.radial=function(){var n=ao.svg.diagonal(),t=Yu,e=n.projection;return n.projection=function(n){return arguments.length?e(Zu(t=n)):t},n},ao.svg.symbol=function(){function n(n,r){return(Ul.get(t.call(this,n,r))||$u)(e.call(this,n,r))}var t=Xu,e=Vu;return n.type=function(e){return arguments.length?(t=En(e),n):t},n.size=function(t){return arguments.length?(e=En(t),n):e},n};var Ul=ao.map({circle:$u,cross:function(n){var t=Math.sqrt(n/5)/2;return"M"+-3*t+","+-t+"H"+-t+"V"+-3*t+"H"+t+"V"+-t+"H"+3*t+"V"+t+"H"+t+"V"+3*t+"H"+-t+"V"+t+"H"+-3*t+"Z"},diamond:function(n){var t=Math.sqrt(n/(2*Fl)),e=t*Fl;return"M0,"+-t+"L"+e+",0 0,"+t+" "+-e+",0Z"},square:function(n){var t=Math.sqrt(n)/2;return"M"+-t+","+-t+"L"+t+","+-t+" "+t+","+t+" "+-t+","+t+"Z"},"triangle-down":function(n){var t=Math.sqrt(n/jl),e=t*jl/2;return"M0,"+e+"L"+t+","+-e+" "+-t+","+-e+"Z"},"triangle-up":function(n){var t=Math.sqrt(n/jl),e=t*jl/2;return"M0,"+-e+"L"+t+","+e+" "+-t+","+e+"Z"}});ao.svg.symbolTypes=Ul.keys();var jl=Math.sqrt(3),Fl=Math.tan(30*Yo);Co.transition=function(n){for(var t,e,r=Hl||++Zl,i=Ku(n),u=[],o=Ol||{time:Date.now(),ease:Nr,delay:0,duration:250},a=-1,l=this.length;++au;u++){i.push(t=[]);for(var e=this[u],a=0,l=e.length;l>a;a++)(r=e[a])&&n.call(r,r.__data__,a,u)&&t.push(r)}return Wu(i,this.namespace,this.id)},Yl.tween=function(n,t){var e=this.id,r=this.namespace;return arguments.length<2?this.node()[r][e].tween.get(n):Y(this,null==t?function(t){t[r][e].tween.remove(n)}:function(i){i[r][e].tween.set(n,t)})},Yl.attr=function(n,t){function e(){this.removeAttribute(a)}function r(){this.removeAttributeNS(a.space,a.local)}function i(n){return null==n?e:(n+="",function(){var t,e=this.getAttribute(a);return e!==n&&(t=o(e,n),function(n){this.setAttribute(a,t(n))})})}function u(n){return null==n?r:(n+="",function(){var t,e=this.getAttributeNS(a.space,a.local);return e!==n&&(t=o(e,n),function(n){this.setAttributeNS(a.space,a.local,t(n))})})}if(arguments.length<2){for(t in n)this.attr(t,n[t]);return this}var o="transform"==n?$r:Mr,a=ao.ns.qualify(n);return Ju(this,"attr."+n,t,a.local?u:i)},Yl.attrTween=function(n,t){function e(n,e){var r=t.call(this,n,e,this.getAttribute(i));return r&&function(n){this.setAttribute(i,r(n))}}function r(n,e){var r=t.call(this,n,e,this.getAttributeNS(i.space,i.local));return r&&function(n){this.setAttributeNS(i.space,i.local,r(n))}}var i=ao.ns.qualify(n);return this.tween("attr."+n,i.local?r:e)},Yl.style=function(n,e,r){function i(){this.style.removeProperty(n)}function u(e){return null==e?i:(e+="",function(){var i,u=t(this).getComputedStyle(this,null).getPropertyValue(n);return u!==e&&(i=Mr(u,e),function(t){this.style.setProperty(n,i(t),r)})})}var o=arguments.length;if(3>o){if("string"!=typeof n){2>o&&(e="");for(r in n)this.style(r,n[r],e);return this}r=""}return Ju(this,"style."+n,e,u)},Yl.styleTween=function(n,e,r){function i(i,u){var o=e.call(this,i,u,t(this).getComputedStyle(this,null).getPropertyValue(n));return o&&function(t){this.style.setProperty(n,o(t),r)}}return arguments.length<3&&(r=""),this.tween("style."+n,i)},Yl.text=function(n){return Ju(this,"text",n,Gu)},Yl.remove=function(){var n=this.namespace;return this.each("end.transition",function(){var t;this[n].count<2&&(t=this.parentNode)&&t.removeChild(this)})},Yl.ease=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].ease:("function"!=typeof n&&(n=ao.ease.apply(ao,arguments)),Y(this,function(r){r[e][t].ease=n}))},Yl.delay=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].delay:Y(this,"function"==typeof n?function(r,i,u){r[e][t].delay=+n.call(r,r.__data__,i,u)}:(n=+n,function(r){r[e][t].delay=n}))},Yl.duration=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].duration:Y(this,"function"==typeof n?function(r,i,u){r[e][t].duration=Math.max(1,n.call(r,r.__data__,i,u))}:(n=Math.max(1,n),function(r){r[e][t].duration=n}))},Yl.each=function(n,t){var e=this.id,r=this.namespace;if(arguments.length<2){var i=Ol,u=Hl;try{Hl=e,Y(this,function(t,i,u){Ol=t[r][e],n.call(t,t.__data__,i,u)})}finally{Ol=i,Hl=u}}else Y(this,function(i){var u=i[r][e];(u.event||(u.event=ao.dispatch("start","end","interrupt"))).on(n,t)});return this},Yl.transition=function(){for(var n,t,e,r,i=this.id,u=++Zl,o=this.namespace,a=[],l=0,c=this.length;c>l;l++){a.push(n=[]);for(var t=this[l],f=0,s=t.length;s>f;f++)(e=t[f])&&(r=e[o][i],Qu(e,f,o,u,{time:r.time,ease:r.ease,delay:r.delay+r.duration,duration:r.duration})),n.push(e)}return Wu(a,o,u)},ao.svg.axis=function(){function n(n){n.each(function(){var n,c=ao.select(this),f=this.__chart__||e,s=this.__chart__=e.copy(),h=null==l?s.ticks?s.ticks.apply(s,a):s.domain():l,p=null==t?s.tickFormat?s.tickFormat.apply(s,a):m:t,g=c.selectAll(".tick").data(h,s),v=g.enter().insert("g",".domain").attr("class","tick").style("opacity",Uo),d=ao.transition(g.exit()).style("opacity",Uo).remove(),y=ao.transition(g.order()).style("opacity",1),M=Math.max(i,0)+o,x=Zi(s),b=c.selectAll(".domain").data([0]),_=(b.enter().append("path").attr("class","domain"),ao.transition(b));v.append("line"),v.append("text");var w,S,k,N,E=v.select("line"),A=y.select("line"),C=g.select("text").text(p),z=v.select("text"),L=y.select("text"),q="top"===r||"left"===r?-1:1;if("bottom"===r||"top"===r?(n=no,w="x",k="y",S="x2",N="y2",C.attr("dy",0>q?"0em":".71em").style("text-anchor","middle"),_.attr("d","M"+x[0]+","+q*u+"V0H"+x[1]+"V"+q*u)):(n=to,w="y",k="x",S="y2",N="x2",C.attr("dy",".32em").style("text-anchor",0>q?"end":"start"),_.attr("d","M"+q*u+","+x[0]+"H0V"+x[1]+"H"+q*u)),E.attr(N,q*i),z.attr(k,q*M),A.attr(S,0).attr(N,q*i),L.attr(w,0).attr(k,q*M),s.rangeBand){var T=s,R=T.rangeBand()/2;f=s=function(n){return T(n)+R}}else f.rangeBand?f=s:d.call(n,s,f);v.call(n,f,s),y.call(n,s,s)})}var t,e=ao.scale.linear(),r=Vl,i=6,u=6,o=3,a=[10],l=null;return n.scale=function(t){return arguments.length?(e=t,n):e},n.orient=function(t){return arguments.length?(r=t in Xl?t+"":Vl,n):r},n.ticks=function(){return arguments.length?(a=co(arguments),n):a},n.tickValues=function(t){return arguments.length?(l=t,n):l},n.tickFormat=function(e){return arguments.length?(t=e,n):t},n.tickSize=function(t){var e=arguments.length;return e?(i=+t,u=+arguments[e-1],n):i},n.innerTickSize=function(t){return arguments.length?(i=+t,n):i},n.outerTickSize=function(t){return arguments.length?(u=+t,n):u},n.tickPadding=function(t){return arguments.length?(o=+t,n):o},n.tickSubdivide=function(){return arguments.length&&n},n};var Vl="bottom",Xl={top:1,right:1,bottom:1,left:1};ao.svg.brush=function(){function n(t){t.each(function(){var t=ao.select(this).style("pointer-events","all").style("-webkit-tap-highlight-color","rgba(0,0,0,0)").on("mousedown.brush",u).on("touchstart.brush",u),o=t.selectAll(".background").data([0]);o.enter().append("rect").attr("class","background").style("visibility","hidden").style("cursor","crosshair"),t.selectAll(".extent").data([0]).enter().append("rect").attr("class","extent").style("cursor","move");var a=t.selectAll(".resize").data(v,m);a.exit().remove(),a.enter().append("g").attr("class",function(n){return"resize "+n}).style("cursor",function(n){return $l[n]}).append("rect").attr("x",function(n){return/[ew]$/.test(n)?-3:null}).attr("y",function(n){return/^[ns]/.test(n)?-3:null}).attr("width",6).attr("height",6).style("visibility","hidden"),a.style("display",n.empty()?"none":null);var l,s=ao.transition(t),h=ao.transition(o);c&&(l=Zi(c),h.attr("x",l[0]).attr("width",l[1]-l[0]),r(s)),f&&(l=Zi(f),h.attr("y",l[0]).attr("height",l[1]-l[0]),i(s)),e(s)})}function e(n){n.selectAll(".resize").attr("transform",function(n){return"translate("+s[+/e$/.test(n)]+","+h[+/^s/.test(n)]+")"})}function r(n){n.select(".extent").attr("x",s[0]),n.selectAll(".extent,.n>rect,.s>rect").attr("width",s[1]-s[0])}function i(n){n.select(".extent").attr("y",h[0]),n.selectAll(".extent,.e>rect,.w>rect").attr("height",h[1]-h[0])}function u(){function u(){32==ao.event.keyCode&&(C||(M=null,L[0]-=s[1],L[1]-=h[1],C=2),S())}function v(){32==ao.event.keyCode&&2==C&&(L[0]+=s[1],L[1]+=h[1],C=0,S())}function d(){var n=ao.mouse(b),t=!1;x&&(n[0]+=x[0],n[1]+=x[1]),C||(ao.event.altKey?(M||(M=[(s[0]+s[1])/2,(h[0]+h[1])/2]),L[0]=s[+(n[0]f?(i=r,r=f):i=f),v[0]!=r||v[1]!=i?(e?a=null:o=null,v[0]=r,v[1]=i,!0):void 0}function m(){d(),k.style("pointer-events","all").selectAll(".resize").style("display",n.empty()?"none":null),ao.select("body").style("cursor",null),q.on("mousemove.brush",null).on("mouseup.brush",null).on("touchmove.brush",null).on("touchend.brush",null).on("keydown.brush",null).on("keyup.brush",null),z(),w({type:"brushend"})}var M,x,b=this,_=ao.select(ao.event.target),w=l.of(b,arguments),k=ao.select(b),N=_.datum(),E=!/^(n|s)$/.test(N)&&c,A=!/^(e|w)$/.test(N)&&f,C=_.classed("extent"),z=W(b),L=ao.mouse(b),q=ao.select(t(b)).on("keydown.brush",u).on("keyup.brush",v);if(ao.event.changedTouches?q.on("touchmove.brush",d).on("touchend.brush",m):q.on("mousemove.brush",d).on("mouseup.brush",m),k.interrupt().selectAll("*").interrupt(),C)L[0]=s[0]-L[0],L[1]=h[0]-L[1];else if(N){var T=+/w$/.test(N),R=+/^n/.test(N);x=[s[1-T]-L[0],h[1-R]-L[1]],L[0]=s[T],L[1]=h[R]}else ao.event.altKey&&(M=L.slice());k.style("pointer-events","none").selectAll(".resize").style("display",null),ao.select("body").style("cursor",_.style("cursor")),w({type:"brushstart"}),d()}var o,a,l=N(n,"brushstart","brush","brushend"),c=null,f=null,s=[0,0],h=[0,0],p=!0,g=!0,v=Bl[0];return n.event=function(n){n.each(function(){var n=l.of(this,arguments),t={x:s,y:h,i:o,j:a},e=this.__chart__||t;this.__chart__=t,Hl?ao.select(this).transition().each("start.brush",function(){o=e.i,a=e.j,s=e.x,h=e.y,n({type:"brushstart"})}).tween("brush:brush",function(){var e=xr(s,t.x),r=xr(h,t.y);return o=a=null,function(i){s=t.x=e(i),h=t.y=r(i),n({type:"brush",mode:"resize"})}}).each("end.brush",function(){o=t.i,a=t.j,n({type:"brush",mode:"resize"}),n({type:"brushend"})}):(n({type:"brushstart"}),n({type:"brush",mode:"resize"}),n({type:"brushend"}))})},n.x=function(t){return arguments.length?(c=t,v=Bl[!c<<1|!f],n):c},n.y=function(t){return arguments.length?(f=t,v=Bl[!c<<1|!f],n):f},n.clamp=function(t){return arguments.length?(c&&f?(p=!!t[0],g=!!t[1]):c?p=!!t:f&&(g=!!t),n):c&&f?[p,g]:c?p:f?g:null},n.extent=function(t){var e,r,i,u,l;return arguments.length?(c&&(e=t[0],r=t[1],f&&(e=e[0],r=r[0]),o=[e,r],c.invert&&(e=c(e),r=c(r)),e>r&&(l=e,e=r,r=l),e==s[0]&&r==s[1]||(s=[e,r])),f&&(i=t[0],u=t[1],c&&(i=i[1],u=u[1]),a=[i,u],f.invert&&(i=f(i),u=f(u)),i>u&&(l=i,i=u,u=l),i==h[0]&&u==h[1]||(h=[i,u])),n):(c&&(o?(e=o[0],r=o[1]):(e=s[0],r=s[1],c.invert&&(e=c.invert(e),r=c.invert(r)),e>r&&(l=e,e=r,r=l))),f&&(a?(i=a[0],u=a[1]):(i=h[0],u=h[1],f.invert&&(i=f.invert(i),u=f.invert(u)),i>u&&(l=i,i=u,u=l))),c&&f?[[e,i],[r,u]]:c?[e,r]:f&&[i,u])},n.clear=function(){return n.empty()||(s=[0,0],h=[0,0],o=a=null),n},n.empty=function(){return!!c&&s[0]==s[1]||!!f&&h[0]==h[1]},ao.rebind(n,l,"on")};var $l={n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},Bl=[["n","e","s","w","nw","ne","se","sw"],["e","w"],["n","s"],[]],Wl=ga.format=xa.timeFormat,Jl=Wl.utc,Gl=Jl("%Y-%m-%dT%H:%M:%S.%LZ");Wl.iso=Date.prototype.toISOString&&+new Date("2000-01-01T00:00:00.000Z")?eo:Gl,eo.parse=function(n){var t=new Date(n);return isNaN(t)?null:t},eo.toString=Gl.toString,ga.second=On(function(n){return new va(1e3*Math.floor(n/1e3))},function(n,t){n.setTime(n.getTime()+1e3*Math.floor(t))},function(n){return n.getSeconds()}),ga.seconds=ga.second.range,ga.seconds.utc=ga.second.utc.range,ga.minute=On(function(n){return new va(6e4*Math.floor(n/6e4))},function(n,t){n.setTime(n.getTime()+6e4*Math.floor(t))},function(n){return n.getMinutes()}),ga.minutes=ga.minute.range,ga.minutes.utc=ga.minute.utc.range,ga.hour=On(function(n){var t=n.getTimezoneOffset()/60;return new va(36e5*(Math.floor(n/36e5-t)+t))},function(n,t){n.setTime(n.getTime()+36e5*Math.floor(t))},function(n){return n.getHours()}),ga.hours=ga.hour.range,ga.hours.utc=ga.hour.utc.range,ga.month=On(function(n){return n=ga.day(n),n.setDate(1),n},function(n,t){n.setMonth(n.getMonth()+t)},function(n){return n.getMonth()}),ga.months=ga.month.range,ga.months.utc=ga.month.utc.range;var Kl=[1e3,5e3,15e3,3e4,6e4,3e5,9e5,18e5,36e5,108e5,216e5,432e5,864e5,1728e5,6048e5,2592e6,7776e6,31536e6],Ql=[[ga.second,1],[ga.second,5],[ga.second,15],[ga.second,30],[ga.minute,1],[ga.minute,5],[ga.minute,15],[ga.minute,30],[ga.hour,1],[ga.hour,3],[ga.hour,6],[ga.hour,12],[ga.day,1],[ga.day,2],[ga.week,1],[ga.month,1],[ga.month,3],[ga.year,1]],nc=Wl.multi([[".%L",function(n){return n.getMilliseconds()}],[":%S",function(n){return n.getSeconds()}],["%I:%M",function(n){return n.getMinutes()}],["%I %p",function(n){return n.getHours()}],["%a %d",function(n){return n.getDay()&&1!=n.getDate()}],["%b %d",function(n){return 1!=n.getDate()}],["%B",function(n){return n.getMonth()}],["%Y",zt]]),tc={range:function(n,t,e){return ao.range(Math.ceil(n/e)*e,+t,e).map(io)},floor:m,ceil:m};Ql.year=ga.year,ga.scale=function(){return ro(ao.scale.linear(),Ql,nc)};var ec=Ql.map(function(n){return[n[0].utc,n[1]]}),rc=Jl.multi([[".%L",function(n){return n.getUTCMilliseconds()}],[":%S",function(n){return n.getUTCSeconds()}],["%I:%M",function(n){return n.getUTCMinutes()}],["%I %p",function(n){return n.getUTCHours()}],["%a %d",function(n){return n.getUTCDay()&&1!=n.getUTCDate()}],["%b %d",function(n){return 1!=n.getUTCDate()}],["%B",function(n){return n.getUTCMonth()}],["%Y",zt]]);ec.year=ga.year.utc,ga.scale.utc=function(){return ro(ao.scale.linear(),ec,rc)},ao.text=An(function(n){return n.responseText}),ao.json=function(n,t){return Cn(n,"application/json",uo,t)},ao.html=function(n,t){return Cn(n,"text/html",oo,t)},ao.xml=An(function(n){return n.responseXML}),"function"==typeof define&&define.amd?(this.d3=ao,define(ao)):"object"==typeof module&&module.exports?module.exports=ao:this.d3=ao}(); \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/liquidFillGauge.js b/core/src/main/resources/org/apache/spark/ui/static/snappydata/liquidFillGauge.js new file mode 100644 index 0000000000000..7ab04e4c55734 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/snappydata/liquidFillGauge.js @@ -0,0 +1,268 @@ +/*! + * @license Open source under BSD 2-clause (http://choosealicense.com/licenses/bsd-2-clause/) + * Copyright (c) 2015, Curtis Bratton + * All rights reserved. + * + * Liquid Fill Gauge v1.1 + */ +function liquidFillGaugeDefaultSettings(){ + return { + minValue: 0, // The gauge minimum value. + maxValue: 100, // The gauge maximum value. + circleThickness: 0.05, // The outer circle thickness as a percentage of it's radius. + circleFillGap: 0.05, // The size of the gap between the outer circle and wave circle as a percentage of the outer circles radius. + circleColor: "#178BCA", // The color of the outer circle. + waveHeight: 0.05, // The wave height as a percentage of the radius of the wave circle. + waveCount: 1, // The number of full waves per width of the wave circle. + waveRiseTime: 1000, // The amount of time in milliseconds for the wave to rise from 0 to it's final height. + waveAnimateTime: 18000, // The amount of time in milliseconds for a full wave to enter the wave circle. + waveRise: true, // Control if the wave should rise from 0 to it's full height, or start at it's full height. + waveHeightScaling: true, // Controls wave size scaling at low and high fill percentages. When true, wave height reaches it's maximum at 50% fill, and minimum at 0% and 100% fill. This helps to prevent the wave from making the wave circle from appear totally full or empty when near it's minimum or maximum fill. + waveAnimate: true, // Controls if the wave scrolls or is static. + waveColor: "#178BCA", // The color of the fill wave. + waveOffset: 0, // The amount to initially offset the wave. 0 = no offset. 1 = offset of one full wave. + textVertPosition: .5, // The height at which to display the percentage text withing the wave circle. 0 = bottom, 1 = top. + textSize: 1, // The relative height of the text to display in the wave circle. 1 = 50% + valueCountUp: true, // If true, the displayed value counts up from 0 to it's final value upon loading. If false, the final value is displayed. + displayPercent: true, // If true, a % symbol is displayed after the value. + textColor: "#045681", // The color of the value text when the wave does not overlap it. + waveTextColor: "#A4DBf8" // The color of the value text when the wave overlaps it. + }; +} + +function loadLiquidFillGauge(elementId, value, config) { + if(config == null) config = liquidFillGaugeDefaultSettings(); + + var gauge = d3.select("#" + elementId); + var radius = Math.min(parseInt(gauge.style("width")), parseInt(gauge.style("height")))/2; + var locationX = parseInt(gauge.style("width"))/2 - radius; + var locationY = parseInt(gauge.style("height"))/2 - radius; + var fillPercent = Math.max(config.minValue, Math.min(config.maxValue, value))/config.maxValue; + + var waveHeightScale; + if(config.waveHeightScaling){ + waveHeightScale = d3.scale.linear() + .range([0,config.waveHeight,0]) + .domain([0,50,100]); + } else { + waveHeightScale = d3.scale.linear() + .range([config.waveHeight,config.waveHeight]) + .domain([0,100]); + } + + var textPixels = (config.textSize*radius/2); + var textFinalValue = parseFloat(value).toFixed(2); + var textStartValue = config.valueCountUp?config.minValue:textFinalValue; + var percentText = config.displayPercent?"%":""; + var circleThickness = config.circleThickness * radius; + var circleFillGap = config.circleFillGap * radius; + var fillCircleMargin = circleThickness + circleFillGap; + var fillCircleRadius = radius - fillCircleMargin; + var waveHeight = fillCircleRadius*waveHeightScale(fillPercent*100); + + var waveLength = fillCircleRadius*2/config.waveCount; + var waveClipCount = 1+config.waveCount; + var waveClipWidth = waveLength*waveClipCount; + + // Rounding functions so that the correct number of decimal places is always displayed as the value counts up. + var textRounder = function(value){ return Math.round(value); }; + if(parseFloat(textFinalValue) != parseFloat(textRounder(textFinalValue))){ + textRounder = function(value){ return parseFloat(value).toFixed(1); }; + } + if(parseFloat(textFinalValue) != parseFloat(textRounder(textFinalValue))){ + textRounder = function(value){ return parseFloat(value).toFixed(2); }; + } + + // Data for building the clip wave area. + var data = []; + for(var i = 0; i <= 40*waveClipCount; i++){ + data.push({x: i/(40*waveClipCount), y: (i/(40))}); + } + + // Scales for drawing the outer circle. + var gaugeCircleX = d3.scale.linear().range([0,2*Math.PI]).domain([0,1]); + var gaugeCircleY = d3.scale.linear().range([0,radius]).domain([0,radius]); + + // Scales for controlling the size of the clipping path. + var waveScaleX = d3.scale.linear().range([0,waveClipWidth]).domain([0,1]); + var waveScaleY = d3.scale.linear().range([0,waveHeight]).domain([0,1]); + + // Scales for controlling the position of the clipping path. + var waveRiseScale = d3.scale.linear() + // The clipping area size is the height of the fill circle + the wave height, so we position the clip wave + // such that the it will overlap the fill circle at all when at 0%, and will totally cover the fill + // circle at 100%. + .range([(fillCircleMargin+fillCircleRadius*2+waveHeight),(fillCircleMargin-waveHeight)]) + .domain([0,1]); + var waveAnimateScale = d3.scale.linear() + .range([0, waveClipWidth-fillCircleRadius*2]) // Push the clip area one full wave then snap back. + .domain([0,1]); + + // Scale for controlling the position of the text within the gauge. + var textRiseScaleY = d3.scale.linear() + .range([fillCircleMargin+fillCircleRadius*2,(fillCircleMargin+textPixels*0.7)]) + .domain([0,1]); + + // Center the gauge within the parent SVG. + var gaugeGroup = gauge.append("g") + .attr('transform','translate('+locationX+','+locationY+')'); + + // Draw the outer circle. + var gaugeCircleArc = d3.svg.arc() + .startAngle(gaugeCircleX(0)) + .endAngle(gaugeCircleX(1)) + .outerRadius(gaugeCircleY(radius)) + .innerRadius(gaugeCircleY(radius-circleThickness)); + gaugeGroup.append("path") + .attr("d", gaugeCircleArc) + .style("fill", config.circleColor) + .attr('transform','translate('+radius+','+radius+')'); + + // Text where the wave does not overlap. + var text1 = gaugeGroup.append("text") + .text(textRounder(textStartValue) + percentText) + .attr("class", "liquidFillGaugeText") + .attr("text-anchor", "middle") + .attr("font-size", textPixels + "px") + .style("fill", config.textColor) + .attr('transform','translate('+radius+','+textRiseScaleY(config.textVertPosition)+')'); + + // The clipping wave area. + var clipArea = d3.svg.area() + .x(function(d) { return waveScaleX(d.x); } ) + .y0(function(d) { return waveScaleY(Math.sin(Math.PI*2*config.waveOffset*-1 + Math.PI*2*(1-config.waveCount) + d.y*2*Math.PI));} ) + .y1(function(d) { return (fillCircleRadius*2 + waveHeight); } ); + var waveGroup = gaugeGroup.append("defs") + .append("clipPath") + .attr("id", "clipWave" + elementId); + var wave = waveGroup.append("path") + .datum(data) + .attr("d", clipArea) + .attr("T", 0); + + // The inner circle with the clipping wave attached. + var fillCircleGroup = gaugeGroup.append("g") + .attr("clip-path", "url(#clipWave" + elementId + ")"); + fillCircleGroup.append("circle") + .attr("cx", radius) + .attr("cy", radius) + .attr("r", fillCircleRadius) + .style("fill", config.waveColor); + + // Text where the wave does overlap. + var text2 = fillCircleGroup.append("text") + .text(textRounder(textStartValue) + percentText) + .attr("class", "liquidFillGaugeText") + .attr("text-anchor", "middle") + .attr("font-size", textPixels + "px") + .style("fill", config.waveTextColor) + .attr('transform','translate('+radius+','+textRiseScaleY(config.textVertPosition)+')'); + + // Make the value count up. + if(config.valueCountUp){ + var textTween = function(){ + var i = d3.interpolate(this.textContent, textFinalValue); + return function(t) { this.textContent = textRounder(i(t)) + percentText; } + }; + text1.transition() + .duration(config.waveRiseTime) + .tween("text", textTween); + text2.transition() + .duration(config.waveRiseTime) + .tween("text", textTween); + } + + // Make the wave rise. wave and waveGroup are separate so that horizontal and vertical movement can be controlled independently. + var waveGroupXPosition = fillCircleMargin+fillCircleRadius*2-waveClipWidth; + if(config.waveRise){ + waveGroup.attr('transform','translate('+waveGroupXPosition+','+waveRiseScale(0)+')') + .transition() + .duration(config.waveRiseTime) + .attr('transform','translate('+waveGroupXPosition+','+waveRiseScale(fillPercent)+')') + .each("start", function(){ wave.attr('transform','translate(1,0)'); }); // This transform is necessary to get the clip wave positioned correctly when waveRise=true and waveAnimate=false. The wave will not position correctly without this, but it's not clear why this is actually necessary. + } else { + waveGroup.attr('transform','translate('+waveGroupXPosition+','+waveRiseScale(fillPercent)+')'); + } + + if(config.waveAnimate) animateWave(); + + function animateWave() { + wave.attr('transform','translate('+waveAnimateScale(wave.attr('T'))+',0)'); + wave.transition() + .duration(config.waveAnimateTime * (1-wave.attr('T'))) + .ease('linear') + .attr('transform','translate('+waveAnimateScale(1)+',0)') + .attr('T', 1) + .each('end', function(){ + wave.attr('T', 0); + animateWave(config.waveAnimateTime); + }); + } + + function GaugeUpdater(){ + this.update = function(value){ + var newFinalValue = parseFloat(value).toFixed(2); + var textRounderUpdater = function(value){ return Math.round(value); }; + if(parseFloat(newFinalValue) != parseFloat(textRounderUpdater(newFinalValue))){ + textRounderUpdater = function(value){ return parseFloat(value).toFixed(1); }; + } + if(parseFloat(newFinalValue) != parseFloat(textRounderUpdater(newFinalValue))){ + textRounderUpdater = function(value){ return parseFloat(value).toFixed(2); }; + } + + var textTween = function(){ + var i = d3.interpolate(this.textContent, parseFloat(value).toFixed(2)); + return function(t) { this.textContent = textRounderUpdater(i(t)) + percentText; } + }; + + text1.transition() + .duration(config.waveRiseTime) + .tween("text", textTween); + text2.transition() + .duration(config.waveRiseTime) + .tween("text", textTween); + + var fillPercent = Math.max(config.minValue, Math.min(config.maxValue, value))/config.maxValue; + var waveHeight = fillCircleRadius*waveHeightScale(fillPercent*100); + var waveRiseScale = d3.scale.linear() + // The clipping area size is the height of the fill circle + the wave height, so we position the clip wave + // such that the it will overlap the fill circle at all when at 0%, and will totally cover the fill + // circle at 100%. + .range([(fillCircleMargin+fillCircleRadius*2+waveHeight),(fillCircleMargin-waveHeight)]) + .domain([0,1]); + var newHeight = waveRiseScale(fillPercent); + var waveScaleX = d3.scale.linear().range([0,waveClipWidth]).domain([0,1]); + var waveScaleY = d3.scale.linear().range([0,waveHeight]).domain([0,1]); + var newClipArea; + if(config.waveHeightScaling){ + newClipArea = d3.svg.area() + .x(function(d) { return waveScaleX(d.x); } ) + .y0(function(d) { return waveScaleY(Math.sin(Math.PI*2*config.waveOffset*-1 + Math.PI*2*(1-config.waveCount) + d.y*2*Math.PI));} ) + .y1(function(d) { return (fillCircleRadius*2 + waveHeight); } ); + } else { + newClipArea = clipArea; + } + + var newWavePosition = config.waveAnimate?waveAnimateScale(1):0; + wave.transition() + .duration(0) + .transition() + .duration(config.waveAnimate?(config.waveAnimateTime * (1-wave.attr('T'))):(config.waveRiseTime)) + .ease('linear') + .attr('d', newClipArea) + .attr('transform','translate('+newWavePosition+',0)') + .attr('T','1') + .each("end", function(){ + if(config.waveAnimate){ + wave.attr('transform','translate('+waveAnimateScale(0)+',0)'); + animateWave(config.waveAnimateTime); + } + }); + waveGroup.transition() + .duration(config.waveRiseTime) + .attr('transform','translate('+waveGroupXPosition+','+newHeight+')') + } + } + + return new GaugeUpdater(); +} \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css similarity index 87% rename from core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css rename to core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css index 73e740086cb16..e4871befd3bfd 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/snappy-dashboard.css +++ b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css @@ -4,7 +4,7 @@ ========================================================================== */ - +/* .keyStates{ float: left; padding: 5px; @@ -58,6 +58,28 @@ min-height: 25px; text-align: center; padding: 10px; +} */ + +.keyStates{ + float:left; + height:150px; + width:150px; + margin: 0px 20px; +} + +.keyStatsValue{ + width:100%; + height:100px; + padding: 5px 0px; + background: white none repeat scroll 0% 0%; +} + +.keyStatesText{ + height:30px; + min-height: 25px; + padding: 5px; + text-align: center; + font-weight: bolder; } .clusterHealthImageBox{ diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.js b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.js new file mode 100644 index 0000000000000..b004f46f36a74 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.js @@ -0,0 +1,49 @@ + + +function createStatusBlock() { + + var avgMemoryUsage = $( "div#avgMemoryUsage" ).data( "value" ); + var avgHeapUsageGauge = $( "div#avgHeapUsage" ).data( "value" ); + var avgOffHeapUsageGauge = $( "div#avgOffHeapUsage" ).data( "value" ); + var avgJVMHeapUsageGauge = $( "div#avgJvmHeapUsage" ).data( "value" ); + + var config = liquidFillGaugeDefaultSettings(); + config.circleThickness = 0.15; + config.circleColor = "#3EC0FF"; + config.textColor = "#3EC0FF"; + config.waveTextColor = "#3EC0FF"; + config.waveColor = "#A0DFFF"; + config.textVertPosition = 0.8; + config.waveAnimateTime = 1000; + config.waveHeight = 0.05; + config.waveAnimate = true; + config.waveRise = false; + config.waveHeightScaling = false; + config.waveOffset = 0.25; + config.textSize = 0.75; + config.waveCount = 2; + + var memoryGauge = loadLiquidFillGauge("memoryUsageGauge", avgMemoryUsage, config); + var heapGauge = loadLiquidFillGauge("heapUsageGauge", avgHeapUsageGauge, config); + var offHeapGauge = loadLiquidFillGauge("offHeapUsageGauge", avgOffHeapUsageGauge, config); + var jvmGauge = loadLiquidFillGauge("jvmHeapUsageGauge", avgJVMHeapUsageGauge, config); + + + /* function NewValue(){ + if(Math.random() > .5){ + return Math.round(Math.random()*100); + } else { + return (Math.random()*100).toFixed(1); + } + } */ +} + +$(document).ready(function() { + + createStatusBlock() + + $.ajaxSetup({ + cache : false + }); + +}); diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 7813d44738c42..9ab89a2d8279c 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -165,7 +165,7 @@ private[spark] object UIUtils extends Logging { - + @@ -179,6 +179,27 @@ private[spark] object UIUtils extends Logging { } + def commonHeaderNodes_2: Seq[Node] = { + + + + + + + + + + + + + + + + + + + } + def vizHeaderNodes: Seq[Node] = { @@ -316,6 +337,54 @@ private[spark] object UIUtils extends Logging { } + /** Returns a simple spark page with correctly formatted tabs */ + def simpleSparkPageWithTabs_2( + title: String, + content: => Seq[Node], + activeTab: SparkUITab, + refreshInterval: Option[Int] = None, + helpText: Option[String] = None, + showVisualization: Boolean = false): Seq[Node] = { + + val appName = activeTab.appName + val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..." + val header = activeTab.headerTabs.map { tab => +
  • + {tab.name} +
    • + } + // val helpButton: Seq[Node] = helpText.map(tooltip(_, "bottom")).getOrElse(Seq.empty) + + + + {commonHeaderNodes_2} + {if (showVisualization) vizHeaderNodes else Seq.empty} + {appName} - {title} + + +
    +
    +
    + + + + {getProductVersionNode} + +
    +

    + {shortAppName} application UI +

    + {getProductDocLinkNode()} +
      {header}
    +
    +
    +
    + {content} +
    + + + } + /** Returns a page with the spark css/js and a simple format. Used for scheduler UI. */ def basicSparkPage( content: => Seq[Node], From b91dfcee7a8a2ac7539d0e67f68bcefb23df1224 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sun, 4 Jun 2017 16:14:30 +0530 Subject: [PATCH 069/142] [SNAPPYDATA] fixing scalastyle errors introduced in previous commits Conflicts: core/src/main/scala/org/apache/spark/SparkEnv.scala --- core/src/main/scala/org/apache/spark/SparkEnv.scala | 2 ++ core/src/main/scala/org/apache/spark/ui/UIUtils.scala | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index 97c0f5bcc2c7f..8f23214f43d4e 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -44,6 +44,8 @@ import scala.util.Properties import com.google.common.collect.MapMaker +import org.slf4j.LoggerFactory + import org.apache.spark.annotation.DeveloperApi import org.apache.spark.api.python.PythonWorkerFactory import org.apache.spark.broadcast.BroadcastManager diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 9ab89a2d8279c..f2e13c8afb8bc 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -165,7 +165,8 @@ private[spark] object UIUtils extends Logging { - + @@ -185,7 +186,8 @@ private[spark] object UIUtils extends Logging { - + From 6cd99a0570b03cd7dcfd66d35f14bd9379521d05 Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Wed, 7 Jun 2017 18:44:05 +0530 Subject: [PATCH 070/142] SNAP-1698: Snappy Dashboard UI Enhancements (#55) * SNAP-1698: Snappy Dashboard UI Enhancements Changes: - CSS styling and JavaScript code changes for displaying Snappy cluster CPU usage widget. - Removed Heap and Off-Heap usage widgets. - Adding icons/styling for displaying drop down and pull up carets/pointers to expand cell details. - Adding handler for toggling expand and collapse cell details. --- .../ui/static/snappydata/snappy-dashboard.css | 89 ++++++++++++++----- .../ui/static/snappydata/snappy-dashboard.js | 41 +++++---- 2 files changed, 91 insertions(+), 39 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css index e4871befd3bfd..53d4983da4b29 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css +++ b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css @@ -5,7 +5,7 @@ */ /* -.keyStates{ +.keyStates { float: left; padding: 5px; margin: 5px 10px; @@ -18,7 +18,7 @@ min-height: 100px; } -.keyStatesLeft{ +.keyStatesLeft { float: left; padding: 5px; margin: 5px 10px; @@ -31,7 +31,7 @@ min-height: 100px; } -.keyStatesRight{ +.keyStatesRight { float: left; padding: 5px; margin: 5px 10px; @@ -44,7 +44,7 @@ min-height: 100px; } -.keyStatsValue{ +.keyStatsValue { padding-bottom: 10px; font-weight: bolder; vertical-align: middle; @@ -53,28 +53,28 @@ font-size: 24px; } -.keyStatesText{ +.keyStatesText { font-weight: bolder; min-height: 25px; text-align: center; padding: 10px; } */ -.keyStates{ +.keyStates { float:left; height:150px; width:150px; margin: 0px 20px; } -.keyStatsValue{ +.keyStatsValue { width:100%; height:100px; padding: 5px 0px; background: white none repeat scroll 0% 0%; } -.keyStatesText{ +.keyStatesText { height:30px; min-height: 25px; padding: 5px; @@ -82,60 +82,105 @@ font-weight: bolder; } -.clusterHealthImageBox{ +.clusterHealthImageBox { float: left; width: 94px; border-right: thin inset; height: 100px; } -.clusterHealthTextBox{ +.clusterHealthTextBox { text-align: center; float: left; - width: 200px;} + width: 200px; +} -.statusTextNormal{ +.statusTextNormal { color: #87B025; } -.statusTextWarning{ +.statusTextWarning { color: #FDB406; } -.statusTextError{ +.statusTextError { color: #FD063A; } -.divClass2{ +.divClass2 { } -.div-width-100{ +.div-width-100 { width: 100px; } -.div-width-200{ +.div-width-200 { width: 200px; } -.div-width-300{ +.div-width-300 { width: 300px; } -.progressBar{ +.progressBar { height: 19px; width: 100%; border-radius: 5px; border: thin solid #3EC0FF; background: #A0DFFF none repeat scroll 0 0; } -.completedProgress{ +.completedProgress { float: left; border-radius: inherit; background: #3EC0FF none repeat scroll 0px 0px; } /* -.remainingProgress{ +.remainingProgress { float: left; border-radius: inherit; background: #A0DFFF none repeat scroll 0px 0px; }*/ -.progressValue{ +.progressValue { float:right; /* width:20%; */ text-align:center; +} + +.titleNodeCount { + font-weight: bold; + display: inline-block; + line-height: 20px; + margin: 10px 0; + font-size: 17.5px; +} +.titleNodeCount2 { + font-weight: bold; + display: inline-block; + line-height: 20px; + margin: 10px 0; + font-size: 17.5px; +} +.cellDetailsBox { + float: left; + padding: 0px 10px; + display: none; + border: 1px solid #dbd9cf; + margin: 5px auto 2px; +} +.caret-downward { + display: inline-block; + width: 0; + height: 0; + vertical-align: middle; + content: ""; + border: 5px solid; + border-right-color: transparent; + border-bottom-color: transparent; + border-left-color: transparent; +} +.caret-upward { + display: inline-block; + width: 0; + height: 0; + vertical-align: middle; + content: ""; + border: 5px solid; + border-right-color: transparent; + border-top-color: transparent; + border-left-color: transparent; } \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.js b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.js index b004f46f36a74..710d154c7089b 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.js +++ b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.js @@ -1,17 +1,31 @@ +function toggleCellDetails(detailsId) { + + $("#"+detailsId).toggle(); + + var spanId = $("#"+detailsId+"-btn"); + if(spanId.hasClass("caret-downward")) { + spanId.addClass("caret-upward"); + spanId.removeClass("caret-downward"); + } else { + spanId.addClass("caret-downward"); + spanId.removeClass("caret-upward"); + } +} function createStatusBlock() { - var avgMemoryUsage = $( "div#avgMemoryUsage" ).data( "value" ); - var avgHeapUsageGauge = $( "div#avgHeapUsage" ).data( "value" ); - var avgOffHeapUsageGauge = $( "div#avgOffHeapUsage" ).data( "value" ); - var avgJVMHeapUsageGauge = $( "div#avgJvmHeapUsage" ).data( "value" ); + var cpuUsage = $( "div#cpuUsage" ).data( "value" ); + var memoryUsage = $( "div#memoryUsage" ).data( "value" ); + // var heapUsageGauge = $( "div#heapUsage" ).data( "value" ); + // var offHeapUsageGauge = $( "div#offHeapUsage" ).data( "value" ); + var jvmHeapUsageGauge = $( "div#jvmHeapUsage" ).data( "value" ); var config = liquidFillGaugeDefaultSettings(); config.circleThickness = 0.15; config.circleColor = "#3EC0FF"; config.textColor = "#3EC0FF"; - config.waveTextColor = "#3EC0FF"; + config.waveTextColor = "#00B0FF"; config.waveColor = "#A0DFFF"; config.textVertPosition = 0.8; config.waveAnimateTime = 1000; @@ -23,19 +37,12 @@ function createStatusBlock() { config.textSize = 0.75; config.waveCount = 2; - var memoryGauge = loadLiquidFillGauge("memoryUsageGauge", avgMemoryUsage, config); - var heapGauge = loadLiquidFillGauge("heapUsageGauge", avgHeapUsageGauge, config); - var offHeapGauge = loadLiquidFillGauge("offHeapUsageGauge", avgOffHeapUsageGauge, config); - var jvmGauge = loadLiquidFillGauge("jvmHeapUsageGauge", avgJVMHeapUsageGauge, config); - + var cpuGauge = loadLiquidFillGauge("cpuUsageGauge", cpuUsage, config); + var memoryGauge = loadLiquidFillGauge("memoryUsageGauge", memoryUsage, config); + // var heapGauge = loadLiquidFillGauge("heapUsageGauge", heapUsageGauge, config); + // var offHeapGauge = loadLiquidFillGauge("offHeapUsageGauge", offHeapUsageGauge, config); + var jvmGauge = loadLiquidFillGauge("jvmHeapUsageGauge", jvmHeapUsageGauge, config); - /* function NewValue(){ - if(Math.random() > .5){ - return Math.round(Math.random()*100); - } else { - return (Math.random()*100).toFixed(1); - } - } */ } $(document).ready(function() { From a7d038b8be816d19ada67aa74613047d6dd5db96 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sun, 2 Jul 2017 13:44:15 -0700 Subject: [PATCH 071/142] [SNAPPYDATA] reduce a byte copy reading from ColumnVector When creating a UTF8String from a dictionary item from ColumnVector, avoid a copy by creating it over the range of bytes directly. Conflicts: sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVector.java Conflicts: sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java --- .../org/apache/spark/unsafe/types/UTF8String.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index b45022fc728bd..1e48fd3f00d30 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -106,6 +106,17 @@ public static UTF8String fromAddress(Object base, long offset, int numBytes) { return new UTF8String(base, offset, numBytes); } + public static UTF8String fromBuffer(ByteBuffer buffer) { + if (buffer.isDirect()) { + sun.nio.ch.DirectBuffer directBuffer = (sun.nio.ch.DirectBuffer)buffer; + return fromAddress(null, directBuffer.address() + buffer.position(), + buffer.remaining()); + } else { + return fromBytes(buffer.array(), buffer.arrayOffset() + buffer.position(), + buffer.remaining()); + } + } + /** * Creates an UTF8String from String. */ From 33d5a2a823694fded89257bae64c66344ac91a6d Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Mon, 3 Jul 2017 14:57:33 -0700 Subject: [PATCH 072/142] [SNAPPYDATA] moved UTF8String.fromBuffer to Utils.stringFromBuffer This is done to maintain full compatibility with upstream spark-unsafe module. Conflicts: sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVector.java Conflicts: core/src/main/scala/org/apache/spark/util/Utils.scala sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java --- .../apache/spark/unsafe/types/UTF8String.java | 11 ----------- .../scala/org/apache/spark/util/Utils.scala | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index 1e48fd3f00d30..b45022fc728bd 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -106,17 +106,6 @@ public static UTF8String fromAddress(Object base, long offset, int numBytes) { return new UTF8String(base, offset, numBytes); } - public static UTF8String fromBuffer(ByteBuffer buffer) { - if (buffer.isDirect()) { - sun.nio.ch.DirectBuffer directBuffer = (sun.nio.ch.DirectBuffer)buffer; - return fromAddress(null, directBuffer.address() + buffer.position(), - buffer.remaining()); - } else { - return fromBytes(buffer.array(), buffer.arrayOffset() + buffer.position(), - buffer.remaining()); - } - } - /** * Creates an UTF8String from String. */ diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 32574f8147f54..9b2bfd750ca82 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You @@ -83,6 +83,7 @@ import org.apache.spark.network.util.JavaUtils import org.apache.spark.serializer.{DeserializationStream, SerializationStream, SerializerInstance} import org.apache.spark.util.logging.RollingFileAppender import org.apache.spark.storage.StorageUtils +import org.apache.spark.unsafe.types.UTF8String; /** CallSite represents a place in user code. It can have a short and a long form. */ private[spark] case class CallSite(shortForm: String, longForm: String) @@ -2881,6 +2882,20 @@ private[spark] object Utils extends Logging { s"k8s://$resolvedURL" } + + /** + * Creates a UTF8String from given ByteBuffer using its position and length. + */ + def stringFromBuffer(buffer: ByteBuffer): UTF8String = { + if (buffer.isDirect) { + val directBuffer = buffer.asInstanceOf[sun.nio.ch.DirectBuffer] + UTF8String.fromAddress(null, directBuffer.address + buffer.position, + buffer.remaining()) + } else { + UTF8String.fromBytes(buffer.array, buffer.arrayOffset + buffer.position, + buffer.remaining()) + } + } } private[util] object CallerContext extends Logging { From 89c9fab7ed17ebaef4db5a4cb4da61968f1138f1 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 23 Feb 2017 17:18:03 +0530 Subject: [PATCH 073/142] [SNAPPYDATA] handle "prepare" in answer comparison inside Map types too Conflicts: sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala --- .../test/scala/org/apache/spark/sql/QueryTest.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala index 73949c682780b..aaaf4c3acb667 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala @@ -24,6 +24,9 @@ import scala.collection.JavaConverters._ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.execution.columnar.InMemoryRelation +import org.apache.spark.sql.execution.datasources.LogicalRelation +import org.apache.spark.sql.execution.streaming.MemoryPlan +import org.apache.spark.sql.types.{Decimal, Metadata, ObjectType} abstract class QueryTest extends PlanTest { @@ -287,15 +290,18 @@ object QueryTest { // We need to call prepareRow recursively to handle schemas with struct types. def prepareRow(row: Row): Row = { - Row.fromSeq(row.toSeq.map { + def prepareValue(v: Any): Any = v match { case null => null case d: java.math.BigDecimal => BigDecimal(d) + case d: Decimal => d.toBigDecimal // to use BigDecimal.compareTo == 0 case d: Double => math.floor(d * 1000.0 + 0.5) // round to three digits // Convert array to Seq for easy equality check. case b: Array[_] => b.toSeq case r: Row => prepareRow(r) + case m: Map[_, _] => m.mapValues(prepareValue) case o => o - }) + } + Row.fromSeq(row.toSeq.map(prepareValue)) } private def genError( From 83ae1ec0376b2bacbb9435aeb76439f74f1ee0f7 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 23 Feb 2017 17:19:13 +0530 Subject: [PATCH 074/142] [SNAPPYDATA] reverting changes to increase DECIMAL precision to 127 The changes to DECIMAL precision were incomplete and broken in more ways than one. The other reason being that future DECIMAL optimization for operations in generated code will depend on value to fit in two longs and there does not seem to be a practical use-case of having precision >38 (which is not supported by most mainstream databases either) Renamed UnsafeRow.todata to toData for consistency. Conflicts: sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java Conflicts: sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala --- .../sql/catalyst/expressions/UnsafeRow.java | 16 +++++++++++++ .../codegen/UnsafeArrayWriter.java | 19 --------------- .../apache/spark/sql/types/DecimalType.scala | 23 ++----------------- .../datasources/json/JsonInferSchema.scala | 22 ++---------------- .../parquet/ParquetSchemaConverter.scala | 21 +---------------- 5 files changed, 21 insertions(+), 80 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java index 71c086029cc5b..ee9a8de9c5107 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java @@ -685,4 +685,20 @@ public void read(Kryo kryo, Input in) { this.baseObject = new byte[sizeInBytes]; in.read((byte[]) baseObject); } + + public void toData(DataOutput out) throws IOException { + byte[] bytes = getBytes(); + out.writeInt(bytes.length); + out.writeInt(this.numFields); + out.write(bytes); + } + + public void fromData(DataInput in) throws IOException, ClassNotFoundException { + this.baseOffset = BYTE_ARRAY_OFFSET; + this.sizeInBytes = in.readInt(); + this.numFields = in.readInt(); + this.bitSetWidthInBytes = calculateBitSetWidthInBytes(numFields); + this.baseObject = new byte[sizeInBytes]; + in.readFully((byte[])baseObject); + } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java index c58e90cad50d9..d7f98e6ac12f3 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java @@ -14,24 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ package org.apache.spark.sql.catalyst.expressions.codegen; @@ -211,7 +193,6 @@ public void write(int ordinal, Decimal input, int precision, int scale) { // assert numBytes <= 16; int roundedSize = ByteArrayMethods.roundNumberOfBytesToNearestWord(numBytes); holder.grow(roundedSize); - zeroOutPaddingBytes(numBytes); // Write the bytes to the variable length portion. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala index 117158aa500a2..fbe24995dc51b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala @@ -14,24 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ package org.apache.spark.sql.types @@ -49,7 +31,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} * A Decimal that must have fixed precision (the maximum number of digits) and scale (the number * of digits on right side of dot). * - * The precision can be up to 127, scale can also be up to 127 (less or equal to precision). + * The precision can be up to 38, scale can also be up to 38 (less or equal to precision). * * The default precision and scale is (10, 0). * @@ -66,8 +48,7 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType { } if (precision > DecimalType.MAX_PRECISION) { - throw new AnalysisException( - s"DecimalType can only support precision up to ${DecimalType.MAX_PRECISION}") + throw new AnalysisException(s"DecimalType can only support precision up to 38") } // default constructor for Java diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala index f3f0d7046de06..a270a6451d5dd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonInferSchema.scala @@ -14,24 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ package org.apache.spark.sql.execution.datasources.json @@ -297,8 +279,8 @@ private[sql] object JsonInferSchema { case (t1: DecimalType, t2: DecimalType) => val scale = math.max(t1.scale, t2.scale) val range = math.max(t1.precision - t1.scale, t2.precision - t2.scale) - if (range + scale > DecimalType.MAX_PRECISION) { - // DecimalType can't support precision > DecimalType.MAX_PRECISION + if (range + scale > 38) { + // DecimalType can't support precision > 38 DoubleType } else { DecimalType(range + scale, scale) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala index 530ee180218a3..c61be077d309f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala @@ -14,24 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ package org.apache.spark.sql.execution.datasources.parquet @@ -612,8 +594,7 @@ private[sql] object ParquetSchemaConverter { } // Returns the minimum number of bytes needed to store a decimal with a given `precision`. - val minBytesForPrecision = Array.tabulate[Int](DecimalType.MAX_PRECISION + 1)( - computeMinBytesForPrecision) + val minBytesForPrecision = Array.tabulate[Int](39)(computeMinBytesForPrecision) // Max precision of a decimal value stored in `numBytes` bytes def maxPrecisionForBytes(numBytes: Int): Int = { From 747fe5ef3235611e4eba1963f1da735213ecc45e Mon Sep 17 00:00:00 2001 From: ymahajan Date: Thu, 17 Nov 2016 22:31:49 +0530 Subject: [PATCH 075/142] [SNAPPYDATA][MERGE-2.1] Some fixes after the merge - Fix for SnappyResourceEventsDUnitTest from Rishi - Scala style fixes from Sachin J - deleting unwanted files - reverting some changes that crept in inadvertently More code changes: - adding dependency for org.fusesource.leveldbjni, com.fasterxml.jackson.core, io.dropwizard.metrics, io.netty and org.apache.commons - fixing compilation issues after merge - adding dependency for jetty-client, jetty-proxy and mllib-local for graphx - bumped up parquetVersion and scalanlp breeze - fixed nettyAllVersion, removed hardcoded value - bumped up version - Implement Kryo.read/write for subclasses of Task - Do not implement KryoSerializable in Task - spark.sql.warehouse.dir moved to StaticSQLConf - moved VECTORIZED_AGG_MAP_MAX_COLUMNS from StaticSQLConf to SQLConf - corrected jackson-databind version Conflicts: core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala --- build.gradle | 6 +- common/network-common/build.gradle | 4 + common/network-shuffle/build.gradle | 1 + core/build.gradle | 5 + .../org/apache/spark/scheduler/Task.scala | 5 +- .../spark/scheduler/TaskResultGetter.scala | 4 +- .../spark/scheduler/TaskSetManager.scala | 47 +- .../spark/storage/memory/MemoryStore.scala | 4 +- .../scheduler/NotSerializableFakeTask.scala | 1 + .../spark/scheduler/TaskSetManagerSuite.scala | 3 + graphx/build.gradle | 1 + mllib/build.gradle | 2 +- .../plans/logical/basicLogicalOperators.scala | 22 +- .../spark/sql/types/AbstractDataType.scala | 5 - .../spark/sql/execution/SparkPlan.scala | 6 +- .../aggregate/HashAggregateExec.scala | 2 +- .../sql/execution/datasources/rules.scala | 8 +- .../streaming/CompactibleFileStreamLog.scala | 1 - .../apache/spark/sql/internal/SQLConf.scala | 982 ++++++++++++++++++ 19 files changed, 1051 insertions(+), 58 deletions(-) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala diff --git a/build.gradle b/build.gradle index 00424bcf2287c..4e24047752c95 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.0.3-2' + version = '2.1.0-2' ext { scalaBinaryVersion = '2.11' @@ -64,13 +64,13 @@ allprojects { chillVersion = '0.8.1' kryoVersion = '4.0.0' nettyVersion = '3.8.0.Final' - nettyAllVersion = '4.0.29.Final' + nettyAllVersion = '4.0.42.Final' derbyVersion = '10.12.1.1' httpClientVersion = '4.5.2' httpCoreVersion = '4.4.4' fasterXmlVersion = '2.6.5' snappyJavaVersion = '1.1.2.6' - parquetVersion = '1.7.0' + parquetVersion = '1.8.1' hiveParquetVersion = '1.6.0' metricsVersion = '3.1.2' thriftVersion = '0.9.3' diff --git a/common/network-common/build.gradle b/common/network-common/build.gradle index 63c23210f801e..62300cac099f5 100644 --- a/common/network-common/build.gradle +++ b/common/network-common/build.gradle @@ -23,4 +23,8 @@ dependencies { compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: '1.8' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.6.5' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.8.5' + compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.5' } diff --git a/common/network-shuffle/build.gradle b/common/network-shuffle/build.gradle index 0ffbc3414ad14..400a225bc8618 100644 --- a/common/network-shuffle/build.gradle +++ b/common/network-shuffle/build.gradle @@ -25,6 +25,7 @@ dependencies { compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: fasterXmlVersion compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.0' testCompile project(path: subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion, configuration: 'testOutput') } diff --git a/core/build.gradle b/core/build.gradle index ebeff567df64d..ec31c88b52e1a 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -96,10 +96,15 @@ dependencies { compile group: 'org.eclipse.jetty', name: 'jetty-servlets', version: jettyVersion compile group: 'org.eclipse.jetty', name: 'jetty-security', version: jettyVersion compile group: 'org.eclipse.jetty', name: 'jetty-continuation', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-client', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-proxy', version: jettyVersion compile group: 'javax.servlet', name: 'javax.servlet-api', version: javaxServletVersion compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.2' compile group: 'org.apache.commons', name: 'commons-math3', version: '3.4.1' compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version + compile group: 'org.apache.commons', name: 'commons-crypto', version: '1.0.0' + compile group: 'io.netty', name: 'netty', version: nettyVersion + compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion compile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion compile group: 'org.slf4j', name: 'jcl-over-slf4j', version: slf4jVersion compile group: 'com.ning', name: 'compress-lzf', version: '1.0.3' diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index ee5a65aa79219..4a9994b90fef5 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -23,7 +23,7 @@ import java.util.Properties import scala.collection.mutable import scala.collection.mutable.HashMap -import com.esotericsoftware.kryo.{KryoSerializable, Kryo} +import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.spark._ @@ -72,8 +72,7 @@ private[spark] abstract class Task[T]( SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array(), val jobId: Option[Int] = None, val appId: Option[String] = None, - val appAttemptId: Option[String] = None) extends Serializable - with KryoSerializable { + val appAttemptId: Option[String] = None) extends Serializable { final def stageId: Int = _stageId diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala index a284f7956cd31..fa5f6592ba92d 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala @@ -69,7 +69,7 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul // deserialize "value" without holding any lock so that it won't block other threads. // We should call it here, so that when it's called again in // "TaskSetManager.handleSuccessfulTask", it does not need to deserialize the value. - directResult.value(taskResultSerializer.get()) + directResult.value() (directResult, serializedData.limit()) case IndirectTaskResult(blockId, size) => if (!taskSetManager.canFetchMoreResults(size)) { @@ -91,7 +91,7 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul val deserializedResult = serializer.get().deserialize[DirectTaskResult[_]]( serializedTaskResult.get.toByteBuffer) // force deserialization of referenced value - deserializedResult.value(taskResultSerializer.get()) + deserializedResult.value() sparkEnv.blockManager.master.removeBlock(blockId) (deserializedResult, size) } diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 25029ec9f48b5..44f6245d27029 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -351,9 +351,7 @@ private[spark] class TaskSetManager( // Check for node-local tasks if (TaskLocality.isAllowed(locality, TaskLocality.NODE_LOCAL)) { - for (index <- speculatableTasks if canRunOnHost(index) && - // don't return executor-local tasks that are still alive - canRunOnExecutor(execId, index)) { + for (index <- speculatableTasks if canRunOnHost(index)) { val locations = tasks(index).preferredLocations.map(_.host) if (locations.contains(host)) { speculatableTasks -= index @@ -376,9 +374,7 @@ private[spark] class TaskSetManager( // Check for rack-local tasks if (TaskLocality.isAllowed(locality, TaskLocality.RACK_LOCAL)) { for (rack <- sched.getRackForHost(host)) { - for (index <- speculatableTasks if canRunOnHost(index) - // don't return executor-local tasks that are still alive - if canRunOnExecutor(execId, index)) { + for (index <- speculatableTasks if canRunOnHost(index)) { val racks = tasks(index).preferredLocations.map(_.host).flatMap(sched.getRackForHost) if (racks.contains(rack)) { speculatableTasks -= index @@ -390,9 +386,7 @@ private[spark] class TaskSetManager( // Check for non-local tasks if (TaskLocality.isAllowed(locality, TaskLocality.ANY)) { - for (index <- speculatableTasks if canRunOnHost(index) && - // don't return executor-local tasks that are still alive - canRunOnExecutor(execId, index)) { + for (index <- speculatableTasks if canRunOnHost(index)) { speculatableTasks -= index return Some((index, TaskLocality.ANY)) } @@ -402,34 +396,21 @@ private[spark] class TaskSetManager( None } - private def canRunOnExecutor(execId: String, taskId: Int): Boolean = { - val locations = tasks(taskId).preferredLocations - locations.isEmpty || locations.exists { - case e: ExecutorCacheTaskLocation => execId == e.executorId - case _ => false - } || locations.collectFirst { - case e: ExecutorCacheTaskLocation if sched.isExecutorAlive(e.executorId) - && !executorIsBlacklisted(e.executorId, taskId) => false - }.getOrElse(true) - } - /** - * Dequeue a pending task for a given node and return its index and locality level. - * Only search for tasks matching the given locality constraint. - * - * @return An option containing (task index within the task set, locality, is speculative?) - */ + * Dequeue a pending task for a given node and return its index and locality level. + * Only search for tasks matching the given locality constraint. + * + * @return An option containing (task index within the task set, locality, is speculative?) + */ private def dequeueTask(execId: String, host: String, maxLocality: TaskLocality.Value) - : Option[(Int, TaskLocality.Value, Boolean)] = + : Option[(Int, TaskLocality.Value, Boolean)] = { for (index <- dequeueTaskFromList(execId, host, getPendingTasksForExecutor(execId))) { return Some((index, TaskLocality.PROCESS_LOCAL, false)) } if (TaskLocality.isAllowed(maxLocality, TaskLocality.NODE_LOCAL)) { - for (index <- dequeueTaskFromList(execId, host, getPendingTasksForHost(host)) - // don't return executor-local tasks that are still alive - if canRunOnExecutor(execId, index)) { + for (index <- dequeueTaskFromList(execId, host, getPendingTasksForHost(host))) { return Some((index, TaskLocality.NODE_LOCAL, false)) } } @@ -445,17 +426,13 @@ private[spark] class TaskSetManager( for { rack <- sched.getRackForHost(host) index <- dequeueTaskFromList(execId, host, getPendingTasksForRack(rack)) - // don't return executor-local tasks that are still alive - if canRunOnExecutor(execId, index) } { - return Some((index, TaskLocality.RACK_LOCAL, false)) + return Some((index, TaskLocality.RACK_LOCAL, false)) } } if (TaskLocality.isAllowed(maxLocality, TaskLocality.ANY)) { - for (index <- dequeueTaskFromList(execId, host, allPendingTasks) - // don't return executor-local tasks that are still alive - if canRunOnExecutor(execId, index)) { + for (index <- dequeueTaskFromList(execId, host, allPendingTasks)) { return Some((index, TaskLocality.ANY, false)) } } diff --git a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala index 17f7a69ad6ba1..ace6ccb3d20a2 100644 --- a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala +++ b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala @@ -27,16 +27,16 @@ import scala.reflect.ClassTag import com.google.common.io.ByteStreams -import org.apache.spark.{SparkConf, TaskContext} import org.apache.spark.internal.Logging import org.apache.spark.internal.config.{UNROLL_MEMORY_CHECK_PERIOD, UNROLL_MEMORY_GROWTH_FACTOR} import org.apache.spark.memory.{MemoryManager, MemoryMode} import org.apache.spark.serializer.{SerializationStream, SerializerManager} import org.apache.spark.storage._ import org.apache.spark.unsafe.Platform -import org.apache.spark.util.{SizeEstimator, Utils} import org.apache.spark.util.collection.SizeTrackingVector import org.apache.spark.util.io.{ChunkedByteBuffer, ChunkedByteBufferOutputStream} +import org.apache.spark.util.{SizeEstimator, Utils} +import org.apache.spark.{SparkConf, TaskContext} private sealed trait MemoryEntry[T] { def size: Long diff --git a/core/src/test/scala/org/apache/spark/scheduler/NotSerializableFakeTask.scala b/core/src/test/scala/org/apache/spark/scheduler/NotSerializableFakeTask.scala index 255be6f46b06b..1af7b853261a8 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/NotSerializableFakeTask.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/NotSerializableFakeTask.scala @@ -39,4 +39,5 @@ private[spark] class NotSerializableFakeTask(myId: Int, stageId: Int) @throws(classOf[IOException]) private def readObject(in: ObjectInputStream): Unit = {} + } diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala index 92bc89722b872..a43839f9de33a 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala @@ -27,6 +27,9 @@ import org.mockito.Mockito.{mock, never, spy, times, verify, when} import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.io.{Output, Input} + import org.apache.spark._ import org.apache.spark.internal.Logging import org.apache.spark.internal.config diff --git a/graphx/build.gradle b/graphx/build.gradle index 64ee2e856d38c..fac71c09aafb5 100644 --- a/graphx/build.gradle +++ b/graphx/build.gradle @@ -19,6 +19,7 @@ description = 'Spark Project GraphX' dependencies { compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-mllib-local_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' diff --git a/mllib/build.gradle b/mllib/build.gradle index 0bcbd130afece..eb4c7dca1aa8e 100644 --- a/mllib/build.gradle +++ b/mllib/build.gradle @@ -25,7 +25,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.11.2') { + compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.12') { exclude(group: 'junit', module: 'junit') exclude(group: 'org.apache.commons', module: 'commons-math3') } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index 28232ab47525b..603f350533899 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -467,8 +467,28 @@ case class View( override def simpleString: String = { s"View (${desc.identifier}, ${output.mkString("[", ",", "]")})" } -} + lazy val expectedColumns = { + if (table.output.isEmpty) { + None + } else { + // Note: The parser (visitPartitionSpec in AstBuilder) already turns + // keys in partition to their lowercase forms. + val staticPartCols = partition.filter(_._2.isDefined).keySet + Some(table.output.filterNot(a => staticPartCols.contains(a.name))) + } + } + + assert(overwrite.enabled || !ifNotExists) + assert(partition.values.forall(_.nonEmpty) || !ifNotExists) + override lazy val resolved: Boolean = + childrenResolved && table.resolved && expectedColumns.forall { expected => + child.output.size == expected.size && child.output.zip(expected).forall { + case (childAttr, tableAttr) => + DataType.equalsIgnoreCompatibleNullability(childAttr.dataType, tableAttr.dataType) + } + } +} /** * A container for holding named common table expressions (CTEs) and a query plan. * This operator will be removed during analysis and the relations will be substituted into child. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala index f405f1ea2e128..95c8aee819fac 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala @@ -133,11 +133,6 @@ protected[sql] abstract class AtomicType extends DataType { private[sql] type InternalType private[sql] val tag: TypeTag[InternalType] private[sql] val ordering: Ordering[InternalType] - - @transient private[sql] lazy val classTag = ScalaReflectionLock.synchronized { - val mirror = runtimeMirror(Utils.getSparkClassLoader) - ClassTag[InternalType](mirror.runtimeClass(tag.tpe)) - } } object AtomicType { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala index b237960deddc6..41817bfa670ce 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala @@ -27,17 +27,17 @@ import org.codehaus.janino.InternalCompilerException import org.apache.spark.{broadcast, SparkEnv} import org.apache.spark.internal.Logging -import org.apache.spark.io.CompressionCodec import org.apache.spark.rdd.{RDD, RDDOperationScope} -import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen.{Predicate => GenPredicate, _} import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.physical._ +import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.util.ThreadUtils +import org.apache.spark.{SparkEnv, broadcast} /** * The base class for physical operators. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala index 23dfc8a4310a3..f138160a61f5d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala @@ -299,7 +299,7 @@ case class HashAggregateExec( private val bufferSchema = StructType.fromAttributes(aggregateBufferAttributes) // The name for Fast HashMap -b private var fastHashMapTerm: String = _ + private var fastHashMapTerm: String = _ private var isFastHashMapEnabled: Boolean = false // whether a vectorized hashmap is used instead diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala index 5cc21eeaeaa94..d37bc302725f2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala @@ -18,8 +18,12 @@ package org.apache.spark.sql.execution.datasources import java.util.Locale +import java.util.regex.Pattern + +import scala.util.control.NonFatal import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast, Expression, InputFileBlockLength, InputFileBlockStart, InputFileName, RowOrdering} @@ -30,6 +34,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.InsertableRelation import org.apache.spark.sql.types.{AtomicType, StructType} import org.apache.spark.sql.util.SchemaUtils +import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession} /** * Replaces [[UnresolvedRelation]]s if the plan is for direct query on files. @@ -317,7 +322,8 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi * table. It also does data type casting and field renaming, to make sure that the columns to be * inserted have the correct data type and fields have the correct names. */ -case class PreprocessTableInsertion(conf: SQLConf) extends Rule[LogicalPlan] { +case class +PreprocessTableInsertion(conf: SQLConf) extends Rule[LogicalPlan] { private def preprocess( insert: InsertIntoTable, tblName: String, diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala index 77bc0ba5548dd..f2969ca12eff9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala @@ -328,4 +328,3 @@ object CompactibleFileStreamLog { } } } - diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala new file mode 100644 index 0000000000000..f5ca04587932d --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -0,0 +1,982 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.internal + +import java.util.{NoSuchElementException, Properties} +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ +import scala.collection.immutable + +import org.apache.hadoop.fs.Path +import org.apache.parquet.hadoop.ParquetOutputCommitter + +import org.apache.spark.internal.Logging +import org.apache.spark.internal.config._ +import org.apache.spark.network.util.ByteUnit +import org.apache.spark.sql.catalyst.CatalystConf +import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol +import org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol +import org.apache.spark.util.Utils + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// This file defines the configuration options for Spark SQL. +//////////////////////////////////////////////////////////////////////////////////////////////////// + + +object SQLConf { + + private val sqlConfEntries = java.util.Collections.synchronizedMap( + new java.util.HashMap[String, ConfigEntry[_]]()) + + private[sql] def register(entry: ConfigEntry[_]): Unit = sqlConfEntries.synchronized { + require(!sqlConfEntries.containsKey(entry.key), + s"Duplicate SQLConfigEntry. ${entry.key} has been registered") + sqlConfEntries.put(entry.key, entry) + } + + private[sql] object SQLConfigBuilder { + + def apply(key: String): ConfigBuilder = new ConfigBuilder(key).onCreate(register) + + } + + + val OPTIMIZER_MAX_ITERATIONS = SQLConfigBuilder("spark.sql.optimizer.maxIterations") + .internal() + .doc("The max number of iterations the optimizer and analyzer runs.") + .intConf + .createWithDefault(100) + + val OPTIMIZER_INSET_CONVERSION_THRESHOLD = + SQLConfigBuilder("spark.sql.optimizer.inSetConversionThreshold") + .internal() + .doc("The threshold of set size for InSet conversion.") + .intConf + .createWithDefault(10) + + val COMPRESS_CACHED = SQLConfigBuilder("spark.sql.inMemoryColumnarStorage.compressed") + .internal() + .doc("When set to true Spark SQL will automatically select a compression codec for each " + + "column based on statistics of the data.") + .booleanConf + .createWithDefault(true) + + val COLUMN_BATCH_SIZE = SQLConfigBuilder("spark.sql.inMemoryColumnarStorage.batchSize") + .internal() + .doc("Controls the size of batches for columnar caching. Larger batch sizes can improve " + + "memory utilization and compression, but risk OOMs when caching data.") + .intConf + .createWithDefault(10000) + + val IN_MEMORY_PARTITION_PRUNING = + SQLConfigBuilder("spark.sql.inMemoryColumnarStorage.partitionPruning") + .internal() + .doc("When true, enable partition pruning for in-memory columnar tables.") + .booleanConf + .createWithDefault(true) + + val PREFER_SORTMERGEJOIN = SQLConfigBuilder("spark.sql.join.preferSortMergeJoin") + .internal() + .doc("When true, prefer sort merge join over shuffle hash join.") + .booleanConf + .createWithDefault(true) + + val RADIX_SORT_ENABLED = SQLConfigBuilder("spark.sql.sort.enableRadixSort") + .internal() + .doc("When true, enable use of radix sort when possible. Radix sort is much faster but " + + "requires additional memory to be reserved up-front. The memory overhead may be " + + "significant when sorting very small rows (up to 50% more in this case).") + .booleanConf + .createWithDefault(true) + + val AUTO_BROADCASTJOIN_THRESHOLD = SQLConfigBuilder("spark.sql.autoBroadcastJoinThreshold") + .doc("Configures the maximum size in bytes for a table that will be broadcast to all worker " + + "nodes when performing a join. By setting this value to -1 broadcasting can be disabled. " + + "Note that currently statistics are only supported for Hive Metastore tables where the " + + "command ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan has been " + + "run, and file-based data source tables where the statistics are computed directly on " + + "the files of data.") + .longConf + .createWithDefault(10L * 1024 * 1024) + + val LIMIT_SCALE_UP_FACTOR = SQLConfigBuilder("spark.sql.limit.scaleUpFactor") + .internal() + .doc("Minimal increase rate in number of partitions between attempts when executing a take " + + "on a query. Higher values lead to more partitions read. Lower values might lead to " + + "longer execution times as more jobs will be run") + .intConf + .createWithDefault(4) + + val ENABLE_FALL_BACK_TO_HDFS_FOR_STATS = + SQLConfigBuilder("spark.sql.statistics.fallBackToHdfs") + .doc("If the table statistics are not available from table metadata enable fall back to hdfs." + + " This is useful in determining if a table is small enough to use auto broadcast joins.") + .booleanConf + .createWithDefault(false) + + val DEFAULT_SIZE_IN_BYTES = SQLConfigBuilder("spark.sql.defaultSizeInBytes") + .internal() + .doc("The default table size used in query planning. By default, it is set to Long.MaxValue " + + "which is larger than `spark.sql.autoBroadcastJoinThreshold` to be more conservative. " + + "That is to say by default the optimizer will not choose to broadcast a table unless it " + + "knows for sure its size is small enough.") + .longConf + .createWithDefault(Long.MaxValue) + + val SHUFFLE_PARTITIONS = SQLConfigBuilder("spark.sql.shuffle.partitions") + .doc("The default number of partitions to use when shuffling data for joins or aggregations.") + .intConf + .createWithDefault(200) + + val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE = + SQLConfigBuilder("spark.sql.adaptive.shuffle.targetPostShuffleInputSize") + .doc("The target post-shuffle input size in bytes of a task.") + .bytesConf(ByteUnit.BYTE) + .createWithDefault(64 * 1024 * 1024) + + val ADAPTIVE_EXECUTION_ENABLED = SQLConfigBuilder("spark.sql.adaptive.enabled") + .doc("When true, enable adaptive query execution.") + .booleanConf + .createWithDefault(false) + + val SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS = + SQLConfigBuilder("spark.sql.adaptive.minNumPostShufflePartitions") + .internal() + .doc("The advisory minimal number of post-shuffle partitions provided to " + + "ExchangeCoordinator. This setting is used in our test to make sure we " + + "have enough parallelism to expose issues that will not be exposed with a " + + "single partition. When the value is a non-positive value, this setting will " + + "not be provided to ExchangeCoordinator.") + .intConf + .createWithDefault(-1) + + val SUBEXPRESSION_ELIMINATION_ENABLED = + SQLConfigBuilder("spark.sql.subexpressionElimination.enabled") + .internal() + .doc("When true, common subexpressions will be eliminated.") + .booleanConf + .createWithDefault(true) + + val CASE_SENSITIVE = SQLConfigBuilder("spark.sql.caseSensitive") + .internal() + .doc("Whether the query analyzer should be case sensitive or not. " + + "Default to case insensitive. It is highly discouraged to turn on case sensitive mode.") + .booleanConf + .createWithDefault(false) + + val PARQUET_SCHEMA_MERGING_ENABLED = SQLConfigBuilder("spark.sql.parquet.mergeSchema") + .doc("When true, the Parquet data source merges schemas collected from all data files, " + + "otherwise the schema is picked from the summary file or a random data file " + + "if no summary file is available.") + .booleanConf + .createWithDefault(false) + + val PARQUET_SCHEMA_RESPECT_SUMMARIES = SQLConfigBuilder("spark.sql.parquet.respectSummaryFiles") + .doc("When true, we make assumption that all part-files of Parquet are consistent with " + + "summary files and we will ignore them when merging schema. Otherwise, if this is " + + "false, which is the default, we will merge all part-files. This should be considered " + + "as expert-only option, and shouldn't be enabled before knowing what it means exactly.") + .booleanConf + .createWithDefault(false) + + val PARQUET_BINARY_AS_STRING = SQLConfigBuilder("spark.sql.parquet.binaryAsString") + .doc("Some other Parquet-producing systems, in particular Impala and older versions of " + + "Spark SQL, do not differentiate between binary data and strings when writing out the " + + "Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide " + + "compatibility with these systems.") + .booleanConf + .createWithDefault(false) + + val PARQUET_INT96_AS_TIMESTAMP = SQLConfigBuilder("spark.sql.parquet.int96AsTimestamp") + .doc("Some Parquet-producing systems, in particular Impala, store Timestamp into INT96. " + + "Spark would also store Timestamp as INT96 because we need to avoid precision lost of the " + + "nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to " + + "provide compatibility with these systems.") + .booleanConf + .createWithDefault(true) + + val PARQUET_CACHE_METADATA = SQLConfigBuilder("spark.sql.parquet.cacheMetadata") + .doc("Turns on caching of Parquet schema metadata. Can speed up querying of static data.") + .booleanConf + .createWithDefault(true) + + val PARQUET_COMPRESSION = SQLConfigBuilder("spark.sql.parquet.compression.codec") + .doc("Sets the compression codec use when writing Parquet files. Acceptable values include: " + + "uncompressed, snappy, gzip, lzo.") + .stringConf + .transform(_.toLowerCase()) + .checkValues(Set("uncompressed", "snappy", "gzip", "lzo")) + .createWithDefault("snappy") + + val PARQUET_FILTER_PUSHDOWN_ENABLED = SQLConfigBuilder("spark.sql.parquet.filterPushdown") + .doc("Enables Parquet filter push-down optimization when set to true.") + .booleanConf + .createWithDefault(true) + + val PARQUET_WRITE_LEGACY_FORMAT = SQLConfigBuilder("spark.sql.parquet.writeLegacyFormat") + .doc("Whether to follow Parquet's format specification when converting Parquet schema to " + + "Spark SQL schema and vice versa.") + .booleanConf + .createWithDefault(false) + + val PARQUET_OUTPUT_COMMITTER_CLASS = SQLConfigBuilder("spark.sql.parquet.output.committer.class") + .doc("The output committer class used by Parquet. The specified class needs to be a " + + "subclass of org.apache.hadoop.mapreduce.OutputCommitter. Typically, it's also a subclass " + + "of org.apache.parquet.hadoop.ParquetOutputCommitter.") + .internal() + .stringConf + .createWithDefault(classOf[ParquetOutputCommitter].getName) + + val PARQUET_VECTORIZED_READER_ENABLED = + SQLConfigBuilder("spark.sql.parquet.enableVectorizedReader") + .doc("Enables vectorized parquet decoding.") + .booleanConf + .createWithDefault(true) + + val ORC_FILTER_PUSHDOWN_ENABLED = SQLConfigBuilder("spark.sql.orc.filterPushdown") + .doc("When true, enable filter pushdown for ORC files.") + .booleanConf + .createWithDefault(false) + + val HIVE_VERIFY_PARTITION_PATH = SQLConfigBuilder("spark.sql.hive.verifyPartitionPath") + .doc("When true, check all the partition paths under the table\'s root directory " + + "when reading data stored in HDFS.") + .booleanConf + .createWithDefault(false) + + val HIVE_METASTORE_PARTITION_PRUNING = + SQLConfigBuilder("spark.sql.hive.metastorePartitionPruning") + .doc("When true, some predicates will be pushed down into the Hive metastore so that " + + "unmatching partitions can be eliminated earlier. This only affects Hive tables " + + "not converted to filesource relations (see HiveUtils.CONVERT_METASTORE_PARQUET and " + + "HiveUtils.CONVERT_METASTORE_ORC for more information).") + .booleanConf + .createWithDefault(true) + + val HIVE_MANAGE_FILESOURCE_PARTITIONS = + SQLConfigBuilder("spark.sql.hive.manageFilesourcePartitions") + .doc("When true, enable metastore partition management for file source tables as well. " + + "This includes both datasource and converted Hive tables. When partition managment " + + "is enabled, datasource tables store partition in the Hive metastore, and use the " + + "metastore to prune partitions during query planning.") + .booleanConf + .createWithDefault(true) + + val HIVE_FILESOURCE_PARTITION_FILE_CACHE_SIZE = + SQLConfigBuilder("spark.sql.hive.filesourcePartitionFileCacheSize") + .doc("When nonzero, enable caching of partition file metadata in memory. All tables share " + + "a cache that can use up to specified num bytes for file metadata. This conf only " + + "has an effect when hive filesource partition management is enabled.") + .longConf + .createWithDefault(250 * 1024 * 1024) + + val OPTIMIZER_METADATA_ONLY = SQLConfigBuilder("spark.sql.optimizer.metadataOnly") + .doc("When true, enable the metadata-only query optimization that use the table's metadata " + + "to produce the partition columns instead of table scans. It applies when all the columns " + + "scanned are partition columns and the query has an aggregate operator that satisfies " + + "distinct semantics.") + .booleanConf + .createWithDefault(true) + + val COLUMN_NAME_OF_CORRUPT_RECORD = SQLConfigBuilder("spark.sql.columnNameOfCorruptRecord") + .doc("The name of internal column for storing raw/un-parsed JSON records that fail to parse.") + .stringConf + .createWithDefault("_corrupt_record") + + val BROADCAST_TIMEOUT = SQLConfigBuilder("spark.sql.broadcastTimeout") + .doc("Timeout in seconds for the broadcast wait time in broadcast joins.") + .intConf + .createWithDefault(5 * 60) + + // This is only used for the thriftserver + val THRIFTSERVER_POOL = SQLConfigBuilder("spark.sql.thriftserver.scheduler.pool") + .doc("Set a Fair Scheduler pool for a JDBC client session.") + .stringConf + .createOptional + + val THRIFTSERVER_UI_STATEMENT_LIMIT = + SQLConfigBuilder("spark.sql.thriftserver.ui.retainedStatements") + .doc("The number of SQL statements kept in the JDBC/ODBC web UI history.") + .intConf + .createWithDefault(200) + + val THRIFTSERVER_UI_SESSION_LIMIT = SQLConfigBuilder("spark.sql.thriftserver.ui.retainedSessions") + .doc("The number of SQL client sessions kept in the JDBC/ODBC web UI history.") + .intConf + .createWithDefault(200) + + // This is used to set the default data source + val DEFAULT_DATA_SOURCE_NAME = SQLConfigBuilder("spark.sql.sources.default") + .doc("The default data source to use in input/output.") + .stringConf + .createWithDefault("parquet") + + val CONVERT_CTAS = SQLConfigBuilder("spark.sql.hive.convertCTAS") + .internal() + .doc("When true, a table created by a Hive CTAS statement (no USING clause) " + + "without specifying any storage property will be converted to a data source table, " + + "using the data source set by spark.sql.sources.default.") + .booleanConf + .createWithDefault(false) + + val GATHER_FASTSTAT = SQLConfigBuilder("spark.sql.hive.gatherFastStats") + .internal() + .doc("When true, fast stats (number of files and total size of all files) will be gathered" + + " in parallel while repairing table partitions to avoid the sequential listing in Hive" + + " metastore.") + .booleanConf + .createWithDefault(true) + + val PARTITION_COLUMN_TYPE_INFERENCE = + SQLConfigBuilder("spark.sql.sources.partitionColumnTypeInference.enabled") + .doc("When true, automatically infer the data types for partitioned columns.") + .booleanConf + .createWithDefault(true) + + val BUCKETING_ENABLED = SQLConfigBuilder("spark.sql.sources.bucketing.enabled") + .doc("When false, we will treat bucketed table as normal table") + .booleanConf + .createWithDefault(true) + + val CROSS_JOINS_ENABLED = SQLConfigBuilder("spark.sql.crossJoin.enabled") + .doc("When false, we will throw an error if a query contains a cartesian product without " + + "explicit CROSS JOIN syntax.") + .booleanConf + .createWithDefault(false) + + val ORDER_BY_ORDINAL = SQLConfigBuilder("spark.sql.orderByOrdinal") + .doc("When true, the ordinal numbers are treated as the position in the select list. " + + "When false, the ordinal numbers in order/sort by clause are ignored.") + .booleanConf + .createWithDefault(true) + + val GROUP_BY_ORDINAL = SQLConfigBuilder("spark.sql.groupByOrdinal") + .doc("When true, the ordinal numbers in group by clauses are treated as the position " + + "in the select list. When false, the ordinal numbers are ignored.") + .booleanConf + .createWithDefault(true) + + // The output committer class used by data sources. The specified class needs to be a + // subclass of org.apache.hadoop.mapreduce.OutputCommitter. + val OUTPUT_COMMITTER_CLASS = + SQLConfigBuilder("spark.sql.sources.outputCommitterClass").internal().stringConf.createOptional + + val FILE_COMMIT_PROTOCOL_CLASS = + SQLConfigBuilder("spark.sql.sources.commitProtocolClass") + .internal() + .stringConf + .createWithDefault(classOf[SQLHadoopMapReduceCommitProtocol].getName) + + val PARALLEL_PARTITION_DISCOVERY_THRESHOLD = + SQLConfigBuilder("spark.sql.sources.parallelPartitionDiscovery.threshold") + .doc("The maximum number of files allowed for listing files at driver side. If the number " + + "of detected files exceeds this value during partition discovery, it tries to list the " + + "files with another Spark distributed job. This applies to Parquet, ORC, CSV, JSON and " + + "LibSVM data sources.") + .intConf + .createWithDefault(32) + + // Whether to automatically resolve ambiguity in join conditions for self-joins. + // See SPARK-6231. + val DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY = + SQLConfigBuilder("spark.sql.selfJoinAutoResolveAmbiguity") + .internal() + .booleanConf + .createWithDefault(true) + + // Whether to retain group by columns or not in GroupedData.agg. + val DATAFRAME_RETAIN_GROUP_COLUMNS = SQLConfigBuilder("spark.sql.retainGroupColumns") + .internal() + .booleanConf + .createWithDefault(true) + + val DATAFRAME_PIVOT_MAX_VALUES = SQLConfigBuilder("spark.sql.pivotMaxValues") + .doc("When doing a pivot without specifying values for the pivot column this is the maximum " + + "number of (distinct) values that will be collected without error.") + .intConf + .createWithDefault(10000) + + val RUN_SQL_ON_FILES = SQLConfigBuilder("spark.sql.runSQLOnFiles") + .internal() + .doc("When true, we could use `datasource`.`path` as table in SQL query.") + .booleanConf + .createWithDefault(true) + + val WHOLESTAGE_CODEGEN_ENABLED = SQLConfigBuilder("spark.sql.codegen.wholeStage") + .internal() + .doc("When true, the whole stage (of multiple operators) will be compiled into single java" + + " method.") + .booleanConf + .createWithDefault(true) + + val WHOLESTAGE_MAX_NUM_FIELDS = SQLConfigBuilder("spark.sql.codegen.maxFields") + .internal() + .doc("The maximum number of fields (including nested fields) that will be supported before" + + " deactivating whole-stage codegen.") + .intConf + .createWithDefault(100) + + val WHOLESTAGE_FALLBACK = SQLConfigBuilder("spark.sql.codegen.fallback") + .internal() + .doc("When true, whole stage codegen could be temporary disabled for the part of query that" + + " fail to compile generated code") + .booleanConf + .createWithDefault(true) + + val MAX_CASES_BRANCHES = SQLConfigBuilder("spark.sql.codegen.maxCaseBranches") + .internal() + .doc("The maximum number of switches supported with codegen.") + .intConf + .createWithDefault(20) + + val FILES_MAX_PARTITION_BYTES = SQLConfigBuilder("spark.sql.files.maxPartitionBytes") + .doc("The maximum number of bytes to pack into a single partition when reading files.") + .longConf + .createWithDefault(128 * 1024 * 1024) // parquet.block.size + + val FILES_OPEN_COST_IN_BYTES = SQLConfigBuilder("spark.sql.files.openCostInBytes") + .internal() + .doc("The estimated cost to open a file, measured by the number of bytes could be scanned in" + + " the same time. This is used when putting multiple files into a partition. It's better to" + + " over estimated, then the partitions with small files will be faster than partitions with" + + " bigger files (which is scheduled first).") + .longConf + .createWithDefault(4 * 1024 * 1024) + + val EXCHANGE_REUSE_ENABLED = SQLConfigBuilder("spark.sql.exchange.reuse") + .internal() + .doc("When true, the planner will try to find out duplicated exchanges and re-use them.") + .booleanConf + .createWithDefault(true) + + val STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT = + SQLConfigBuilder("spark.sql.streaming.stateStore.minDeltasForSnapshot") + .internal() + .doc("Minimum number of state store delta files that needs to be generated before they " + + "consolidated into snapshots.") + .intConf + .createWithDefault(10) + + val CHECKPOINT_LOCATION = SQLConfigBuilder("spark.sql.streaming.checkpointLocation") + .doc("The default location for storing checkpoint data for streaming queries.") + .stringConf + .createOptional + + val MIN_BATCHES_TO_RETAIN = SQLConfigBuilder("spark.sql.streaming.minBatchesToRetain") + .internal() + .doc("The minimum number of batches that must be retained and made recoverable.") + .intConf + .createWithDefault(100) + + val UNSUPPORTED_OPERATION_CHECK_ENABLED = + SQLConfigBuilder("spark.sql.streaming.unsupportedOperationCheck") + .internal() + .doc("When true, the logical plan for streaming query will be checked for unsupported" + + " operations.") + .booleanConf + .createWithDefault(true) + + val VARIABLE_SUBSTITUTE_ENABLED = + SQLConfigBuilder("spark.sql.variable.substitute") + .doc("This enables substitution using syntax like ${var} ${system:var} and ${env:var}.") + .booleanConf + .createWithDefault(true) + + val VARIABLE_SUBSTITUTE_DEPTH = + SQLConfigBuilder("spark.sql.variable.substitute.depth") + .internal() + .doc("Deprecated: The maximum replacements the substitution engine will do.") + .intConf + .createWithDefault(40) + + val VECTORIZED_AGG_MAP_MAX_COLUMNS = + SQLConfigBuilder("spark.sql.codegen.aggregate.map.columns.max") + .internal() + .doc("Sets the maximum width of schema (aggregate keys + values) for " + + "which aggregate with" + + "keys uses an in-memory columnar map to speed up execution. " + + "Setting this to 0 effectively" + + "disables the columnar map") + .intConf + .createWithDefault(3) + + val ENABLE_TWOLEVEL_AGG_MAP = + SQLConfigBuilder("spark.sql.codegen.aggregate.map.twolevel.enable") + .internal() + .doc("Enable two-level aggregate hash map. When enabled, records will first be " + + "inserted/looked-up at a 1st-level, small, fast map, and then fallback to a " + + "2nd-level, larger, slower map when 1st level is full or keys cannot be found. " + + "When disabled, records go directly to the 2nd level. Defaults to true.") + .booleanConf + .createWithDefault(true) + + val STREAMING_FILE_COMMIT_PROTOCOL_CLASS = + SQLConfigBuilder("spark.sql.streaming.commitProtocolClass") + .internal() + .stringConf + .createWithDefault(classOf[ManifestFileCommitProtocol].getName) + + val FILE_SINK_LOG_DELETION = SQLConfigBuilder("spark.sql.streaming.fileSink.log.deletion") + .internal() + .doc("Whether to delete the expired log files in file stream sink.") + .booleanConf + .createWithDefault(true) + + val FILE_SINK_LOG_COMPACT_INTERVAL = + SQLConfigBuilder("spark.sql.streaming.fileSink.log.compactInterval") + .internal() + .doc("Number of log files after which all the previous files " + + "are compacted into the next log file.") + .intConf + .createWithDefault(10) + + val FILE_SINK_LOG_CLEANUP_DELAY = + SQLConfigBuilder("spark.sql.streaming.fileSink.log.cleanupDelay") + .internal() + .doc("How long that a file is guaranteed to be visible for all readers.") + .timeConf(TimeUnit.MILLISECONDS) + .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes + + val FILE_SOURCE_LOG_DELETION = SQLConfigBuilder("spark.sql.streaming.fileSource.log.deletion") + .internal() + .doc("Whether to delete the expired log files in file stream source.") + .booleanConf + .createWithDefault(true) + + val FILE_SOURCE_LOG_COMPACT_INTERVAL = + SQLConfigBuilder("spark.sql.streaming.fileSource.log.compactInterval") + .internal() + .doc("Number of log files after which all the previous files " + + "are compacted into the next log file.") + .intConf + .createWithDefault(10) + + val FILE_SOURCE_LOG_CLEANUP_DELAY = + SQLConfigBuilder("spark.sql.streaming.fileSource.log.cleanupDelay") + .internal() + .doc("How long in milliseconds a file is guaranteed to be visible for all readers.") + .timeConf(TimeUnit.MILLISECONDS) + .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes + + val STREAMING_SCHEMA_INFERENCE = + SQLConfigBuilder("spark.sql.streaming.schemaInference") + .internal() + .doc("Whether file-based streaming sources will infer its own schema") + .booleanConf + .createWithDefault(false) + + val STREAMING_POLLING_DELAY = + SQLConfigBuilder("spark.sql.streaming.pollingDelay") + .internal() + .doc("How long to delay polling new data when no data is available") + .timeConf(TimeUnit.MILLISECONDS) + .createWithDefault(10L) + + val STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL = + SQLConfigBuilder("spark.sql.streaming.noDataProgressEventInterval") + .internal() + .doc("How long to wait between two progress events when there is no data") + .timeConf(TimeUnit.MILLISECONDS) + .createWithDefault(10000L) + + val STREAMING_METRICS_ENABLED = + SQLConfigBuilder("spark.sql.streaming.metricsEnabled") + .doc("Whether Dropwizard/Codahale metrics will be reported for active streaming queries.") + .booleanConf + .createWithDefault(false) + + val STREAMING_PROGRESS_RETENTION = + SQLConfigBuilder("spark.sql.streaming.numRecentProgressUpdates") + .doc("The number of progress updates to retain for a streaming query") + .intConf + .createWithDefault(100) + + val NDV_MAX_ERROR = + SQLConfigBuilder("spark.sql.statistics.ndv.maxError") + .internal() + .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " + + "column level statistics.") + .doubleConf + .createWithDefault(0.05) + + val IGNORE_CORRUPT_FILES = SQLConfigBuilder("spark.sql.files.ignoreCorruptFiles") + .doc("Whether to ignore corrupt files. If true, the Spark jobs will continue to run when " + + "encountering corrupted or non-existing and contents that have been read will still be " + + "returned.") + .booleanConf + .createWithDefault(false) + + object Deprecated { + val MAPRED_REDUCE_TASKS = "mapred.reduce.tasks" + } +} + +/** + * A class that enables the setting and getting of mutable config parameters/hints. + * + * In the presence of a SQLContext, these can be set and queried by passing SET commands + * into Spark SQL's query functions (i.e. sql()). Otherwise, users of this class can + * modify the hints by programmatically calling the setters and getters of this class. + * + * SQLConf is thread-safe (internally synchronized, so safe to be used in multiple threads). + */ +private[sql] class SQLConf extends Serializable with CatalystConf with Logging { + import SQLConf._ + + /** Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap. */ + @transient protected[spark] val settings = java.util.Collections.synchronizedMap( + new java.util.HashMap[String, String]()) + + @transient private val reader = new ConfigReader(settings) + + /** ************************ Spark SQL Params/Hints ******************* */ + + def optimizerMaxIterations: Int = getConf(OPTIMIZER_MAX_ITERATIONS) + + def optimizerInSetConversionThreshold: Int = getConf(OPTIMIZER_INSET_CONVERSION_THRESHOLD) + + def stateStoreMinDeltasForSnapshot: Int = getConf(STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT) + + def checkpointLocation: Option[String] = getConf(CHECKPOINT_LOCATION) + + def isUnsupportedOperationCheckEnabled: Boolean = getConf(UNSUPPORTED_OPERATION_CHECK_ENABLED) + + def streamingFileCommitProtocolClass: String = getConf(STREAMING_FILE_COMMIT_PROTOCOL_CLASS) + + def fileSinkLogDeletion: Boolean = getConf(FILE_SINK_LOG_DELETION) + + def fileSinkLogCompactInterval: Int = getConf(FILE_SINK_LOG_COMPACT_INTERVAL) + + def fileSinkLogCleanupDelay: Long = getConf(FILE_SINK_LOG_CLEANUP_DELAY) + + def fileSourceLogDeletion: Boolean = getConf(FILE_SOURCE_LOG_DELETION) + + def fileSourceLogCompactInterval: Int = getConf(FILE_SOURCE_LOG_COMPACT_INTERVAL) + + def fileSourceLogCleanupDelay: Long = getConf(FILE_SOURCE_LOG_CLEANUP_DELAY) + + def streamingSchemaInference: Boolean = getConf(STREAMING_SCHEMA_INFERENCE) + + def streamingPollingDelay: Long = getConf(STREAMING_POLLING_DELAY) + + def streamingNoDataProgressEventInterval: Long = + getConf(STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL) + + def streamingMetricsEnabled: Boolean = getConf(STREAMING_METRICS_ENABLED) + + def streamingProgressRetention: Int = getConf(STREAMING_PROGRESS_RETENTION) + + def filesMaxPartitionBytes: Long = getConf(FILES_MAX_PARTITION_BYTES) + + def filesOpenCostInBytes: Long = getConf(FILES_OPEN_COST_IN_BYTES) + + def useCompression: Boolean = getConf(COMPRESS_CACHED) + + def parquetCompressionCodec: String = getConf(PARQUET_COMPRESSION) + + def parquetCacheMetadata: Boolean = getConf(PARQUET_CACHE_METADATA) + + def parquetVectorizedReaderEnabled: Boolean = getConf(PARQUET_VECTORIZED_READER_ENABLED) + + def columnBatchSize: Int = getConf(COLUMN_BATCH_SIZE) + + def numShufflePartitions: Int = getConf(SHUFFLE_PARTITIONS) + + def targetPostShuffleInputSize: Long = + getConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) + + def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED) + + def minNumPostShufflePartitions: Int = + getConf(SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS) + + def minBatchesToRetain: Int = getConf(MIN_BATCHES_TO_RETAIN) + + def parquetFilterPushDown: Boolean = getConf(PARQUET_FILTER_PUSHDOWN_ENABLED) + + def orcFilterPushDown: Boolean = getConf(ORC_FILTER_PUSHDOWN_ENABLED) + + def verifyPartitionPath: Boolean = getConf(HIVE_VERIFY_PARTITION_PATH) + + def metastorePartitionPruning: Boolean = getConf(HIVE_METASTORE_PARTITION_PRUNING) + + def manageFilesourcePartitions: Boolean = getConf(HIVE_MANAGE_FILESOURCE_PARTITIONS) + + def filesourcePartitionFileCacheSize: Long = getConf(HIVE_FILESOURCE_PARTITION_FILE_CACHE_SIZE) + + def gatherFastStats: Boolean = getConf(GATHER_FASTSTAT) + + def optimizerMetadataOnly: Boolean = getConf(OPTIMIZER_METADATA_ONLY) + + def wholeStageEnabled: Boolean = getConf(WHOLESTAGE_CODEGEN_ENABLED) + + def wholeStageMaxNumFields: Int = getConf(WHOLESTAGE_MAX_NUM_FIELDS) + + def wholeStageFallback: Boolean = getConf(WHOLESTAGE_FALLBACK) + + def maxCaseBranchesForCodegen: Int = getConf(MAX_CASES_BRANCHES) + + def exchangeReuseEnabled: Boolean = getConf(EXCHANGE_REUSE_ENABLED) + + def caseSensitiveAnalysis: Boolean = getConf(SQLConf.CASE_SENSITIVE) + + def subexpressionEliminationEnabled: Boolean = + getConf(SUBEXPRESSION_ELIMINATION_ENABLED) + + def autoBroadcastJoinThreshold: Long = getConf(AUTO_BROADCASTJOIN_THRESHOLD) + + def limitScaleUpFactor: Int = getConf(LIMIT_SCALE_UP_FACTOR) + + def fallBackToHdfsForStatsEnabled: Boolean = getConf(ENABLE_FALL_BACK_TO_HDFS_FOR_STATS) + + def preferSortMergeJoin: Boolean = getConf(PREFER_SORTMERGEJOIN) + + def enableRadixSort: Boolean = getConf(RADIX_SORT_ENABLED) + + def defaultSizeInBytes: Long = getConf(DEFAULT_SIZE_IN_BYTES) + + def isParquetSchemaMergingEnabled: Boolean = getConf(PARQUET_SCHEMA_MERGING_ENABLED) + + def isParquetSchemaRespectSummaries: Boolean = getConf(PARQUET_SCHEMA_RESPECT_SUMMARIES) + + def parquetOutputCommitterClass: String = getConf(PARQUET_OUTPUT_COMMITTER_CLASS) + + def isParquetBinaryAsString: Boolean = getConf(PARQUET_BINARY_AS_STRING) + + def isParquetINT96AsTimestamp: Boolean = getConf(PARQUET_INT96_AS_TIMESTAMP) + + def writeLegacyParquetFormat: Boolean = getConf(PARQUET_WRITE_LEGACY_FORMAT) + + def inMemoryPartitionPruning: Boolean = getConf(IN_MEMORY_PARTITION_PRUNING) + + def columnNameOfCorruptRecord: String = getConf(COLUMN_NAME_OF_CORRUPT_RECORD) + + def broadcastTimeout: Int = getConf(BROADCAST_TIMEOUT) + + def defaultDataSourceName: String = getConf(DEFAULT_DATA_SOURCE_NAME) + + def convertCTAS: Boolean = getConf(CONVERT_CTAS) + + def partitionColumnTypeInferenceEnabled: Boolean = + getConf(SQLConf.PARTITION_COLUMN_TYPE_INFERENCE) + + def fileCommitProtocolClass: String = getConf(SQLConf.FILE_COMMIT_PROTOCOL_CLASS) + + def parallelPartitionDiscoveryThreshold: Int = + getConf(SQLConf.PARALLEL_PARTITION_DISCOVERY_THRESHOLD) + + def bucketingEnabled: Boolean = getConf(SQLConf.BUCKETING_ENABLED) + + def dataFrameSelfJoinAutoResolveAmbiguity: Boolean = + getConf(DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY) + + def dataFrameRetainGroupColumns: Boolean = getConf(DATAFRAME_RETAIN_GROUP_COLUMNS) + + def dataFramePivotMaxValues: Int = getConf(DATAFRAME_PIVOT_MAX_VALUES) + + override def runSQLonFile: Boolean = getConf(RUN_SQL_ON_FILES) + + def enableTwoLevelAggMap: Boolean = getConf(ENABLE_TWOLEVEL_AGG_MAP) + + def variableSubstituteEnabled: Boolean = getConf(VARIABLE_SUBSTITUTE_ENABLED) + + def variableSubstituteDepth: Int = getConf(VARIABLE_SUBSTITUTE_DEPTH) + + def warehousePath: String = new Path(getConf(StaticSQLConf.WAREHOUSE_PATH)).toString + + def ignoreCorruptFiles: Boolean = getConf(IGNORE_CORRUPT_FILES) + + override def orderByOrdinal: Boolean = getConf(ORDER_BY_ORDINAL) + + override def groupByOrdinal: Boolean = getConf(GROUP_BY_ORDINAL) + + override def crossJoinEnabled: Boolean = getConf(SQLConf.CROSS_JOINS_ENABLED) + + def ndvMaxError: Double = getConf(NDV_MAX_ERROR) + /** ********************** SQLConf functionality methods ************ */ + + /** Set Spark SQL configuration properties. */ + def setConf(props: Properties): Unit = settings.synchronized { + props.asScala.foreach { case (k, v) => setConfString(k, v) } + } + + /** Set the given Spark SQL configuration property using a `string` value. */ + def setConfString(key: String, value: String): Unit = { + require(key != null, "key cannot be null") + require(value != null, s"value cannot be null for key: $key") + val entry = sqlConfEntries.get(key) + if (entry != null) { + // Only verify configs in the SQLConf object + entry.valueConverter(value) + } + setConfWithCheck(key, value) + } + + /** Set the given Spark SQL configuration property. */ + def setConf[T](entry: ConfigEntry[T], value: T): Unit = { + require(entry != null, "entry cannot be null") + require(value != null, s"value cannot be null for key: ${entry.key}") + require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") + setConfWithCheck(entry.key, entry.stringConverter(value)) + } + + /** Return the value of Spark SQL configuration property for the given key. */ + @throws[NoSuchElementException]("if key is not set") + def getConfString(key: String): String = { + Option(settings.get(key)). + orElse { + // Try to use the default value + Option(sqlConfEntries.get(key)).map(_.defaultValueString) + }. + getOrElse(throw new NoSuchElementException(key)) + } + + /** + * Return the value of Spark SQL configuration property for the given key. If the key is not set + * yet, return `defaultValue`. This is useful when `defaultValue` in ConfigEntry is not the + * desired one. + */ + def getConf[T](entry: ConfigEntry[T], defaultValue: T): T = { + require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") + Option(settings.get(entry.key)).map(entry.valueConverter).getOrElse(defaultValue) + } + + /** + * Return the value of Spark SQL configuration property for the given key. If the key is not set + * yet, return `defaultValue` in [[ConfigEntry]]. + */ + def getConf[T](entry: ConfigEntry[T]): T = { + require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") + entry.readFrom(reader) + } + + /** + * Return the value of an optional Spark SQL configuration property for the given key. If the key + * is not set yet, returns None. + */ + def getConf[T](entry: OptionalConfigEntry[T]): Option[T] = { + require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") + entry.readFrom(reader) + } + + /** + * Return the `string` value of Spark SQL configuration property for the given key. If the key is + * not set yet, return `defaultValue`. + */ + def getConfString(key: String, defaultValue: String): String = { + val entry = sqlConfEntries.get(key) + if (entry != null && defaultValue != "") { + // Only verify configs in the SQLConf object + entry.valueConverter(defaultValue) + } + Option(settings.get(key)).getOrElse(defaultValue) + } + + /** + * Return all the configuration properties that have been set (i.e. not the default). + * This creates a new copy of the config properties in the form of a Map. + */ + def getAllConfs: immutable.Map[String, String] = + settings.synchronized { settings.asScala.toMap } + + /** + * Return all the configuration definitions that have been defined in [[SQLConf]]. Each + * definition contains key, defaultValue and doc. + */ + def getAllDefinedConfs: Seq[(String, String, String)] = sqlConfEntries.synchronized { + sqlConfEntries.values.asScala.filter(_.isPublic).map { entry => + (entry.key, getConfString(entry.key, entry.defaultValueString), entry.doc) + }.toSeq + } + + /** + * Return whether a given key is set in this [[SQLConf]]. + */ + def contains(key: String): Boolean = { + settings.containsKey(key) + } + + private def setConfWithCheck(key: String, value: String): Unit = { + settings.put(key, value) + } + + def unsetConf(key: String): Unit = { + settings.remove(key) + } + + def unsetConf(entry: ConfigEntry[_]): Unit = { + settings.remove(entry.key) + } + + def clear(): Unit = { + settings.clear() + } +} + +/** + * Static SQL configuration is a cross-session, immutable Spark configuration. External users can + * see the static sql configs via `SparkSession.conf`, but can NOT set/unset them. + */ +object StaticSQLConf { + val globalConfKeys = java.util.Collections.synchronizedSet(new java.util.HashSet[String]()) + + private def buildConf(key: String): ConfigBuilder = { + ConfigBuilder(key).onCreate { entry => + globalConfKeys.add(entry.key) + SQLConf.register(entry) + } + } + + val WAREHOUSE_PATH = buildConf("spark.sql.warehouse.dir") + .doc("The default location for managed databases and tables.") + .stringConf + .createWithDefault(Utils.resolveURI("spark-warehouse").toString) + + val CATALOG_IMPLEMENTATION = buildConf("spark.sql.catalogImplementation") + .internal() + .stringConf + .checkValues(Set("hive", "in-memory")) + .createWithDefault("in-memory") + + val GLOBAL_TEMP_DATABASE = buildConf("spark.sql.globalTempDatabase") + .internal() + .stringConf + .createWithDefault("global_temp") + + // This is used to control when we will split a schema's JSON string to multiple pieces + // in order to fit the JSON string in metastore's table property (by default, the value has + // a length restriction of 4000 characters, so do not use a value larger than 4000 as the default + // value of this property). We will split the JSON string of a schema to its length exceeds the + // threshold. Note that, this conf is only read in HiveExternalCatalog which is cross-session, + // that's why this conf has to be a static SQL conf. + val SCHEMA_STRING_LENGTH_THRESHOLD = buildConf("spark.sql.sources.schemaStringLengthThreshold") + .doc("The maximum length allowed in a single cell when " + + "storing additional schema information in Hive's metastore.") + .internal() + .intConf + .createWithDefault(4000) + + // When enabling the debug, Spark SQL internal table properties are not filtered out; however, + // some related DDL commands (e.g., ANALYZE TABLE and CREATE TABLE LIKE) might not work properly. + val DEBUG_MODE = buildConf("spark.sql.debug") + .internal() + .doc("Only used for internal debugging. Not all functions are supported when it is enabled.") + .booleanConf + .createWithDefault(false) +} From 50e7c81e0517e2232cdaae37e0864e1c83bea633 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Sat, 18 Feb 2017 08:47:22 +0530 Subject: [PATCH 076/142] [SNAPPYDATA][MERGE-2.1] - Removed SimplifyCasts, RemoveDispensableExpressions - Fixed precheckin failuers - Fixed Task serialization issues - Serialize new TaskMetrics using Kryo serializer - Pass extraOptions in case of saveAsTable - removed debug statement - SnappySink for structured streaming query result Conflicts: core/src/main/scala/org/apache/spark/executor/Executor.scala core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala core/src/main/scala/org/apache/spark/scheduler/Task.scala core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala --- .../org/apache/spark/executor/Executor.scala | 3 +- .../apache/spark/executor/TaskMetrics.scala | 4 + .../apache/spark/scheduler/DAGScheduler.scala | 13 +- .../apache/spark/scheduler/ResultTask.scala | 2 +- .../org/apache/spark/scheduler/Task.scala | 155 ++++++++++-------- .../org/apache/spark/util/AccumulatorV2.scala | 3 +- .../sql/catalyst/optimizer/Optimizer.scala | 51 +----- .../sql/execution/streaming/SnappySink.scala | 8 + 8 files changed, 113 insertions(+), 126 deletions(-) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/SnappySink.scala diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index e724f824d15c4..ff205d4933bd0 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -334,7 +334,6 @@ private[spark] class Executor( taskDescription.serializedTask, Thread.currentThread.getContextClassLoader) task.localProperties = taskDescription.properties task.setTaskMemoryManager(taskMemoryManager) - // If this task has been killed before we deserialized it, let's quit now. Otherwise, // continue executing the task. val killReason = reasonIfKilled @@ -466,7 +465,7 @@ private[spark] class Executor( // Note: accumulator updates must be collected after TaskMetrics is updated val accumUpdates = task.collectAccumulatorUpdates() val directResult = new DirectTaskResult(value, accumUpdates, - Some(task.metrics.resultSerializationTimeMetric)) + Some(task.metrics.resultSerializationTimeMetric)) val serializedDirectResult = ser.serialize(directResult) val resultSize = serializedDirectResult.limit() diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala index 445ca611118cf..134c2706cfb40 100644 --- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala @@ -270,7 +270,9 @@ class TaskMetrics private[spark] () extends Serializable with KryoSerializable { override def write(kryo: Kryo, output: Output): Unit = { _executorDeserializeTime.write(kryo, output) + _executorDeserializeCpuTime.write(kryo, output) _executorRunTime.write(kryo, output) + _executorCpuTime.write(kryo, output) _resultSize.write(kryo, output) _jvmGCTime.write(kryo, output) _resultSerializationTime.write(kryo, output) @@ -288,7 +290,9 @@ class TaskMetrics private[spark] () extends Serializable with KryoSerializable { // read the TaskContext thread-local once val taskContext = TaskContext.get() _executorDeserializeTime.read(kryo, input, taskContext) + _executorDeserializeCpuTime.read(kryo, input, taskContext) _executorRunTime.read(kryo, input, taskContext) + _executorCpuTime.read(kryo, input, taskContext) _resultSize.read(kryo, input, taskContext) _jvmGCTime.read(kryo, input, taskContext) _resultSerializationTime.read(kryo, input, taskContext) diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 7e206626f35ef..ab5672a8ced9d 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -1018,7 +1018,6 @@ class DAGScheduler( try { // For ShuffleMapTask, serialize and broadcast (rdd, shuffleDep). // For ResultTask, serialize and broadcast (rdd, func). -<<<<<<< HEAD var taskBinaryBytes: Array[Byte] = null // taskBinaryBytes and partitions are both effected by the checkpoint status. We need // this synchronization in case another concurrent job is checkpointing this RDD, so we get a @@ -1074,9 +1073,9 @@ class DAGScheduler( val locs = taskIdToLocations(id) val part = partitions(id) stage.pendingPartitions += id - new ShuffleMapTask(stage.id, stage.latestInfo.attemptNumber, - taskBinary, part, locs, properties, serializedTaskMetrics, Option(jobId), - Option(sc.applicationId), sc.applicationAttemptId) + new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, taskData, + taskBinary, part, locs, stage.latestInfo.taskMetrics, properties, + Option(jobId), Option(sc.applicationId), Option(sc.applicationId)) } case stage: ResultStage => @@ -1084,9 +1083,9 @@ class DAGScheduler( val p: Int = stage.partitions(id) val part = partitions(p) val locs = taskIdToLocations(id) - new ResultTask(stage.id, stage.latestInfo.attemptNumber, - taskBinary, part, locs, id, properties, serializedTaskMetrics, - Option(jobId), Option(sc.applicationId), sc.applicationAttemptId) + new ResultTask(stage.id, stage.latestInfo.attemptId, taskData, + taskBinary, part, locs, id, properties, stage.latestInfo.taskMetrics, + Option(jobId), Option(sc.applicationId), Option(sc.applicationId)) } } } catch { diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index f94aad960e41a..73b4662a0c949 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -53,7 +53,7 @@ import org.apache.spark.rdd.RDD * @param jobId id of the job this task belongs to * @param appId id of the app this task belongs to * @param appAttemptId attempt id of the app this task belongs to - */ + */ private[spark] class ResultTask[T, U]( stageId: Int, stageAttemptId: Int, diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index 4a9994b90fef5..6ab0140fc0353 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -20,12 +20,8 @@ package org.apache.spark.scheduler import java.nio.ByteBuffer import java.util.Properties -import scala.collection.mutable -import scala.collection.mutable.HashMap - import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{Input, Output} - import org.apache.spark._ import org.apache.spark.broadcast.Broadcast import org.apache.spark.executor.TaskMetrics @@ -34,30 +30,31 @@ import org.apache.spark.memory.{MemoryMode, TaskMemoryManager} import org.apache.spark.metrics.MetricsSystem import org.apache.spark.util._ +import scala.collection.mutable +import scala.collection.mutable.HashMap + /** - * A unit of execution. We have two kinds of Task's in Spark: - * - * - [[org.apache.spark.scheduler.ShuffleMapTask]] - * - [[org.apache.spark.scheduler.ResultTask]] - * - * A Spark job consists of one or more stages. The very last stage in a job consists of multiple - * ResultTasks, while earlier stages consist of ShuffleMapTasks. A ResultTask executes the task - * and sends the task output back to the driver application. A ShuffleMapTask executes the task - * and divides the task output to multiple buckets (based on the task's partitioner). - * - * @param _stageId id of the stage this task belongs to - * @param _stageAttemptId attempt id of the stage this task belongs to - * @param _partitionId index of the number in the RDD - * @param _metrics a [[TaskMetrics]] that is created at driver side and sent to executor side. - * @param localProperties copy of thread-local properties set by the user on the driver side. - * @param serializedTaskMetrics a `TaskMetrics` that is created and serialized on the driver side - * and sent to executor side. - * - * The parameters below are optional: - * @param jobId id of the job this task belongs to - * @param appId id of the app this task belongs to - * @param appAttemptId attempt id of the app this task belongs to - */ + * A unit of execution. We have two kinds of Task's in Spark: + * + * - [[org.apache.spark.scheduler.ShuffleMapTask]] + * - [[org.apache.spark.scheduler.ResultTask]] + * + * A Spark job consists of one or more stages. The very last stage in a job consists of multiple + * ResultTasks, while earlier stages consist of ShuffleMapTasks. A ResultTask executes the task + * and sends the task output back to the driver application. A ShuffleMapTask executes the task + * and divides the task output to multiple buckets (based on the task's partitioner). + * + * @param _stageId id of the stage this task belongs to + * @param _stageAttemptId attempt id of the stage this task belongs to + * @param _partitionId index of the number in the RDD + * @param _metrics a [[TaskMetrics]] that is created at driver side and sent to executor side. + * @param localProperties copy of thread-local properties set by the user on the driver side. + * + * The parameters below are optional: + * @param _jobId id of the job this task belongs to + * @param _appId id of the app this task belongs to + * @param _appAttemptId attempt id of the app this task belongs to + */ private[spark] abstract class Task[T]( private var _stageId: Int, private var _stageAttemptId: Int, @@ -70,9 +67,9 @@ private[spark] abstract class Task[T]( // The default value is only used in tests. serializedTaskMetrics: Array[Byte] = SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array(), - val jobId: Option[Int] = None, - val appId: Option[String] = None, - val appAttemptId: Option[String] = None) extends Serializable { + private var _jobId: Option[Int] = None, + private var _appId: Option[String] = None, + private var _appAttemptId: Option[String] = None) extends Serializable { final def stageId: Int = _stageId @@ -83,24 +80,29 @@ private[spark] abstract class Task[T]( @transient lazy val metrics: TaskMetrics = SparkEnv.get.closureSerializer.newInstance().deserialize(ByteBuffer.wrap(serializedTaskMetrics)) + final def jobId: Int = _jobId.get + + final def appId: String = _appId.get + + final def appAttemptId: String = _appAttemptId.get + @transient private[spark] var taskDataBytes: Array[Byte] = _ protected final def getTaskBytes: Array[Byte] = { val bytes = taskDataBytes if ((bytes ne null) && bytes.length > 0) bytes else taskBinary.get.value } - /** - * Called by [[org.apache.spark.executor.Executor]] to run this task. - * - * @param taskAttemptId an identifier for this task attempt that is unique within a SparkContext. - * @param attemptNumber how many times this task has been attempted (0 for the first attempt) - * @return the result of the task along with updates of Accumulators. - */ + * Called by [[org.apache.spark.executor.Executor]] to run this task. + * + * @param taskAttemptId an identifier for this task attempt that is unique within a SparkContext. + * @param attemptNumber how many times this task has been attempted (0 for the first attempt) + * @return the result of the task along with updates of Accumulators. + */ final def run( - taskAttemptId: Long, - attemptNumber: Int, - metricsSystem: MetricsSystem): T = { + taskAttemptId: Long, + attemptNumber: Int, + metricsSystem: MetricsSystem): T = { SparkEnv.get.blockManager.registerTask(taskAttemptId) context = new TaskContextImpl( stageId, @@ -122,9 +124,9 @@ private[spark] abstract class Task[T]( new CallerContext( "TASK", SparkEnv.get.conf.get(APP_CALLER_CONTEXT), - appId, - appAttemptId, - jobId, + _appId, + _appAttemptId, + _jobId, Option(stageId), Option(stageAttemptId), Option(taskAttemptId), @@ -202,17 +204,18 @@ private[spark] abstract class Task[T]( * If defined, this task has been killed and this option contains the reason. */ def reasonIfKilled: Option[String] = Option(_reasonIfKilled) + def killed: Boolean = _killed /** - * Returns the amount of time spent deserializing the RDD and function to be run in nanos. - */ + * Returns the amount of time spent deserializing the RDD and function to be run. + */ def executorDeserializeTime: Long = _executorDeserializeTime def executorDeserializeCpuTime: Long = _executorDeserializeCpuTime /** - * Collect the latest values of accumulators used in this task. If the task failed, - * filter out the accumulators whose values should not be included on failures. - */ + * Collect the latest values of accumulators used in this task. If the task failed, + * filter out the accumulators whose values should not be included on failures. + */ def collectAccumulatorUpdates(taskFailed: Boolean = false): Seq[AccumulatorV2[_, _]] = { if (context != null) { // Note: internal accumulators representing task metrics always count failed values @@ -234,6 +237,7 @@ private[spark] abstract class Task[T]( def kill(interruptThread: Boolean, reason: String) { require(reason != null) _reasonIfKilled = reason + killed = true if (context != null) { context.markInterrupted(reason) } @@ -256,6 +260,9 @@ private[spark] abstract class Task[T]( kryo.writeClassAndObject(output, taskBinary.get) } _metrics.write(kryo, output) + output.writeInt(_jobId.get) + output.writeString(_appId.get) + output.writeString(_appAttemptId.get) } def readKryo(kryo: Kryo, input: Input): Unit = { @@ -270,30 +277,33 @@ private[spark] abstract class Task[T]( taskBinary = None } else { taskBinary = Some(kryo.readClassAndObject(input) - .asInstanceOf[Broadcast[Array[Byte]]]) + .asInstanceOf[Broadcast[Array[Byte]]]) } _metrics = new TaskMetrics _metrics.read(kryo, input) + _jobId = Some(input.readInt()) + _appId = Some(input.readString()) + _appAttemptId = Some(input.readString()) } } /** - * Handles transmission of tasks and their dependencies, because this can be slightly tricky. We - * need to send the list of JARs and files added to the SparkContext with each task to ensure that - * worker nodes find out about it, but we can't make it part of the Task because the user's code in - * the task might depend on one of the JARs. Thus we serialize each task as multiple objects, by - * first writing out its dependencies. - */ + * Handles transmission of tasks and their dependencies, because this can be slightly tricky. We + * need to send the list of JARs and files added to the SparkContext with each task to ensure that + * worker nodes find out about it, but we can't make it part of the Task because the user's code in + * the task might depend on one of the JARs. Thus we serialize each task as multiple objects, by + * first writing out its dependencies. + */ private[spark] object Task { /** - * Serialize a task and the current app dependencies (files and JARs added to the SparkContext) - */ + * Serialize a task and the current app dependencies (files and JARs added to the SparkContext) + */ def serializeWithDependencies( - task: Task[_], - currentFiles: mutable.Map[String, Long], - currentJars: mutable.Map[String, Long], - serializer: SerializerInstance) - : ByteBuffer = { + task: Task[_], + currentFiles: mutable.Map[String, Long], + currentJars: mutable.Map[String, Long], + serializer: SerializerInstance) + : ByteBuffer = { val out = new ByteBufferOutputStream(4096) val dataOut = new DataOutputStream(out) @@ -326,14 +336,14 @@ private[spark] object Task { } /** - * Deserialize the list of dependencies in a task serialized with serializeWithDependencies, - * and return the task itself as a serialized ByteBuffer. The caller can then update its - * ClassLoaders and deserialize the task. - * - * @return (taskFiles, taskJars, taskProps, taskBytes) - */ + * Deserialize the list of dependencies in a task serialized with serializeWithDependencies, + * and return the task itself as a serialized ByteBuffer. The caller can then update its + * ClassLoaders and deserialize the task. + * + * @return (taskFiles, taskJars, taskProps, taskBytes) + */ def deserializeWithDependencies(serializedTask: ByteBuffer) - : (HashMap[String, Long], HashMap[String, Long], Properties, ByteBuffer) = { + : (HashMap[String, Long], HashMap[String, Long], Properties, ByteBuffer) = { val in = new ByteBufferInputStream(serializedTask) val dataIn = new DataInputStream(in) @@ -364,7 +374,7 @@ private[spark] object Task { } private[spark] final class TaskData private(var compressedBytes: Array[Byte], - var uncompressedLen: Int, var reference: Int) extends Serializable { + var uncompressedLen: Int, var reference: Int) extends Serializable { def this(compressedBytes: Array[Byte], uncompressedLen: Int) = this(compressedBytes, uncompressedLen, TaskData.NO_REF) @@ -387,8 +397,8 @@ private[spark] final class TaskData private(var compressedBytes: Array[Byte], override def equals(obj: Any): Boolean = obj match { case d: TaskData => uncompressedLen == d.uncompressedLen && - reference == d.reference && - java.util.Arrays.equals(compressedBytes, d.compressedBytes) + reference == d.reference && + java.util.Arrays.equals(compressedBytes, d.compressedBytes) case _ => false } } @@ -430,3 +440,4 @@ private[spark] object TaskData { } } } + diff --git a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala index 42010a261c553..7f1626d706ca5 100644 --- a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala +++ b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala @@ -55,7 +55,8 @@ abstract class AccumulatorV2[IN, OUT] extends Serializable { if (this.metadata != null) { throw new IllegalStateException("Cannot register an Accumulator twice.") } - this.metadata = AccumulatorMetadata(AccumulatorContext.newId(), name, countFailedValues) + val id = AccumulatorContext.newId() + this.metadata = AccumulatorMetadata(id, name, countFailedValues) AccumulatorContext.register(this) sc.cleaner.foreach(_.registerAccumulatorForCleanup(this)) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index b0c13ac931329..dd6f5f509c57d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -17,7 +17,12 @@ package org.apache.spark.sql.catalyst.optimizer +import scala.annotation.tailrec +import scala.collection.immutable.HashSet import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer + +import org.apache.spark.api.java.function.FilterFunction import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis._ @@ -28,9 +33,12 @@ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules._ import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf} import org.apache.spark.sql.types._ import org.apache.spark.util.Utils +import scala.collection.mutable + /** * Abstract class all optimizers should inherit of, contains the standard batches (extending * Optimizers can override this. @@ -1071,49 +1079,6 @@ object PushPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper { } } -/** - * Removes [[Cast Casts]] that are unnecessary because the input is already the correct type. - */ -object SimplifyCasts extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case Cast(e, dataType) if e.dataType == dataType || - (e.dataType.getClass == dataType.getClass && - e.dataType.asNullable == dataType) => e - } -} - -/** - * Removes nodes that are not necessary. - */ -object RemoveDispensableExpressions extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case UnaryPositive(child) => child - case PromotePrecision(child) => child - } -} - -/** -======= - * Removes [[Cast Casts]] that are unnecessary because the input is already the correct type. - */ -object SimplifyCasts extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case Cast(e, dataType) if e.dataType == dataType || - (e.dataType.getClass == dataType.getClass && - e.dataType.asNullable == dataType) => e - } -} - -/** - * Removes nodes that are not necessary. - */ -object RemoveDispensableExpressions extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case UnaryPositive(child) => child - case PromotePrecision(child) => child - } -} - /** * Combines two adjacent [[Limit]] operators into one, merging the * expressions into one single expression. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/SnappySink.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/SnappySink.scala new file mode 100644 index 0000000000000..dc737004db23c --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/SnappySink.scala @@ -0,0 +1,8 @@ +package org.apache.spark.sql.execution.streaming + +/** + * Created by ymahajan on 10/4/17. + */ +class SnappySink { + +} From f5f47613d22c1b11773b4394181c4234958caad4 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Sat, 20 May 2017 20:53:24 -0700 Subject: [PATCH 077/142] [SNAPPYDATA][MERGE-2.1] removed struct streaming classes Conflicts: sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala --- .../apache/spark/sql/execution/streaming/SnappySink.scala | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/SnappySink.scala diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/SnappySink.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/SnappySink.scala deleted file mode 100644 index dc737004db23c..0000000000000 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/SnappySink.scala +++ /dev/null @@ -1,8 +0,0 @@ -package org.apache.spark.sql.execution.streaming - -/** - * Created by ymahajan on 10/4/17. - */ -class SnappySink { - -} From 17001ab807cdff67007cd5d4469c69331f8acc8b Mon Sep 17 00:00:00 2001 From: ymahajan Date: Thu, 22 Jun 2017 21:40:44 -0700 Subject: [PATCH 078/142] [SNAPPYDATA][MERGE-2.1] - Avoid splitExpressions for DynamicFoldableExpressions. This used to create a lot of codegen issues - Bump up the Hadoop version, to avoid issues in IDEA. - Modified AnalysisException to use getSimpleMessage Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4e24047752c95..e074b296e1c0c 100644 --- a/build.gradle +++ b/build.gradle @@ -50,7 +50,7 @@ allprojects { ext { scalaBinaryVersion = '2.11' scalaVersion = scalaBinaryVersion + '.8' - hadoopVersion = '2.7.2' + hadoopVersion = '2.7.3' protobufVersion = '2.6.1' jerseyVersion = '2.22.2' sunJerseyVersion = '1.19.1' From 6915b5aa849d689c652a0d2383cc9ac811ab5900 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Thu, 6 Jul 2017 16:15:55 -0700 Subject: [PATCH 079/142] [SNAPPYDATA][MERGE-2.1] - Handled Array[Decimal] type in ScalaReflection, fixes SNAP-1772 (SplitSnappyClusterDUnitTest#testComplexTypesForColumnTables_SNAP643) - Fixing scalaStyle issues - updated .gitignore; gitignore build-artifacts and .gradle Conflicts: core/src/main/scala/org/apache/spark/scheduler/Task.scala core/src/main/scala/org/apache/spark/storage/BlockManager.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala --- .gitignore | 1 + .../org/apache/spark/scheduler/Task.scala | 102 +++--- .../spark/scheduler/TaskSetManager.scala | 10 +- .../CoarseGrainedSchedulerBackend.scala | 4 +- .../apache/spark/storage/BlockManager.scala | 304 +++++++++--------- .../spark/storage/memory/MemoryStore.scala | 4 +- .../scala/org/apache/spark/util/Utils.scala | 4 +- .../spark/sql/catalyst/ScalaReflection.scala | 2 +- .../codegen/GenerateUnsafeProjection.scala | 1 - .../sql/catalyst/optimizer/Optimizer.scala | 3 +- .../spark/sql/execution/SparkPlan.scala | 6 +- .../sql/execution/WholeStageCodegenExec.scala | 3 +- .../sql/streaming/DataStreamWriter.scala | 5 +- 13 files changed, 227 insertions(+), 222 deletions(-) diff --git a/.gitignore b/.gitignore index ae8da5c979079..15fd113d1d77f 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ spark-*-bin-*.tgz spark-tests.log src_managed/ streaming-tests.log +target/ build-artifacts/ unit-tests.log work/ diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index 6ab0140fc0353..edc541cdc2841 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -20,8 +20,12 @@ package org.apache.spark.scheduler import java.nio.ByteBuffer import java.util.Properties +import scala.collection.mutable +import scala.collection.mutable.HashMap + import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark._ import org.apache.spark.broadcast.Broadcast import org.apache.spark.executor.TaskMetrics @@ -30,31 +34,27 @@ import org.apache.spark.memory.{MemoryMode, TaskMemoryManager} import org.apache.spark.metrics.MetricsSystem import org.apache.spark.util._ -import scala.collection.mutable -import scala.collection.mutable.HashMap - /** - * A unit of execution. We have two kinds of Task's in Spark: - * - * - [[org.apache.spark.scheduler.ShuffleMapTask]] - * - [[org.apache.spark.scheduler.ResultTask]] - * - * A Spark job consists of one or more stages. The very last stage in a job consists of multiple - * ResultTasks, while earlier stages consist of ShuffleMapTasks. A ResultTask executes the task - * and sends the task output back to the driver application. A ShuffleMapTask executes the task - * and divides the task output to multiple buckets (based on the task's partitioner). - * - * @param _stageId id of the stage this task belongs to - * @param _stageAttemptId attempt id of the stage this task belongs to - * @param _partitionId index of the number in the RDD - * @param _metrics a [[TaskMetrics]] that is created at driver side and sent to executor side. - * @param localProperties copy of thread-local properties set by the user on the driver side. - * - * The parameters below are optional: - * @param _jobId id of the job this task belongs to - * @param _appId id of the app this task belongs to - * @param _appAttemptId attempt id of the app this task belongs to - */ + * A unit of execution. We have two kinds of Task's in Spark: + * + * - [[org.apache.spark.scheduler.ShuffleMapTask]] + * - [[org.apache.spark.scheduler.ResultTask]] + * + * A Spark job consists of one or more stages. The very last stage in a job consists of multiple + * ResultTasks, while earlier stages consist of ShuffleMapTasks. A ResultTask executes the task + * and sends the task output back to the driver application. A ShuffleMapTask executes the task + * and divides the task output to multiple buckets (based on the task's partitioner). + * + * @param _stageId id of the stage this task belongs to + * @param _stageAttemptId attempt id of the stage this task belongs to + * @param _partitionId index of the number in the RDD + * @param _metrics a [[TaskMetrics]] that is created at driver side and sent to executor side. + * @param localProperties copy of thread-local properties set by the user on the driver side. + * The parameters below are optional: + * @param _jobId id of the job this task belongs to + * @param _appId id of the app this task belongs to + * @param _appAttemptId attempt id of the app this task belongs to + */ private[spark] abstract class Task[T]( private var _stageId: Int, private var _stageAttemptId: Int, @@ -92,13 +92,14 @@ private[spark] abstract class Task[T]( val bytes = taskDataBytes if ((bytes ne null) && bytes.length > 0) bytes else taskBinary.get.value } + /** - * Called by [[org.apache.spark.executor.Executor]] to run this task. - * - * @param taskAttemptId an identifier for this task attempt that is unique within a SparkContext. - * @param attemptNumber how many times this task has been attempted (0 for the first attempt) - * @return the result of the task along with updates of Accumulators. - */ + * Called by [[org.apache.spark.executor.Executor]] to run this task. + * + * @param taskAttemptId an identifier for this task attempt that is unique within a SparkContext. + * @param attemptNumber how many times this task has been attempted (0 for the first attempt) + * @return the result of the task along with updates of Accumulators. + */ final def run( taskAttemptId: Long, attemptNumber: Int, @@ -207,15 +208,15 @@ private[spark] abstract class Task[T]( def killed: Boolean = _killed /** - * Returns the amount of time spent deserializing the RDD and function to be run. - */ + * Returns the amount of time spent deserializing the RDD and function to be run in nanos. + */ def executorDeserializeTime: Long = _executorDeserializeTime def executorDeserializeCpuTime: Long = _executorDeserializeCpuTime /** - * Collect the latest values of accumulators used in this task. If the task failed, - * filter out the accumulators whose values should not be included on failures. - */ + * Collect the latest values of accumulators used in this task. If the task failed, + * filter out the accumulators whose values should not be included on failures. + */ def collectAccumulatorUpdates(taskFailed: Boolean = false): Seq[AccumulatorV2[_, _]] = { if (context != null) { // Note: internal accumulators representing task metrics always count failed values @@ -288,16 +289,16 @@ private[spark] abstract class Task[T]( } /** - * Handles transmission of tasks and their dependencies, because this can be slightly tricky. We - * need to send the list of JARs and files added to the SparkContext with each task to ensure that - * worker nodes find out about it, but we can't make it part of the Task because the user's code in - * the task might depend on one of the JARs. Thus we serialize each task as multiple objects, by - * first writing out its dependencies. - */ + * Handles transmission of tasks and their dependencies, because this can be slightly tricky. We + * need to send the list of JARs and files added to the SparkContext with each task to ensure that + * worker nodes find out about it, but we can't make it part of the Task because the user's code in + * the task might depend on one of the JARs. Thus we serialize each task as multiple objects, by + * first writing out its dependencies. + */ private[spark] object Task { /** - * Serialize a task and the current app dependencies (files and JARs added to the SparkContext) - */ + * Serialize a task and the current app dependencies (files and JARs added to the SparkContext) + */ def serializeWithDependencies( task: Task[_], currentFiles: mutable.Map[String, Long], @@ -336,12 +337,12 @@ private[spark] object Task { } /** - * Deserialize the list of dependencies in a task serialized with serializeWithDependencies, - * and return the task itself as a serialized ByteBuffer. The caller can then update its - * ClassLoaders and deserialize the task. - * - * @return (taskFiles, taskJars, taskProps, taskBytes) - */ + * Deserialize the list of dependencies in a task serialized with serializeWithDependencies, + * and return the task itself as a serialized ByteBuffer. The caller can then update its + * ClassLoaders and deserialize the task. + * + * @return (taskFiles, taskJars, taskBytes) + */ def deserializeWithDependencies(serializedTask: ByteBuffer) : (HashMap[String, Long], HashMap[String, Long], Properties, ByteBuffer) = { @@ -374,7 +375,7 @@ private[spark] object Task { } private[spark] final class TaskData private(var compressedBytes: Array[Byte], - var uncompressedLen: Int, var reference: Int) extends Serializable { + var uncompressedLen: Int, var reference: Int) extends Serializable { def this(compressedBytes: Array[Byte], uncompressedLen: Int) = this(compressedBytes, uncompressedLen, TaskData.NO_REF) @@ -440,4 +441,3 @@ private[spark] object TaskData { } } } - diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 44f6245d27029..c7faeb398a3ca 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -397,11 +397,11 @@ private[spark] class TaskSetManager( } /** - * Dequeue a pending task for a given node and return its index and locality level. - * Only search for tasks matching the given locality constraint. - * - * @return An option containing (task index within the task set, locality, is speculative?) - */ + * Dequeue a pending task for a given node and return its index and locality level. + * Only search for tasks matching the given locality constraint. + * + * @return An option containing (task index within the task set, locality, is speculative?) + */ private def dequeueTask(execId: String, host: String, maxLocality: TaskLocality.Value) : Option[(Int, TaskLocality.Value, Boolean)] = { diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index 12929002b4a8f..8b41331770856 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -42,7 +42,7 @@ import javax.annotation.concurrent.GuardedBy import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet} import scala.concurrent.Future -import org.apache.spark.{SparkEnv, ExecutorAllocationClient, SparkException, TaskState} +import org.apache.spark.{ExecutorAllocationClient, SparkEnv, SparkException, TaskState} import org.apache.spark.internal.Logging import org.apache.spark.rpc._ import org.apache.spark.scheduler._ @@ -644,7 +644,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp protected def doRequestTotalExecutors(requestedTotal: Int): Future[Boolean] = Future.successful(false) - /**W + /** * Request that the cluster manager kill the specified executors. * * When asking the executor to be replaced, the executor loss is considered a failure, and diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index 54780b380e26d..d40d1d3c036f8 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -29,6 +29,7 @@ import scala.collection.mutable.HashMap import scala.concurrent.{Await, ExecutionContext, Future} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.duration._ import scala.reflect.ClassTag import scala.util.Random import scala.util.control.NonFatal @@ -215,14 +216,14 @@ private[spark] class BlockManager( private val maxRemoteBlockToMem = conf.get(config.MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM) /** - * Initializes the BlockManager with the given appId. This is not performed in the constructor as - * the appId may not be known at BlockManager instantiation time (in particular for the driver, - * where it is only learned after registration with the TaskScheduler). - * - * This method initializes the BlockTransferService and ShuffleClient, registers with the - * BlockManagerMaster, starts the BlockManagerWorker endpoint, and registers with a local shuffle - * service if configured. - */ + * Initializes the BlockManager with the given appId. This is not performed in the constructor as + * the appId may not be known at BlockManager instantiation time (in particular for the driver, + * where it is only learned after registration with the TaskScheduler). + * + * This method initializes the BlockTransferService and ShuffleClient, registers with the + * BlockManagerMaster, starts the BlockManagerWorker endpoint, and registers with a local shuffle + * service if configured. + */ def initialize(appId: String): Unit = { blockTransferService.init(this) shuffleClient.init(appId) @@ -301,15 +302,15 @@ private[spark] class BlockManager( } /** - * Report all blocks to the BlockManager again. This may be necessary if we are dropped - * by the BlockManager and come back or if we become capable of recovering blocks on disk after - * an executor crash. - * - * This function deliberately fails silently if the master returns false (indicating that - * the slave needs to re-register). The error condition will be detected again by the next - * heart beat attempt or new block registration and another try to re-register all blocks - * will be made then. - */ + * Report all blocks to the BlockManager again. This may be necessary if we are dropped + * by the BlockManager and come back or if we become capable of recovering blocks on disk after + * an executor crash. + * + * This function deliberately fails silently if the master returns false (indicating that + * the slave needs to re-register). The error condition will be detected again by the next + * heart beat attempt or new block registration and another try to re-register all blocks + * will be made then. + */ private def reportAllBlocks(): Unit = { logInfo(s"Reporting ${blockInfoManager.size} blocks to the master.") for ((blockId, info) <- blockInfoManager.entries) { @@ -322,11 +323,11 @@ private[spark] class BlockManager( } /** - * Re-register with the master and report all blocks to it. This will be called by the heart beat - * thread if our heartbeat to the block manager indicates that we were not registered. - * - * Note that this method must be called without any BlockInfo locks held. - */ + * Re-register with the master and report all blocks to it. This will be called by the heart beat + * thread if our heartbeat to the block manager indicates that we were not registered. + * + * Note that this method must be called without any BlockInfo locks held. + */ def reregister(): Unit = { // TODO: We might need to rate limit re-registering. logInfo(s"BlockManager $blockManagerId re-registering with master") @@ -335,8 +336,8 @@ private[spark] class BlockManager( } /** - * Re-register with the master sometime soon. - */ + * Re-register with the master sometime soon. + */ private def asyncReregister(): Unit = { asyncReregisterLock.synchronized { if (asyncReregisterTask == null) { @@ -353,8 +354,8 @@ private[spark] class BlockManager( } /** - * For testing. Wait for any pending asynchronous re-registration; otherwise, do nothing. - */ + * For testing. Wait for any pending asynchronous re-registration; otherwise, do nothing. + */ def waitForAsyncReregister(): Unit = { val task = asyncReregisterTask if (task != null) { @@ -368,9 +369,9 @@ private[spark] class BlockManager( } /** - * Interface to get local block data. Throws an exception if the block cannot be found or - * cannot be read successfully. - */ + * Interface to get local block data. Throws an exception if the block cannot be found or + * cannot be read successfully. + */ override def getBlockData(blockId: BlockId): ManagedBuffer = { if (blockId.isShuffle) { shuffleManager.shuffleBlockResolver.getBlockData(blockId.asInstanceOf[ShuffleBlockId]) @@ -403,9 +404,9 @@ private[spark] class BlockManager( } /** - * Get the BlockStatus for the block identified by the given ID, if it exists. - * NOTE: This is mainly for testing. - */ + * Get the BlockStatus for the block identified by the given ID, if it exists. + * NOTE: This is mainly for testing. + */ def getStatus(blockId: BlockId): Option[BlockStatus] = { blockInfoManager.get(blockId).map { info => val memSize = if (memoryStore.contains(blockId)) memoryStore.getSize(blockId) else 0L @@ -415,10 +416,10 @@ private[spark] class BlockManager( } /** - * Get the ids of existing blocks that match the given filter. Note that this will - * query the blocks stored in the disk block manager (that the block manager - * may not know of). - */ + * Get the ids of existing blocks that match the given filter. Note that this will + * query the blocks stored in the disk block manager (that the block manager + * may not know of). + */ def getMatchingBlockIds(filter: BlockId => Boolean): Seq[BlockId] = { // The `toArray` is necessary here in order to force the list to be materialized so that we // don't try to serialize a lazy iterator when responding to client requests. @@ -429,14 +430,14 @@ private[spark] class BlockManager( } /** - * Tell the master about the current storage status of a block. This will send a block update - * message reflecting the current status, *not* the desired storage level in its block info. - * For example, a block with MEMORY_AND_DISK set might have fallen out to be only on disk. - * - * droppedMemorySize exists to account for when the block is dropped from memory to disk (so - * it is still valid). This ensures that update in master will compensate for the increase in - * memory on slave. - */ + * Tell the master about the current storage status of a block. This will send a block update + * message reflecting the current status, *not* the desired storage level in its block info. + * For example, a block with MEMORY_AND_DISK set might have fallen out to be only on disk. + * + * droppedMemorySize exists to account for when the block is dropped from memory to disk (so + * it is still valid). This ensures that update in master will compensate for the increase in + * memory on slave. + */ private def reportBlockStatus( blockId: BlockId, status: BlockStatus, @@ -451,10 +452,10 @@ private[spark] class BlockManager( } /** - * Actually send a UpdateBlockInfo message. Returns the master's response, - * which will be true if the block was successfully recorded and false if - * the slave needs to re-register. - */ + * Actually send a UpdateBlockInfo message. Returns the master's response, + * which will be true if the block was successfully recorded and false if + * the slave needs to re-register. + */ private def tryToReportBlockStatus( blockId: BlockId, status: BlockStatus, @@ -466,10 +467,10 @@ private[spark] class BlockManager( } /** - * Return the updated storage status of the block with the given ID. More specifically, if - * the block is dropped from memory and possibly added to disk, return the new storage level - * and the updated in-memory and on-disk sizes. - */ + * Return the updated storage status of the block with the given ID. More specifically, if + * the block is dropped from memory and possibly added to disk, return the new storage level + * and the updated in-memory and on-disk sizes. + */ private def getCurrentBlockStatus(blockId: BlockId, info: BlockInfo): BlockStatus = { info.synchronized { info.level match { @@ -494,8 +495,8 @@ private[spark] class BlockManager( } /** - * Get locations of an array of blocks. - */ + * Get locations of an array of blocks. + */ private def getLocationBlockIds(blockIds: Array[BlockId]): Array[Seq[BlockManagerId]] = { val startTimeMs = if (isDebugEnabled) System.currentTimeMillis else 0L val locations = master.getLocations(blockIds).toArray @@ -504,9 +505,9 @@ private[spark] class BlockManager( } /** - * Cleanup code run in response to a failed local read. - * Must be called while holding a read lock on the block. - */ + * Cleanup code run in response to a failed local read. + * Must be called while holding a read lock on the block. + */ private def handleLocalReadFailure(blockId: BlockId): Nothing = { releaseLock(blockId) // Remove the missing block so that its unavailability is reported to the driver @@ -515,8 +516,8 @@ private[spark] class BlockManager( } /** - * Get block from local block manager as an iterator of Java objects. - */ + * Get block from local block manager as an iterator of Java objects. + */ def getLocalValues(blockId: BlockId): Option[BlockResult] = { logDebug(s"Getting local block $blockId") blockInfoManager.lockForReading(blockId) match { @@ -627,10 +628,10 @@ private[spark] class BlockManager( } /** - * Get block from remote block managers. - * - * This does not acquire a lock on this block in this JVM. - */ + * Get block from remote block managers. + * + * This does not acquire a lock on this block in this JVM. + */ private def getRemoteValues[T: ClassTag](blockId: BlockId): Option[BlockResult] = { val ct = implicitly[ClassTag[T]] getRemoteBytes(blockId).map { data => @@ -658,8 +659,8 @@ private[spark] class BlockManager( } /** - * Get block from remote block managers as serialized bytes. - */ + * Get block from remote block managers as serialized bytes. + */ def getRemoteBytes(blockId: BlockId): Option[ChunkedByteBuffer] = { logDebug(s"Getting remote block $blockId") require(blockId != null, "BlockId is null") @@ -734,12 +735,12 @@ private[spark] class BlockManager( } /** - * Get a block from the block manager (either local or remote). - * - * This acquires a read lock on the block if the block was stored locally and does not acquire - * any locks if the block was fetched from a remote block manager. The read lock will - * automatically be freed once the result's `data` iterator is fully consumed. - */ + * Get a block from the block manager (either local or remote). + * + * This acquires a read lock on the block if the block was stored locally and does not acquire + * any locks if the block was fetched from a remote block manager. The read lock will + * automatically be freed once the result's `data` iterator is fully consumed. + */ def get[T: ClassTag](blockId: BlockId): Option[BlockResult] = { val local = getLocalValues(blockId) if (local.isDefined) { @@ -755,8 +756,8 @@ private[spark] class BlockManager( } /** - * Downgrades an exclusive write lock to a shared read lock. - */ + * Downgrades an exclusive write lock to a shared read lock. + */ def downgradeLock(blockId: BlockId): Unit = { blockInfoManager.downgradeLock(blockId) } @@ -772,28 +773,28 @@ private[spark] class BlockManager( } /** - * Registers a task with the BlockManager in order to initialize per-task bookkeeping structures. - */ + * Registers a task with the BlockManager in order to initialize per-task bookkeeping structures. + */ def registerTask(taskAttemptId: Long): Unit = { blockInfoManager.registerTask(taskAttemptId) } /** - * Release all locks for the given task. - * - * @return the blocks whose locks were released. - */ + * Release all locks for the given task. + * + * @return the blocks whose locks were released. + */ def releaseAllLocksForTask(taskAttemptId: Long): Seq[BlockId] = { blockInfoManager.releaseAllLocksForTask(taskAttemptId) } /** - * Retrieve the given block if it exists, otherwise call the provided `makeIterator` method - * to compute the block, persist it, and return its values. - * - * @return either a BlockResult if the block was successfully cached, or an iterator if the block - * could not be cached. - */ + * Retrieve the given block if it exists, otherwise call the provided `makeIterator` method + * to compute the block, persist it, and return its values. + * + * @return either a BlockResult if the block was successfully cached, or an iterator if the block + * could not be cached. + */ def getOrElseUpdate[T]( blockId: BlockId, level: StorageLevel, @@ -832,8 +833,8 @@ private[spark] class BlockManager( } /** - * @return true if the block was stored or false if an error occurred. - */ + * @return true if the block was stored or false if an error occurred. + */ def putIterator[T: ClassTag]( blockId: BlockId, values: Iterator[T], @@ -852,10 +853,10 @@ private[spark] class BlockManager( } /** - * A short circuited method to get a block writer that can write data directly to disk. - * The Block will be appended to the File specified by filename. Callers should handle error - * cases. - */ + * A short circuited method to get a block writer that can write data directly to disk. + * The Block will be appended to the File specified by filename. Callers should handle error + * cases. + */ def getDiskWriter( blockId: BlockId, file: File, @@ -985,11 +986,11 @@ private[spark] class BlockManager( } /** - * Helper method used to abstract common code from [[doPutBytes()]] and [[doPutIterator()]]. - * - * @param putBody a function which attempts the actual put() and returns None on success - * or Some on failure. - */ + * Helper method used to abstract common code from [[doPutBytes()]] and [[doPutIterator()]]. + * + * @param putBody a function which attempts the actual put() and returns None on success + * or Some on failure. + */ private def doPut[T]( blockId: BlockId, level: StorageLevel, @@ -1064,17 +1065,17 @@ private[spark] class BlockManager( } /** - * Put the given block according to the given level in one of the block stores, replicating - * the values if necessary. - * - * If the block already exists, this method will not overwrite it. - * - * @param keepReadLock if true, this method will hold the read lock when it returns (even if the - * block already exists). If false, this method will hold no locks when it - * returns. - * @return None if the block was already present or if the put succeeded, or Some(iterator) - * if the put failed. - */ + * Put the given block according to the given level in one of the block stores, replicating + * the values if necessary. + * + * If the block already exists, this method will not overwrite it. + * + * @param keepReadLock if true, this method will hold the read lock when it returns (even if the + * block already exists). If false, this method will hold no locks when it + * returns. + * @return None if the block was already present or if the put succeeded, or Some(iterator) + * if the put failed. + */ private def doPutIterator[T]( blockId: BlockId, iterator: () => Iterator[T], @@ -1171,14 +1172,14 @@ private[spark] class BlockManager( } /** - * Attempts to cache spilled bytes read from disk into the MemoryStore in order to speed up - * subsequent reads. This method requires the caller to hold a read lock on the block. - * - * @return a copy of the bytes from the memory store if the put succeeded, otherwise None. - * If this returns bytes from the memory store then the original disk store bytes will - * automatically be disposed and the caller should not continue to use them. Otherwise, - * if this returns None then the original disk store bytes will be unaffected. - */ + * Attempts to cache spilled bytes read from disk into the MemoryStore in order to speed up + * subsequent reads. This method requires the caller to hold a read lock on the block. + * + * @return a copy of the bytes from the memory store if the put succeeded, otherwise None. + * If this returns bytes from the memory store then the original disk store bytes will + * automatically be disposed and the caller should not continue to use them. Otherwise, + * if this returns None then the original disk store bytes will be unaffected. + */ private def maybeCacheDiskBytesInMemory( blockInfo: BlockInfo, blockId: BlockId, @@ -1218,12 +1219,12 @@ private[spark] class BlockManager( } /** - * Attempts to cache spilled values read from disk into the MemoryStore in order to speed up - * subsequent reads. This method requires the caller to hold a read lock on the block. - * - * @return a copy of the iterator. The original iterator passed this method should no longer - * be used after this method returns. - */ + * Attempts to cache spilled values read from disk into the MemoryStore in order to speed up + * subsequent reads. This method requires the caller to hold a read lock on the block. + * + * @return a copy of the iterator. The original iterator passed this method should no longer + * be used after this method returns. + */ private def maybeCacheDiskValuesInMemory[T]( blockInfo: BlockInfo, blockId: BlockId, @@ -1255,8 +1256,8 @@ private[spark] class BlockManager( } /** - * Get peer block managers in the system. - */ + * Get peer block managers in the system. + */ private def getPeers(forceFetch: Boolean): Seq[BlockManagerId] = { peerFetchLock.synchronized { val cachedPeersTtl = conf.getInt("spark.storage.cachedPeersTtl", 60 * 1000) // milliseconds @@ -1388,18 +1389,18 @@ private[spark] class BlockManager( } /** - * Read a block consisting of a single object. - */ + * Read a block consisting of a single object. + */ def getSingle[T: ClassTag](blockId: BlockId): Option[T] = { get[T](blockId).map(_.data.next().asInstanceOf[T]) } /** - * Write a block consisting of a single object. - * - * @return true if the block was stored or false if the block was already stored or an - * error occurred. - */ + * Write a block consisting of a single object. + * + * @return true if the block was stored or false if the block was already stored or an + * error occurred. + */ def putSingle[T: ClassTag]( blockId: BlockId, value: T, @@ -1409,16 +1410,16 @@ private[spark] class BlockManager( } /** - * Drop a block from memory, possibly putting it on disk if applicable. Called when the memory - * store reaches its limit and needs to free up space. - * - * If `data` is not put on disk, it won't be created. - * - * The caller of this method must hold a write lock on the block before calling this method. - * This method does not release the write lock. - * - * @return the block's new effective StorageLevel. - */ + * Drop a block from memory, possibly putting it on disk if applicable. Called when the memory + * store reaches its limit and needs to free up space. + * + * If `data` is not put on disk, it won't be created. + * + * The caller of this method must hold a write lock on the block before calling this method. + * This method does not release the write lock. + * + * @return the block's new effective StorageLevel. + */ private[storage] override def dropFromMemory[T: ClassTag]( blockId: BlockId, data: () => Either[Array[T], ChunkedByteBuffer]): StorageLevel = { @@ -1466,10 +1467,10 @@ private[spark] class BlockManager( } /** - * Remove all blocks belonging to the given RDD. - * - * @return The number of blocks removed. - */ + * Remove all blocks belonging to the given RDD. + * + * @return The number of blocks removed. + */ def removeRdd(rddId: Int): Int = { // TODO: Avoid a linear scan by creating another mapping of RDD.id to blocks. logInfo(s"Removing RDD $rddId") @@ -1479,8 +1480,8 @@ private[spark] class BlockManager( } /** - * Remove all blocks belonging to the given broadcast. - */ + * Remove all blocks belonging to the given broadcast. + */ def removeBroadcast(broadcastId: Long, tellMaster: Boolean): Int = { logDebug(s"Removing broadcast $broadcastId") val blocksToRemove = blockInfoManager.entries.map(_._1).collect { @@ -1491,8 +1492,8 @@ private[spark] class BlockManager( } /** - * Remove a block from both memory and disk. - */ + * Remove a block from both memory and disk. + */ def removeBlock(blockId: BlockId, tellMaster: Boolean = true): Unit = { logDebug(s"Removing block $blockId") blockInfoManager.lockForWriting(blockId) match { @@ -1506,9 +1507,9 @@ private[spark] class BlockManager( } /** - * Internal version of [[removeBlock()]] which assumes that the caller already holds a write - * lock on the block. - */ + * Internal version of [[removeBlock()]] which assumes that the caller already holds a write + * lock on the block. + */ private def removeBlockInternal(blockId: BlockId, tellMaster: Boolean): Unit = { // Removals are idempotent in disk store and memory store. At worst, we get a warning. val removedFromMemory = memoryStore.remove(blockId) @@ -1577,6 +1578,7 @@ private[spark] object BlockManager { } blockManagers.toMap } +<<<<<<< HEAD private class ShuffleMetricsSource( override val sourceName: String, diff --git a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala index ace6ccb3d20a2..17f7a69ad6ba1 100644 --- a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala +++ b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala @@ -27,16 +27,16 @@ import scala.reflect.ClassTag import com.google.common.io.ByteStreams +import org.apache.spark.{SparkConf, TaskContext} import org.apache.spark.internal.Logging import org.apache.spark.internal.config.{UNROLL_MEMORY_CHECK_PERIOD, UNROLL_MEMORY_GROWTH_FACTOR} import org.apache.spark.memory.{MemoryManager, MemoryMode} import org.apache.spark.serializer.{SerializationStream, SerializerManager} import org.apache.spark.storage._ import org.apache.spark.unsafe.Platform +import org.apache.spark.util.{SizeEstimator, Utils} import org.apache.spark.util.collection.SizeTrackingVector import org.apache.spark.util.io.{ChunkedByteBuffer, ChunkedByteBufferOutputStream} -import org.apache.spark.util.{SizeEstimator, Utils} -import org.apache.spark.{SparkConf, TaskContext} private sealed trait MemoryEntry[T] { def size: Long diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 9b2bfd750ca82..56e159746b518 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -81,9 +81,9 @@ import org.apache.spark.internal.config._ import org.apache.spark.launcher.SparkLauncher import org.apache.spark.network.util.JavaUtils import org.apache.spark.serializer.{DeserializationStream, SerializationStream, SerializerInstance} -import org.apache.spark.util.logging.RollingFileAppender import org.apache.spark.storage.StorageUtils -import org.apache.spark.unsafe.types.UTF8String; +import org.apache.spark.unsafe.types.UTF8String +import org.apache.spark.util.logging.RollingFileAppender /** CallSite represents a place in user code. It can have a short and a long form. */ private[spark] case class CallSite(shortForm: String, longForm: String) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala index cd34a18754377..9c6186a790eef 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala @@ -103,6 +103,7 @@ object ScalaReflection extends ScalaReflection { case t if t <:< definitions.ShortTpe => classOf[Array[Short]] case t if t <:< definitions.ByteTpe => classOf[Array[Byte]] case t if t <:< definitions.BooleanTpe => classOf[Array[Boolean]] + case t if t <:< localTypeOf[Decimal] => classOf[Array[Decimal]] case other => // There is probably a better way to do this, but I couldn't find it... val elementType = dataTypeFor(other).asInstanceOf[ObjectType].cls @@ -821,7 +822,6 @@ trait ScalaReflection { // The Predef.Map is scala.collection.immutable.Map. // Since the map values can be mutable, we explicitly import scala.collection.Map at here. - import scala.collection.Map /** * Any codes calling `scala.reflect.api.Types.TypeApi.<:<` should be wrapped by this method to diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala index b8ccf6e8e4ad7..c5390839d53d1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala @@ -111,7 +111,6 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro s""" // Remember the current cursor so that we can calculate how many bytes are // written later. - final int $tmpCursor = $bufferHolder.cursor; ${writeStructToBuffer(ctx, input.value, t.map(_.dataType), bufferHolder)} $rowWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index dd6f5f509c57d..2a1fafbd4c7d0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -25,6 +25,7 @@ import scala.collection.mutable.ArrayBuffer import org.apache.spark.api.java.function.FilterFunction import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf} import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog} import org.apache.spark.sql.catalyst.expressions._ @@ -37,8 +38,6 @@ import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf} import org.apache.spark.sql.types._ import org.apache.spark.util.Utils -import scala.collection.mutable - /** * Abstract class all optimizers should inherit of, contains the standard batches (extending * Optimizers can override this. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala index 41817bfa670ce..abd5b3fee1e81 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala @@ -28,16 +28,16 @@ import org.codehaus.janino.InternalCompilerException import org.apache.spark.{broadcast, SparkEnv} import org.apache.spark.internal.Logging import org.apache.spark.rdd.{RDD, RDDOperationScope} +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen.{Predicate => GenPredicate, _} import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.physical._ -import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types.DataType -import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.util.ThreadUtils -import org.apache.spark.{SparkEnv, broadcast} + /** * The base class for physical operators. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala index 2f3f5dc71cb53..03a6b6c536158 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala @@ -26,8 +26,9 @@ import org.apache.spark.broadcast import org.apache.spark.rdd.RDD import com.esotericsoftware.kryo.io.{Input, Output} import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import com.esotericsoftware.kryo.io.{Input, Output} -import org.apache.spark.{Partition, SparkContext, TaskContext, broadcast} +import org.apache.spark.{broadcast, Partition, SparkContext, TaskContext} import org.apache.spark.rdd.{RDD, ZippedPartitionsBaseRDD, ZippedPartitionsPartition} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala index 2fc903168cfa0..e7997537d8f39 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala @@ -20,7 +20,10 @@ package org.apache.spark.sql.streaming import java.util.Locale import scala.collection.JavaConverters._ - +import org.apache.spark.annotation.Experimental +import org.apache.spark.sql.execution.datasources.DataSource +import org.apache.spark.sql.execution.streaming._ +import org.apache.spark.sql.{AnalysisException, Dataset, ForeachWriter} import org.apache.spark.annotation.InterfaceStability import org.apache.spark.sql.{AnalysisException, Dataset, ForeachWriter} import org.apache.spark.sql.catalyst.streaming.InternalOutputModes From 40ea70c3c78aa881172c74eac3dcd703c9b4c402 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 11 Jul 2017 11:21:01 +0530 Subject: [PATCH 080/142] [SNAP-1790] Fix one case of incorrect offset in ByteArrayMethods.arrayEquals The endOffset incorrectly uses current leftOffset+length when the leftOffset may already have been incremented for word alignment. Conflicts: common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java --- .../spark/unsafe/array/ByteArrayMethods.java | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java index 058939c4c0e8a..0b04a5a4d022a 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java @@ -53,14 +53,29 @@ public static int roundNumberOfBytesToNearestWord(int numBytes) { * Optimized byte array equality check for byte arrays. * @return true if the arrays are equal, false otherwise */ - public static boolean arrayEquals( - final Object leftBase, long leftOffset, final Object rightBase, - long rightOffset, final long length) { - long endOffset = leftOffset + length - 8; - while (leftOffset <= endOffset) { - if (Platform.getLong(leftBase, leftOffset) != - Platform.getLong(rightBase, rightOffset)) { - return false; + public static boolean arrayEquals(final Object leftBase, long leftOffset, + final Object rightBase, long rightOffset, final long length) { + long endOffset = leftOffset + length; + // try to align at least one side + if ((rightOffset & 0x7) != 0 && (leftOffset & 0x7) != 0) { // mod 8 + final long alignedOffset = Math.min(((leftOffset + 7) >>> 3) << 3, endOffset); + if (Platform.unaligned()) { + if (leftOffset <= (alignedOffset - 4)) { + if (Platform.getInt(leftBase, leftOffset) != + Platform.getInt(rightBase, rightOffset)) { + return false; + } + leftOffset += 4; + rightOffset += 4; + } + } + while (leftOffset < alignedOffset) { + if (Platform.getByte(leftBase, leftOffset) != + Platform.getByte(rightBase, rightOffset)) { + return false; + } + leftOffset++; + rightOffset++; } leftOffset += 8; rightOffset += 8; From 0b1e9934ddaa256d8b2d9f1aa1f5c3074b90732c Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sun, 9 Jul 2017 23:10:16 +0530 Subject: [PATCH 081/142] [SNAPPYDATA][MERGE-2.1] Missing patches and version changes - updated optimized ByteArrayMethods.arrayEquals as per the code in Spark 2.1 - adapt the word alignment code and optimize it a bit - in micro-benchmarks the new method is 30-60% faster than upstream version; at larger sizes it is 40-50% faster meaning its base word comparison loop itself is faster - increase default locality time from 3s to 10s since the previous code to force executor-specific routing if it is alive has been removed - added back cast removal optimization when types differ only in nullability - add serialization and proper nanoTime handling from *CpuTime added in Spark 2.1.x; use DoubleAccumulator for these new fields like done for others to get more accurate results; also avoid the rare conditions where these cpu times could be negative - cleanup handling of jobId and related new fields in Task with kryo serialization - reverted change to AnalysisException with null check for plan since it is transient now - reverted old Spark 2.0 code that was retained in InsertIntoTable and changed to Spark 2.1 code - updated library versions and make them uniform as per upstream Spark for commons-lang3, metrics-core, py4j, breeze, univocity; also updated exclusions as per the changes to Spark side between 2.0.2 to 2.1.0 - added gradle build for the new mesos sub-project Conflicts: common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java core/src/main/scala/org/apache/spark/executor/Executor.scala core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala core/src/main/scala/org/apache/spark/scheduler/Task.scala core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala --- assembly/build.gradle | 3 ++ build.gradle | 2 + common/network-common/build.gradle | 6 +-- common/network-shuffle/build.gradle | 2 +- common/unsafe/build.gradle | 2 +- .../spark/unsafe/array/ByteArrayMethods.java | 41 ++++++++++------- core/build.gradle | 10 +++-- .../org/apache/spark/executor/Executor.scala | 9 ++-- .../apache/spark/executor/TaskMetrics.scala | 12 ++--- .../apache/spark/scheduler/DAGScheduler.scala | 7 +-- .../apache/spark/scheduler/ResultTask.scala | 6 +-- .../spark/scheduler/ShuffleMapTask.scala | 7 ++- .../org/apache/spark/scheduler/Task.scala | 24 +++++----- .../spark/scheduler/TaskSetManager.scala | 1 + examples/build.gradle | 2 +- mesos/build.gradle | 35 +++++++++++++++ mllib-local/build.gradle | 4 +- mllib/build.gradle | 2 +- settings.gradle | 2 + sql/catalyst/build.gradle | 1 + .../codegen/UnsafeArrayWriter.java | 3 +- .../sql/catalyst/optimizer/expressions.scala | 6 ++- .../plans/logical/basicLogicalOperators.scala | 44 ++++++++++--------- .../spark/sql/types/AbstractDataType.scala | 18 -------- sql/core/build.gradle | 2 +- sql/hive/build.gradle | 2 + yarn/build.gradle | 2 + 27 files changed, 154 insertions(+), 101 deletions(-) create mode 100644 mesos/build.gradle diff --git a/assembly/build.gradle b/assembly/build.gradle index 0d81f789eb05b..ddd4ed6a99955 100644 --- a/assembly/build.gradle +++ b/assembly/build.gradle @@ -31,6 +31,9 @@ dependencies { compile project(subprojectBase + 'snappy-spark-yarn_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-mllib_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) + if (rootProject.hasProperty('mesos')) { + compile project(subprojectBase + 'snappy-spark-mesos_' + scalaBinaryVersion) + } if (rootProject.hasProperty('ganglia')) { compile project(subprojectBase + 'snappy-spark-ganglia-lgpl_' + scalaBinaryVersion) } diff --git a/build.gradle b/build.gradle index e074b296e1c0c..ad119c2ab7945 100644 --- a/build.gradle +++ b/build.gradle @@ -79,6 +79,8 @@ allprojects { seleniumVersion = '2.52.0' curatorVersion = '2.7.1' commonsCodecVersion = '1.10' + commonsLang3Version = '3.5' + commonsMath3Version = '3.4.1' avroVersion = '1.7.7' jsr305Version = '3.0.1' jlineVersion = '2.14.2' diff --git a/common/network-common/build.gradle b/common/network-common/build.gradle index 62300cac099f5..8fba6a3379a4e 100644 --- a/common/network-common/build.gradle +++ b/common/network-common/build.gradle @@ -24,7 +24,7 @@ dependencies { compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version compile group: 'com.google.guava', name: 'guava', version: guavaVersion compile group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: '1.8' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.6.5' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.8.5' - compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.5' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: fasterXmlVersion + compile group: 'org.apache.commons', name: 'commons-lang3', version: commonsLang3Version } diff --git a/common/network-shuffle/build.gradle b/common/network-shuffle/build.gradle index 400a225bc8618..9c916a8532d9d 100644 --- a/common/network-shuffle/build.gradle +++ b/common/network-shuffle/build.gradle @@ -25,7 +25,7 @@ dependencies { compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: fasterXmlVersion compile group: 'com.google.guava', name: 'guava', version: guavaVersion - compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion testCompile project(path: subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion, configuration: 'testOutput') } diff --git a/common/unsafe/build.gradle b/common/unsafe/build.gradle index b14fed1ab31d7..6f7c9d1ef98fd 100644 --- a/common/unsafe/build.gradle +++ b/common/unsafe/build.gradle @@ -27,7 +27,7 @@ dependencies { compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version compile group: 'com.google.guava', name: 'guava', version: guavaVersion - testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.2' + testCompile group: 'org.apache.commons', name: 'commons-lang3', version: commonsLang3Version } // reset the srcDirs to allow javac compilation with specific args below diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java index 0b04a5a4d022a..c4d4ef5f50bce 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java @@ -55,12 +55,11 @@ public static int roundNumberOfBytesToNearestWord(int numBytes) { */ public static boolean arrayEquals(final Object leftBase, long leftOffset, final Object rightBase, long rightOffset, final long length) { - long endOffset = leftOffset + length; // try to align at least one side if ((rightOffset & 0x7) != 0 && (leftOffset & 0x7) != 0) { // mod 8 - final long alignedOffset = Math.min(((leftOffset + 7) >>> 3) << 3, endOffset); + final long endOffset = Math.min(((leftOffset + 7) >>> 3) << 3, leftOffset + length); if (Platform.unaligned()) { - if (leftOffset <= (alignedOffset - 4)) { + if (leftOffset <= (endOffset - 4)) { if (Platform.getInt(leftBase, leftOffset) != Platform.getInt(rightBase, rightOffset)) { return false; @@ -69,7 +68,7 @@ public static boolean arrayEquals(final Object leftBase, long leftOffset, rightOffset += 4; } } - while (leftOffset < alignedOffset) { + while (leftOffset < endOffset) { if (Platform.getByte(leftBase, leftOffset) != Platform.getByte(rightBase, rightOffset)) { return false; @@ -77,25 +76,35 @@ public static boolean arrayEquals(final Object leftBase, long leftOffset, leftOffset++; rightOffset++; } - leftOffset += 8; - rightOffset += 8; } + long endOffset = leftOffset + length; // for architectures that support unaligned accesses, chew it up 8 bytes at a time - if (unaligned || (((leftOffset + i) % 8 == 0) && ((rightOffset + i) % 8 == 0))) { - while (i <= length - 8) { - if (Platform.getLong(leftBase, leftOffset + i) != - Platform.getLong(rightBase, rightOffset + i)) { - return false; + if (Platform.unaligned() || (((leftOffset & 0x7) == 0) && ((rightOffset & 0x7) == 0))) { + endOffset -= 8; + while (leftOffset <= endOffset) { + if (Platform.getLong(leftBase, leftOffset) != + Platform.getLong(rightBase, rightOffset)) { + return false; + } + leftOffset += 8; + rightOffset += 8; + } + endOffset += 4; + if (leftOffset <= endOffset) { + if (Platform.getInt(leftBase, leftOffset) != + Platform.getInt(rightBase, rightOffset)) { + return false; } - i += 8; + leftOffset += 4; + rightOffset += 4; } - leftOffset += 4; - rightOffset += 4; + endOffset += 4; } - endOffset += 4; + // this will finish off the unaligned comparisons, or do the entire aligned + // comparison whichever is needed. while (leftOffset < endOffset) { if (Platform.getByte(leftBase, leftOffset) != - Platform.getByte(rightBase, rightOffset)) { + Platform.getByte(rightBase, rightOffset)) { return false; } leftOffset++; diff --git a/core/build.gradle b/core/build.gradle index ec31c88b52e1a..901058f46a4c4 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -99,10 +99,12 @@ dependencies { compile group: 'org.eclipse.jetty', name: 'jetty-client', version: jettyVersion compile group: 'org.eclipse.jetty', name: 'jetty-proxy', version: jettyVersion compile group: 'javax.servlet', name: 'javax.servlet-api', version: javaxServletVersion - compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.2' - compile group: 'org.apache.commons', name: 'commons-math3', version: '3.4.1' + compile group: 'org.apache.commons', name: 'commons-lang3', version: commonsLang3Version + compile group: 'org.apache.commons', name: 'commons-math3', version: commonsMath3Version compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version - compile group: 'org.apache.commons', name: 'commons-crypto', version: '1.0.0' + compile(group: 'org.apache.commons', name: 'commons-crypto', version: '1.0.0') { + exclude(group: 'net.java.dev.jna', module: 'jna') + } compile group: 'io.netty', name: 'netty', version: nettyVersion compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion compile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion @@ -137,7 +139,7 @@ dependencies { compile(group: 'net.razorvine', name: 'pyrolite', version: '4.13') { exclude(group: 'net.razorvine', module: 'serpent') } - compile group: 'net.sf.py4j', name: 'py4j', version: '0.10.1' + compile group: 'net.sf.py4j', name: 'py4j', version: '0.10.4' testCompile group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion, classifier: 'tests' testCompile "org.apache.derby:derby:${derbyVersion}" diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index ff205d4933bd0..cbff59036b63b 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -412,13 +412,14 @@ private[spark] class Executor( // includes the Partition. Second, Task.run() deserializes the RDD and function to be run. task.metrics.setExecutorDeserializeTime(math.max( taskStart - deserializeStartTime + task.executorDeserializeTime, 0L) / 1000000.0) - task.metrics.setExecutorDeserializeCpuTime( - (taskStartCpu - deserializeStartCpuTime) + task.executorDeserializeCpuTime) + task.metrics.setExecutorDeserializeCpuTime(math.max( + taskStartCpu - deserializeStartCpuTime + task.executorDeserializeCpuTime, 0L) / + 1000000.0) // We need to subtract Task.run()'s deserialization time to avoid double-counting task.metrics.setExecutorRunTime(math.max( taskFinish - taskStart - task.executorDeserializeTime, 0L) / 1000000.0) - task.metrics.setExecutorCpuTime( - (taskFinishCpu - taskStartCpu) - task.executorDeserializeCpuTime) + task.metrics.setExecutorCpuTime(math.max( + taskFinishCpu - taskStartCpu - task.executorDeserializeCpuTime, 0L) / 1000000.0) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) // Expose task metrics using the Dropwizard metrics system. diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala index 134c2706cfb40..e126d9b495d65 100644 --- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala @@ -48,9 +48,9 @@ import org.apache.spark.util._ class TaskMetrics private[spark] () extends Serializable with KryoSerializable { // Each metric is internally represented as an accumulator private val _executorDeserializeTime = new DoubleAccumulator - private val _executorDeserializeCpuTime = new LongAccumulator + private val _executorDeserializeCpuTime = new DoubleAccumulator private val _executorRunTime = new DoubleAccumulator - private val _executorCpuTime = new LongAccumulator + private val _executorCpuTime = new DoubleAccumulator private val _resultSize = new LongAccumulator private val _jvmGCTime = new LongAccumulator private val _resultSerializationTime = new DoubleAccumulator @@ -67,7 +67,7 @@ class TaskMetrics private[spark] () extends Serializable with KryoSerializable { /** * CPU Time taken on the executor to deserialize this task in nanoseconds. */ - def executorDeserializeCpuTime: Long = _executorDeserializeCpuTime.sum + def executorDeserializeCpuTime: Long = _executorDeserializeCpuTime.sum.toLong /** * Time the executor spends actually running the task (including fetching shuffle data). @@ -78,7 +78,7 @@ class TaskMetrics private[spark] () extends Serializable with KryoSerializable { * CPU Time the executor spends actually running the task * (including fetching shuffle data) in nanoseconds. */ - def executorCpuTime: Long = _executorCpuTime.sum + def executorCpuTime: Long = _executorCpuTime.sum.toLong /** * The number of bytes this task transmitted back to the driver as the TaskResult. @@ -131,10 +131,10 @@ class TaskMetrics private[spark] () extends Serializable with KryoSerializable { // Setters and increment-ers private[spark] def setExecutorDeserializeTime(v: Double): Unit = _executorDeserializeTime.setValue(v) - private[spark] def setExecutorDeserializeCpuTime(v: Long): Unit = + private[spark] def setExecutorDeserializeCpuTime(v: Double): Unit = _executorDeserializeCpuTime.setValue(v) private[spark] def setExecutorRunTime(v: Double): Unit = _executorRunTime.setValue(v) - private[spark] def setExecutorCpuTime(v: Long): Unit = _executorCpuTime.setValue(v) + private[spark] def setExecutorCpuTime(v: Double): Unit = _executorCpuTime.setValue(v) private[spark] def setResultSize(v: Long): Unit = _resultSize.setValue(v) private[spark] def setJvmGCTime(v: Long): Unit = _jvmGCTime.setValue(v) private[spark] def setResultSerializationTime(v: Double): Unit = diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index ab5672a8ced9d..adcfb4eaf932d 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -793,7 +793,8 @@ class DAGScheduler( // That should take care of at least part of the priority inversion problem with // cross-job dependencies. private def activeJobForStage(stage: Stage): Option[Int] = { - val jobsThatUseStage: Array[Int] = stage.jobIds.toArray.sorted + val jobsThatUseStage: Array[Int] = stage.jobIds.toArray + java.util.Arrays.sort(jobsThatUseStage) jobsThatUseStage.find(jobIdToActiveJob.contains) } @@ -1075,7 +1076,7 @@ class DAGScheduler( stage.pendingPartitions += id new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, taskData, taskBinary, part, locs, stage.latestInfo.taskMetrics, properties, - Option(jobId), Option(sc.applicationId), Option(sc.applicationId)) + jobId, Option(sc.applicationId), Option(sc.applicationId)) } case stage: ResultStage => @@ -1085,7 +1086,7 @@ class DAGScheduler( val locs = taskIdToLocations(id) new ResultTask(stage.id, stage.latestInfo.attemptId, taskData, taskBinary, part, locs, id, properties, stage.latestInfo.taskMetrics, - Option(jobId), Option(sc.applicationId), Option(sc.applicationId)) + jobId, Option(sc.applicationId), Option(sc.applicationId)) } } } catch { diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index 73b4662a0c949..5661a19ae65dc 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -63,8 +63,8 @@ private[spark] class ResultTask[T, U]( locs: Seq[TaskLocation], private var _outputId: Int, localProperties: Properties, - serializedTaskMetrics: Array[Byte], - jobId: Option[Int] = None, + metrics: TaskMetrics, + jobId: Int = -1, appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[U](stageId, stageAttemptId, partition.index, _taskData, @@ -89,7 +89,7 @@ private[spark] class ResultTask[T, U]( ByteBuffer.wrap(getTaskBytes), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = math.max(System.nanoTime() - deserializeStartTime, 0L) _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { - threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime + math.max(threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime, 0L) } else 0L func(context, rdd.iterator(partition, context)) diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index 46e03c033f36c..5c99931eb3109 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -63,13 +63,12 @@ private[spark] class ShuffleMapTask( private var partition: Partition, @transient private var locs: Seq[TaskLocation], localProperties: Properties, - serializedTaskMetrics: Array[Byte], - jobId: Option[Int] = None, + jobId: Int = -1, appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[MapStatus](stageId, stageAttemptId, partition.index, _taskData, _taskBinary, metrics, localProperties, jobId, appId, appAttemptId) -with KryoSerializable with Logging { + with KryoSerializable with Logging { /** A constructor used only in test suites. This does not require passing in an RDD. */ def this(partitionId: Int) { @@ -93,7 +92,7 @@ with KryoSerializable with Logging { ByteBuffer.wrap(getTaskBytes), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = math.max(System.nanoTime() - deserializeStartTime, 0L) _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { - threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime + math.max(threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime, 0L) } else 0L var writer: ShuffleWriter[Any, Any] = null diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index edc541cdc2841..ac0586cb6066d 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -67,7 +67,7 @@ private[spark] abstract class Task[T]( // The default value is only used in tests. serializedTaskMetrics: Array[Byte] = SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array(), - private var _jobId: Option[Int] = None, + private var _jobId: Int = -1, private var _appId: Option[String] = None, private var _appAttemptId: Option[String] = None) extends Serializable { @@ -80,11 +80,13 @@ private[spark] abstract class Task[T]( @transient lazy val metrics: TaskMetrics = SparkEnv.get.closureSerializer.newInstance().deserialize(ByteBuffer.wrap(serializedTaskMetrics)) - final def jobId: Int = _jobId.get + final def jobId: Int = _jobId - final def appId: String = _appId.get + final def metrics: TaskMetrics = _metrics - final def appAttemptId: String = _appAttemptId.get + final def appId: String = if (_appId.isDefined) _appId.get else null + + final def appAttemptId: String = if (_appAttemptId.isDefined) _appAttemptId.get else null @transient private[spark] var taskDataBytes: Array[Byte] = _ @@ -251,8 +253,10 @@ private[spark] abstract class Task[T]( output.writeInt(_stageId) output.writeVarInt(_stageAttemptId, true) output.writeVarInt(_partitionId, true) + output.writeVarInt(_jobId, true) output.writeLong(epoch) output.writeLong(_executorDeserializeTime) + output.writeLong(_executorDeserializeCpuTime) if ((taskData ne null) && taskData.uncompressedLen > 0) { // actual bytes will be shipped in TaskDescription output.writeBoolean(true) @@ -261,17 +265,18 @@ private[spark] abstract class Task[T]( kryo.writeClassAndObject(output, taskBinary.get) } _metrics.write(kryo, output) - output.writeInt(_jobId.get) - output.writeString(_appId.get) - output.writeString(_appAttemptId.get) + output.writeString(appId) + output.writeString(appAttemptId) } def readKryo(kryo: Kryo, input: Input): Unit = { _stageId = input.readInt() _stageAttemptId = input.readVarInt(true) _partitionId = input.readVarInt(true) + _jobId = input.readVarInt(true) epoch = input.readLong() _executorDeserializeTime = input.readLong() + _executorDeserializeCpuTime = input.readLong() // actual bytes are shipped in TaskDescription taskData = TaskData.EMPTY if (input.readBoolean()) { @@ -282,9 +287,8 @@ private[spark] abstract class Task[T]( } _metrics = new TaskMetrics _metrics.read(kryo, input) - _jobId = Some(input.readInt()) - _appId = Some(input.readString()) - _appAttemptId = Some(input.readString()) + _appId = Option(input.readString()) + _appAttemptId = Option(input.readString()) } } diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index c7faeb398a3ca..25a1f6b477f8a 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -1005,6 +1005,7 @@ private[spark] class TaskSetManager( private def getLocalityWait(level: TaskLocality.TaskLocality): Long = { val defaultWait = conf.get(config.LOCALITY_WAIT) + // val defaultWait = conf.get("spark.locality.wait", "10s") val localityWaitKey = level match { case TaskLocality.PROCESS_LOCAL => "spark.locality.wait.process" case TaskLocality.NODE_LOCAL => "spark.locality.wait.node" diff --git a/examples/build.gradle b/examples/build.gradle index eeeee87812fed..325c31cbbb89d 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -26,7 +26,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-streaming-flume_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion) - compile group: 'org.apache.commons', name: 'commons-math3', version: '3.4.1' + compile group: 'org.apache.commons', name: 'commons-math3', version: commonsMath3Version compile group: 'com.github.scopt', name: 'scopt_' + scalaBinaryVersion, version: '3.3.0' compile group: 'com.twitter', name: 'parquet-hadoop-bundle', version: hiveParquetVersion diff --git a/mesos/build.gradle b/mesos/build.gradle new file mode 100644 index 0000000000000..94e0d8ebed48f --- /dev/null +++ b/mesos/build.gradle @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Project Mesos' + +dependencies { + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + + compile(group: 'org.apache.mesos', name: 'mesos', version: '1.0.0', classifier: 'shaded-protobuf') { + exclude(group: 'com.google.protobuf', module: 'protobuf-java') + } + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile group: 'org.eclipse.jetty', name: 'jetty-server', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-plus', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-util', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-http', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion + compile group: 'org.eclipse.jetty', name: 'jetty-servlets', version: jettyVersion + + testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') +} diff --git a/mllib-local/build.gradle b/mllib-local/build.gradle index c4183a09ba733..2796bca9de463 100644 --- a/mllib-local/build.gradle +++ b/mllib-local/build.gradle @@ -19,11 +19,11 @@ description = 'Spark Project ML Local Library' dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.11.2') { + compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.12') { exclude(group: 'junit', module: 'junit') exclude(group: 'org.apache.commons', module: 'commons-math3') } - compile group: 'org.apache.commons', name: 'commons-math3', version: '3.4.1' + compile group: 'org.apache.commons', name: 'commons-math3', version: commonsMath3Version testCompile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' } diff --git a/mllib/build.gradle b/mllib/build.gradle index eb4c7dca1aa8e..cc47140680c60 100644 --- a/mllib/build.gradle +++ b/mllib/build.gradle @@ -29,7 +29,7 @@ dependencies { exclude(group: 'junit', module: 'junit') exclude(group: 'org.apache.commons', module: 'commons-math3') } - compile group: 'org.apache.commons', name: 'commons-math3', version: '3.4.1' + compile group: 'org.apache.commons', name: 'commons-math3', version: commonsMath3Version compile(group: 'org.jpmml', name: 'pmml-model', version: '1.2.15') { exclude(group: 'org.jpmml', module: 'pmml-agent') } diff --git a/settings.gradle b/settings.gradle index 7150a225b3de3..faea562f5b934 100644 --- a/settings.gradle +++ b/settings.gradle @@ -34,6 +34,7 @@ include ':snappy-spark-catalyst_' + scalaBinaryVersion include ':snappy-spark-sql_' + scalaBinaryVersion include ':snappy-spark-hive_' + scalaBinaryVersion include ':snappy-spark-hive-thriftserver_' + scalaBinaryVersion +include ':snappy-spark-mesos_' + scalaBinaryVersion include ':snappy-spark-unsafe_' + scalaBinaryVersion include ':snappy-spark-assembly_' + scalaBinaryVersion include ':snappy-spark-streaming-flume_' + scalaBinaryVersion @@ -62,6 +63,7 @@ project(':snappy-spark-catalyst_' + scalaBinaryVersion).projectDir = "$rootDir/s project(':snappy-spark-sql_' + scalaBinaryVersion).projectDir = "$rootDir/sql/core" as File project(':snappy-spark-hive_' + scalaBinaryVersion).projectDir = "$rootDir/sql/hive" as File project(':snappy-spark-hive-thriftserver_' + scalaBinaryVersion).projectDir = "$rootDir/sql/hive-thriftserver" as File +project(':snappy-spark-mesos_' + scalaBinaryVersion).projectDir = "$rootDir/mesos" as File project(':snappy-spark-unsafe_' + scalaBinaryVersion).projectDir = "$rootDir/common/unsafe" as File project(':snappy-spark-assembly_' + scalaBinaryVersion).projectDir = "$rootDir/assembly" as File project(':snappy-spark-streaming-flume_' + scalaBinaryVersion).projectDir = "$rootDir/external/flume" as File diff --git a/sql/catalyst/build.gradle b/sql/catalyst/build.gradle index aef0611699b4d..0f2340d49e819 100644 --- a/sql/catalyst/build.gradle +++ b/sql/catalyst/build.gradle @@ -27,6 +27,7 @@ dependencies { compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion compile group: 'org.scala-lang.modules', name: 'scala-parser-combinators_' + scalaBinaryVersion, version: '1.0.4' compile group: 'org.codehaus.janino', name: 'janino', version: '3.0.6' + compile group: 'org.codehaus.janino', name: 'commons-compiler', version: '3.0.6' compile group: 'org.antlr', name: 'antlr4-runtime', version: antlrVersion compile group: 'commons-codec', name: 'commons-codec', version: commonsCodecVersion antlr group: 'org.antlr', name: 'antlr4', version: antlrVersion diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java index d7f98e6ac12f3..791e8d80e6cba 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java @@ -190,9 +190,10 @@ public void write(int ordinal, Decimal input, int precision, int scale) { } else { final byte[] bytes = input.toJavaBigDecimal().unscaledValue().toByteArray(); final int numBytes = bytes.length; - // assert numBytes <= 16; + assert numBytes <= 16; int roundedSize = ByteArrayMethods.roundNumberOfBytesToNearestWord(numBytes); holder.grow(roundedSize); + zeroOutPaddingBytes(numBytes); // Write the bytes to the variable length portion. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala index 1c0b7bd806801..a8a1c662e14f5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala @@ -607,8 +607,10 @@ object FoldablePropagation extends Rule[LogicalPlan] { */ object SimplifyCasts extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case Cast(e, dataType, _) if e.dataType == dataType => e - case c @ Cast(e, dataType, _) => (e.dataType, dataType) match { + case Cast(e, dataType) if e.dataType == dataType || + (e.dataType.getClass == dataType.getClass && + e.dataType.asNullable == dataType) => e + case c @ Cast(e, dataType) => (e.dataType, dataType) match { case (ArrayType(from, false), ArrayType(to, true)) if from == to => e case (MapType(fromKey, fromValue, false), MapType(toKey, toValue, true)) if fromKey == toKey && fromValue == toValue => e diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index 603f350533899..fbf1b6f8783f6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -14,6 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark.sql.catalyst.plans.logical @@ -433,8 +451,9 @@ case class InsertIntoDir( storage: CatalogStorageFormat, provider: Option[String], child: LogicalPlan, - overwrite: Boolean = true) - extends UnaryNode { + overwrite: Boolean = true, + ifNotExists: Boolean) + extends LogicalPlan { override def output: Seq[Attribute] = Seq.empty override lazy val resolved: Boolean = false @@ -468,27 +487,12 @@ case class View( s"View (${desc.identifier}, ${output.mkString("[", ",", "]")})" } - lazy val expectedColumns = { - if (table.output.isEmpty) { - None - } else { - // Note: The parser (visitPartitionSpec in AstBuilder) already turns - // keys in partition to their lowercase forms. - val staticPartCols = partition.filter(_._2.isDefined).keySet - Some(table.output.filterNot(a => staticPartCols.contains(a.name))) - } - } - assert(overwrite.enabled || !ifNotExists) assert(partition.values.forall(_.nonEmpty) || !ifNotExists) - override lazy val resolved: Boolean = - childrenResolved && table.resolved && expectedColumns.forall { expected => - child.output.size == expected.size && child.output.zip(expected).forall { - case (childAttr, tableAttr) => - DataType.equalsIgnoreCompatibleNullability(childAttr.dataType, tableAttr.dataType) - } - } + + override lazy val resolved: Boolean = childrenResolved && table.resolved } + /** * A container for holding named common table expressions (CTEs) and a query plan. * This operator will be removed during analysis and the relations will be substituted into child. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala index 95c8aee819fac..3041f44b116ea 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala @@ -14,24 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ package org.apache.spark.sql.types diff --git a/sql/core/build.gradle b/sql/core/build.gradle index c6b3f5038db8d..2f943eeeab845 100644 --- a/sql/core/build.gradle +++ b/sql/core/build.gradle @@ -23,7 +23,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-sketch_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile group: 'com.univocity', name: 'univocity-parsers', version: '2.1.1' + compile group: 'com.univocity', name: 'univocity-parsers', version: '2.2.1' compile group: 'org.apache.parquet', name: 'parquet-column', version: parquetVersion compile group: 'org.apache.parquet', name: 'parquet-hadoop', version: parquetVersion compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion diff --git a/sql/hive/build.gradle b/sql/hive/build.gradle index 64ba9c361ea65..e632625149210 100644 --- a/sql/hive/build.gradle +++ b/sql/hive/build.gradle @@ -47,6 +47,7 @@ dependencies { exclude(group: 'commons-logging', module: 'commons-logging') exclude(group: 'org.codehaus.groovy', module: 'groovy-all') exclude(group: 'jline', module: 'jline') + exclude(group: 'org.json', module: 'json') } compile(group: 'org.spark-project.hive', name: 'hive-metastore', version: hiveVersion) { exclude(group: 'org.spark-project.hive', module: 'hive-serde') @@ -57,6 +58,7 @@ dependencies { exclude(group: 'com.google.guava', module: 'guava') exclude(group: 'org.slf4j', module: 'slf4j-api') exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + exclude(group: 'log4j', module: 'log4j') exclude(group: 'org.apache.derby', module: 'derby') } diff --git a/yarn/build.gradle b/yarn/build.gradle index 5fb389cba38c0..5d57c8927ff71 100644 --- a/yarn/build.gradle +++ b/yarn/build.gradle @@ -92,6 +92,7 @@ dependencies { exclude(group: 'commons-logging', module: 'commons-logging') exclude(group: 'org.codehaus.groovy', module: 'groovy-all') exclude(group: 'jline', module: 'jline') + exclude(group: 'org.json', module: 'json') } compile(group: 'org.spark-project.hive', name: 'hive-metastore', version: hiveVersion) { exclude(group: 'org.spark-project.hive', module: 'hive-serde') @@ -102,6 +103,7 @@ dependencies { exclude(group: 'com.google.guava', module: 'guava') exclude(group: 'org.slf4j', module: 'slf4j-api') exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + exclude(group: 'log4j', module: 'log4j') exclude(group: 'org.apache.derby', module: 'derby') } compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) { From ffa86a7708e9a617dc71f02903086f5878a7d224 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 19 Jul 2017 04:54:56 +0530 Subject: [PATCH 082/142] [SNAP-1389] Optimized UTF8String.compareTo (#62) - use unsigned long comparisons, followed by unsigned int comparison if possible, before finishing with unsigned byte comparisons for better performance - use big-endian long/int for comparison since it requires the lower-index characters to be MSB positions - no alignment attempted since we expect most cases to fail early in first long comparison itself Detailed performance results in https://github.com/SnappyDataInc/spark/pull/62 Conflicts: common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java --- .../apache/spark/unsafe/types/UTF8String.java | 75 ++++++++++++++----- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index b45022fc728bd..27461adcfbc62 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -1225,28 +1225,65 @@ public int compareTo(@Nonnull final UTF8String other) { return compare(other); } - public int compare(final UTF8String other) { - int len = Math.min(numBytes, other.numBytes); - int wordMax = (len / 8) * 8; - long roffset = other.offset; - Object rbase = other.base; - for (int i = 0; i < wordMax; i += 8) { - long left = getLong(base, offset + i); - long right = getLong(rbase, roffset + i); - if (left != right) { - if (IS_LITTLE_ENDIAN) { - return Long.compareUnsigned(Long.reverseBytes(left), Long.reverseBytes(right)); - } else { - return Long.compareUnsigned(left, right); - } + /** Read integer in big-endian format */ + static int getIntBigEndian(final Object base, final long offset) { + return isLittleEndian ? Integer.reverseBytes(Platform.getInt(base, offset)) + : Platform.getInt(base, offset); + } + + /** Read long in big-endian format */ + static long getLongBigEndian(final Object base, final long offset) { + return isLittleEndian ? Long.reverseBytes(Platform.getLong(base, offset)) + : Platform.getLong(base, offset); + } + + public final int compare(final UTF8String other) { + final Object rightBase = other.getBaseObject(); + long rightOffset = other.getBaseOffset(); + final Object leftBase = base; + long leftOffset = offset; + + final int len = Math.min(numBytes, other.numBytes); + long endOffset = leftOffset + len; + // for architectures that support unaligned accesses, read 8 bytes at a time + if (Platform.unaligned() || (((leftOffset & 0x7) == 0) && ((rightOffset & 0x7) == 0))) { + endOffset -= 8; + while (leftOffset <= endOffset) { + // In UTF-8, the byte should be unsigned, so we should compare them as unsigned long. + final long ll = getLongBigEndian(leftBase, leftOffset); + final long rl = getLongBigEndian(rightBase, rightOffset); + final long res = ll - rl; + // If the sign of both values is same then "res" is with correct sign. + // If the sign of values is different then "res" has opposite sign. + // The XOR operations will revert the sign bit of res if sign of values is different. + // After that converting to signum is "(1 + ((v >> 63) << 1))" + // where (v >> 63) will flow the sign to give -1 or 0, and (1 + 2 times) + // of that will give -1 or 1 respectively. + if (res != 0) return (int)(1 + (((ll ^ rl ^ res) >> 63) << 1)); + leftOffset += 8; + rightOffset += 8; } + endOffset += 4; + if (leftOffset <= endOffset) { + // In UTF-8, the byte should be unsigned, so we should compare them as unsigned int + // which is done by converting to unsigned longs. + // After that conversion to signed integer is "(1 + ((v >> 63) << 1))" as above. + final long res = (getIntBigEndian(leftBase, leftOffset) & 0xffffffffL) - + (getIntBigEndian(rightBase, rightOffset) & 0xffffffffL); + if (res != 0) return (int)(1 + ((res >> 63) << 1)); + leftOffset += 4; + rightOffset += 4; + } + endOffset += 4; } - for (int i = wordMax; i < len; i++) { + // finish the remaining bytes + while (leftOffset < endOffset) { // In UTF-8, the byte should be unsigned, so we should compare them as unsigned int. - int res = (getByte(i) & 0xFF) - (Platform.getByte(rbase, roffset + i) & 0xFF); - if (res != 0) { - return res; - } + final int res = (Platform.getByte(leftBase, leftOffset) & 0xff) - + (Platform.getByte(rightBase, rightOffset) & 0xff); + if (res != 0) return res; + leftOffset++; + rightOffset++; } return numBytes - other.numBytes; } From 8293c86d8cd2607d782daa690bdb7fe7dc336bc0 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sun, 23 Jul 2017 20:07:45 +0530 Subject: [PATCH 083/142] [SNAPPYDATA][PERF] optimized pattern matching for byte/time strings also added slf4j excludes to some imports Conflicts: common/network-common/src/main/java/org/apache/spark/network/util/JavaUtils.java external/spark-ganglia-lgpl/build.gradle --- .../apache/spark/network/util/JavaUtils.java | 26 +++++++++++-------- common/network-shuffle/build.gradle | 5 +++- core/build.gradle | 22 +++++++++++++--- external/spark-ganglia-lgpl/build.gradle | 7 ++++- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/JavaUtils.java b/common/network-common/src/main/java/org/apache/spark/network/util/JavaUtils.java index afc59efaef810..5ac74a9d66ec2 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/JavaUtils.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/JavaUtils.java @@ -42,7 +42,9 @@ */ public class JavaUtils { private static final Logger logger = LoggerFactory.getLogger(JavaUtils.class); - + private static final Pattern timePattern = Pattern.compile("(-?[0-9]+)([a-zA-Z]+)?"); + private static final Pattern byteAsStringPattern = Pattern.compile("([0-9]+)([a-zA-Z]+)?"); + private static final Pattern fractionPattern = Pattern.compile("([0-9]+\\.[0-9]+)([a-zA-Z]+)?"); /** * Define a default value for driver memory here since this value is referenced across the code * base and nearly all files already use Utils.scala @@ -211,10 +213,10 @@ private static boolean isSymlink(File file) throws IOException { * The unit is also considered the default if the given string does not specify a unit. */ public static long timeStringAs(String str, TimeUnit unit) { - String lower = str.toLowerCase(Locale.ROOT).trim(); + String s = str.trim(); try { - Matcher m = Pattern.compile("(-?[0-9]+)([a-z]+)?").matcher(lower); + Matcher m = timePattern.matcher(s); if (!m.matches()) { throw new NumberFormatException("Failed to parse time string: " + str); } @@ -223,12 +225,13 @@ public static long timeStringAs(String str, TimeUnit unit) { String suffix = m.group(2); // Check for invalid suffixes - if (suffix != null && !timeSuffixes.containsKey(suffix)) { + TimeUnit target = unit; + if (suffix != null && (target = timeSuffixes.get(suffix.toLowerCase())) == null) { throw new NumberFormatException("Invalid suffix: \"" + suffix + "\""); } // If suffix is valid use that, otherwise none was provided and use the default passed - return unit.convert(val, suffix != null ? timeSuffixes.get(suffix) : unit); + return unit.convert(val, target); } catch (NumberFormatException e) { String timeError = "Time must be specified as seconds (s), " + "milliseconds (ms), microseconds (us), minutes (m or min), hour (h), or day (d). " + @@ -259,24 +262,25 @@ public static long timeStringAsSec(String str) { * provided, a direct conversion to the provided unit is attempted. */ public static long byteStringAs(String str, ByteUnit unit) { - String lower = str.toLowerCase(Locale.ROOT).trim(); + String s = str.trim(); try { - Matcher m = Pattern.compile("([0-9]+)([a-z]+)?").matcher(lower); - Matcher fractionMatcher = Pattern.compile("([0-9]+\\.[0-9]+)([a-z]+)?").matcher(lower); + Matcher m = byteAsStringPattern.matcher(s); + Matcher fractionMatcher; if (m.matches()) { long val = Long.parseLong(m.group(1)); String suffix = m.group(2); // Check for invalid suffixes - if (suffix != null && !byteSuffixes.containsKey(suffix)) { + ByteUnit target = unit; + if (suffix != null && (target = byteSuffixes.get(suffix.toLowerCase())) == null) { throw new NumberFormatException("Invalid suffix: \"" + suffix + "\""); } // If suffix is valid use that, otherwise none was provided and use the default passed - return unit.convertFrom(val, suffix != null ? byteSuffixes.get(suffix) : unit); - } else if (fractionMatcher.matches()) { + return unit.convertFrom(val, target); + } else if ((fractionMatcher = fractionPattern.matcher(s)).matches()) { throw new NumberFormatException("Fractional values are not supported. Input was: " + fractionMatcher.group(1)); } else { diff --git a/common/network-shuffle/build.gradle b/common/network-shuffle/build.gradle index 9c916a8532d9d..a36492bb087dd 100644 --- a/common/network-shuffle/build.gradle +++ b/common/network-shuffle/build.gradle @@ -25,7 +25,10 @@ dependencies { compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: fasterXmlVersion compile group: 'com.google.guava', name: 'guava', version: guavaVersion - compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion + compile(group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + } testCompile project(path: subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion, configuration: 'testOutput') } diff --git a/core/build.gradle b/core/build.gradle index 901058f46a4c4..a9cef0e2a24ec 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -54,6 +54,8 @@ dependencies { compile(group: 'org.apache.zookeeper', name: 'zookeeper', version: '3.4.8') { exclude(group: 'org.jboss.netty', module: 'netty') exclude(group: 'jline', module: 'jline') + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') } compile group: 'com.google.protobuf', name: 'protobuf-java', version: protobufVersion compile(group: 'org.apache.hadoop', name: 'hadoop-client', version: hadoopVersion) { @@ -126,10 +128,22 @@ dependencies { compile(group: 'com.clearspring.analytics', name: 'stream', version: '2.7.0') { exclude(group: 'it.unimi.dsi', module: 'fastutil') } - compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion - compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: metricsVersion - compile group: 'io.dropwizard.metrics', name: 'metrics-json', version: metricsVersion - compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: metricsVersion + compile(group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + } + compile(group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: metricsVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + } + compile(group: 'io.dropwizard.metrics', name: 'metrics-json', version: metricsVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + } + compile(group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: metricsVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + } compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion compile(group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_' + scalaBinaryVersion, version: fasterXmlVersion) { exclude(group: 'com.google.guava', module: 'guava') diff --git a/external/spark-ganglia-lgpl/build.gradle b/external/spark-ganglia-lgpl/build.gradle index 39e0a747ce432..31773cc30c3e1 100644 --- a/external/spark-ganglia-lgpl/build.gradle +++ b/external/spark-ganglia-lgpl/build.gradle @@ -20,5 +20,10 @@ description = 'Spark Ganglia Integration' dependencies { compile project(subprojectBase + 'spark-core_' + scalaBinaryVersion) - compile group: 'io.dropwizard.metrics', name: 'metrics-ganglia', version: metricsVersion + compile(group: 'io.dropwizard.metrics', name: 'metrics-ganglia', version: metricsVersion) { + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + } + + testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') } From 5fd52dd865cffb95c133056063e1d27db138791f Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Tue, 25 Jul 2017 16:51:04 +0530 Subject: [PATCH 084/142] SNAP-1792: Display snappy members logs on Snappy Pulse UI (#58) Changes: - Adding snappy member details javascript for new UI view named SnappyData Member Details Page --- .../static/snappydata/snappy-memberdetails.js | 114 ++++++++++++++++++ .../scala/org/apache/spark/ui/UIUtils.scala | 1 + 2 files changed, 115 insertions(+) create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-memberdetails.js diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-memberdetails.js b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-memberdetails.js new file mode 100644 index 0000000000000..d481d5fbe5469 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-memberdetails.js @@ -0,0 +1,114 @@ + + +var baseParams; + +var curLogLength; +var startByte; +var endByte; +var totalLogLength; + +var byteLength; + +function setLogScroll(oldHeight) { + var logContent = $(".log-content"); + logContent.scrollTop(logContent[0].scrollHeight - oldHeight); +} + +function tailLog() { + var logContent = $(".log-content"); + logContent.scrollTop(logContent[0].scrollHeight); +} + +function setLogData() { + $('#log-data').html("Showing " + curLogLength + " Bytes: " + startByte + + " - " + endByte + " of " + totalLogLength); +} + +function disableMoreButton() { + var moreBtn = $(".log-more-btn"); + moreBtn.attr("disabled", "disabled"); + moreBtn.html("Top of Log"); +} + +function noNewAlert() { + var alert = $(".no-new-alert"); + alert.css("display", "block"); + window.setTimeout(function () {alert.css("display", "none");}, 4000); +} + +function loadMore() { + var offset = Math.max(startByte - byteLength, 0); + var moreByteLength = Math.min(byteLength, startByte); + + $.ajax({ + type: "GET", + url: "/dashboard/memberDetails/log" + baseParams + "&offset=" + offset + "&byteLength=" + moreByteLength, + success: function (data) { + var oldHeight = $(".log-content")[0].scrollHeight; + var newlineIndex = data.indexOf('\n'); + var dataInfo = data.substring(0, newlineIndex).match(/\d+/g); + var retStartByte = dataInfo[0]; + var retLogLength = dataInfo[2]; + + var cleanData = data.substring(newlineIndex + 1); + if (retStartByte == 0) { + disableMoreButton(); + } + $("pre", ".log-content").prepend(cleanData); + + curLogLength = curLogLength + (startByte - retStartByte); + startByte = retStartByte; + totalLogLength = retLogLength; + setLogScroll(oldHeight); + setLogData(); + } + }); +} + +function loadNew() { + $.ajax({ + type: "GET", + url: "/dashboard/memberDetails/log" + baseParams + "&byteLength=0", + success: function (data) { + var dataInfo = data.substring(0, data.indexOf('\n')).match(/\d+/g); + var newDataLen = dataInfo[2] - totalLogLength; + if (newDataLen != 0) { + $.ajax({ + type: "GET", + url: "/dashboard/memberDetails/log" + baseParams + "&byteLength=" + newDataLen, + success: function (data) { + var newlineIndex = data.indexOf('\n'); + var dataInfo = data.substring(0, newlineIndex).match(/\d+/g); + var retStartByte = dataInfo[0]; + var retEndByte = dataInfo[1]; + var retLogLength = dataInfo[2]; + + var cleanData = data.substring(newlineIndex + 1); + $("pre", ".log-content").append(cleanData); + + curLogLength = curLogLength + (retEndByte - retStartByte); + endByte = retEndByte; + totalLogLength = retLogLength; + tailLog(); + setLogData(); + } + }); + } else { + noNewAlert(); + } + } + }); +} + +function initLogPage(params, logLen, start, end, totLogLen, defaultLen) { + baseParams = params; + curLogLength = logLen; + startByte = start; + endByte = end; + totalLogLength = totLogLen; + byteLength = defaultLen; + tailLog(); + if (startByte == 0) { + disableMoreButton(); + } +} \ No newline at end of file diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index f2e13c8afb8bc..861a210c47d1a 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -200,6 +200,7 @@ private[spark] object UIUtils extends Logging { + } def vizHeaderNodes: Seq[Node] = { From e7392ba3a40d99569b5742e269055d0c04eb55e1 Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Tue, 1 Aug 2017 00:24:42 +0530 Subject: [PATCH 085/142] SNAP-1744: UI itself needs to consistently refer to itself as "SnappyData Pulse" (#64) * SNAP-1744: UI itself needs to consistently refer to itself as "SnappyData Pulse" Changes: - SnappyData Dashboard UI is named as SnappyData Pulse now. - Code refactoring and code clean up. Conflicts: core/src/main/scala/org/apache/spark/ui/UIUtils.scala --- .../ui/static/snappydata/snappy-dashboard.css | 10 ++ .../scala/org/apache/spark/ui/UIUtils.scala | 99 +++++-------------- 2 files changed, 36 insertions(+), 73 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css index 53d4983da4b29..69c45348a9e99 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css +++ b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappy-dashboard.css @@ -4,6 +4,16 @@ ========================================================================== */ +.UIName { + line-height: 2.5; + vertical-align: middle; + font-size: 20px; + padding: 0; + margin: 0; + font-weight: bold; + color: #3CA881; +} + /* .keyStates { float: left; diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 861a210c47d1a..419bf927fcaca 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -165,8 +165,6 @@ private[spark] object UIUtils extends Logging { - @@ -180,23 +178,9 @@ private[spark] object UIUtils extends Logging { } - def commonHeaderNodes_2: Seq[Node] = { - - - - - + def commonHeaderNodesSnappy: Seq[Node] = { - - - - - - - - - @@ -256,8 +240,8 @@ private[spark] object UIUtils extends Logging {
    - - + + {getProductUINameNode} {getProductVersionNode}
    @@ -307,6 +291,7 @@ private[spark] object UIUtils extends Logging { {commonHeaderNodes} + {commonHeaderNodesSnappy} {if (showVisualization) vizHeaderNodes else Seq.empty} {appName} - {title} @@ -318,60 +303,6 @@ private[spark] object UIUtils extends Logging { {getProductUINameNode} {getProductVersionNode} - - -
    -
    - - - - {getProductVersionNode} - - {getProductVersionNode} -
    - {getProductDocLinkNode()} -
      {header}
    -
    -
    -
    - {content} -
    - - - } - - /** Returns a simple spark page with correctly formatted tabs */ - def simpleSparkPageWithTabs_2( - title: String, - content: => Seq[Node], - activeTab: SparkUITab, - refreshInterval: Option[Int] = None, - helpText: Option[String] = None, - showVisualization: Boolean = false): Seq[Node] = { - - val appName = activeTab.appName - val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..." - val header = activeTab.headerTabs.map { tab => -
  • - {tab.name} -
    • - } - // val helpButton: Seq[Node] = helpText.map(tooltip(_, "bottom")).getOrElse(Seq.empty) - - - - {commonHeaderNodes_2} - {if (showVisualization) vizHeaderNodes else Seq.empty} - {appName} - {title} - - -
    -
    -
    - - - - {getProductVersionNode}

    @@ -685,6 +616,7 @@ private[spark] object UIUtils extends Logging { def getProductVersionNode(): Node = { val versionDetails = SparkUI.getProductVersion val versionTooltipText = +<<<<<<< HEAD "SnappyData Ver. " + versionDetails.getOrElse("productVersion", "") + " ( Underlying Spark Ver. " + org.apache.spark.SPARK_VERSION + " )" @@ -705,6 +637,27 @@ private[spark] object UIUtils extends Logging { Source Revision : {versionDetails.getOrElse("sourceRevision", "")}

    +||||||| parent of c1acf05... SNAP-1744: UI itself needs to consistently refer to itself as "SnappyData Pulse" (#64) + "SnappyData Ver. " + SparkUI.getProductVersion + " ( Underlying Spark Ver. " + + org.apache.spark.SPARK_VERSION + " )" + + {SparkUI.getProductVersion} +======= + "SnappyData Ver. " + SparkUI.getProductVersion + " ( Underlying Spark Ver. " + + org.apache.spark.SPARK_VERSION + " )" + + {SparkUI.getProductVersion} +>>>>>>> c1acf05... SNAP-1744: UI itself needs to consistently refer to itself as "SnappyData Pulse" (#64) + } + + def getProductUINameNode(): Node = { + Pulse } def getProductDocLinkNode(): Node = { From 08fc9b62d573e8540b0619dcb642b8c5311877dc Mon Sep 17 00:00:00 2001 From: Sachin Janani Date: Wed, 9 Aug 2017 22:08:45 +0530 Subject: [PATCH 086/142] [SNAP-1377,SNAP-902] Proper handling of exception in case of Lead and Server HA (#65) * [SNAP-1377] Added callback used for checking CacheClosedException * [SNAP-1377] Added check for GemfirexdRuntimeException and GemfireXDException * Added license header in source file * Fix issue seen during precheckin Conflicts: core/src/main/scala/org/apache/spark/executor/Executor.scala --- .../apache/spark/SnappySparkCallback.scala | 24 ++++++++ .../apache/spark/SparkCallBackFactory.scala | 30 ++++++++++ .../org/apache/spark/executor/Executor.scala | 59 +++++++++++++------ 3 files changed, 94 insertions(+), 19 deletions(-) create mode 100644 core/src/main/scala/org/apache/spark/SnappySparkCallback.scala create mode 100644 core/src/main/scala/org/apache/spark/SparkCallBackFactory.scala diff --git a/core/src/main/scala/org/apache/spark/SnappySparkCallback.scala b/core/src/main/scala/org/apache/spark/SnappySparkCallback.scala new file mode 100644 index 0000000000000..979ef2e8f46d8 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/SnappySparkCallback.scala @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark + +trait SnappySparkCallback { + + def checkCacheClosing(t: Throwable): Boolean + + def checkRuntimeOrGemfireException(t: Throwable): Boolean +} diff --git a/core/src/main/scala/org/apache/spark/SparkCallBackFactory.scala b/core/src/main/scala/org/apache/spark/SparkCallBackFactory.scala new file mode 100644 index 0000000000000..2ad8be9fc5f07 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/SparkCallBackFactory.scala @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark + +object SparkCallBackFactory { + + var snappySparkCallbackImpl: SnappySparkCallback = _ + + def setSnappySparkCallback(snappySparkCallback: SnappySparkCallback): Unit = { + snappySparkCallbackImpl = snappySparkCallback + } + + def getSnappySparkCallback(): SnappySparkCallback = { + snappySparkCallbackImpl + } +} diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index cbff59036b63b..26ce8405ea926 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -529,28 +529,46 @@ private[spark] class Executor( execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) case t: Throwable => - // Attempt to exit cleanly by informing the driver of our failure. - // If anything goes wrong (or this was a fatal exception), we will delegate to - // the default uncaught exception handler, which will terminate the Executor. - logError(s"Exception in $taskName (TID $taskId)", t) - - // SPARK-20904: Do not report failure to driver if if happened during shut down. Because - // libraries may set up shutdown hooks that race with running tasks during shutdown, - // spurious failures may occur and can result in improper accounting in the driver (e.g. - // the task failure would not be ignored if the shutdown happened because of premption, - // instead of an app issue). - if (!ShutdownHookManager.inShutdown()) { + //Check if cache is closing + val snappyCallBack = SparkCallBackFactory.getSnappySparkCallback() + if (snappyCallBack != null && snappyCallBack.checkCacheClosing(t)) { + logError(s"Cache closed exception in $taskName (TID $taskId)", t) + setTaskFinishedAndClearInterruptStatus() + val reason = new ExecutorLostFailure(executorId, false, Some(t.getMessage)) + execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) + } else if (snappyCallBack != null && snappyCallBack.checkRuntimeOrGemfireException(t)) { + logError(s"Executor killed $taskName (TID $taskId)", t) + setTaskFinishedAndClearInterruptStatus() + val reason = { + try { + new ExceptionFailure(t, null, true) + } catch { + case _:Throwable => new ExceptionFailure(t, null, false) + } + } + execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) + val accUpdates = accums.map(acc => acc.toInfo(Some(acc.value), None)) + } else { + // Attempt to exit cleanly by informing the driver of our failure. + // If anything goes wrong (or this was a fatal exception), we will delegate to + // the default uncaught exception handler, which will terminate the Executor. + logError(s"Exception in $taskName (TID $taskId)", t) + // Collect latest accumulator values to report back to the driver val accums: Seq[AccumulatorV2[_, _]] = if (task != null) { task.metrics.setExecutorRunTime( math.max(System.nanoTime() - taskStart, 0L) / 1000000.0) + val taskEndCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { + threadMXBean.getCurrentThreadCpuTime + } else 0L + task.metrics.setExecutorCpuTime( + math.max(taskEndCpu - taskStartCpu, 0L) / 1000000.0) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) task.collectAccumulatorUpdates(taskFailed = true) } else { Seq.empty } - val accUpdates = accums.map(acc => acc.toInfo(Some(acc.value), None)) val serializedTaskEndReason = { @@ -564,14 +582,17 @@ private[spark] class Executor( } setTaskFinishedAndClearInterruptStatus() execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskEndReason) - } else { - logInfo("Not reporting error to driver during JVM shutdown.") - } - // Don't forcibly exit unless the exception was inherently fatal, to avoid - // stopping other tasks unnecessarily. - if (!t.isInstanceOf[SparkOutOfMemoryError] && Utils.isFatalError(t)) { - uncaughtExceptionHandler.uncaughtException(Thread.currentThread(), t) + // Don't forcibly exit unless the exception was inherently fatal, to avoid + // stopping other tasks unnecessarily. + if (Utils.isFatalError(t)) { + if (!isLocal) { + Thread.getDefaultUncaughtExceptionHandler. + uncaughtException(Thread.currentThread(), t) + } else { + SparkUncaughtExceptionHandler.uncaughtException(t) + } + } } } finally { runningTasks.remove(taskId) From 0e2181a3079931674693cf449dfcf6e52258a6d7 Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Thu, 10 Aug 2017 00:40:45 +0530 Subject: [PATCH 087/142] Snap 1833 (#67) Added a fallback path for WholeStageCodeGenRDD. As we dynamically change the classloader, generated code compile time classloaders and runtime class loader might be different. There is no clean way to handle this apart from recompiling the generated code. This code path will be executed only in case of components having dynamically changing class loaders i.e Snappy jobs & UDFs. Other sql queries won't be impacted by this. Conflicts: sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala --- .../org/apache/spark/executor/Executor.scala | 6 ++-- .../expressions/codegen/CodeGenerator.scala | 4 +++ .../sql/execution/WholeStageCodegenExec.scala | 30 +++++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 26ce8405ea926..9bd555b52729b 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -68,7 +68,7 @@ private[spark] class Executor( private val EMPTY_BYTE_BUFFER = ByteBuffer.wrap(new Array[Byte](0)) - private val conf = env.conf + protected val conf = env.conf // No ip or host:port - just hostname Utils.checkHost(executorHostname) @@ -126,8 +126,8 @@ private[spark] class Executor( // Create our ClassLoader // do this after SparkEnv creation so can access the SecurityManager - protected var urlClassLoader = createClassLoader() - protected var replClassLoader = addReplClassLoaderIfNeeded(urlClassLoader) + protected val urlClassLoader = createClassLoader() + protected val replClassLoader = addReplClassLoaderIfNeeded(urlClassLoader) // Set the classloader for serializer env.serializer.setDefaultClassLoader(replClassLoader) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 38a410774b271..5f2bc633812ba 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1386,6 +1386,10 @@ object CodeGenerator extends Logging { throw e.getCause } + def invalidate(code: CodeAndComment) : Unit = { + cache.invalidate(code) + } + /** * Compile the Java source code into a Java class, using Janino. */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala index 03a6b6c536158..6d9e9a7fb0a28 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala @@ -15,6 +15,7 @@ * limitations under the License. */ + package org.apache.spark.sql.execution import java.util.Locale @@ -25,6 +26,8 @@ import scala.collection.mutable import org.apache.spark.broadcast import org.apache.spark.rdd.RDD import com.esotericsoftware.kryo.io.{Input, Output} +import scala.util.control.Exception.catching + import com.esotericsoftware.kryo.{Kryo, KryoSerializable} import com.esotericsoftware.kryo.io.{Input, Output} @@ -712,7 +715,14 @@ case class CollapseCodegenStages(conf: SQLConf) extends Rule[SparkPlan] { } } +<<<<<<< HEAD class WholeStageCodegenRDD(sc: SparkContext, var source: CodeAndComment, +||||||| parent of e3f2e3d... Snap 1833 (#67) + +case class WholeStageCodegenRDD(@transient sc: SparkContext, var source: CodeAndComment, +======= +case class WholeStageCodegenRDD(@transient sc: SparkContext, var source: CodeAndComment, +>>>>>>> e3f2e3d... Snap 1833 (#67) var references: Array[Any], var durationMs: SQLMetric, inputRDDs: Seq[RDD[InternalRow]]) extends ZippedPartitionsBaseRDD[InternalRow](sc, inputRDDs) @@ -730,6 +740,26 @@ class WholeStageCodegenRDD(sc: SparkContext, var source: CodeAndComment, override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = { + val catcher = catching(classOf[ClassCastException]) + new Iterator[InternalRow] { + private[this] var i = computeInternal(split, context) + + private[this] def replace() = i = { + logInfo(s"ClassCast Exception, hence recompiling") + CodeGenerator.invalidate(source) + computeInternal(split, context) + } + + override def hasNext: Boolean = catcher.opt(i.hasNext).getOrElse { + replace(); hasNext + } + + override def next(): InternalRow = i.next() + } + } + + def computeInternal(split: Partition, + context: TaskContext): Iterator[InternalRow] = { val clazz = CodeGenerator.compile(source) val buffer = clazz.generate(references).asInstanceOf[BufferedRowIterator] if (rdds.length == 1) { From 638b0fb3689ba4f4b7759cd0b33cdd631acf4dfd Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Tue, 15 Aug 2017 11:04:02 +0530 Subject: [PATCH 088/142] Refactored the executor exception handling for cache (#71) Refactored the executor exception handling for cache closed exception. Conflicts: core/src/main/scala/org/apache/spark/executor/Executor.scala --- .../apache/spark/SnappySparkCallback.scala | 24 ---- .../apache/spark/SparkCallBackFactory.scala | 30 ----- .../org/apache/spark/executor/Executor.scala | 127 ++++++++++-------- 3 files changed, 68 insertions(+), 113 deletions(-) delete mode 100644 core/src/main/scala/org/apache/spark/SnappySparkCallback.scala delete mode 100644 core/src/main/scala/org/apache/spark/SparkCallBackFactory.scala diff --git a/core/src/main/scala/org/apache/spark/SnappySparkCallback.scala b/core/src/main/scala/org/apache/spark/SnappySparkCallback.scala deleted file mode 100644 index 979ef2e8f46d8..0000000000000 --- a/core/src/main/scala/org/apache/spark/SnappySparkCallback.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2017 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package org.apache.spark - -trait SnappySparkCallback { - - def checkCacheClosing(t: Throwable): Boolean - - def checkRuntimeOrGemfireException(t: Throwable): Boolean -} diff --git a/core/src/main/scala/org/apache/spark/SparkCallBackFactory.scala b/core/src/main/scala/org/apache/spark/SparkCallBackFactory.scala deleted file mode 100644 index 2ad8be9fc5f07..0000000000000 --- a/core/src/main/scala/org/apache/spark/SparkCallBackFactory.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2017 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ -package org.apache.spark - -object SparkCallBackFactory { - - var snappySparkCallbackImpl: SnappySparkCallback = _ - - def setSnappySparkCallback(snappySparkCallback: SnappySparkCallback): Unit = { - snappySparkCallbackImpl = snappySparkCallback - } - - def getSnappySparkCallback(): SnappySparkCallback = { - snappySparkCallbackImpl - } -} diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 9bd555b52729b..42702ea3e3fc7 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -528,72 +528,72 @@ private[spark] class Executor( setTaskFinishedAndClearInterruptStatus() execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) + case t: Throwable if isStoreCloseException(t) => + logError(s"Store closed exception in $taskName (TID $taskId)", t) + setTaskFinishedAndClearInterruptStatus() + val reason = new ExecutorLostFailure(executorId, false, Some(t.getMessage)) + val ser = env.closureSerializer.newInstance() + execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) + + case t: Throwable if isStoreException(t) => + logError(s"Executor killed $taskName (TID $taskId)", t) + setTaskFinishedAndClearInterruptStatus() + val reason = { + try { + new ExceptionFailure(t, null, true) + } catch { + case _: Throwable => new ExceptionFailure(t, null, false) + } + } + execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) + case t: Throwable => - //Check if cache is closing - val snappyCallBack = SparkCallBackFactory.getSnappySparkCallback() - if (snappyCallBack != null && snappyCallBack.checkCacheClosing(t)) { - logError(s"Cache closed exception in $taskName (TID $taskId)", t) - setTaskFinishedAndClearInterruptStatus() - val reason = new ExecutorLostFailure(executorId, false, Some(t.getMessage)) - execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) - } else if (snappyCallBack != null && snappyCallBack.checkRuntimeOrGemfireException(t)) { - logError(s"Executor killed $taskName (TID $taskId)", t) - setTaskFinishedAndClearInterruptStatus() - val reason = { - try { - new ExceptionFailure(t, null, true) - } catch { - case _:Throwable => new ExceptionFailure(t, null, false) - } + // Attempt to exit cleanly by informing the driver of our failure. + // If anything goes wrong (or this was a fatal exception), we will delegate to + // the default uncaught exception handler, which will terminate the Executor. + logError(s"Exception in $taskName (TID $taskId)", t) + + // Collect latest accumulator values to report back to the driver + val accums: Seq[AccumulatorV2[_, _]] = + if (task != null) { + task.metrics.setExecutorRunTime( + math.max(System.nanoTime() - taskStart, 0L) / 1000000.0) + val taskEndCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { + threadMXBean.getCurrentThreadCpuTime + } else 0L + task.metrics.setExecutorCpuTime( + math.max(taskEndCpu - taskStartCpu, 0L) / 1000000.0) + task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) + task.collectAccumulatorUpdates(taskFailed = true) + } else { + Seq.empty } - execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) - val accUpdates = accums.map(acc => acc.toInfo(Some(acc.value), None)) - } else { - // Attempt to exit cleanly by informing the driver of our failure. - // If anything goes wrong (or this was a fatal exception), we will delegate to - // the default uncaught exception handler, which will terminate the Executor. - logError(s"Exception in $taskName (TID $taskId)", t) - - // Collect latest accumulator values to report back to the driver - val accums: Seq[AccumulatorV2[_, _]] = - if (task != null) { - task.metrics.setExecutorRunTime( - math.max(System.nanoTime() - taskStart, 0L) / 1000000.0) - val taskEndCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { - threadMXBean.getCurrentThreadCpuTime - } else 0L - task.metrics.setExecutorCpuTime( - math.max(taskEndCpu - taskStartCpu, 0L) / 1000000.0) - task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) - task.collectAccumulatorUpdates(taskFailed = true) - } else { - Seq.empty - } - val accUpdates = accums.map(acc => acc.toInfo(Some(acc.value), None)) - val serializedTaskEndReason = { - try { - ser.serialize(new ExceptionFailure(t, accUpdates).withAccums(accums)) - } catch { - case _: NotSerializableException => - // t is not serializable so just send the stacktrace - ser.serialize(new ExceptionFailure(t, accUpdates, false).withAccums(accums)) - } + val accUpdates = accums.map(acc => acc.toInfo(Some(acc.value), None)) + + val serializedTaskEndReason = { + try { + ser.serialize(new ExceptionFailure(t, accUpdates).withAccums(accums)) + } catch { + case _: NotSerializableException => + // t is not serializable so just send the stacktrace + ser.serialize(new ExceptionFailure(t, accUpdates, false).withAccums(accums)) } - setTaskFinishedAndClearInterruptStatus() - execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskEndReason) - - // Don't forcibly exit unless the exception was inherently fatal, to avoid - // stopping other tasks unnecessarily. - if (Utils.isFatalError(t)) { - if (!isLocal) { - Thread.getDefaultUncaughtExceptionHandler. + } + setTaskFinishedAndClearInterruptStatus() + execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskEndReason) + + // Don't forcibly exit unless the exception was inherently fatal, to avoid + // stopping other tasks unnecessarily. + if (isFatalError(t)) { + if (!isLocal) { + Thread.getDefaultUncaughtExceptionHandler. uncaughtException(Thread.currentThread(), t) - } else { - SparkUncaughtExceptionHandler.uncaughtException(t) - } + } else { + SparkUncaughtExceptionHandler.uncaughtException(t) } } + } finally { runningTasks.remove(taskId) } @@ -854,6 +854,15 @@ private[spark] class Executor( } heartbeater.scheduleAtFixedRate(heartbeatTask, initialDelay, intervalMs, TimeUnit.MILLISECONDS) } + + // Pluggable Throwable handlers for a task related to underlying store + protected def isStoreCloseException(t: Throwable) : Boolean = false + + protected def isStoreException(t: Throwable) : Boolean = false + + protected def isFatalError(t: Throwable) : Boolean = { + Utils.isFatalError(t) + } } private[spark] object Executor { From 4d16611f5f740dc604963996292b113866212110 Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Wed, 16 Aug 2017 12:47:24 +0530 Subject: [PATCH 089/142] [SNAP-1930] Rectified a code in WholeStageCodeGenRdd. (#73) This change will avoid repeatedly calling code compilation incase of a ClassCastException. Conflicts: sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala --- .../sql/execution/WholeStageCodegenExec.scala | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala index 6d9e9a7fb0a28..e8405c798c4ab 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala @@ -715,14 +715,7 @@ case class CollapseCodegenStages(conf: SQLConf) extends Rule[SparkPlan] { } } -<<<<<<< HEAD -class WholeStageCodegenRDD(sc: SparkContext, var source: CodeAndComment, -||||||| parent of e3f2e3d... Snap 1833 (#67) - -case class WholeStageCodegenRDD(@transient sc: SparkContext, var source: CodeAndComment, -======= case class WholeStageCodegenRDD(@transient sc: SparkContext, var source: CodeAndComment, ->>>>>>> e3f2e3d... Snap 1833 (#67) var references: Array[Any], var durationMs: SQLMetric, inputRDDs: Seq[RDD[InternalRow]]) extends ZippedPartitionsBaseRDD[InternalRow](sc, inputRDDs) @@ -740,21 +733,20 @@ case class WholeStageCodegenRDD(@transient sc: SparkContext, var source: CodeAnd override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = { - val catcher = catching(classOf[ClassCastException]) new Iterator[InternalRow] { - private[this] var i = computeInternal(split, context) - - private[this] def replace() = i = { - logInfo(s"ClassCast Exception, hence recompiling") - CodeGenerator.invalidate(source) - computeInternal(split, context) - } - - override def hasNext: Boolean = catcher.opt(i.hasNext).getOrElse { - replace(); hasNext + private[this] var iter = computeInternal(split, context) + + override def hasNext: Boolean = try { + iter.hasNext + } catch { + case _: ClassCastException => + logInfo(s"ClassCastException, hence recompiling") + CodeGenerator.invalidate(source) + iter = computeInternal(split, context) + iter.hasNext } - override def next(): InternalRow = i.next() + override def next(): InternalRow = iter.next() } } From a2ecd8b1d95597a4300f5f6cb47af55e74510c55 Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Thu, 17 Aug 2017 16:59:02 +0530 Subject: [PATCH 090/142] Snap 1813 : Security - Add Server (Jetty web server) level user authentication for Web UI in SnappyData. (#72) * SNAP-1813: Security - Add Server (Jetty web server) level user authentication for Web UI in SnappyData. Changes: - Adding Securty handler in jetty server with Basic Authentication. - Adding LDAP Authentication code changes for Snappy UI. Authenticator (SnappyBasicAuthenticator) is initialized by snappy leader. Conflicts: core/src/main/scala/org/apache/spark/ui/JettyUtils.scala --- .../org/apache/spark/ui/JettyUtils.scala | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index 0adeb4058b6e4..36a537717c465 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -29,10 +29,13 @@ import org.eclipse.jetty.client.api.Response import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP import org.eclipse.jetty.proxy.ProxyServlet import org.eclipse.jetty.server._ +import org.eclipse.jetty.security.authentication.BasicAuthenticator +import org.eclipse.jetty.security.{ConstraintMapping, ConstraintSecurityHandler, HashLoginService, SecurityHandler} import org.eclipse.jetty.server.handler._ import org.eclipse.jetty.server.handler.gzip.GzipHandler import org.eclipse.jetty.servlet._ import org.eclipse.jetty.util.component.LifeCycle +import org.eclipse.jetty.util.security.{Constraint, Credential} import org.eclipse.jetty.util.thread.{QueuedThreadPool, ScheduledExecutorScheduler} import org.json4s.JValue import org.json4s.jackson.JsonMethods.{pretty, render} @@ -50,6 +53,31 @@ private[spark] object JettyUtils extends Logging { val SPARK_CONNECTOR_NAME = "Spark" val REDIRECT_CONNECTOR_NAME = "HttpsRedirect" + val snappyDataRealm = "SnappyDataPulse" + val snappyDataRoles = Array("user") + var customAuthenticator: Option[BasicAuthenticator] = None + + lazy val constraintMapping = { + val constraint = new Constraint() + constraint.setName(Constraint.__BASIC_AUTH); + constraint.setRoles(snappyDataRoles); + constraint.setAuthenticate(true); + + val cm = new ConstraintMapping(); + cm.setConstraint(constraint); + cm.setPathSpec("/*") + cm + } + + lazy val snappyHashLoginService = { + val userName = "snappyuser" + val password = "snappyuser" + val ls = new HashLoginService() + ls.putUser(userName, Credential.getCredential(password), snappyDataRoles) + ls.setName(snappyDataRealm) + ls + } + // Base type for a function that returns something based on an HTTP request. Allows for // implicit conversion from many types of functions to jetty Handlers. type Responder[T] = HttpServletRequest => T @@ -418,6 +446,16 @@ private[spark] object JettyUtils extends Logging { throw e } } + /* Basic Authentication Handler */ + private def basicAuthenticationHandler(): SecurityHandler = { + val csh = new ConstraintSecurityHandler(); + csh.setAuthenticator(customAuthenticator.get); + csh.setRealmName(snappyDataRealm); + csh.addConstraintMapping(constraintMapping); + csh.setLoginService(snappyHashLoginService); + + csh + } private def createRedirectHttpsHandler(securePort: Int, scheme: String): ContextHandler = { val redirectHandler: ContextHandler = new ContextHandler From 61e9811031f262c0d75cf4dedaac054c73e2d74c Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Fri, 18 Aug 2017 05:49:04 -0700 Subject: [PATCH 091/142] [SNAPPYDATA] fixing scalastyle failure introduced by last commit merge of SNAP-1813 in 6b8f59e58f6f21103149ebacebfbaa5b7a5cbf00 introduced scalastyle failure Conflicts: core/src/main/scala/org/apache/spark/ui/JettyUtils.scala --- core/src/main/scala/org/apache/spark/ui/JettyUtils.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index 36a537717c465..9459cc1d3cb7e 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -29,8 +29,9 @@ import org.eclipse.jetty.client.api.Response import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP import org.eclipse.jetty.proxy.ProxyServlet import org.eclipse.jetty.server._ -import org.eclipse.jetty.security.authentication.BasicAuthenticator import org.eclipse.jetty.security.{ConstraintMapping, ConstraintSecurityHandler, HashLoginService, SecurityHandler} +import org.eclipse.jetty.security.authentication.BasicAuthenticator +import org.eclipse.jetty.server.{HttpConnectionFactory, Request, Server, ServerConnector} import org.eclipse.jetty.server.handler._ import org.eclipse.jetty.server.handler.gzip.GzipHandler import org.eclipse.jetty.servlet._ From a9a70db192580a2fa4098c76c554441555b76188 Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Fri, 18 Aug 2017 20:05:08 +0530 Subject: [PATCH 092/142] Resized company logo (#74) * Changes:. - Adding resized SnappyData Logo for UI . - Displaying spark version in version details pop up. - Code/Files(unused logo images) clean up. - Updated CSS Conflicts: core/src/main/scala/org/apache/spark/ui/UIUtils.scala --- .../snappydata/SnappyData-Logo-230X50.png | Bin 5208 -> 0 bytes .../snappydata/pulse-snappydata-130X50.png | Bin 4650 -> 0 bytes .../static/snappydata/snappydata-175X28.png | Bin 0 -> 3479 bytes .../org/apache/spark/ui/static/webui.css | 1 + .../scala/org/apache/spark/ui/UIUtils.scala | 53 ++++-------------- 5 files changed, 11 insertions(+), 43 deletions(-) delete mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/SnappyData-Logo-230X50.png delete mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/pulse-snappydata-130X50.png create mode 100644 core/src/main/resources/org/apache/spark/ui/static/snappydata/snappydata-175X28.png diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/SnappyData-Logo-230X50.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/SnappyData-Logo-230X50.png deleted file mode 100644 index f4520e17965f3e22cb4c0abe60f20d6e24f17bac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5208 zcmai2XIK+kv>l2N0RcfJND&YOLJ=f}9_cOg-VqQ9z1IY!NbjhCAf14M5u(BcX;Opq zCOshi(tDNia^KJQ=k0IKob#O@=bKr3_FijEoUXR&Evoxe007)lS5wj_&MU<61Vl=_ zsIgk~a*T+Ol#qHN@Y~lXN`!DU2A2jnmMR zzi~(K`AwxpP!XxWsD>dFW+&(AN6Eanxhh&xEa^X>A_@M1wq zT2=?i|GS=lMP*&_)Z1Buy_vzJn}?vxmMiob#bV}J+&TZ4eAMf_`tQ)d+Jb`Qrw9N5 z0c0Gnxtdy}OqxkvX3R`@ouc&oQP$4EW}I{*3zDhOYTK^YDN>VOZTR_}g&y}ltvX0( zk4bu|5mZ@Bvsf&AKF8zo)bco+1y1`a)5`xMiomT!0wAN=hx$VGb(p9Ne=UwL;}gDK zA6W1dd5fo=h0ll2fV@vP+A68<-hjlvrY8ktu;-zYu2jJ_u&pKAEpVW1er*#4pgZ}~ ziez#3q+qkL+zwy~83Rz_p!xF+Ua|J5o^w>+zl*TW&NaDE3W~xH`pdtKByxO*Y$o4j zDh~x^&URt$DAI+@!JI!n6DMK40&HrGImrN0;)CH}+kCu6pE`oT?fsY8)Iimfpp=)$ zw+ey3bt{@nWBisab}mW^)E+j4-{cHDEJb&B$htYZ9Ah`z+uIp=wU^*smvf2z2de{% zi;D;ijVu0)W4b~w(WdzAA2;%4g1+(w4`4SfXr7DE zkO8`PqHIkots5H}8dmyp%d92&`JGPk2!7Rm+Wp!$_imcoWZ9%PdoOFmDLoPQbl26@ zMFa8X<+(&?c+;;{$=UHL56u69hSF zcz&ga=>)}5Yf({=WVnx7I=A}p?CjO)34WR}6k8Vpx%QJJ2LO=>S2RUBaw?Ej z%ZU?(bCz?yrM)?(tS4IU$=RL*fkV0#1?oo%*|zr%*(5uH3us#w78b(701Bi@iwF(H zd=iWJWIDe=Swr9+X~=J!J^rl^e;)>oM$e8mgwtzqr(eXJna^38nXNq|*kJT=@n1b& zx?qE@F3ys2VE4h`?-K#hw?X74&R-(sc#;^N|5Lo!~_x0aWe69Iz0THO$@)RX&UkHdV>(Sy8O&oNLaI+m@s zHmA$UfGX?lKjM3*aTp3J)O?g9;bb?hw6q_`$V&?R@Lg-I8PVmltVP-$z?4)~SC^Ly z9Y4g-u}bLb2!W%8Y;ZDMb_vP+633Uge|%f2c$Jlpj*eX8Fc?f64yRD43nS6GRgo!V z!gnXbYECj7!p|RMU{GwGj{e~{SA4f8SzfSge>~@L-jlEvj&{yyMNLBOx5$0RKQnbc z`k&qotcuatySkzm{w^AijR9d77Z-&3W`Xl%CX9ZZt!WCo?Dvx-e7)yhQT|+@iOG0? zAi^NW4+<9+rj_KRX)AF4_3PI+_O@#RRPBvJ=qpa{%7_QrLL|%QmTbw;a+-{ zs1wTzBGr5G9gnoIm=5^-KKmZX_NZc=SS7>5`gNAHoKPrKCNtg6V@HJ8JyseKT3Xoh z@+VNJ;oP1650YZ2W7isPfsOODhpKAvIwyaYMELV$arvlRhyKQbQPS!Dp|y3MDdzLJ^eEG@2YHXn^!|g3HQUr^{%vQ zXZ83}CrVVz+DO?qp2EVyG`%!RI(BYQ!~n0bRYUEP)wo%OnxUcLOiQgw8iFZP*la

    4!BciR`lG+zd`#o$Z|*<3&EJ{qG(NkdV{2Mp7=0@>2s@(kDVpcQur) zt=W?oxz%Z}k+sdQ5`qdJgr0_TfJPfyTNP;VuN7>O$bWym;ho+AUsUUUGDO}~0Qo#X zo_mgsjS&+z(qiH-VD;^#R?cCbhLnRGn4=i|=O;;a5dbI@diG?12odYB+wVV$90WGR3MG}I7fX{|cNnYlbMs6nwqwD zKL(MhTd2epH-jT<^SYdVi9~|g8Y!_!GGVXL~yr2J()9d1Nni zadJpg!`Z@pUw$^SyE}6eGOWsd67HY24I_}fq~J5s_&zk-+r-S;K;yDMNu)+`WGccU zXz@u`M{8@uEh_%_V0@Zveta+igJ9z2Z+>)8e(^<# zK0&6=e1bdirj$(Z{QMAgO7P!3=`(Ij6W%FlgopdnJ@JZ5rKiVc)k>emL7A4 z629)95tpqWk4^1=Y)o5Wk8!oNr~dtJAWp53NQxha+XNQi&)Qnt(;K-`8B}c|_e(U@ z)IjD9;IoK3%13L2vNPv#dUZ`r8Y+FpwS(lYBIN{h=Xn9UEjU`du8OT!`m=d;Ioqhb zllj=qPai)%Z#HN9F{S$)1R#;fi6%Gc^39En1fMc5gH=2P=JKKvs74 z`Pq5l>l97l1Om%n6hA`rP~UMNrJl|owiO(BND7Z81Jtr>U_p4jDb}zR@L+{{?rrlwI7o09XtS;@$+`f(-UtkVX?JS&#Ea8n5eQA!)b|ezsv7m6Y1}6#=Q{ncm@$}!#_xV4mHmS< z>NBN++Kx5^W1}ouR)n3PdbWF46|0P>6LN z`qt>^Xt(Kq)&`GicSVa5>gs~zI_=OCnly&D%`8%@ggH5!+t`gbKBe7O$##= zU$X8UGK&X@8#{+kBh++sDoa}+fVV@wR)0~NSvn~1-^Kan$&{K@QKFc=?RHGQetWjvbh!K`+7IyCC^T+wX1 zv3u3R@(~#zeD=j%hK~zlWBCl%>4)|x)}Euy(fg9_yUaocGzrx9w(Vh#*|i^8h2~Xt z((ePnz>CU=P&gF_he%%DTZj4O|01e3Co5|=zf4E(?45v#ukJmdDqwEb?Ay!2#(DgM zSM3(DdfItXJZWn5PdC=rVWD+;mj#i1THP8e{{9$AHEU_tOUR-p_4GzuX2ygp-)md2)viyw(Jh$eW^R+T( z8&TrY(iXBFvC$v})5D%O{(K=*P%jzw$5K+}1Uq{2mf+Q!H}|&61cR6RBM_MgDh?o= zIVL&z;_dRrf`IDM+b7%yrn_N7_Ha)*Gy#3RiOn1UQ^-_?+thosrTa|Y{?U?O6CqhK#oq+Lu zsHDQ@ijaAQsh-TolRWW@fr!XQX`{TnJX%K5!+0_+wJj#{$Ggl z5yFG=Kc$@`Cj}MCaez`Lyq9*5mfF2lTYBy9_bPT!?l@*7OP~T{;rll~g`HvEdarc&%mdzWhN{OtJ3X|NJPXbSGo#APod%LezF%JeTL~o2|R+_pak$iWk?CtHLd=`41{t=?Q68INEeWx=bhE z+T1w)iw-vruU1YulS{zXRMhr~?w#uAR}mRwc3dO)fY2`77MaFI=v`5W@MIrtNLUJBr72kJK6JN}{8aZiCxI-aOJCUNyW4 zq^5g^Zcd24&EDx+smWt{rdsc3gKsiFqXIkw0*K$MYDWSUAm?CY9HK=3+6!(FnMyw7HS%6WKr9PSMkDsrpafmQTIOVr<(2d|D{ zEgUI&+r2(;#FiKc=^Gd4j(`1{S^;5*^PR{bI=MW!&;9rHL`&YA|Loes>-@JKhjVp* zU4H-j_u07dWTMN&;c!H&gTsm1-PIa-fZh#14-)TzVZSrRDOD&c{_b_KKQS25*P|)X z<$F1hjV)GhYHHG=yj9-H8wCIl3vWZ@`#-`%n5d#Y$9{V6B(0LtxEZs5VZMp)j>)nIW!(Xi?-7BKtqUi-n({#st|@Vis#VF`V}pJ9VTpO^D*Fx?EaCa*Cxufa7P~beYJWS9K$ziXh$}8$>}3`^VkYrR zBWkoQaGNqU)BEgf$h&SGQn=HvM}a6*3K*^+oH_pc_iy$4uhusUW2Nt*xw02y>e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{01=W&L_t(|+U=ZaU{po6$A8tG z1(J{r2w^8|Q5bdwK|n-BKvWFrx}pQ9<0$9|sJH=&qo9rgOvbYVpEvNQRUb{OzxNRokp=oidiU&Az3^&1QLK&z>mNhQ>jCh}0g??V zgDWvhhZ|Dv1daf20#90fIA};#?zMgdc-!72d80KXVg_NZW>-POP(-oXy6Xa*ZnP5_ny zZGq*$5X;*b1kA$|ka`FxG^D(?ycums*$2o13?R<}payUgFctV7Pta<1FH=w z#{u_SJU$J4RmP&+418>5Ng&x&s??{Q6~GKb%4lFKFbr62DmAbsEfxOU$HA5o`~b8v zmCE(g;Zjqnxxg*f^S=!#gD>K28Dm+m2TY}M{d~8?;<1sb)W9mftRdKZcR0vHK6}T+JEkMeSi1gLxlZa1#(}1)IiMuHzs)XBf00<-5Rx zmUrQ;6Kgfo{a3PaswP{6Ce?K3yn&^{i{(Q4q=go)dCriszGanq0?z{X0;2(GD&;-e zR$!(5f$ysM{nIslX=qtZA9<*i9xRiuD}hPMQc#(?RS{6GA!V+q)CS-+L&_I{CxCw$ zQbw6dxh&ThpkYlC=zjw&-c<>}RUVoKSYM{dAuEll0?gj)D(tc9h}PXYh5mXDEE zk!_Ze-BX<*A>}nyRQL~pnU{(p#2Hd9smMR~QG%|(t9Z@`+)>Wf^GfUSHABk4O3dV! zt;gvX5X=pRlnpAfTs})&7b|P4&+BWgRB?fmP(FQq!m=s@YLX&|GM#M>900xm>f_mp zU2Q5gsDc6N1FW;!y4O@{zMpdMSfDKx=xi!gTuzw)JR3;EES%BORO$>y@ZE;aPU>O@ z_E@iO0Y34%yk+r?6+B9_yoN|TVQ25VzXKETL{M7*hy7VCL(1{MKY#*Lsqjld5!U0m zUY2U1?_-v7ya#LoVu0(d^|g~#-C-(~R36CmnN6SK*~iEO-U41XmC7)rj0FCSXP3~+ zLwi%HBHwn(o<}gG^xZJpYw$Xm}M%p#wybc zxZUEdg{4gGOr;Lgq)jKkieM_$$I6>w38|rX=IP2ewxhw3$jhYAPkHnYszjy@hQS#)vePx~)7VuxJ&Ykn$0q_nD?r zBds>?!E;U`4>%4i1Nxdu^|F?)^JcJ5EaH7rspeLIDcPD3$1UEb0(MiWkv>tynoEY1 z%?&BFA?1Jkpyl5r8d#n87vr*PO+S}deMJ~DgsJzh`2WS^@D%i8=Vvc)w5U||i3u$# z{hf2j;Yr#$rkq_)@2=kc+o3XH++Rnw5zz@BSF3(oc}w&}N7vgfMr6@LpMJ{pq5Xoc zmw#T>faBNLbWz3?`rDfyNy^^xf2Ig2!x6wR>CmBwQbCEkkyzfBR5^|+>#(!#vqQHG7kM@++H4yBmfu$&jz+9d z$(^2EmsJW7E)JeKI(Ow?%4=)dd0A=d5n)xd2!Kr& zN4v%8V+)roECdR_0OAv$=rGi_ZT3DL85At27eYbI?7X3US0)u5>>wX`rHf7Ib9ESB$Tc^m0XwQ6ob|PYf2X87yg*_mxf(OxMzm8sQ;!z z_P>(-FS|uRmR-BrwsNnPFZ}!Hx~Af;;XAqNxRmv}N4fXBvR(9=-h!N17j;9TIlTWJ z=T}3t264{N+C?}&W&R+dAF@0XE*BGKMd+fXB|;|vbZ|V-yDdjmh!6+dxSG_}Y0v!A zdH=#b^3KKmbzaFn!kfDYY}QC*>tQn zGb0}#=CFTNPmobUX~R&x_q7Gizs~6;$2>ifLxF3EaWv3--uX%oT|P`KHD@|^bh|up zQs*JIA79$1BkKeT!b2%&FgtJ9)g%=ij&(dcsK3}TOB1}O6jwro{^i^E3Rev%UUH(S z%8Ia&hwp91^V=UzY#7>99R1rQ=k8+yTj^n4eB(#XMFQY*`QWEPqT?vo^1@Cv`>~y# zGYPbDO#0}$fUMhK^J^eZXS(-&zAEYaw*an@rF6QTISC3^@TMW@*BkK*x z&QdEwv*q16Uyj()d*G6j>%-+|!`kuUs+G$*ClFP*G#d~)tIOTl$EXvP=A4snRMJ`>m{nq#qu6G>=md?laOF9KTmGjf?09VOT zId>QCipFy%1Q!8pb-u8mxg7fL$7<yACQjtk8Uj(fN{2lA2~`@24+IUP&(25+ST9h2%3fnflRM!epw zr~TX6b#Zml91bhefnR!!%K%%}_Em!}gH^lvS%1D6okcp4 zpcr@G7w6i4dJNVjbmP5Qm+4(k`d{PR_7XG!w|y8hDB!cXA>wpD8~ekG2~6`!RU^(U zggITKsjd8FQsCCxb&oeDh;8Fqxn6p)dr-#WmA1MiXBX>(^@Xy| zVf-LUe{O%@zWqcubYly=jP%vsaz(mc`iTHt;lzt3{SRIx0NAf%Hvd5LHg4?p)}*DB zy9|J6OxF6dvS^rEY{?==xA;8U*6qoJ#JI3xap1NXoj1Qy7{R&~be$D}Bh&du0NOVn zcyhg(Ib$0!YPsnACpR{1=<~L2QxV}EH=dg2dE{k^b>}y_cwr+!T{T`j_uh!bhKG`Okg1y2so@M)`gE<;mPv@l9*KJnsvPGU~&Vl}Jg!g#3yF|2>ckXN* zeuI3tep`{B(Ak|fA0U_5CM%3~-3@t1y*OzOYDI3INb&@3j?sGIylm9%Qiipc?_b`Ytjn(>ZTE%R@ONONm3_~@ z8tJU1Hf+l?bhGTY9i2j*PUz92s$N4KA@YcrkR_sWBJ7!YSBo7hW~$x)5NI3VdlfAR z2;QAHXWMSNZ~e)3>{JJxwJeMK?5?5eM?m|p_XIuO1)J@1k2TzP}brVIM%!jv6c+1Dl9`9t*S2TzFhfA=G} z0B=7FDP7T$-Dy3SP>AO)MF_y!qPMSqOJ6_Mg~%Vk{P&-6t!*(vxr(eY3GyAy^1hMf z?lo#xK*xZ8(WTax^nPr?+2BUTb*)#W0eiW@oh*r zrBMZ<;Xtjmagvxx>xujcVDoSB#{o}|zUHiJ=ZF(=jwxp~Q~N(mG(Z|fvm6mumJ_&% zxX*FrPJi{9JEmNFkG%TOg#t>-5liUJzdA%wZb0eo5k1fRo;;=(7YZn~A3oQ2jEM6I zLd_d>wp`@!^mFRDleF}wThvOo(z6Z@CBK?|NVkm55n-}4ElmGw#07*qoM6N<$f`f_a$p8QV diff --git a/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappydata-175X28.png b/core/src/main/resources/org/apache/spark/ui/static/snappydata/snappydata-175X28.png new file mode 100644 index 0000000000000000000000000000000000000000..5d20ba7fdcf4374eab61ec9fc464166337d3053f GIT binary patch literal 3479 zcmV;I4QTR-P)S2LTv5PbfPv|Gukzv5>$%hPHiH;Gbe;1K)7|ghTUGa8 zb!)jmE=tt5{e0o>l&MS`8&$LgZ5)y*Kq^VZdjOq5ml4QHz;FG%OlA82ldS7?<3m6% z0@(ym1Nd7&l4NReS!HF~SjfwG-0ke=I`!@%9t;TX-S^H)B9z1U%-UoQt!22c1)Nsbu4@Tu$;AD$??*js070J}# zl4k$)zA$kzT^70D_?FHPia- zj&^X&`pRE;FYpQV;t2%${mR%i2917k38HF4jq}}S0H{m?nt@{>m<3*od1{Zbbrnol z&{CAKiSr>5-x_p^L17ul0MO~c2_=fx3t^BShA**2-!55Pf-OtBScw}B+$ zeu>~4XrV~6!}OLU;{8dc{#`^_R{&=NwFZPJ)KP@5fSXZfIqyFX>o8KFy?wc5Tr2Dl;zoC07i!gs(uC{r9TZU@XY*|Mn_Sc&j= z;8I|@2saOxuSiGY>j%_B0tHwtUC&i0LPy?x59nDC5&Y|pSkN#Ru6C=YmElwYYwAOS;FxER>Vz_lXJ6l+y@8l^wbkmWyI zS@VSzN7>vFKDI>|it<<<1C0!O4)BJpvn_Bmu(`={62fCB^PMpQd;#2Qfa?q#0_(%Ahpra`*INATTQc1<%gm#7W56pGi&q=S zZU%^X9Xn!e{v++A5nzGE=utj!XZ|{Gk;;e@BV~{6oa6f%dtb9ytOhm-;~ct|u=abI z8C$rH)4KTv=D!_%F z@}iv>3iJbN{b0SE)um36?>0|))9EjCLO6_A`@kCmJ^@a%cttWLDR|dvz^k*qpJ9x+ zMV=bqPoSrOo(6g{@L`b+{G~D9)x4l@3OyBg+;SeHc)ujZC@X1=u*9MYNRdpPO)~W!G5QA5MWjv2RX~}R+T2m3M(}|YW2%p-nZ5^@`DVZl}DqzA`kQq zwKY7erG9}>>xakL9GiCya9keFo6YE+uxGX)>YLWNaiG(0io&j_^$wbr;wo6*)I^zb?sh>pxu z){$!wI@uaeIhBpw5$Z%r1Fwh5Rg6nDk*H6QO!-vma9}LS)a4}VYZ)8Aig7Qm#;ZIU zm`pM?g+zTn67iv2H}50haD?-iF!w1E@qNQgR~`r1qNG?zQHad}_6BwYIRtpq_!a}p z=pyqRlZl;#&CRo>^LT)%PI+rr;2pwJ+q;&mM5oF12$IJH(q_y}2K2yAf?1ZC=aaU{ zdD151GT@+`p@y;qxFrIcC5j75s;=fvTe|~4jhuuk-xyLf;~A zY7n%0?E!=hDAT*%KzNIBq1C}F9jyMs%{M9vZo5)JB0dY~7lO|pLh|KjN!0fSc>txB zMEu8$e{K};2#I)qD{FKn5${JbH3?W6UbEyX6VxlQy&1lzErBPV1{$P6KLmy$G8ps@ zn{S6A>S+Rw2OhQ3T1B39>zp!Eq^zoZWjNqOlt;3_5(8edcwKD01t{+WU*~Cfih*;s zjrZ-eScvkUBC@rtllpX|RU7L2ZpTR0*a->B7=*QLx|sQ=xyTc5TA)Eu?;mSL{b-vr z?VPUpWF6#**CWtiS#j`sdMmA6{!K{7W|J5+7IdYuLrztUlM0_*g zZQwEkk(8*&eGF_q!XE;HyV}-j)osJBEI40SuX+u*5y{+|%SGdztzVj>maP_l_Ojo` zSQ*ZxO5h2U2MT7^>w@?5C^YDh&ukx6AXM4d3M)xoW2akmqwpM{*%NjQ!luDe+v{AH zN_?Bc1F8aoVHQplPK(2kq`QV;RdXgx%`HXc$vm^4VR3qkb?WXb#n=>1G+joC;3;`V9btulRlvCjQ&CoV9Z*6;4)6t8 zz;(Z0ll8vm)NpfBS7i$DssSO2{|>T5=>P++7uEy2PeZEK@onOISgW2_!f>lgwkgGG zXbdrHO3cm$+%RF%w9~~}Y4Ni<^+`{1bb|7btCyf7E;MtepisB4HIejMlP z2W?VVol}n@nOa^b7HSMZbrNuZC{LPjUCmQ;1-heLi^mYOLDA+&iQ~*O->J&iZK4N( zw6Yduh*@wN<#Zc&uYvJX@$y`FY2BzBEfk8OT#cd<-u-9>UQ^}a61DLVVVkxB<#O$q z*S!~5WL?PL7FcV%fwY(Vgf|;`$LAW%t*i7IWRlfD|Z)^i1KexS^)8@8b zsE_AOUkTsst|_C!)0}ZA?-y<&$$1vGya758xXi_Ku3_Aw}eDJB

  • {tab.name} @@ -245,9 +245,9 @@ private[spark] object UIUtils extends Logging { {getProductVersionNode}
    • -
    +
    - + {getProductVersionNode}
    @@ -280,7 +280,7 @@ private[spark] object UIUtils extends Logging { showVisualization: Boolean = false): Seq[Node] = { val appName = activeTab.appName - val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..." + // val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..." val header = activeTab.headerTabs.map { tab =>
  • {tab.name} @@ -300,9 +300,12 @@ private[spark] object UIUtils extends Logging {
    - - {getProductUINameNode} - {getProductVersionNode} + + +
    +
    + +

    @@ -616,48 +619,12 @@ private[spark] object UIUtils extends Logging { def getProductVersionNode(): Node = { val versionDetails = SparkUI.getProductVersion val versionTooltipText = -<<<<<<< HEAD - "SnappyData Ver. " + versionDetails.getOrElse("productVersion", "") + - " ( Underlying Spark Ver. " + org.apache.spark.SPARK_VERSION + " )" - -

    - { - versionDetails.getOrElse("productVersion", "") - } - -
    - Product Name : {versionDetails.getOrElse("productName", "")}
    - Product Version : {versionDetails.getOrElse("productVersion", "")}
    - Build : { - versionDetails.getOrElse("buildId", "") + " " + - versionDetails.getOrElse("buildDate", "") - }
    - Source Revision : {versionDetails.getOrElse("sourceRevision", "")} -
    -
    -||||||| parent of c1acf05... SNAP-1744: UI itself needs to consistently refer to itself as "SnappyData Pulse" (#64) - "SnappyData Ver. " + SparkUI.getProductVersion + " ( Underlying Spark Ver. " + - org.apache.spark.SPARK_VERSION + " )" - - {SparkUI.getProductVersion} -======= "SnappyData Ver. " + SparkUI.getProductVersion + " ( Underlying Spark Ver. " + org.apache.spark.SPARK_VERSION + " )" {SparkUI.getProductVersion} ->>>>>>> c1acf05... SNAP-1744: UI itself needs to consistently refer to itself as "SnappyData Pulse" (#64) - } - - def getProductUINameNode(): Node = { - Pulse } def getProductDocLinkNode(): Node = { From 8451e555ea81b16c6b709b52a325623121c69d6b Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 19 Aug 2017 13:44:26 +0530 Subject: [PATCH 093/142] [SNAPPYDATA] update janino to latest release 3.0.7 --- sql/catalyst/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/build.gradle b/sql/catalyst/build.gradle index 0f2340d49e819..c13c15fe09dac 100644 --- a/sql/catalyst/build.gradle +++ b/sql/catalyst/build.gradle @@ -26,8 +26,8 @@ dependencies { compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion compile group: 'org.scala-lang.modules', name: 'scala-parser-combinators_' + scalaBinaryVersion, version: '1.0.4' - compile group: 'org.codehaus.janino', name: 'janino', version: '3.0.6' - compile group: 'org.codehaus.janino', name: 'commons-compiler', version: '3.0.6' + compile group: 'org.codehaus.janino', name: 'janino', version: '3.0.7' + compile group: 'org.codehaus.janino', name: 'commons-compiler', version: '3.0.7' compile group: 'org.antlr', name: 'antlr4-runtime', version: antlrVersion compile group: 'commons-codec', name: 'commons-codec', version: commonsCodecVersion antlr group: 'org.antlr', name: 'antlr4', version: antlrVersion From 1c266557234c01e1e33bba674097e753eb0351bd Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Mon, 21 Aug 2017 14:14:21 +0530 Subject: [PATCH 094/142] [SNAP-1951] move authentication handler bind to be inside connect (#75) When bind to default 5050 port fails, then code clears the loginService inside SecurityHandler.close causing the next attempt on 5051 to fail with "IllegalStateException: No LoginService for SnappyBasicAuthenticator". This change moves the authentication handler setting inside the connect method. Conflicts: core/src/main/scala/org/apache/spark/ui/JettyUtils.scala From 422e72347b2aa2c0d0e2d1ba9dc8f7dca626bc43 Mon Sep 17 00:00:00 2001 From: Amogh Shetkar Date: Thu, 24 Aug 2017 18:24:02 +0530 Subject: [PATCH 095/142] Bump version spark 2.1.1.1-rc1, store 1.5.6-rc1 and sparkJobserver 0.6.2.6-rc1 Conflicts: build.gradle --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ad119c2ab7945..d4e5c7cb5dfd3 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.1.0-2' + version = '2.1.1.1-rc1' ext { scalaBinaryVersion = '2.11' From 547a8d714dc9f7d3010b87e44af8560b24c49f0b Mon Sep 17 00:00:00 2001 From: kneeraj Date: Wed, 30 Aug 2017 17:28:46 +0530 Subject: [PATCH 096/142] Updated the year in the Snappydata copyright header. (#76) Conflicts: core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala --- assembly/build.gradle | 2 +- build.gradle | 2 +- common/network-common/build.gradle | 2 +- common/network-shuffle/build.gradle | 2 +- common/network-yarn/build.gradle | 2 +- common/sketch/build.gradle | 2 +- common/tags/build.gradle | 2 +- common/unsafe/build.gradle | 2 +- core/build.gradle | 2 +- .../scala/org/apache/spark/SparkConf.scala | 2 +- .../scala/org/apache/spark/SparkContext.scala | 2 +- .../scala/org/apache/spark/SparkEnv.scala | 2 +- .../spark/deploy/SparkSubmitArguments.scala | 2 +- .../spark/scheduler/TaskSetManager.scala | 2 +- .../CoarseGrainedSchedulerBackend.scala | 2 +- .../spark/storage/DiskBlockManager.scala | 2 +- .../apache/spark/storage/StorageUtils.scala | 2 +- .../spark/storage/memory/MemoryStore.scala | 19 +++++++++++++++++++ examples/build.gradle | 2 +- .../docker-integration-tests/build.gradle | 2 +- external/flume-sink/build.gradle | 2 +- external/flume/build.gradle | 2 +- external/kafka-0-10-sql/build.gradle | 2 +- external/kafka-0-10/build.gradle | 2 +- external/kafka-0-8/build.gradle | 2 +- external/spark-ganglia-lgpl/build.gradle | 2 +- graphx/build.gradle | 2 +- launcher/build.gradle | 2 +- .../apache/spark/launcher/SparkLauncher.java | 2 +- mllib-local/build.gradle | 2 +- mllib/build.gradle | 2 +- python/pyspark/shell.py | 2 +- repl/build.gradle | 2 +- settings.gradle | 2 +- sql/catalyst/build.gradle | 2 +- .../sql/catalyst/CatalystTypeConverters.scala | 2 +- .../sql/catalyst/expressions/Projection.scala | 19 +++++++++++++++++++ .../expressions/aggregate/interfaces.scala | 2 +- .../expressions/namedExpressions.scala | 2 +- .../spark/sql/catalyst/plans/QueryPlan.scala | 2 +- .../plans/physical/partitioning.scala | 2 +- sql/core/build.gradle | 2 +- .../sql/execution/aggregate/AggUtils.scala | 2 +- .../aggregate/SortAggregateExec.scala | 2 +- sql/hive-thriftserver/build.gradle | 2 +- .../sql/hive/thriftserver/CliSuite.scala | 2 +- .../HiveThriftServer2Suites.scala | 2 +- sql/hive/build.gradle | 2 +- .../spark/sql/hive/HiveSparkSubmitSuite.scala | 2 +- streaming/build.gradle | 2 +- .../spark/streaming/dstream/DStream.scala | 2 +- .../streaming/dstream/FileInputDStream.scala | 2 +- .../rdd/WriteAheadLogBackedBlockRDD.scala | 2 +- tools/build.gradle | 2 +- yarn/build.gradle | 2 +- 55 files changed, 91 insertions(+), 53 deletions(-) diff --git a/assembly/build.gradle b/assembly/build.gradle index ddd4ed6a99955..2fb7421ce1b89 100644 --- a/assembly/build.gradle +++ b/assembly/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/build.gradle b/build.gradle index d4e5c7cb5dfd3..a0852d4aa5d62 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/common/network-common/build.gradle b/common/network-common/build.gradle index 8fba6a3379a4e..6ea74750089e4 100644 --- a/common/network-common/build.gradle +++ b/common/network-common/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/common/network-shuffle/build.gradle b/common/network-shuffle/build.gradle index a36492bb087dd..f19af0c6cb085 100644 --- a/common/network-shuffle/build.gradle +++ b/common/network-shuffle/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/common/network-yarn/build.gradle b/common/network-yarn/build.gradle index b447d5aecaeaa..7097acf8f0e89 100644 --- a/common/network-yarn/build.gradle +++ b/common/network-yarn/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/common/sketch/build.gradle b/common/sketch/build.gradle index 28936ece117e9..0192f28b59f45 100644 --- a/common/sketch/build.gradle +++ b/common/sketch/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/common/tags/build.gradle b/common/tags/build.gradle index e272cfbaa6382..b3ca2da966d28 100644 --- a/common/tags/build.gradle +++ b/common/tags/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/common/unsafe/build.gradle b/common/unsafe/build.gradle index 6f7c9d1ef98fd..83ad85298286a 100644 --- a/common/unsafe/build.gradle +++ b/common/unsafe/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/build.gradle b/core/build.gradle index a9cef0e2a24ec..1a8b040699750 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala index d6ece44c7173f..454724b63ce4d 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index ebd6610b81742..f6dc1b22a9482 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index 8f23214f43d4e..d7c1568ad58d7 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index c962b4df84e80..6240831f4f254 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 25a1f6b477f8a..f65b8a7292cc2 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index 8b41331770856..a132859b60c0b 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index 25ba6c2353f54..cde249849b5bb 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala index dab5bbcfece70..4a205bed6c93b 100644 --- a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala +++ b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala index 17f7a69ad6ba1..649cae8de6774 100644 --- a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala +++ b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala @@ -15,6 +15,25 @@ * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package org.apache.spark.storage.memory import java.io.OutputStream diff --git a/examples/build.gradle b/examples/build.gradle index 325c31cbbb89d..c8110ed5f51b1 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/external/docker-integration-tests/build.gradle b/external/docker-integration-tests/build.gradle index 93ae7e08befab..0ff0515f66224 100644 --- a/external/docker-integration-tests/build.gradle +++ b/external/docker-integration-tests/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/external/flume-sink/build.gradle b/external/flume-sink/build.gradle index 3a0060cca4897..9e31f41c64bf0 100644 --- a/external/flume-sink/build.gradle +++ b/external/flume-sink/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/external/flume/build.gradle b/external/flume/build.gradle index 6e2dc5a974d82..2f46b499e0e5c 100644 --- a/external/flume/build.gradle +++ b/external/flume/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/external/kafka-0-10-sql/build.gradle b/external/kafka-0-10-sql/build.gradle index 45108783e735d..38014d5d945ce 100644 --- a/external/kafka-0-10-sql/build.gradle +++ b/external/kafka-0-10-sql/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/external/kafka-0-10/build.gradle b/external/kafka-0-10/build.gradle index 9b6db7851f1d4..6b62fa4e320be 100644 --- a/external/kafka-0-10/build.gradle +++ b/external/kafka-0-10/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/external/kafka-0-8/build.gradle b/external/kafka-0-8/build.gradle index f756511358b54..5ae9720b3c4ac 100644 --- a/external/kafka-0-8/build.gradle +++ b/external/kafka-0-8/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/external/spark-ganglia-lgpl/build.gradle b/external/spark-ganglia-lgpl/build.gradle index 31773cc30c3e1..c7835df3a3bf1 100644 --- a/external/spark-ganglia-lgpl/build.gradle +++ b/external/spark-ganglia-lgpl/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/graphx/build.gradle b/graphx/build.gradle index fac71c09aafb5..f7393440f8c8c 100644 --- a/graphx/build.gradle +++ b/graphx/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/launcher/build.gradle b/launcher/build.gradle index 185fb1bdd875d..cb3ee1809ccd4 100644 --- a/launcher/build.gradle +++ b/launcher/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java index fd6763e13ed35..ca7b99a8fd6f6 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/mllib-local/build.gradle b/mllib-local/build.gradle index 2796bca9de463..408c4153a43e2 100644 --- a/mllib-local/build.gradle +++ b/mllib-local/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/mllib/build.gradle b/mllib/build.gradle index cc47140680c60..f2ccf4f827373 100644 --- a/mllib/build.gradle +++ b/mllib/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py index 8122533f66cb1..2636334b56ad9 100644 --- a/python/pyspark/shell.py +++ b/python/pyspark/shell.py @@ -18,7 +18,7 @@ # # Changes for SnappyData data platform. # -# Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. +# Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You diff --git a/repl/build.gradle b/repl/build.gradle index 018602803e4f2..ea40dc5577b94 100644 --- a/repl/build.gradle +++ b/repl/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/settings.gradle b/settings.gradle index faea562f5b934..6d2282a70d2b5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/catalyst/build.gradle b/sql/catalyst/build.gradle index c13c15fe09dac..27d81efd95bc3 100644 --- a/sql/catalyst/build.gradle +++ b/sql/catalyst/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala index c40d712817791..950ead971924b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala index 98153efd6710f..d039096d64a1e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala @@ -15,6 +15,25 @@ * limitations under the License. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.InternalRow diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala index d6d394f5b8ae9..6870fd71499d3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala index 8f66478011a84..9dc8f7e143425 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala index a06f53b8ab2b3..f4d086e1976f5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index 66fc8772b6773..d628254c6daef 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/core/build.gradle b/sql/core/build.gradle index 2f943eeeab845..b5a98c11dc310 100644 --- a/sql/core/build.gradle +++ b/sql/core/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala index 60d3e70dbb2d4..8cb235d4b3f12 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala index 00de22e54ab07..91f1798663221 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/hive-thriftserver/build.gradle b/sql/hive-thriftserver/build.gradle index 6dd72cdbd08a1..336832ee159d1 100644 --- a/sql/hive-thriftserver/build.gradle +++ b/sql/hive-thriftserver/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 521facc027226..6319627ec5f56 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index b445cc359049f..3af4630076b1d 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/hive/build.gradle b/sql/hive/build.gradle index e632625149210..bed3e2da88f83 100644 --- a/sql/hive/build.gradle +++ b/sql/hive/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index fd7f1a535b0a9..72532e4f0d078 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/streaming/build.gradle b/streaming/build.gradle index bcfeecb82f8ff..fec941a914c55 100644 --- a/streaming/build.gradle +++ b/streaming/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala index 039c401ace9af..f9995a4f52071 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala index cb8a2443bf1f1..2f108fea09680 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala b/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala index 9618ec2507c76..f5442a7fbc8d8 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala @@ -17,7 +17,7 @@ /* * Changes for SnappyData data platform. * - * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Portions Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/tools/build.gradle b/tools/build.gradle index 05b48719a0d3e..4ca8ca53397e6 100644 --- a/tools/build.gradle +++ b/tools/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/yarn/build.gradle b/yarn/build.gradle index 5d57c8927ff71..13424cda13bd2 100644 --- a/yarn/build.gradle +++ b/yarn/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You From a5e767afb2900901eedb9b37d12ad6a3af306700 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 31 Aug 2017 03:18:53 +0530 Subject: [PATCH 097/142] [SNAPPYDATA] upgrade netty versions (SPARK-18971, SPARK-18586) - upgrade netty-all to 4.0.43.Final (SPARK-18971) - upgrade netty-3.8.0.Final to netty-3.9.9.Final for security vulnerabilities (SPARK-18586) --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index a0852d4aa5d62..da3a760dd4f9c 100644 --- a/build.gradle +++ b/build.gradle @@ -63,8 +63,8 @@ allprojects { hiveVersion = '1.2.1.spark2' chillVersion = '0.8.1' kryoVersion = '4.0.0' - nettyVersion = '3.8.0.Final' - nettyAllVersion = '4.0.42.Final' + nettyVersion = '3.9.9.Final' + nettyAllVersion = '4.0.43.Final' derbyVersion = '10.12.1.1' httpClientVersion = '4.5.2' httpCoreVersion = '4.4.4' From 73b9c6d252165ffae5b1295b0fdaf3d8a7020840 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 2 Sep 2017 10:22:42 +0530 Subject: [PATCH 098/142] [SNAPPYDATA] more efficient passing of non-primitive literals Instead of using CodegenFallback, add the value directly as reference object. Avoids an unncessary cast for every loop (and a virtual call) as also serialized object is smaller. Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala --- .../spark/sql/catalyst/expressions/literals.scala | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index cd176d941819f..2c16579aacf23 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -308,10 +308,18 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression { case ByteType | ShortType => ev.copy(code = "", value = s"($javaType)$value") case TimestampType | LongType => - ev.copy(code = "", value = s"${value}L") + ev.isNull = "false" + ev.value = s"${value}L" + ev.copy("") case _ => - ev.copy(code = "", value = ctx.addReferenceObj("literal", value, - ctx.javaType(dataType))) + if (value == null) { + ev.isNull = "true" + ev.value = "null" + } else { + ev.isNull = "false" + ev.value = ctx.addReferenceObj("value", value, ctx.javaType(dataType)) + } + ev.copy("") } } } From 435d321e1de28161e94ce2d806029aca70d20b4f Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 5 Sep 2017 13:25:34 +0530 Subject: [PATCH 099/142] [SNAP-1993] Optimize UTF8String.contains (#78) - Optimized version of UTF8String.contains that improves performance by 40-50%. However, it is still 1.5-3X slower than JDK String.contains (that probably uses JVM intrinsics since the library version is slower than the new UTF8String.contains) - Adding native JNI hooks to UTF8String.contains and ByteArrayMethods.arrayEquals if present. Comparison when searching in decently long strings (100-200 characters from customers.csv treating full line as a single string). Java HotSpot(TM) 64-Bit Server VM 1.8.0_144-b01 on Linux 4.10.0-33-generic Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz compare contains: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------ UTF8String (orig) 241 / 243 4.7 214.4 1.0X UTF8String (opt) 133 / 137 8.4 118.4 1.8X String 97 / 99 11.6 86.4 2.5X Regex 267 / 278 4.2 237.5 0.9X Conflicts: common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java --- .../java/org/apache/spark/unsafe/Native.java | 121 ++++++++++++++++++ .../spark/unsafe/array/ByteArrayMethods.java | 6 + .../apache/spark/unsafe/types/UTF8String.java | 23 +++- 3 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 common/unsafe/src/main/java/org/apache/spark/unsafe/Native.java diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/Native.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/Native.java new file mode 100644 index 0000000000000..bab111d3be609 --- /dev/null +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/Native.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.unsafe; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.net.URLDecoder; +import java.security.CodeSource; +import java.util.Locale; + +import org.apache.log4j.Logger; + +/** + * Optimized JNI calls. + */ +public final class Native { + + public static final int MIN_JNI_SIZE = 32; + + public static final boolean debug; + private static final Logger logger; + + private static final boolean is64Bit; + private static final boolean isSolaris; + private static final boolean nativeLoaded; + + private Native() { + } + + static { + debug = Boolean.getBoolean("spark.native.debug"); + + String arch = System.getProperty("os.arch"); + is64Bit = arch.contains("64") || arch.contains("s390x"); + String os = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); + isSolaris = os.contains("sunos") || os.contains("solaris"); + + logger = Logger.getLogger(Native.class); + + String library = isSolaris() ? "native_sol" : "native"; + if (is64Bit()) { + library += "64"; + } + if (debug) { + library += "_g"; + } + + boolean loaded = false; + CodeSource cs = Native.class.getProtectionDomain().getCodeSource(); + URL jarURL = cs != null ? cs.getLocation() : null; + String libDir; + try { + if (jarURL != null) { + libDir = new File(URLDecoder.decode(jarURL.getFile(), "UTF-8")) + .getParentFile().getCanonicalPath(); + } else { + // try in SNAPPY_HOME and SPARK_HOME + String productHome = System.getenv("SNAPPY_HOME"); + if (productHome == null) { + productHome = System.getenv("SPARK_HOME"); + } + if (productHome == null) { + throw new IllegalStateException("Unable to locate jar location"); + } + libDir = new File(productHome, "jars").getCanonicalPath(); + } + File libraryPath = new File(libDir, System.mapLibraryName(library)); + if (libraryPath.exists()) { + System.load(libraryPath.getPath()); + logger.info("library " + library + " loaded from " + libraryPath); + } else { + System.loadLibrary(library); + logger.info("library " + library + " loaded from system path"); + } + + loaded = true; + } catch (IOException ioe) { + if (logger.isInfoEnabled()) { + logger.info("library " + library + " could not be loaded due to " + ioe); + } + } catch (UnsatisfiedLinkError ule) { + if (logger.isInfoEnabled()) { + logger.info("library " + library + " could not be loaded"); + } + } + nativeLoaded = loaded; + } + + public static boolean is64Bit() { + return is64Bit; + } + + public static boolean isSolaris() { + return isSolaris; + } + + public static boolean isLoaded() { + return nativeLoaded; + } + + public static native boolean arrayEquals(long leftAddress, + long rightAddress, long size); + + public static native boolean containsString(long sourceAddress, + long sourceEnd, long destAddress, int destSize); +} diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java index c4d4ef5f50bce..d892a201d34e5 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java @@ -17,6 +17,7 @@ package org.apache.spark.unsafe.array; +import org.apache.spark.unsafe.Native; import org.apache.spark.unsafe.Platform; public class ByteArrayMethods { @@ -55,6 +56,11 @@ public static int roundNumberOfBytesToNearestWord(int numBytes) { */ public static boolean arrayEquals(final Object leftBase, long leftOffset, final Object rightBase, long rightOffset, final long length) { + if (leftBase == null && rightBase == null && + length > Native.MIN_JNI_SIZE && Native.isLoaded()) { + return Native.arrayEquals(leftOffset, rightOffset, length); + } + long endOffset = leftOffset + length; // try to align at least one side if ((rightOffset & 0x7) != 0 && (leftOffset & 0x7) != 0) { // mod 8 final long endOffset = Math.min(((leftOffset + 7) >>> 3) << 3, leftOffset + length); diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index 27461adcfbc62..62b9c4ac87e21 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -30,6 +30,7 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; +import org.apache.spark.unsafe.Native; import org.apache.spark.unsafe.Platform; import org.apache.spark.unsafe.array.ByteArrayMethods; import org.apache.spark.unsafe.hash.Murmur3_x86_32; @@ -324,15 +325,25 @@ public UTF8String substringSQL(int pos, int length) { * Returns whether this contains `substring` or not. */ public boolean contains(final UTF8String substring) { - if (substring.numBytes == 0) { + final int slen = substring.numBytes; + if (slen == 0) { return true; } - byte first = substring.getByte(0); - for (int i = 0; i <= numBytes - substring.numBytes; i++) { - if (getByte(i) == first && matchAt(substring, i)) { - return true; - } + final Object base = this.base; + final int len = this.numBytes; + // noinspection ConstantConditions + if (base == null && len > Native.MIN_JNI_SIZE && + substring.base == null && Native.isLoaded()) { + return Native.containsString(offset, offset + len, substring.offset, slen); + } + + final byte first = substring.getByte(0); + long offset = this.offset; + final long end = offset + len - slen; + for (; offset <= end; offset++) { + if (Platform.getByte(base, offset) == first && ByteArrayMethods.arrayEquals( + base, offset, substring.base, substring.offset, slen)) return true; } return false; } From ced024bc4435747d0d81ee7e94e392c97ee6e3b5 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sun, 10 Sep 2017 14:39:56 +0530 Subject: [PATCH 100/142] [SNAPPYDATA][AQP-293] Native JNI callback changes for UTF8String (#80) - added MacOSX library handling to Native; made minimum size to use JNI as configurable (system property "spark.utf8.jniSize") - added compareString to Native API for string comparison - commented out JNI for ByteArrayMethods.arrayEquals since it is seen to be less efficient for cases where match fails in first few bytes (JNI overhead of 5-7ns is far more) - made the "memory leak" warning in Executor to be debug level; reason being that it comes from proper MemoryConsumers so its never a leak and it should not be required of MemoryConsumers to always clean up memory (unnecessary additional task listeners for each ParamLiteral) - pass source size in Native to make the API uniform --- .../java/org/apache/spark/unsafe/Native.java | 37 ++++++++++++++++--- .../spark/unsafe/array/ByteArrayMethods.java | 7 +++- .../apache/spark/unsafe/types/UTF8String.java | 16 +++++++- .../org/apache/spark/executor/Executor.scala | 2 +- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/Native.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/Native.java index bab111d3be609..c3a59ba9bef79 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/Native.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/Native.java @@ -30,13 +30,16 @@ */ public final class Native { - public static final int MIN_JNI_SIZE = 32; + public static final int MIN_JNI_SIZE = Integer.getInteger("spark.utf8.jniSize", 32); public static final boolean debug; private static final Logger logger; + private static boolean isMac; + private static boolean isWindows; + private static boolean isSolaris; + private static final boolean is64Bit; - private static final boolean isSolaris; private static final boolean nativeLoaded; private Native() { @@ -45,14 +48,25 @@ private Native() { static { debug = Boolean.getBoolean("spark.native.debug"); + String suffix = ""; + String os = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); + if (os.startsWith("mac") || os.startsWith("darwin")) { + isMac = true; + // no suffix since library extension will be different + } else if (os.startsWith("windows")) { + isWindows = true; + // no suffix since library extension will be different + } else if (os.startsWith("sunos") || os.startsWith("solaris")) { + isSolaris = true; + suffix = "_sol"; + } + String arch = System.getProperty("os.arch"); is64Bit = arch.contains("64") || arch.contains("s390x"); - String os = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); - isSolaris = os.contains("sunos") || os.contains("solaris"); logger = Logger.getLogger(Native.class); - String library = isSolaris() ? "native_sol" : "native"; + String library = "native" + suffix; if (is64Bit()) { library += "64"; } @@ -105,6 +119,14 @@ public static boolean is64Bit() { return is64Bit; } + public static boolean isMac() { + return isMac; + } + + public static boolean isWindows() { + return isWindows; + } + public static boolean isSolaris() { return isSolaris; } @@ -116,6 +138,9 @@ public static boolean isLoaded() { public static native boolean arrayEquals(long leftAddress, long rightAddress, long size); + public static native int compareString(long leftAddress, + long rightAddress, long size); + public static native boolean containsString(long sourceAddress, - long sourceEnd, long destAddress, int destSize); + int sourceSize, long destAddress, int destSize); } diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java index d892a201d34e5..776ff2881f64c 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java @@ -17,7 +17,6 @@ package org.apache.spark.unsafe.array; -import org.apache.spark.unsafe.Native; import org.apache.spark.unsafe.Platform; public class ByteArrayMethods { @@ -56,10 +55,14 @@ public static int roundNumberOfBytesToNearestWord(int numBytes) { */ public static boolean arrayEquals(final Object leftBase, long leftOffset, final Object rightBase, long rightOffset, final long length) { + // for the case that equals will fail in first few bytes itself, the overhead + // of JNI call is too high + /* if (leftBase == null && rightBase == null && - length > Native.MIN_JNI_SIZE && Native.isLoaded()) { + length >= Native.MIN_JNI_SIZE && Native.isLoaded()) { return Native.arrayEquals(leftOffset, rightOffset, length); } + */ long endOffset = leftOffset + length; // try to align at least one side if ((rightOffset & 0x7) != 0 && (leftOffset & 0x7) != 0) { // mod 8 diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index 62b9c4ac87e21..23d834a9efaad 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -333,9 +333,9 @@ public boolean contains(final UTF8String substring) { final Object base = this.base; final int len = this.numBytes; // noinspection ConstantConditions - if (base == null && len > Native.MIN_JNI_SIZE && + if (base == null && len >= Native.MIN_JNI_SIZE && substring.base == null && Native.isLoaded()) { - return Native.containsString(offset, offset + len, substring.offset, slen); + return Native.containsString(offset, len, substring.offset, slen); } final byte first = substring.getByte(0); @@ -1255,6 +1255,18 @@ public final int compare(final UTF8String other) { long leftOffset = offset; final int len = Math.min(numBytes, other.numBytes); + + // for the case that compare will fail in first few bytes itself, the overhead + // of JNI call is too high + /* + // noinspection ConstantConditions + if (leftBase == null && rightBase == null && + len >= Native.MIN_JNI_SIZE && Native.isLoaded()) { + final int result = Native.compareString(leftOffset, rightOffset, len); + return result != 0 ? result : (numBytes - other.numBytes); + } + */ + long endOffset = leftOffset + len; // for architectures that support unaligned accesses, read 8 bytes at a time if (Platform.unaligned() || (((leftOffset & 0x7) == 0) && ((rightOffset & 0x7) == 0))) { diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 42702ea3e3fc7..0277339680359 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -376,7 +376,7 @@ private[spark] class Executor( if (conf.getBoolean("spark.unsafe.exceptionOnMemoryLeak", false)) { throw new SparkException(errMsg) } else { - logWarning(errMsg) + logDebug(errMsg) } } From 90166b80c62800168d7f746b1144ff9411bfe3a2 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 12 Sep 2017 03:56:45 +0530 Subject: [PATCH 101/142] [SNAPPYDATA] update jetty version update jetty to latest 9.2.x version in an attempt to fix occasional "bad request" errors seen currently on dashboard --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index da3a760dd4f9c..65fba838402f4 100644 --- a/build.gradle +++ b/build.gradle @@ -54,7 +54,7 @@ allprojects { protobufVersion = '2.6.1' jerseyVersion = '2.22.2' sunJerseyVersion = '1.19.1' - jettyVersion = '9.2.16.v20160414' + jettyVersion = '9.2.22.v20170606' log4jVersion = '1.2.17' slf4jVersion = '1.7.21' junitVersion = '4.12' From 055753c391b8266ddfc04c36321159f06620707d Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 16 Sep 2017 23:00:23 +0530 Subject: [PATCH 102/142] [SNAP-2033] pass the original number of buckets in table via OrderlessHashPartitioning (#82) also reduced parallel forks in tests to be same as number of processors/cores --- build.gradle | 2 +- .../apache/spark/sql/catalyst/plans/physical/partitioning.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 65fba838402f4..cd3c7ed396f57 100644 --- a/build.gradle +++ b/build.gradle @@ -281,7 +281,7 @@ subprojects { } test { jvmArgs '-Xss4096k' - maxParallelForks = (2 * Runtime.getRuntime().availableProcessors()) + maxParallelForks = Runtime.getRuntime().availableProcessors() systemProperties 'spark.master.rest.enabled': 'false', 'test.src.tables': 'src' diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index d628254c6daef..193cac19adc9c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -222,7 +222,7 @@ case object SinglePartition extends Partitioning { * than this partitioning then also it is considered equal. */ case class OrderlessHashPartitioning(expressions: Seq[Expression], - aliases: Seq[Seq[Attribute]], numPartitions: Int, numBuckets: Int) + aliases: Seq[Seq[Attribute]], numPartitions: Int, numBuckets: Int, tableBuckets: Int) extends Expression with Partitioning with Unevaluable { From bed1dded985e8ab6a207d51e4c0a85e885653806 Mon Sep 17 00:00:00 2001 From: Amogh Shetkar Date: Wed, 20 Sep 2017 16:11:18 +0530 Subject: [PATCH 103/142] Update versions for snappydata 1.0.0, store 1.6.0, spark 2.1.1.1 and spark-jobserver 0.6.2.6 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index cd3c7ed396f57..ebba211340101 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.1.1.1-rc1' + version = '2.1.1.1' ext { scalaBinaryVersion = '2.11' From 525f4d6e0defc6071ae302dfde122902d9cc3461 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 20 Sep 2017 21:25:43 +0530 Subject: [PATCH 104/142] [SNAPPYDATA] use common "vendorName" in build scripts --- build.gradle | 2 ++ common/network-yarn/build.gradle | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ebba211340101..a89571e6bc8d6 100644 --- a/build.gradle +++ b/build.gradle @@ -48,6 +48,8 @@ allprojects { version = '2.1.1.1' ext { + productName = 'SnappyData' + vendorName = 'SnappyData, Inc.' scalaBinaryVersion = '2.11' scalaVersion = scalaBinaryVersion + '.8' hadoopVersion = '2.7.3' diff --git a/common/network-yarn/build.gradle b/common/network-yarn/build.gradle index 7097acf8f0e89..eaa3c542c7b33 100644 --- a/common/network-yarn/build.gradle +++ b/common/network-yarn/build.gradle @@ -89,7 +89,7 @@ shadowJar { 'Created-By' : createdBy, 'Title' : project.name, 'Version' : version, - 'Vendor' : 'SnappyData, Inc.' + 'Vendor' : vendorName ) } From b21cacff2704291a0e84e2ba40d956571fea5a3e Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sat, 7 Oct 2017 03:37:38 +0530 Subject: [PATCH 105/142] [SNAPPYDATA] relax access-level of Executor thread pools to protected Conflicts: core/src/main/scala/org/apache/spark/executor/Executor.scala --- core/src/main/scala/org/apache/spark/executor/Executor.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 0277339680359..0a9f572d698a5 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -86,7 +86,7 @@ private[spark] class Executor( } // Start worker thread pool - private val threadPool = { + protected final val threadPool = { val threadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("Executor task launch worker-%d") @@ -102,7 +102,7 @@ private[spark] class Executor( } private val executorSource = new ExecutorSource(threadPool, executorId) // Pool used for threads that supervise task killing / cancellation - private val taskReaperPool = ThreadUtils.newDaemonCachedThreadPool("Task reaper") + protected final val taskReaperPool = ThreadUtils.newDaemonCachedThreadPool("Task reaper") // For tasks which are in the process of being killed, this map holds the most recently created // TaskReaper. All accesses to this map should be synchronized on the map itself (this isn't // a ConcurrentHashMap because we use the synchronization for purposes other than simply guarding From 7007a35cbf20263fb45e8c820cf526404dcb020f Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 10 Oct 2017 18:21:01 +0530 Subject: [PATCH 106/142] [SNAPPYDATA] version upgrades as per previous cherry-picks Following cherry-picked versions for dependency upgrades that fix various issues: 553aac5, 1a64388, a8567e3, 26a4cba, 55834a8 Some were already updated in snappy-spark while others are handled in this. Conflicts: sql/core/build.gradle --- build.gradle | 2 +- mesos/build.gradle | 1 + mllib-local/build.gradle | 2 +- mllib/build.gradle | 2 +- sql/core/build.gradle | 3 +++ 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index a89571e6bc8d6..949a113740c84 100644 --- a/build.gradle +++ b/build.gradle @@ -72,7 +72,7 @@ allprojects { httpCoreVersion = '4.4.4' fasterXmlVersion = '2.6.5' snappyJavaVersion = '1.1.2.6' - parquetVersion = '1.8.1' + parquetVersion = '1.8.2' hiveParquetVersion = '1.6.0' metricsVersion = '3.1.2' thriftVersion = '0.9.3' diff --git a/mesos/build.gradle b/mesos/build.gradle index 94e0d8ebed48f..6309c04f0af3e 100644 --- a/mesos/build.gradle +++ b/mesos/build.gradle @@ -31,5 +31,6 @@ dependencies { compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion compile group: 'org.eclipse.jetty', name: 'jetty-servlets', version: jettyVersion + testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') } diff --git a/mllib-local/build.gradle b/mllib-local/build.gradle index 408c4153a43e2..5e7b48acb710d 100644 --- a/mllib-local/build.gradle +++ b/mllib-local/build.gradle @@ -19,7 +19,7 @@ description = 'Spark Project ML Local Library' dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.12') { + compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.13.1') { exclude(group: 'junit', module: 'junit') exclude(group: 'org.apache.commons', module: 'commons-math3') } diff --git a/mllib/build.gradle b/mllib/build.gradle index f2ccf4f827373..cc780b9d5a5c7 100644 --- a/mllib/build.gradle +++ b/mllib/build.gradle @@ -25,7 +25,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.12') { + compile(group: 'org.scalanlp', name: 'breeze_' + scalaBinaryVersion, version: '0.13.1') { exclude(group: 'junit', module: 'junit') exclude(group: 'org.apache.commons', module: 'commons-math3') } diff --git a/sql/core/build.gradle b/sql/core/build.gradle index b5a98c11dc310..e83a8b2697419 100644 --- a/sql/core/build.gradle +++ b/sql/core/build.gradle @@ -36,6 +36,9 @@ dependencies { testCompile group: 'org.postgresql', name: 'postgresql', version: '9.4.1207.jre7' testCompile group: 'org.apache.parquet', name: 'parquet-avro', version: parquetVersion testCompile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' + // different avro version from parent (1.7.7) since parquet-avro depends on 1.8.x + // which is used by ParquetAvroCompatibilitySuite that uses AvroParquetWriter + testCompile group: 'org.apache.avro', name: 'avro', version: '1.8.1' } // fix scala+java test ordering From cca3704500bd15c4ef940712cb7b0f42460fc88c Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Mon, 23 Oct 2017 12:04:15 +0530 Subject: [PATCH 107/142] Snap 2044 (#85) * Corrected SnappySession code. --- python/pyspark/shell.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py index 2636334b56ad9..8fc111f5693ed 100644 --- a/python/pyspark/shell.py +++ b/python/pyspark/shell.py @@ -50,7 +50,7 @@ from pyspark import SparkConf from pyspark.context import SparkContext from pyspark.sql import SparkSession, SQLContext -from pyspark.sql.snappy import SnappyContext +from pyspark.sql.snappy import SnappySession from pyspark.storagelevel import StorageLevel if os.environ.get("SPARK_EXECUTOR_URI"): @@ -59,8 +59,6 @@ SparkContext._ensure_initialized() try: - sqlContext = SnappyContext(sc) -except py4j.protocol.Py4JError: # Try to access HiveConf, it will raise exception if Hive is not added conf = SparkConf() if conf.get('spark.sql.catalogImplementation', 'hive').lower() == 'hive': @@ -81,12 +79,14 @@ "please make sure you build spark with hive") spark = SparkSession.builder.getOrCreate() + sc = spark.sparkContext -sql = spark.sql +snappy = SnappySession(sc) +sql = snappy.sql atexit.register(lambda: sc.stop()) # for compatibility -sqlContext = spark._wrapped +sqlContext = snappy._wrapped sqlCtx = sqlContext print("""Welcome to @@ -101,6 +101,7 @@ platform.python_build()[0], platform.python_build()[1])) print("SparkSession available as 'spark'.") +print("SnappySession available as 'snappy'.") # The ./bin/pyspark script stores the old PYTHONSTARTUP value in OLD_PYTHONSTARTUP, # which allows us to execute the user's PYTHONSTARTUP file: From d7140c3970e07c5ee0ae999261042cf9c3376950 Mon Sep 17 00:00:00 2001 From: ahshahid Date: Mon, 23 Oct 2017 18:31:09 -0700 Subject: [PATCH 108/142] Snap 2061 (#83) * added previous code for reference * added data validation in the test * Incorporated review comments. added test for dataset encoder conversion to dataframe. Conflicts: sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala --- .../org/apache/spark/sql/SQLContext.scala | 45 +++++- .../apache/spark/sql/SQLContextSuite.scala | 151 ++++++++++++++++++ 2 files changed, 193 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index af6018472cb03..b73de9724d48f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -17,11 +17,12 @@ package org.apache.spark.sql +import java.beans.{BeanInfo, Introspector, PropertyDescriptor} +import java.lang.reflect.Method import java.util.Properties import scala.collection.immutable import scala.reflect.runtime.universe.TypeTag - import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.annotation.{DeveloperApi, Experimental, InterfaceStability} import org.apache.spark.api.java.{JavaRDD, JavaSparkContext} @@ -31,7 +32,7 @@ import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.command.ShowTablesCommand -import org.apache.spark.sql.internal.{SessionState, SharedState, SQLConf} +import org.apache.spark.sql.internal.{SQLConf, SessionState, SharedState} import org.apache.spark.sql.sources.BaseRelation import org.apache.spark.sql.streaming.{DataStreamReader, StreamingQueryManager} import org.apache.spark.sql.types._ @@ -1095,6 +1096,7 @@ object SQLContext { * method for internal use. */ private[sql] def beansToRows( +<<<<<<< HEAD data: Iterator[_], beanClass: Class[_], attrs: Seq[AttributeReference]): Iterator[InternalRow] = { @@ -1104,12 +1106,49 @@ object SQLContext { (e, CatalystTypeConverters.createToCatalystConverter(attr.dataType)) } data.map { element => +||||||| parent of 61e5899... Snap 2061 (#83) + data: Iterator[_], + beanInfo: BeanInfo, + attrs: Seq[AttributeReference]): Iterator[InternalRow] = { + val extractors = + beanInfo.getPropertyDescriptors.filterNot(_.getName == "class").map(_.getReadMethod) + val methodsToConverts = extractors.zip(attrs).map { case (e, attr) => + (e, CatalystTypeConverters.createToCatalystConverter(attr.dataType)) + } + data.map{ element => +======= + data: Iterator[_], + beanInfo: BeanInfo, + attrs: Seq[AttributeReference]): Iterator[InternalRow] = { + val converters = getExtractors(beanInfo, attrs) + data.map{ element => +>>>>>>> 61e5899... Snap 2061 (#83) new GenericInternalRow( - methodsToConverts.map { case (e, convert) => convert(e.invoke(element)) } + converters.map { case (e, convert) => convert(e.getReadMethod.invoke(element)) } ): InternalRow } } + def getExtractors( beanInfo: BeanInfo, + attrs: Seq[AttributeReference]): Array[(PropertyDescriptor, Any => Any)] = { + val methodsToConverts = beanInfo.getPropertyDescriptors. + filterNot(_.getName == "class").zip(attrs) + methodsToConverts.map { case (desc, attr) => + attr.dataType match { + case strct: StructType => { + val extractors = getExtractors(Introspector.getBeanInfo(desc.getPropertyType), + strct.toAttributes) + (desc, (x: Any) => { + val arr = Array.tabulate[Any](strct.length)(i => + extractors(i)._2(extractors(i)._1.getReadMethod.invoke(x))) + new GenericInternalRow(arr) + }) + } + case _ => (desc, CatalystTypeConverters.createToCatalystConverter(attr.dataType)) + } + } + } + /** * Extract `spark.sql.*` properties from the conf and return them as a [[Properties]]. */ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala index a1799829932b8..2efa9669fcfd1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql +import java.sql.{Date, Timestamp} + import org.apache.spark.{SharedSparkContext, SparkFunSuite} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -142,4 +144,153 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext { } } + test("Bug SNAP-2061 Nested POJO object not handled when creating DataFrame from RDD") { + val sqlContext = SQLContext.getOrCreate(sc) + val personsCollection = for (k <- 1 until 100) yield { + new Person(k, "name_" + k, k.toLong, k.toShort, + k.toByte, k.toDouble *86.7543d, k.toFloat *7.31f, + true, Array.fill[Byte](k)(k.toByte), + new java.sql.Date(7836*k*1000), new Timestamp(7896*k*1000), + new Address("12320 sw horizon," + k, 97007*k)) + } + + val personsRDD = sc.parallelize(personsCollection) + val df = sqlContext.createDataFrame(personsRDD, classOf[Person]) + val rows = df.collect() + val keys = scala.collection.mutable.Set[Int]() + for(i <- 1 until 100) keys.add(i) + for(row <- rows) { + assert(keys.remove(row.getAs[Int]("id"))) + val k = row.getAs[Int]("id") + assert("name_" + k == row.getAs[String]("name"), "String field match not as expected") + assert(k.toLong == row.getAs[Long]("longField"), "Long field match not as expected") + assert(k.toShort == row.getAs[Short]("shortField"), "Short field match not as expected") + assert(k.toByte == row.getAs[Byte]("byteField"), "Byte field match not as expected") + assert(k*86.7543d == row.getAs[Double]("doubleField"), "Double field match not as expected") + assert(k*7.31f == row.getAs[Float]("floatField"), "Float field match not as expected") + assert(true == row.getAs[Boolean]("booleanField"), "Boolean field match not as expected") + assertResult(Array.fill[Byte](k)(k.toByte).seq) {row.getAs[Array[Byte]]("binaryField").toSeq} + assert(new java.sql.Date(7836*k*1000).toString == row.getAs[Date]("datee").toString, + "Date field match not as expected") + assert(new Timestamp(7896*k*1000).toString == row.getAs[Timestamp]("timeeStamp").toString, + "TimeStamp field match not as expected") + val addressStruct = row.getAs[Row]("address") + assert("12320 sw horizon," + k == addressStruct.getAs[String]("street"), + "struct field match not as expected") + assert(97007*k == addressStruct.getAs[Int]("zip"), "struct field match not as expected") + } + assert(keys.isEmpty) + } + + test("Bug SNAP-2061 Nested POJO object not handled when creating DataSet from RDD") { + val sqlContext = SQLContext.getOrCreate(sc) + val spark = sqlContext.sparkSession + + val personsCollection = for (k <- 1 until 100) yield { + new Person(k, "name_" + k, k.toLong, k.toShort, + k.toByte, k.toDouble *86.7543d, k.toFloat *7.31f, + true, Array.fill[Byte](k)(k.toByte), + new java.sql.Date(7836*k*1000), new Timestamp(7896*k*1000), + new Address("12320 sw horizon," + k, 97007*k)) + } + + + + val personsDataSet = spark.createDataset(personsCollection)(Encoders.bean(classOf[Person])) + + var rows = personsDataSet.toDF().collect() + val keys = scala.collection.mutable.Set[Int]() + for(i <- 1 until 100) keys.add(i) + for(row <- rows) { + assert(keys.remove(row.getAs[Int]("id"))) + val k = row.getAs[Int]("id") + assert("name_" + k == row.getAs[String]("name"), "String field match not as expected") + assert(k.toLong == row.getAs[Long]("longField"), "Long field match not as expected") + assert(k.toShort == row.getAs[Short]("shortField"), "Short field match not as expected") + assert(k.toByte == row.getAs[Byte]("byteField"), "Byte field match not as expected") + assert(k*86.7543d == row.getAs[Double]("doubleField"), "Double field match not as expected") + assert(k*7.31f == row.getAs[Float]("floatField"), "Float field match not as expected") + assert(true == row.getAs[Boolean]("booleanField"), "Boolean field match not as expected") + assertResult(Array.fill[Byte](k)(k.toByte).seq) {row.getAs[Array[Byte]]("binaryField").toSeq} + assert(new java.sql.Date(7836*k*1000).toString == row.getAs[Date]("datee").toString, + "Date field match not as expected") + assert(new Timestamp(7896*k*1000).toString == row.getAs[Timestamp]("timeeStamp").toString, + "TimeStamp field match not as expected") + val addressStruct = row.getAs[Row]("address") + assert("12320 sw horizon," + k == addressStruct.getAs[String]("street"), + "struct field match not as expected") + assert(97007*k == addressStruct.getAs[Int]("zip"), "struct field match not as expected") + } + assert(keys.isEmpty) + personsDataSet.createOrReplaceTempView("tempPersonsTable") + rows = spark.sql("select * from tempPersonsTable").collect() + for(i <- 1 until 100) keys.add(i) + for(row <- rows) { + assert(keys.remove(row.getAs[Int]("id"))) + val k = row.getAs[Int]("id") + assert("name_" + k == row.getAs[String]("name"), "String field match not as expected") + assert(k.toLong == row.getAs[Long]("longField"), "Long field match not as expected") + assert(k.toShort == row.getAs[Short]("shortField"), "Short field match not as expected") + assert(k.toByte == row.getAs[Byte]("byteField"), "Byte field match not as expected") + assert(k*86.7543d == row.getAs[Double]("doubleField"), "Double field match not as expected") + assert(k*7.31f == row.getAs[Float]("floatField"), "Float field match not as expected") + assert(true == row.getAs[Boolean]("booleanField"), "Boolean field match not as expected") + assertResult(Array.fill[Byte](k)(k.toByte).seq) {row.getAs[Array[Byte]]("binaryField").toSeq} + assert(new java.sql.Date(7836*k*1000).toString == row.getAs[Date]("datee").toString, + "Date field match not as expected") + assert(new Timestamp(7896*k*1000).toString == row.getAs[Timestamp]("timeeStamp").toString, + "TimeStamp field match not as expected") + val addressStruct = row.getAs[Row]("address") + assert("12320 sw horizon," + k == addressStruct.getAs[String]("street"), + "struct field match not as expected") + assert(97007*k == addressStruct.getAs[Int]("zip"), "struct field match not as expected") + } + assert(keys.isEmpty) + sqlContext.dropTempTable("tempPersonsTable") + } + + +} + +class Person(var id: Int, var name: String, var longField: Long, var shortField: Short, + var byteField: Byte, var doubleField: Double, var floatField: Float, + var booleanField: Boolean, var binaryField: Array[Byte], + var datee: Date, var timeeStamp: Timestamp, + var address: Address ) extends java.io.Serializable{ + def this() = this(0, null, 0, 0, 0, 0d, 0f, false, null, null, null, null) + def getName: String = name + def getId: Int = id + def getLongField: Long = longField + def getShortField: Short = shortField + def getByteField: Byte = byteField + def getDoubleField: Double = doubleField + def getFloatField: Float = floatField + def getBooleanField: Boolean = booleanField + def getBinaryField: Array[Byte] = binaryField + def getDatee: Date = datee + def getTimeeStamp: Timestamp = timeeStamp + def getAddress: Address = address + + def setName(name: String): Unit = {this.name = name} + def setId(id: Int): Unit = {this.id = id} + def setLongField(longField: Long): Unit = {this.longField = longField} + def setShortField(shortField: Short): Unit = {this.shortField = shortField} + def setByteField(byteField: Byte): Unit = {this.byteField = byteField} + def setDoubleField(doubleField: Double): Unit = {this.doubleField = doubleField} + def setFloatField(floatField: Float): Unit = {this.floatField = floatField} + def setBooleanField(booleanField: Boolean): Unit = {this.booleanField = booleanField} + def setBinaryField(binaryField: Array[Byte]): Unit = {this.binaryField = binaryField} + def setDatee(datee: Date): Unit = {this.datee = datee} + def setTimeeStamp(ts: Timestamp): Unit = {this.timeeStamp = ts} + def setAddress(address: Address): Unit = {this.address = address} +} + + + +class Address(var street: String, var zip: Int) extends java.io.Serializable { + def this() = this(null, -1) + def getStreet: String = this.street + def getZip: Int = this.zip + def setStreet(street: String): Unit = {this.street = street} + def setZip(zip: Int): Unit = {this.zip = zip} } From 560cafa67c4f399bbc7e9e091f3236dec982b52d Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 24 Oct 2017 12:02:59 +0530 Subject: [PATCH 109/142] [SNAPPYDATA] build changes/fixes (#81) - update gradle to 3.5 - updated many dependencies to latest bugfix releases - changed provided dependencies to compile/compileOnly - changed deprecated "<<" with doLast - changed deprecated JavaCompile.forkOptions.executable with javaHome - gradlew* script changes as from upstream release (as updated by ./gradlew wrapper --gradle-version 3.5.1) Conflicts: sql/core/build.gradle --- build.gradle | 68 ++++++++++------------- common/network-common/build.gradle | 4 +- common/network-shuffle/build.gradle | 4 +- common/network-yarn/build.gradle | 8 +-- common/sketch/build.gradle | 2 +- common/unsafe/build.gradle | 2 +- core/build.gradle | 24 ++++---- examples/build.gradle | 2 +- external/kafka-0-10-sql/build.gradle | 14 +++-- external/kafka-0-10/build.gradle | 4 +- external/kafka-0-8/build.gradle | 3 +- gradle/wrapper/gradle-wrapper.jar | Bin 53638 -> 54711 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 68 +++++++++++++---------- gradlew.bat | 12 +--- graphx/build.gradle | 2 +- repl/build.gradle | 2 +- sql/catalyst/build.gradle | 2 +- sql/core/build.gradle | 11 +++- sql/hive/build.gradle | 16 ++++-- 20 files changed, 132 insertions(+), 120 deletions(-) diff --git a/build.gradle b/build.gradle index 949a113740c84..3f5af8c43bef2 100644 --- a/build.gradle +++ b/build.gradle @@ -15,6 +15,8 @@ * LICENSE file. */ +import org.gradle.api.tasks.testing.logging.* + apply plugin: 'wrapper' // TODO: profiles and allow changing hadoopVersion @@ -25,7 +27,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'io.snappydata:gradle-scalatest:0.13.1' + classpath 'io.snappydata:gradle-scalatest:0.16' classpath 'org.github.ngbinh.scalastyle:gradle-scalastyle-plugin_2.11:0.8.2' } } @@ -54,38 +56,42 @@ allprojects { scalaVersion = scalaBinaryVersion + '.8' hadoopVersion = '2.7.3' protobufVersion = '2.6.1' - jerseyVersion = '2.22.2' - sunJerseyVersion = '1.19.1' + jerseyVersion = '2.26' + sunJerseyVersion = '1.19.4' jettyVersion = '9.2.22.v20170606' log4jVersion = '1.2.17' - slf4jVersion = '1.7.21' + slf4jVersion = '1.7.25' junitVersion = '4.12' javaxServletVersion = '3.1.0' guavaVersion = '14.0.1' hiveVersion = '1.2.1.spark2' chillVersion = '0.8.1' - kryoVersion = '4.0.0' - nettyVersion = '3.9.9.Final' - nettyAllVersion = '4.0.43.Final' + kryoVersion = '4.0.1' + nettyVersion = '3.10.6.Final' + nettyAllVersion = '4.0.51.Final' derbyVersion = '10.12.1.1' - httpClientVersion = '4.5.2' - httpCoreVersion = '4.4.4' - fasterXmlVersion = '2.6.5' - snappyJavaVersion = '1.1.2.6' + httpClientVersion = '4.5.3' + httpCoreVersion = '4.4.7' + jackson1Version = '1.9.13' + jacksonVersion = '2.9.1' + snappyJavaVersion = '1.1.4' + lz4Version = '1.4.0' + lzfVersion = '1.0.4' parquetVersion = '1.8.2' - hiveParquetVersion = '1.6.0' - metricsVersion = '3.1.2' + // hiveParquetVersion = '1.6.0' + metricsVersion = '3.2.5' thriftVersion = '0.9.3' antlrVersion = '4.5.3' jpamVersion = '1.1' seleniumVersion = '2.52.0' curatorVersion = '2.7.1' commonsCodecVersion = '1.10' - commonsLang3Version = '3.5' - commonsMath3Version = '3.4.1' + commonsLang3Version = '3.6' + commonsMath3Version = '3.6.1' avroVersion = '1.7.7' - jsr305Version = '3.0.1' + jsr305Version = '3.0.2' jlineVersion = '2.14.2' + xbeanAsm5Version = '4.5' scalatestVersion = '2.2.6' pegdownVersion = '1.6.0' @@ -120,16 +126,16 @@ def getStackTrace(def t) { return sw.toString() } -task cleanSparkScalaTest << { +task cleanSparkScalaTest { doLast { def workingDir = "${testResultsBase}/scalatest" delete workingDir file(workingDir).mkdirs() -} -task cleanSparkJUnit << { +} } +task cleanSparkJUnit { doLast { def workingDir = "${testResultsBase}/junit" delete workingDir file(workingDir).mkdirs() -} +} } subprojects { apply plugin: 'scala' @@ -171,10 +177,6 @@ subprojects { } configurations { - provided { - description 'a dependency that is provided externally at runtime' - visible true - } testOutput { extendsFrom testCompile description 'a dependency that exposes test artifacts' @@ -189,21 +191,6 @@ subprojects { artifacts { testOutput packageTests } - - idea { - module { - scopes.PROVIDED.plus += [ configurations.provided ] - } - } - - sourceSets { - main.compileClasspath += configurations.provided - main.runtimeClasspath -= configurations.provided - test.compileClasspath += configurations.provided - test.runtimeClasspath += configurations.provided - } - - javadoc.classpath += configurations.provided } task packageScalaDocs(type: Jar, dependsOn: scaladoc) { classifier = 'javadoc' @@ -243,6 +230,9 @@ subprojects { task scalaTest(type: Test) { actions = [ new com.github.maiflai.ScalaTestAction() ] + testLogging.exceptionFormat = TestExceptionFormat.FULL + testLogging.events = TestLogEvent.values() as Set + List suites = [] extensions.add(com.github.maiflai.ScalaTestAction.SUITES, suites) extensions.add('suite', { String name -> suites.add(name) } ) diff --git a/common/network-common/build.gradle b/common/network-common/build.gradle index 6ea74750089e4..52dd2ec59b72e 100644 --- a/common/network-common/build.gradle +++ b/common/network-common/build.gradle @@ -24,7 +24,7 @@ dependencies { compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version compile group: 'com.google.guava', name: 'guava', version: guavaVersion compile group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: '1.8' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion - compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: fasterXmlVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion compile group: 'org.apache.commons', name: 'commons-lang3', version: commonsLang3Version } diff --git a/common/network-shuffle/build.gradle b/common/network-shuffle/build.gradle index f19af0c6cb085..de3cdc886be68 100644 --- a/common/network-shuffle/build.gradle +++ b/common/network-shuffle/build.gradle @@ -22,8 +22,8 @@ dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) compile group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: '1.8' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion - compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: fasterXmlVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion compile group: 'com.google.guava', name: 'guava', version: guavaVersion compile(group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion) { exclude(group: 'org.slf4j', module: 'slf4j-api') diff --git a/common/network-yarn/build.gradle b/common/network-yarn/build.gradle index eaa3c542c7b33..c04629191cc78 100644 --- a/common/network-yarn/build.gradle +++ b/common/network-yarn/build.gradle @@ -16,7 +16,7 @@ */ plugins { - id 'com.github.johnrengelman.shadow' version '1.2.3' + id 'com.github.johnrengelman.shadow' version '2.0.1' } description = 'Spark Project YARN Shuffle Service' @@ -26,7 +26,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion - provided (group: 'org.apache.hadoop', name: 'hadoop-client', version: hadoopVersion) { + compileOnly (group: 'org.apache.hadoop', name: 'hadoop-client', version: hadoopVersion) { exclude(group: 'asm', module: 'asm') exclude(group: 'org.codehaus.jackson', module: 'jackson-core-asl') exclude(group: 'org.codehaus.jackson', module: 'jackson-mapper-asl') @@ -49,8 +49,8 @@ dependencies { runtimeJar project(subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion) runtimeJar project(subprojectBase + 'snappy-spark-network-shuffle_' + scalaBinaryVersion) runtimeJar group: 'io.netty', name: 'netty-all', version: nettyAllVersion - runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion - runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: fasterXmlVersion + runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion */ } diff --git a/common/sketch/build.gradle b/common/sketch/build.gradle index 0192f28b59f45..4b5712ed8d2e1 100644 --- a/common/sketch/build.gradle +++ b/common/sketch/build.gradle @@ -24,5 +24,5 @@ dependencies { tasks.withType(JavaCompile) { options.compilerArgs << '-XDignore.symbol.file' options.fork = true - options.forkOptions.executable = "${System.properties['java.home']}/../bin/javac" + options.forkOptions.javaHome = file(System.properties['java.home']) } diff --git a/common/unsafe/build.gradle b/common/unsafe/build.gradle index 83ad85298286a..17cf13459060b 100644 --- a/common/unsafe/build.gradle +++ b/common/unsafe/build.gradle @@ -37,5 +37,5 @@ sourceSets.main.java.srcDirs = [ 'src/main/java' ] tasks.withType(JavaCompile) { options.compilerArgs << '-XDignore.symbol.file' options.fork = true - options.forkOptions.executable = "${System.properties['java.home']}/../bin/javac" + options.forkOptions.javaHome = file(System.properties['java.home']) } diff --git a/core/build.gradle b/core/build.gradle index 1a8b040699750..a4973db71a839 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -47,11 +47,11 @@ dependencies { compile(group: 'com.twitter', name: 'chill-java', version: chillVersion) { exclude(group: 'com.esotericsoftware', module: 'kryo-shaded') } - compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' + compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version // explicitly include netty from akka-remote to not let zookeeper override it compile group: 'io.netty', name: 'netty', version: nettyVersion // explicitly exclude old netty from zookeeper - compile(group: 'org.apache.zookeeper', name: 'zookeeper', version: '3.4.8') { + compile(group: 'org.apache.zookeeper', name: 'zookeeper', version: '3.4.10') { exclude(group: 'org.jboss.netty', module: 'netty') exclude(group: 'jline', module: 'jline') exclude(group: 'org.slf4j', module: 'slf4j-api') @@ -76,7 +76,8 @@ dependencies { exclude(group: 'com.sun.jersey.contribs') exclude(group: 'com.google.protobuf', module: 'protobuf-java') } - compile(group: 'net.java.dev.jets3t', name: 'jets3t', version: '0.9.3') { + compile(group: 'net.java.dev.jets3t', name: 'jets3t', version: '0.9.4') { + exclude(group: 'commons-codec', module: 'commons-codec') exclude(group: 'commons-logging', module: 'commons-logging') } compile(group: 'org.apache.curator', name: 'curator-recipes', version: curatorVersion) { @@ -87,7 +88,6 @@ dependencies { } compile 'org.scala-lang:scalap:' + scalaVersion - compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded' , version: '4.4' compile group: 'org.roaringbitmap', name: 'RoaringBitmap' , version: '0.5.11' compile group: 'org.eclipse.jetty', name: 'jetty-server', version: jettyVersion @@ -111,10 +111,10 @@ dependencies { compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion compile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion compile group: 'org.slf4j', name: 'jcl-over-slf4j', version: slf4jVersion - compile group: 'com.ning', name: 'compress-lzf', version: '1.0.3' compile group: 'org.xerial.snappy', name: 'snappy-java', version: snappyJavaVersion - compile group: 'net.jpountz.lz4', name: 'lz4', version: '1.3.0' - compile group: 'commons-net', name: 'commons-net', version: '2.2' + compile group: 'org.lz4', name: 'lz4-java', version: lz4Version + compile group: 'com.ning', name: 'compress-lzf', version: lzfVersion + compile group: 'commons-net', name: 'commons-net', version: '3.6' compile group: 'org.json4s', name: 'json4s-jackson_' + scalaBinaryVersion, version: '3.2.11' compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: jerseyVersion compile group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion @@ -125,7 +125,7 @@ dependencies { exclude(group: 'com.google.protobuf', module: 'protobuf-java') } compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion - compile(group: 'com.clearspring.analytics', name: 'stream', version: '2.7.0') { + compile(group: 'com.clearspring.analytics', name: 'stream', version: '2.8.0') { exclude(group: 'it.unimi.dsi', module: 'fastutil') } compile(group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion) { @@ -144,13 +144,13 @@ dependencies { exclude(group: 'org.slf4j', module: 'slf4j-api') exclude(group: 'org.slf4j', module: 'slf4j-log4j12') } - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion - compile(group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_' + scalaBinaryVersion, version: fasterXmlVersion) { + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + compile(group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_' + scalaBinaryVersion, version: jacksonVersion) { exclude(group: 'com.google.guava', module: 'guava') } compile group: 'org.apache.ivy', name: 'ivy', version: '2.4.0' compile group: 'oro', name: 'oro', version: '2.0.8' - compile(group: 'net.razorvine', name: 'pyrolite', version: '4.13') { + compile(group: 'net.razorvine', name: 'pyrolite', version: '4.20') { exclude(group: 'net.razorvine', module: 'serpent') } compile group: 'net.sf.py4j', name: 'py4j', version: '0.10.4' @@ -163,7 +163,7 @@ dependencies { testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-htmlunit-driver', version: seleniumVersion) { exclude(group: 'com.google.guava', module: 'guava') } - testCompile group: 'xml-apis', name: 'xml-apis', version: '1.4.01' + testCompile group: 'xml-apis', name: 'xml-apis', version: '1.0.b2' testCompile group: 'org.hamcrest', name: 'hamcrest-core', version: '1.3' testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '1.3' testCompile(group: 'org.apache.curator', name: 'curator-test', version: curatorVersion) { diff --git a/examples/build.gradle b/examples/build.gradle index c8110ed5f51b1..359c464579ea8 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -28,7 +28,7 @@ dependencies { compile group: 'org.apache.commons', name: 'commons-math3', version: commonsMath3Version compile group: 'com.github.scopt', name: 'scopt_' + scalaBinaryVersion, version: '3.3.0' - compile group: 'com.twitter', name: 'parquet-hadoop-bundle', version: hiveParquetVersion + // compile group: 'com.twitter', name: 'parquet-hadoop-bundle', version: hiveParquetVersion runtimeJar group: 'com.github.scopt', name: 'scopt_' + scalaBinaryVersion, version: '3.3.0' } diff --git a/external/kafka-0-10-sql/build.gradle b/external/kafka-0-10-sql/build.gradle index 38014d5d945ce..c613b6f98f6ca 100644 --- a/external/kafka-0-10-sql/build.gradle +++ b/external/kafka-0-10-sql/build.gradle @@ -19,15 +19,19 @@ description = 'Kafka 0.10 Source for Structured Streaming' dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - provided project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) - provided project(subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion) - provided project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) - compile group: 'org.apache.kafka', name: 'kafka-clients', version: '0.10.0.1' + compile(group: 'org.apache.kafka', name: 'kafka-clients', version: '0.10.0.1') { + exclude(group: 'net.jpountz.lz4', module: 'lz4') + } testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion, configuration: 'testOutput') - testCompile group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.1' + testCompile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.1') { + exclude(group: 'net.jpountz.lz4', module: 'lz4') + } testCompile group: 'net.sf.jopt-simple', name: 'jopt-simple', version: '3.2' } diff --git a/external/kafka-0-10/build.gradle b/external/kafka-0-10/build.gradle index 6b62fa4e320be..7f84cdbdb2711 100644 --- a/external/kafka-0-10/build.gradle +++ b/external/kafka-0-10/build.gradle @@ -19,8 +19,8 @@ description = 'Spark Integration for Kafka 0.10' dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - provided project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) - provided project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) compile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.1') { exclude(group: 'com.sun.jmx', module: 'jmxri') diff --git a/external/kafka-0-8/build.gradle b/external/kafka-0-8/build.gradle index 5ae9720b3c4ac..a43428c1ef737 100644 --- a/external/kafka-0-8/build.gradle +++ b/external/kafka-0-8/build.gradle @@ -20,12 +20,13 @@ description = 'Spark Integration for Kafka 0.8' dependencies { compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) - compile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.8.2.1') { + compile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.8.2.2') { exclude(group: 'com.sun.jmx', module: 'jmxri') exclude(group: 'com.sun.jdmk ', module: 'jmxtools') exclude(group: 'net.sf.jopt-simple', module: 'jopt-simple') exclude(group: 'org.slf4j', module: 'slf4j-simple') exclude(group: 'org.apache.zookeeper', module: 'zookeeper') + exclude(group: 'net.jpountz.lz4', module: 'lz4') } testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 5ccda13e9cb94678ba179b32452cf3d60dc36353..f808147c25e097ea8fc879b4336725042a14bfe3 100644 GIT binary patch delta 28620 zcmZ6yV{j!*^zNI9F|lpiwr$%sceImC%!zH=JI+izv2EM-eNUZp>)ikCs?}AE)t|ba zRqNNQ!;2x>sv(e6((f{uVWIZP?`ro`(qKKNndFC1Jo!hP3 z53v8~^4~fb`2XtuYmon$vzs+D#Q$$k;v^~j|2;PGg%;=kj;?+6(y#w>T!aGyV+2cX z?88hJ9Kr*}Y3X{;YoUB$jHi)HU|Iiwk@-nFIAx|TXDe$1-wp?vBVrEAS%@*z>jl}A zGR8)+iQV~(e6yC`n}(g8rtbT;2AmZZB$mIXx*hI#3JMBOF>aP1^JiP^0gIdC)Dc{&RCw5)mEz2H(OCpLQ5 zEsjfIl43fgCi@)7ZgnCHbBFo1IH-8Gb;0nlH)}?-__xe@niYnXVYC$$7Dw(R-))6f zu)sqrDi%o>6LXeibZND!u1NWKPl+Bzz!HZ8$dcN#RYNY*E*_Qj!VYIb$-r24v9aAP zvB*mX-1pp{pW-U(K0Twy^a%3jBAgCGeRB;k*9AbqyOPm4uyHyMUWgF`Lb~r4uQe zkQQD7{R@%s%}gK$%a^cUe3mtLLucd%2oMlvB%;irwG9)~97kzTBU`3Np-#a&qXBu1 z`hcC0yT!Cxv$Eov7iNW6ZkQRM*_t6bKzf7-Gd+*c@)BA>|I&&WQ@N1f*9^3b!N0ru zM07HwL+%~Cu*mLU!3c=zQp%lb?PI}m%#~vVv!ChKbp^#u>BT5rSfX2g$M|qea_;v6 z61HwmeYu^;Ioc}|&1xyCywjc|f@aHj=b>woQGCmCrl~DCme?F9!2i_T!ENe<=hAM? zE0E_F*{O^F9t!rN?Ufa|F9S-7DT=HgG$YOsPCLfc&so2+1mwTwBZKDyT8K*ZL%f2$ zz9lgG&G|e`VwZx%HxkTS=<2|RQA=})*r;Tp>4m9zqn8RqSp!}1VEHXw6Pbe(=Bgu zhOe+MX@hI-pRhg`kkiHd;YxZD|K5LqyP+gM912I78vKRTLhW+|ACDX;{&OOw6O)%u z#d=6GW{&47hW3|GPCUmUfh=nOL0^i(p8DilQ*6>PZ_u8?vnmZC{h9A}izHemW!@sJ zm!vgCYt%gGXjI7L}CV4QO+y663KQ9ROM;4XL$furzgIL|bywxDwe5KhO1x>=(@c5m%W1Bgo10Ki2*~Vmao_=Z^fJ z+KT)qm>mC0EQyj?`f-4%>W0o}q8Ne@DMOaFu9^)?a*p<4RSYkfh{K^K3JQ_7YK~@a z9AR)Zt5?5Gf8WI@_|IXl+w39VNaJ3EX^O!u24wgh=eb`CdRsjm*Ep>sVJTgZZz8_4(`)_hCT6a2-W1r~$Q2#&Q6;DgD@W+=L!yyti1R{SVo||V_lV;D;nxJ_LKe3Rb0j= z`y*fNqu40UH+zB%=IF={NnZ~6mivEm2>ty}tYpUjqDU0&{c%1bV_Nvg!3_*Z@x`H6 zUENZPpvMEHA9C^e`$F)&Ytbos_xf2crqvIH6jR3I$51o+TU|TKEy&Fo9YARlkt|DQ z-IxZ%Wwp#muiP$~HS$JiPzE7!)fS0LAJG3ZNHbCQ(IFv$fl(2Ffsy{NdqW6hGIua_ zchA*@GtpS~6x7bj{C<3FWeGC*yVDnhsuvg6hXws>km`5n49UNs#?QpwlHghY*J)y` zHqU*Y5~cY;MSN>lb$-{mPQONcDGHv(Y*4$RzPIxlmEjzh;UFOfuitjh@s@Y~%Lcd5BQ6+F|DZd})u4?fGW~qzy~R$bI7?gOzDn;CfS%uagYY5= z@yDXC9}%H`H=XeVUZ_0FSd-&VS(b5|3ycx>4L)?>e81X9S{O~>djIu~_u}1k<sBHY>(=i27d!9NaYoEt;XEM^%QhXO*OQ#(AX_g*StA76ZI8z&ByfAoj56- z#`x5o*e!e3On#B#BX(^5Rs@H=_)-{te4kU|I=6zFHQny=u~lZuaAOy>n}8EAp5YO2 zJNWyrS7NWQ_gAra1k@a}&B;C6>fcewkz9|dT1_vEh3ELI?LW@IgF=b#(SV$_1%}9y zo>AHKt{S47kI^gHZCvH9O>BQ_tKr8kn1?;2;J(}6?3Zd`!Oc^Ea{fyTsxI{4{4oW$ z)xA5{<>|8S*aE=tGgSb0Hld6q1a~plbq(m%=vs{Gl6QXq;a`XGL9YfC<^|dVh^c64 zGHXMi`L&&7;W!SceqR=>F-$n7nD6>&K9$)b6vKT~lE5D*7Z;OUXhJ8cUazZQS#`o# zwc6&s!R;5TKioYbVaImdg`iq6X|!GyIdNNr(^p#?7Qp5-Bgi*ryuo?sd97S(*i*b4 zn?SD_S@>EKI@X?zsFu066?YFfy$Qe`*}!d;IXX(3_)-Awyx~$STDk@L_SHdH*iTp* z0;fy#Ln#RZsJK}RQPkY4(%&rH=E>{kdV;M>JwtwQv*RkvdQ&Pq0(dDyNrWZn=F%t5 zlISIv?7H0YbfCqC3Dv9Ag7uiOVfDsFc#{yU+0IkC69X1xZaNY~b za7#=L7e~ObrTts=@D?>R#*K3eb#sKC){ZK7CoSEMQ9SyDQxnVnuZhr+1DD~^&UMh$ zc&x4e_Ld$L_hL~iU=9&p;i~{&0noBh1<$vgE2FOGZ5sCgs*aLxkup7*o`#a1%AriY zYs~6;kNh)zJ5MXw+8t8$GaGI93BEt_iJYIFjtA&*g+GIEpxf_5I9{6jmyj`g7j(9j z0km=i#ZH+V{q*kMc@>cJ*z&^52(hfy^ln(#aOGyNb8`wXRhjUSKS~>OM`hNR$?!A+ zxw$0>O9|-1F&`&%&}G5yqlI@Z2yrp(ur78l4*w)-+$FasS5ZVdcJ$}ol3Q1V>Qh*b z9Rothxr=kH&xIdD8OB)TA#!gMrI25Sd#udS8isqUyG1xO9x8Q=4=4L%$06_H>Vw8M z6@*JGP8LbeW)I22m7 z9cSv;PTOo6`E8{CW>HW2?y$-Xi;Rn<+ba|vrh56V}*ont?`uFv!fa>H}q-J{RL~cJT}Ja}-rP+>w`#Jeg92o%=zf`jGO3q^vD|IO#M>A%d z-3+>021hZODMj6XEEEs(4$XKN4Le8 z%^UXck>j%2j;FAw>$z8k%leCItOMP&rZ5xLM|%q=W_4sZN4i6lBvn)4Mt-+2&iA8- zN+(=iQ~VXk%#_wPGt{F*fZMZsGFu)x^;pi-u&;nr2*p;jTdQ5U_keR6+tKt-jx&~lQjd8NN{zGbCdr$eobXvN6#!0k$SdI7 zijTl>S`eF_(Ln;VS^sI`+;(OP^BO34jCuDm>SDU^xqId^AF+1v&>fYR^}F~w4c{~w zhPBu>ayzfhW;#3$J3J3NbbFe6MSQh&6xqv@@Fp=NC1+??)jUlz*6T79I_@5&ie|Q_ zz~#hX)>RDlQ7-8#Xgr2epaV&t;h0Xx%~gNAfjiNT>!|qZ>vFH0EAe!2T*ob9KFZ{ylWursOZCZp!ZQ-{|0Js7$45%Ii7TU7FP{#^mrC zMt{Vz;gQd4tD4v7k(e35Evf~CMV7EiZAiN@5FB50Rm!QV^>jOyNCN5LnaWDwlPBs2 zHGa%<{}CyPd!wHR9!ilKFQ@D-srN)zfTNF;eVBD*-lsCl?Y~lrn5MbmtF3-@j2Abl z5opmC)ojnv&8O7g!**0H+K!oT-wBE`RG5a7T~}CKWxIWZHu+dhiaW0T zi5r1!rQc+Lqv|(EBLlJ}pVi1?Yv@KG(dMRW4&WsC_9;V2w2CG_KhyW?^|JE{r#&@Q zQ58X;o2%E2(L=w(^f;)TAhPdb`o-T>k8aK8RJ+x|Z+!l@v$nb=lN+ig+xqPr=y3KE zHET`Q8r!Eq=cNke(qA3Z=hjM7(pIH+Wr6pxw1S}rVEWlr3`8NqY;AzQ$^JKHd3RTZ z(9-4RpgJ-9Exwv$a#*c)`lkzix-#2`&b#B3wee2BcDA>20&q=0>gtU7o>ygdXE%*( z`EN@;_{~vPz7@U7+iHDp#kAWG+vDcM+S`8HOh}_5$1g^LN&vV9hec2rU0Ov@C==$1 zGeQE%M5Ytg4CEOJ#1la*3FHFy1oR!EClJC(DgeKYNPIkuu^+7{sf)V0v#cq*CVAi;w@J~ar7FYvs;<; z5%|+Qb@i7lYlW=hu3q)97uEH`c*fJ!RI5MrI1BOHFQz z;(M6UcD6cn{nbuk^dWF2_w37P&FZZrPY4&lQ*LSXo+z^2_40P>>RQ>ZYKy_v8I!i{ zNB8N%0?vkx`@r!2iTFgZmRbQeQPfza=-8BfKx?@P?mpSuOqt6nAMh28p6g_l>{vGI z2w}t=%+8_P*(ZcqHHM@at(p&t&FW12GuLED89B?lNm86-&K4}g+iaSwS~oIo>mkL8 z&ji%SI#pK#kmi>gxDxb*Ll2ori44<69G4q~fVi71F>&=R91Qhc7Y@!gmCM_2X>XS8 z0i9O?(?va(=oN9GqsG_JJdd_zcPcjs)kj-nCqpOO7m78qli1AG@$)+y;BNNyoIs1Q zHG@Bd6+@1+_Az3M=h5sZ_|h|nrZM+|Z_Q~cHs_Mj2tLOu{!8>1mcloL1iA~dv|vd) zFyUxh*5HDy_o^J|Jx}Y`0MYZY9uaskz=E3e_XojI zb8PM=?2Gb$A}PR}S z_^xC?hmDde8>9JMFU_gK-uKq)3?O4#e1Rg|DXq{6z7G^lQQg4^*->kt>R|H_P@$!| zHUR7Sr)O{YOw6p`IxT%dp0$bn$aTR!f6=qy$kYLYm*n-dxi^0E5>ep&31ty>S>n&f z8=D5)tg#XJEG{MPD~sq1dE2hB#`2N|LP3LIZd>H6QCg*pk0CUIdUcmU-SKB?r{eYz>-ZZS$lGWSp*`UD{1{NT1i zsA=fw(jWjv>rQ1;fmTpr8}>GxWN!-1JptWW8O3%?0(s`1UiOxitS4;&cz#QM5ysPu zCLE1@($dUfZ+ZpC_cn5~e?f_3D%_VR=ct@I`JTc>$!sz9DJzB8SP@99Hu3NJK&Hc7 z{(*7g->+GU^}=}k_SD;+&J$PYudAax0VOXifxxgA9?pmktN=y<#f0wsD*qtaw_iW6 ztm*uFN^2&t;-t2wZAPZgfal{@u!gg}b*g`&=&Bx}t}1_f@V$Xy;zi~yjR{ty#0U+7 zs4)U`0{T^@1IT9CC(FE$f8fheh<}8l%lh&r#tyEqn2=@ zpL?O741T6p&4|k@F~9GU_3p}9O65cG6j;^^S2e!R!!T;Tq5fw=xuC?nS@<9A9sAE% z|93*E;^u7aX6f#(VC(K->165lzY|Q)xIXw05e&&|`guD{iSZ}NnVC*YR0^@gg}}K8 z+P!}h8Q)7q$S1{QaiGniT_>dD$3TG}H7qwffD?48#;O>DQJ$7LEi3nA$$nN-0R%Qc zdXWkJRQc!H^D%Zmi$-d^Px_HUh7<5iMYUR@nN`$|0lOx>R#t~5H&SAp$6D21H?3w! zIE>}(=#GjRX*rhs91WJRg*;*=Y+mRA&ub^}|A(prBUw<5puxZn5t2z@`GNZG=*PH$ zLb`Kn=bT(fJg6g_GZJhoArB&ST@+G%8YJ#buKS^4OLGxDKCRVtT0!#4vhr&sj=pkh z;coNUM`ZJvR#d1c)1d~f>^)B9e+tq&P8F?vRp0!(ni7%Q(WhVddn1kmwvYX^fuGmX z#1K58>wJ0r9Y_LEaGVNuJivPLcv?i``NToqR)A=2+4^l!ma!T&_QH)Pr-8-*B_ea% zfZ8}Np&-pEhvpK=x~uMp7G*DHGvAR(bv8ZU(QZp|(M8AQ4WH@|x5rDxDXfaJ$fqZ9 zKpncatKxtsA_877?TorWt}wv1Mn%BOcJE_FXBR4XYY+3hD%F$08_30v`U2wTXP8NS z-q17Y=u0_VC>4=5(Js+O9<~QBB59~R<5lWIECDXMEtt9udR^{BijW!7Sd>BT0qPZo zu8Jya!p$Jz%~PWtvAT5mVSufSr3eeOUu*^mzHW6P%k=bUBgGC~&>T2lcz zTgqZp$y=_rv7RN4j_#5#EbEf#B09K*Yh*T98`4+2FAx)FPSh$sBu;Bo{gVwRZx*Eg zCClcz;YBc%4)O?3T{CFJva;B^wYziJObZ_HcocjTU}r4u-6?SLj>tCTirpGL^CHS>Na&sxlF4{ZcXWo`G88r*gR3OB}oL3FZt zsSWmaQ5#@{_%nd0b!Pg&Xo?(I=G4dCk`+z*tBlMj7mgh(Y$JK9CU68#Qb`xn$Vt|E z$v#sSc(~1ouIYREs24{CwJ?C(eIjFW-N|eoMp~q(b)>r-`NbS=I9>;};kDM97Cx)^ zx0s!Cx)SCt7IOm5!4Mhe@-$_F*)!&=^n~jKo1YuQqz)Wdh~qVFn9#CxkY6QnYTIy2->)96T~AJxG5#C%`ymfGb_aU2VZ@p zS65y!{ncC+!xGsrS&&bBUj9PsBo1$DLBjIa@F$lPyE_$`U|e2T4l z{!K7H3dFjwgoDXQgpHrDh7d;85bu-w#xJCcA*zdkc>z{(PJbff$UultCy92ocryba z5Y@N{V{>p!rT_tDnt@EH=TGn;S<<|Bf$Qp)h;;o_Vek|`ixT1;7q;>c{z+$;xZTzR z65rxR0K#X?Nv6};lT#U5sb9JWaTl{2T%tT%3L^wDiPt<|O_YIj)jl(hrggxYew(Wg zs(v|)-}}|htpQo8%}$g7TdD&ICcMxK(2ov)y{-m_#4+o2&jkAOzJqRQ*0A`9^rpdt zx3bz5bN)$hLeL(u3C=4x1i{WBqfw{`?tem3`c1ps(W`L$nKMgfvtj-99i=SE?Y_du zRgG(jik|86$9Qt0rE1x}22&ebkB zeP|f&4dY66@DGZuEZXnk<|l`*HPz(VBmnwWCZ75gD%O z1=g3Sfb?h)+gG?i-tqLTxdwL{kaMI0&@FpCy<1HRMEE$CE|+~QAK)+eWO{2eQ9eu< zCr}PQJ{F&lqL?2nuU1x3|7JMWdC~Zatqb;Bp}t|-8+ry1l$&60N*%rZhBekAV7~nz zb7s0(?cOl!#V>c!lbn{IMX~<=f3g4nq5hDSzMBUBVSd$SAE*@Y|Idr;d5v9i8T^0j zKe)@lCc7-Pto~|T^6#mZQ>EuE`c1!t?=JrFCXXQCCzSB!`&QfXyHVc9^gzOo z6%ww>mGCZ2he7#+JGG8(g+5zMN;HFzbb6mKVqpElzNXu=9cF!>VBp62h&^Vvl4R;& zRsZ?nF1|)LsJKYlP+yv5PqxL z40Eg840npgM`Z|a^iW8kWal%`Uqq5nsDdm8?v;}^9eMQ7RJiQJM%U#t-yYA8NaCgxbE{?@D^{qw7r|{&yHb7Z^%h{Q?(Q zwPQ^Oed`{)M)-M|n^-6Q4&C{Ne0gzx3gla8vu^0B*Cs*Xts-6?a24dmBWh4my7KlC^ktAO_J{j|XQ&g_acE*RRE|qBZXIb1I&<49Er+BdjLpE^B8tHm zs3lO&6lgJ%(i4MBNwL+-W!+1QKd)@*OIUpMmD+D~iO7^W8AY|dAi|G`9V@cB;|E4jyU5x8A`uIRIU3&Oj z7B}%vLS8b&WfvE#t;Td0p)?}PY9ESqD&ft>TA~5tu)n@mb>=P&sf&kOco*OP$rK_( zYAsF24`1Z|tT~Hx^BW2GOrmqsFDv52*iVBl+ZGMpTA@pxij??DoZR}x? zY!%_u;2!y>)IwyOJ}*ahKra>kIdcw``_J#ecza)#cBG=cqMw)o}M*Bl?VEYys(RQP97bCB- zuip^B>HH3WkLfC4>dK10Ct-^Gw{nAv|dL0bh4!5 z$LC0~qN~OGMEu zhW1Co3u~fVw&yw?=!NsQN)-s?OH>pY%-K3%yPw`hSIkJ1R;UevD>7jGkkqg*l44LB zG*euNru0|?*d=}C0Y}rn6y~BSV>b#l$SWdCv)X0O&iTdIIU5xiuveA}sguhv@mNpd zA<;=O;?7SWcm(nJ%*w>-I!U((3dw2NQKRpY_-o4JR5sT<499CYF8TczBK5L#NVVjm zNEK_6_Cid%lCss6Xs7RH?Oa*K*X_d{QhvvvWnH0ZkR}rp0#oqn)9Ru)&1$KZqWFTx z{|#Jn7M5%}h@HBUOs6d9o-w#Q$t7Wc5Y5n6?NGz)TEqH7z)aDY@% z6|;1hvvlJ_90s!5A=~K9e$^!CpN6_~q09ED8X@1d1I&!?hAGAioZsej)VEpEW7-xx z9(y+21$dBSfW4V{u`T5JrS%rQXJ#00`5$!2RX|v)?3Q01eF*0lCW~0aLU9(?zT!B9tGmgk_D_>Ld7L zSUZQ;W0*7?-MHR{u}?Gqrk(q&3`_Z{1;`TWS-)`^*b^}o7S;sU_0=8Q!xVNpd=22M z9$5#+7yyNo5kHI}E2B>J>;sNGnOEGHo(Lx44QEdJnr`woM>V3xZ_BfIeRgphcz#`M zEUMS=Qy*D&^mH>IR%@pZ6sEBa?{^Z&so4@aROYMW^bCVKI}+zUs&H*0x_JojZF7gJ zg9rQ%6xaV?6|kXLz(qildvEYmRnK(@CTdPMCA?zO+UCwaPL9d2F3W1?Z3Xt*(!sQeGs zVKTqAjsCz7vW5e6zI~rCX%-~k86POOM3`Ejp|}^guDMeCTTJ*JG;^o>&od!w8-2BF z08?q|bjy~y|6N}8v%_Jv;0r6-Y&$J;#PI#`$%P2UmXoAmPT%e&SHL`=Gmdrjtnw|) z=lH7M(|zFJJ{HOS`#5ZwrptVZpUW(Z+iDm$;tc{(Tqi7U1^RCUNq9s4zzyfdq&P0{ zc>v&-`VIkjhrdETx}=vSJ%9*5e+&&Jg|ROvaRuKK#Xm$@R>N-x=XO~nfIQ)u*k+Sp z6=Oh)na9vpTG>y|z_!gA{2*fL-J(tTsa(*qi2Yz$c!B&TM-&G12`q3hS;1C;X`-1H z%l@G)<5CKFoiBz-i`|k2DNk|PUr-NZjS;awrKr+REfBLg#juNPqKO&OU4RBOl`IlJ za4VD1aLenDFDdU8X>nF?DVeUiG&sPIcR*_?m>oijsq1hOJSdwyLNev`p zd6WM2WZdkIPg|NbM68e`Pb-GYZY1KL4e9Y%<6IHhb_fwYku?ZkZ-DYHz+?vYYvl~5 z^2B(pd+)~sBfSTaw#fat;%De58Of6PkZ^~b4BJ7p9fa3=imd^o^33&)3e zM7PO++|)l0$+{C^N2)9_6QKrH%}S;_w8iONH1sQKfIls(uG47th8hN^cLFq(ycDPY zutTjb=B67Sf{aD5+75JDx_nwe{kk>B#_k1_C{ue$a1JyU%gG~kTqE)I9_gE(9=QvC z+MHZgjkGb7x&~Q8mR;B4k{0m#;n`F!L%L%&h%J_lk46lMfB&ZTN;d)>eFFFYQq*Pr zWuNLCYf;`gV+YkhJxm_fQPI;t;E*^(&WJ9U5UX1YC!gSHUH{M@KoRT=0=DhyZY;9+ zOb@&Nv*pb)%rNdvFs#kDVv&F0m#Bpx)b+S>@(0HRSsBLjt!EDMlPNCPTB{fE#~2Ep zm$Q1w$3FH5M04cZrnUkx;NGULK-0duHv!h80hDVecBfZJ@7V3pmNNs_hEMh8t5UvL zA9Tftm<~{_3YDn}W1I>XRKC0lK;yobHr8NoR!?ts$I{lgl$pWUj$cN(Jz2kzmF zWls2m2YCbP0N*x$(i6D1{wLGEDu z-+I8@YMhsa{{RdZa&iJBUh>T#bh6?wEHGF5-48<>H&FSBZQLpdMbAawW>#v20TH24 z1uhsK&QUyF1YZo&+_Q<_0h)^a{DL99>M^_xWLK?KYS*I?<6W6ViQ#x&yY?3I9rWi- z`=#QVyE%K)Ar1Bj@RY;5?VtYazsX3jD21 z783%^9tEjnM?;k;A&NDBXF=&AcQjAdXql1Nm)NIhhE_@;^piQbNya}>1C58A$~u`R z;+_mpgihQiExBvn(8|9C?N=a4>Vo5cj@XG_O{x8tM#FfT-mc_#j5# zUuovAewa)p>1qZUpXAqtP*wq!Q*INpV-qQ-gq&q1NSnO0+GtvYRs1C&jT(lkbQIii2`@O{;!(yJLe){Lx57u(`iwPJd{|L1b`W*|& znw_ns{gb;~1Xc4{yXVWWfmO`j3e74+UekOadkl;PIJOdp9F7Qm+@g0UTI)Pk$EHZ1 zuB@En3VK+<Y1U#Cd#{N0f6- ztkowtfkVa10Wp|ad%yc$qmMJ=^9jwqAZ5B5%$^O8Ggxd}|K) zME^)pJ%22@N8G!$2I2+Q?PI@`#vAW8g=>3!;Qg6(g=*%jYOgX?8sY4&+U< zUN91LSSw>zlA)todYqcywD&O_z~>805Vm57nfLVv1_$^R9fM`YPJ5 z_mLl2-@6NP2;5X-psv=luWSe$HAAx)u*egD_uk+^>N^sg2i`zL(_h%FORaGQNg5kb zFcO*oW*-u8!09P|4?kA0nuF%rX3qxO2I^~>H_OTy&=y}r2STsI$-?~RE`9av-OpOa z1|H|2TwVbqwQiB_l&)9VbX{IU+zB{e9O&i;_g0yPKJ@P{S84C|*^V4;p4Rx*alOT| z)40zJT>TIIz^9FMU$!BHe!1pguj~Rx<6*uv5ViIcBi64LCK~bD2nNmY1;kv|5zzSA z##UX!2;T2^sJoq7aiU6Vhi62gQYk6)u-B0;eYMU8ju3j$r=&({L!uAHLA0?2Q|IsP z2K`v)+8gskHeAlVtDI^j_Oc=NTEiCaRpQxihy zw&MNmiU$q1P8D}O{)`uFi?lPi9=p5Kk$gSO3hfWC?``d3EA8iPH@s5Z@E*?v0eUir zodC}9^(n2vuFiu7NFL|h_qAtn>mKORyz6&VhQ_M9KUMH5%@!qVwhbhoC2I;r zqFDSJeUaJ>kr=y-gRpu+Z7h(NXL&b-IwDBXvz0x0rBb-TII*ztmQ4@_a)zi}el zAUh&N)m*PACDaj8qNfY3_vU~`^?-QI+bg>2exHxq!!j&x+pGwiK~-<(_SlxUT3(VF zpaLPgcY7j$AQH;e+Q8rq1d_W|H zWN4bYT{5)YNc{a5xUV|@O0(4F0Aln-N z44G9aB`y1wC4cXQT9+f_mjx0)eP2TlC>SOEqGhT^NRcIepe=eiLX$~kdeG*Qp%BkD zOy_%Dere8W6n;I7vr9v%DmUEqlvp?-77lHk*ub4uS#*7e_7J~VFwwQkgL?7@dF0nC zq8fYAJ zAkXDBy;{)HlC^!nWpzNk^Qm9GGwn6d%E`$oN%ZeQy0ca9Va>OD?|lH^(OFqgUIO8@ z80FDb1Wz)nfCDfW3cQ;+kS`uPZbwPI^9iGViQEokk|52aF;6a`g}t@NBU3buEO%3l zblSI_3&qSMDxwt-KbqF?(fVy2;e%ukpO02_D!@HhekX8Bn|8zzb{t`Ylz{9CmmXe` zHh$1-kOgl`ci$0RjcJ5(5C+ejgc~Eyk)JN}Jx9WveC&q$4D`U|Czp^kKtI5|6JSr6 z-b*DF%ASVXt<~DqB-KMQ!l_O!Gs=Q4Y|x|0!9U=fo3FS5zyq27DT*4=6Hb!yMJ&rU z3k@FMLx<^$wSCR9`8)5BD-yhxqI=T%>g*ps&=`$l`9b-oj ztC^~(lckbq4YnIPz0~K22_NGa@wqwb)>;}}4e#S0fvxzxlm@v0xt zwM5H=z}2cA8V&;YAOE1HIQGkWl1__~lxNT9IeB|mO0zRRJfkA}I0{E&@h2;FvP~Tq zSZRo4TFz%DJ1xh&JLjFfGjBE&l&AKxZ7}8{))>Xx;(Xa{*>iU*`>>wjBD^wkY317{ zCHy5T&pp^x8M#okTdWhSr}3NNb1-)eob_-E!#%YKkMt*(!p0)lZ z0|x0hd*R9IYk5V^Bd`nL?QhcJdmsvitA^u(J66Uc1^Z5cG$TPEAs^#5&oTiPPfC+4 zE~zTbDY}`ARoj);+kaLAZ2WIcj;*ALVBuJ?bIvus<^}30m+QXdt4^7lb>knX^C5ib zo0vEWTHKlOdFs+)z4d;uPiqw&*6{}*pxV-vo29a}v0?gbFFS3ot{$7#+P>YdzZzzB z*e2pVn@XpuBlCb!i7`8F0)b=vo=vGPM~R$7lZ7x0jrs1~-?df7Z8ZScB*)^*qXazH zp9dK>cX|_3?M~$k<~0aX^A9YeEhVPh58xUg9oHVyp$QAMpjZ_=MaSj_1uK>iU?4WW zl*O+?>5ZI>7V@Z++#f{)zMlQjr=UxP8|lOWk@7CYSDx{T^V(30WldW4GqwIaspJu* z->Bx0Q_Gei&XUlOQ_;gbI(n5(IKqB%{}dtZ(O742j{_Gd%~%EFY1A9ZOlv2778RUW z_gB&jTI)+Z=rI&1vORS(v0014fd|Ut18Nx+6zWrfKLEFw4%Z=OX+La~w~laMP)Sp= zqm8Y^Zh7+>Q!XP7$er{9aE*DqhUe@KN}zEQDWIe++=3vd{wwIv{Y*LzO`**WP{nEt z_o1`0Kx>^w9VceBXj5Urm-Cl2?h&2v_HrT2#b9h=j0Iv1k6SB)9( z*NcZaAfiq(+S}2qmeBYYk8$5}qb#nXH<@8doIr&+YrSIiY96k?0KHPLclZwt5Pxl zd2hVz1=pz=82KF42YjmjaNHVi`OFY20Z{jm6%GkyL(&OPY^&NbZx-ME_^KP*{-xPI zPTM@;Mg7A*3TaY49ElJqI!BFwIlc|NIM3b8H-D-0nyrOvAkrR0be@K&f6jVYSGCu! z{#&1Q@@meJBb;lyc_XjhQy-3@<()&PQ`x6Ordol#mQE0<705}e$H@-2uTw-$fSyR1 zyQqN2A@LXbk*#&3T6 zzjX|lnQe~n2LO# zOeaj~{Obny6R7Z;z>al?5zlN`dXsMLp+?7~Mjb>YIYQas`qHH>it`!$E!T!|VbB1=)dS8=KFh5D!P zCPDKH?oZ{N@+)KPW4mR+nDhO#MqtFYB;uLwd&L9)S4aOJl~+O05$4xVQu&0+S&c7D z|A{|X7r+UE%UDnH=r-F`T3$bykDK6jDE%4YQ}i%ljaP2}xUoNn>xudaJ=6tHglCDr zr61c)4wOg5^E@}w57`&S)QZzMxW?SS;EZ4k>;_TYDL$hpON*T$%i^+9Jcskt_5f8& z@#VPl@yQ)OUKN(%po#~*l#p;A!|7*Hjx3$9ynw)?ChL+#B^};tUxtG6X|~op7<8D` z6@+#5e=n|Gok;z%q4DkeF5KEQJp1ZQY%V6@Z}EY*UMGKM_Dzg@bFzPr5}hKP!X`wT zxUE;sN!g^F#h-vugil;d@ghqkZ_-o75JyWAGevHrT2&R@Ch zi2!|~at}Rti%Xm?J|;fph8jH|vxr*I#oc0wb+^Q6*;i4{L9RkSJ$P(ac?V+)dmU^1 z1RiN0k`%5elaDM}3Xz%c&0}M`Acjk2`bJeX!dFHU*92v|W{ua=6eQF^q_-Sy$rs|5 zV8U+=k{0>j%i>wP%@-C4|A9FD-Xgbp3*bXme1u`b{S$a$g56JZp5JdAba$! zFa(k81eNE0VSn2@cu3+zqkIJ2HN{O;yRwp!h#O3)7xIj8BxyCAp!X$%qfDKPYO|&B zk#8gmJ!EUP)QqQKJ{g+u)0EIYRQNB_b<@>AnGb$uRHeGVq#O=SnS=>-w+W#P~bQf zQN?JLBuo$Cj(T<+qJ*f()$%YxhPIkVBDiTxoa3utVAr>0o;JvfNz0BS45$?L^WKIF zbN`Z~^9Q*llj0kEKtA2KQmVAz&429OGj^Vl-&VE+Gfypmq zc;8YjPWa-wJS58ipil=_05srX`S0Jrcz>{qlO*5}4D)Gz;ke>tR@E&{Y=ljvE~qAz z<5v~QE4cI&CsSK&4*jWAClC(pa1QqYu@hHAtR>VbI3PerU#e}6aK-`pp*#7izovsD zxi_i}EC8YUQwPAkw9|cdHV}7$V0y3?VjDr*?e)8?Ee&}l)@qyWyCmwtqk*bzCcin| zN-Ny`-S&tLSonQa46K7aeuFPOs0ol3t~x1S<(zR-ItR1mjCR&-0J7j93HT8+W=Kd6 z1o!Xk(Id~(asqa{1GCQmMBELG4MKf-oTW988Qy~~knVbmokLRt>4><-?Xp6jz#}*1 z&~4}o8WEYge;>_wWhrC#sqmk8*!|)q^I*bl3FbR7JEKVMJW=a}oM8uR$uRFnqVZwX z+D0HhoCUO`Tou350%_!|Cwv(N!N^I?8qP$0Vbv503|kVO25O*EN}SEZh&hTLSOQ?I z4B_fvZm|BkQwF5>!f;#Szo7bZAZC=alBw7;{W5gVrq3^##}UfnlDXGMko%Zcv8}Ez zI%N5=c__`I7p&35Eoi{&yYft-d8mRX(#rs*Ym@wY723x82T({*0|x&mL=SJVZ8OAA zuPvm0GG3jIaH8T7(QCMes(8$jV-(*e&$II27NzOG;Qg(Ical}~)i?!bc)-$vz`ooT zoFJX{Qj^5JNTaQ}GwG4Nw55%)v?Y-h=BkAQAt`xCW%$fn*T@mt z$bT$Fw;r%SB;a5@%#*W=8A*^3W}^=jG59F4#?N;vkob*Ng(rf)JALYmBkcJ)De$J&gKH2E{syh!^9Qv|xeBkK zymUh#p37IxuTI>X)92eX`hH7{8pU^3Si8NRoY?1$Jn@s)VDmNNy=#DPpJMrkQ2RmQ zw>ilf!$I_~Um6Q~Mh$L3Q%2*HzdV%ecW|p5A2Dk}&f(8?cwXvzErakjK0tV{h;{tW zN_?1bN-@WSrDz*c*D(ae_$)}%b<9p+&05Av_dDcuK0P(0^Y@&^{-w)+jo4mJP zTH#u9P#}_K1(^?dU@Kwo(^cmU#mpSVMF5joL6~`- zey2UcYPLGa3}u7CH_j*`-QI@!Gjb~^op^YIcwTwN`yaq>KKN+_cN(Z5>?t6exz&Gd z2#gPRo?Az1aX(`=~@oGXXm8coX0B(g%mQ?u_Ne{Qpl~UjbEB)4hM`?(S|$K^l|}N$KwH z?knM?LAaE3mvjo!NVn3`T@r${@W1+f-z)h3{&`mQGqY#To;kD6*|X2qVBGt= zvx*sbMQm7JG}X`p&kVMjW?JF2_)LWsy)KSkMS{Z#!D1QilDDyhZo5T%%GcABbbyn? zX)IG1w_SiXBcLl_R}KV(fiXi_cy(iyj&iG}#b0x@Gt%Xj>;|9F=ycrw!m5spBxRG< z{i`?VEk*%bGdjZ6<0Wn{y~1%N->hW9ptA~ivcq2f%c82X+K#t5S}+87~K&98{U+2;kSn0QMJBV zfK7$&AUW9_MS)rX0tkTgLk%Xku`8PvbNdqvMogI-((V&_2H~Z-?(go|hz(VCCvk;u zV(yTtb|i(0q>^3lgU*+8&JC3(ZwZh;wu|lDk3NT%cuI{g-uW{otVj8T#xgl={a{f9 zP#AVMH&8}|#Q?tx9Lqk8n0?KkTcxTIQ_jk#mjFu|^pq5oD){D(XGA#%DF0O=Fs!EB z!h;C&9_6RBfL7^Af_77UZj0>uL)c_){CfsBvX6XDCXx5ZFinxRpLZ!&aH*HxFFIkE z)ggb-2^_s2zxSCqq2QAhxR(aXr+EvP(a16&KF#AgCp*jE&`4tML@l6XY`XEm^zgX& zK}kl|RH=m8rFgJAO6vG5>u~gA%>q)H6Z%a{ROeD%tTF1Ku?lOb>+Y!{{OF4=SD?ds zNip{2J=EhzZMA~<;m{BESldwAysd+RM7@9k0Qdmlz+QpcLG=$%4-h7o)+{@aS5(Bm zcGUhtP=oIXk${FIGg$X*GCJC2_D@U&jbzMU;!r`AF810htR=BDV8bS`^eBqq*WvGn zCDweb$b*& z5AB40AGInJC7w{xr`#bG70=X-MwF?klJHGITZ>ta^ApghUr{@sAh*ig!$f&?&_fcG z#_Z7JM5@baHw4^1u4WV=?`Ak2AQX8+AbDLEG1h-ON2s{&%H?O!V~=MTwid}rd#s9U zh`q@^bd1M&rXJUZ-!o%=CxyElwWiDIN3x8y%`Sbi$k@ifV5o88k0+|^WY=6*LBOEU zS3go8VGB%p0;e-wgX)`GEEu!O$mi}IVCUjv<8Ezi6Yyiu&cVjr*jYr-Rr`hCH7MZK z7_R&Kx7yiI?>S0WjwJ?M^cG!`f0*?B+F1 z;)+d^45p`!O-6#go&-So1*S-iRz5XoD+EORv=k>*jwAJWkDqOcS-r1F0OtnF^f~O$( z>kaRZZriUFYL;K?o#Bv8g{u4ZCa0TK&@aRJGEwNKpFNZ3jV@uGsUnTtdb?Sj_ZE## zel%$KCjAFOKWq2MlMzb{MC~2Dn#dp2ZIue^=DlXz|sJ-LMtGBlfs zdc1m`Y@z9P$4YK9Trv?2SY5s=lR9xlYscJ8{*bqFD_H4NeZqp*KFtxVy|lNmLBkx7 zw+d{$%!pXFy@a_dURAoR4PwKXr5`cx9TA&$i4MxXj%18ls|>P|?mWSNr!oFzHltFj zqRIN;s@5}CjcE%KiUX%mz2XB>JGa!Q^Y>9HUCYOwXr3pjpi&7x%?0;oywA~i z7EHefkM>4t8b2i&NNb1yT6yVXb42J*XPN=;Oo9p$IWi=Ae3qCOsZ&1bs>ozT8(GXa z5bn2NVYH774RjB5ZrKPeO-}W;67S13lj;wtPKvb5C-Ua3NSM5Dmz$HeF(Np zBX&R$AG*shj{VLFk3!L{HKftfUf%eclox!a2!Y)H=l#KK$FuD|2 zdj*RWm$}kdsqCUmtd*}iy3ZyY0-ouGGoz&dgwfXCjfO*9cWPA2uf0qyox9l<L4UN9-^wI!70N{?cOVMO2M5`<_)?)qNx+M78MUqmi@3p4m zjjpPp4ZhFZql@7jeNG|fg-MZ{e=C5BkC*!3Ofksptn9P7tGi zqi#?YN6^&43tgAya|`8DbFKr8My}p8_nq(dUhMjOR$f;&Bk0pqU0gksLDLuf;pBt4 znqtNn%qJ(MM5Qn;KHF7AFpo#!CS3P5g<7}_FCfdnf^FVwkG`JmZPKf%OMpHl4^IAf zBvs79jBc-Z)sR%bbZD+Yy*GS7jQp^w)(9M)^mJ>9wP1?#&LeKRV^SQ?rS7Ef_7Im? zFFstQ)bH`dt56$4cJ|-l;g*h$)6+@AvzkK@T-;`bpGr6if8=Y$rp{d=z7{|z^>Sip zjQk{bi~4lx9%Oli`s%L2e<9kLR2sqb(5&WW_v(p1O4|kJ9ik6sXzZ~B5aH@wA9Qtk zK}#4Ndup7(HP#c~ov#~o7;i=7?U@=Y+-~jgU}AqPz2IvFI~fzJhC*DLQ+u2bdB0xG%w1*pM3 z3$CP0ZH{cna1+`r7bw_S-}N)DJ{07-HXsY-_~Tl~ zkkoQ7*b#k{Z*(%5qd&_3Lb_yGk$Pi#(5&Ws-qzpuTYY=|jN;w(O{^$XOPHhoPEURi z6U83KZnyhOmXwoDnrO$XY$FXtYSoW9z!iHqSL7{!tFC+k6@E6EYQ`P~3biM)US(KA zUPV|eT8V2(3mb(5wjv9?A5~izdTy|ezYv+>Ne%?xOBFd%1xh z?V2|4&zkUgSb}kD7W57gg&Pvap?Bh?rKG&Md*C%vH+Q-%bVWx;#B1Nd`Rsb{Q6R?; zD-T=!#^m}R32edI&aCyhbD(sBXa@#sGg#2}6G1Gw*?B2B4EwJ-0_9(GW*vGb8u9HT zar(xe)~3ofj^NJYG{4%#ku@nlE;$le*sm8Zh0NPkP;sO2YX? zyJ&^_B}err4baTkhoPc~teb1sbrJZ*lSrl(XO4BzZIE9_Yk6`VUB$guGB76oCryll zZ44b)G*M(w+d)CxdmcJk@)KKa(n}5U?qoLpAG_EWoFvA(jO)ICzVd0gQRH0cDleM0+7DU@S;8wDHy?wT?>&hL?Uo+#n9#>7GsJmB`yL>hYP!X$f8jh06*kiFq+?Xf{0bBdQB^we`|cv~}N z6K%!$>4t+ezq=m?cPWSc>&Z)AsTyP2ic*0ZjXuHsDgtPs` zxRHLxlh_KcH)>1iRM7h!2kz4;-xvkg>*PouiawO|sm`&=w-_*+bUI?BaMEKdBA<@D zWkk*Wmv*FItFi((UcZPzBLsGFsOUXu3r>?+X%wF<|JKdJn-Y8@?FAyI=n@zIc}tbR zBXuqnD(zymGSi6QzQBsteltMslNYKygFYc~R?@IVszaE|6WS2xUnfd5nzIhbfS!nh zNyqmJ*;i|Z?OhD;M7Ny3<#!Y9I-b~QBK7GN4#23-8t31**wDA!6$W z5?t*_zYRxl<`=!@1UYkmroo_)6x`JIaz-E7s7^$^9x$ZP-ru$itApi_Q6XVqHO8)u zz?!WSq0GY|`hjp~d<9pzq8YU%t7lRt;)ad4!Df7h>G^J6+zn?=+Q+^nBKeq$Y04>> zMS7R6zAnc3Y|T=y)0XMU8;WJu84h_4l_!dwo{97Ieo%%*=TZi~r-px70#DFCKePnq z??R#aAc64`ATQz>|1+N{p(B8^deDK1n&G}UkmY`P+!JqY)PBM$&Ts&XSUjiahj?6) zR}0t5tRyyHdbQC0xqlmw;r+2p z!zAbIbo*kd<=sWv`pB?u>-Q`IL-;DPd-T<6Omd9%5w5*s2}Gqq#INl>;UO_YN6(XWYXSNW}L9GZ^G}$Jw&C_B&)rs5b!duT5|0@IEzh4)wNRR3Y^C_i_VOMpz_&hGN8U0|aOR0}l(pht zNFyiT`$kN3NJdIXL9|Drgoak=0|H6YNn9(I7EO(&r)KTF)Gf@+5}-7eZH!~ru~`x* z+>%QTnPz@nA-+qFA;$C6-RiP9w5d7borsT>U*Dnd`fgkR()(VYSL?g0z|NdZ9SMvg zcty4psTWUK^;zV!tkVo?&h#t?7a2}TSX8CYNavQVq%r3=SPU5vM!S zkcqAxocG8cDbe003L`Py(^pNY$kHBYjG$xc&D@d_Z7A!g6n=rXPl1zmgJ5`LVWsf* zB~L3THryQBlxq&Y!B9O>&Mx8}vL)1(6jaxILWOk${7#jRv^)@wQ;pM@6i1=To~+Kc zJ+#k-8hP*vA?m;o9 zxCndUG5^eS^(rQh_s&4w+XMCV`E)X6BJ;|zrjL??Heeej&synkA6fD_4U4NvGMH6l zElWsyX@-d3SowUd0KGXGA0A97wW#g;sDtApzN530}u14lx#J z)Tbc4sZrHkyMojbBQ!-PapYA&aL?#Uz>*ZRE~*TG&-?bbUIben7vX)$a+$O zC{{xA8TOf`*y@X!KE#k9BU9c7d`c~)RW0QyU+H^tg8VeCb@;4K&QYt%4hqr~zzK_QL+tooAAiNZYXlAPrGlF;~Y{2Ga%21eQOJia`g zJO7Ta{NGxBI5Ld{=toVmvrdbU51CFK;mh+(32~;Yt3fWLx`L$Wst)34yRP}S#VMW| zhfJLJUb;yyfknM%LFI~UK1gY%!Gfe|VarYR=t^n5M0bi;pM}4q87)A@2y5fJ93X zu%o$t8)_iwmKLI`ejgg;^>&Nv#O3lxQz%yImQcSu&> z9vpX+6mX+MUQ4K%F|!T1Ac}5AOM5{{ng)M@kRb|oetdNz(QN@B8q>hGOE~~%WyxkB zMYGCE*J7?&C6w6`BqQVur+~z@20h9UuUSnU@Q%-%d(5;zOG@K}J9f&vZa8mhN!RqK zKD@R5d`b9xsTDV6M`(I3ZU5T{npoB1(pRe^dmy%N8hft$bFcQEbCJ#^&jhS?kMM)o zHEp5Llyy&nq=^sFEpyLBp&Cu$Je1u$<0l=O2ZcX1#a8&T8^rpjbOxk%LkZjis7OD- zEyNP+%)g>`fy&i1f$fu^-F8YJeO|^F5U=HIGC4sZJjeaL+tXmWO$;AsA5)$oc0d3^)K#Vd5GP@o*lA#H*Ppy1`yl)SAvGKC6 zsMcaXHh(d}pstGRZ}7HjhvFxn)98&^N15ekPWm`FtDi>yGv5uMkY5u|zfW3#r_Ke; zn{6?y%daI3$Lz&0%_*CMBT{L;?_=5 zzd;i43-GT;qw80)eVXU^*B0{&@848;PLH6p=;du#;)z&^%w z4}$&HmKX;@KXC6~_3q_(39%8kk-^JF5xjiLA~rm zf+`6bfdZVh?Ex)O*SHz|=u3`Uq-FgfPs6XtDthvoSwG3{(A~3SWpL;}C(vo9yPjwI zU!5;m9=#L2zx##(ILf11#TLOo2@i>sIo*s2QC~=`0{N!jCS)DT_pdxf%AygBatGR> zn&RmCU727S${kbjh9PvPbZDl}ZVknRRW{vKgoRR*bE4*R>(^=$-p1`b3$C(BPy
    oM>#dGzD2Oj-5j|{uwCro;$7hx zS=hn>d8%~iSgxX$hA1gixVKeTQ!(q{4)prwcfR|~N3rVS27bfF0Uv!bHzUeN8@1qm zug0J?QGmdJ7xDtr0e{MuxKagJ)~;x<(O}jRE(@yu;$hwrO+-|if^`xdR`PbFoCTx8_%xBRmp!?W=SdagX8RgdCrBUlO6z%3p z2wy*)lHW94FwQ9wY``|6Hnf1QuhY6mmY6+Mf_@@6C!sXpg)`F2GHQ<}f#jt2w6Hq*A!_K$Y>r1 zl86kOU{~Df((2+d%!P~k5^4l})O51>^<;C`^*apjzIRQ7#EHJ+OrUF0xNGnzw6nt+ z4KJ{<_wxpeEJJ26T}{8e97^>i%1b^(VZpox)uB`Qw6>%vT!)W0gYAD=bm+EGP!DIL z&>D?Ld3xVrxCiIK+-7SJvbE|hQ7{klcR72|5A)l#0!q1*_|TK$ImgRWuH8n~fw2uICfx;c$kq ztHtQS!3Bk38?@ERr`Iv@fl&ggUb!(v0_f@=%JNF{3iC$aI>>5*>_HMuE~9S`yu>@J zCK4;W1T)Uw0DEuHC6VVDf|1q<$#C*4E^S*6puGJ|JV+lb66P@3Yux19 zS@~hp$I~$GHfsxhMR zo^2E=@=nz&OxP#0BOnAA65KD+n^9~TEv1*kEa>$dAM0vZ=`Xd-wp?OSO{9lGSw!C-j##L+@1q%mA)tw1}M%t3AjAg3xo5q2@-dXY zTY1W5cv0iD<4lAY;hAiWdZWYa4am}21UsVBa>%m(fD9pNKVfmJdX*?}pOX=r6Px;A+5C!{3tX|gHf&x^Br_v$KF)Vx+5>0nY({K;kzQOFA`vQQM8KElc2 z=j}h(atJ`xqn8Z|{BI7ix$o4@#p;vWE;@E?F3gs+vaLQg)7ij9p5c?^qY>wNW+Mj1 zk~(XzaSK>Cy-gPm1s+EuLeZQZtc!WPQOQlje9gIlU?^^*Vs@LDPPeOC-ry+Yz~<_-5IIm<|K@eon=jgw1o3e zkyn&#O-4?n0v4Tx2jfQu6%X5ds&<*NQGFAm%sWeEMaL!+0Xfv9)poxPvkXP5QjC55 ztP@P93~TJMeE;#-*4sKtc zLJ|Dg8!A0vaS#vD^K>`>;3+t#lDU?tAotcg~h3i?%_BQjCN)6**ub9TPnv1bHxnSGS^X#50(Q(O|7jU~(is#e*(&s`I=@kNo<2z-fe^kS;eD4q zNUN=C%t)<8e1JpsRfXS@4LMNezJSlE|EbRDUe_E!+h>m%>kw&XBblSjkU`fAF~g?l zh#4WQHbmhpDs6ZDT}@If*m$Qjo-xN+G(%E>&lQ4$^mea4gf`EjtRj3KTTCw*Hkzi1 zj2PXv?ORiujd3FJK+rz?fm@2ZE;%Qc>W{HvmN&vwc?gxUm|l4;M?y77$VDg5`9LUeLm4zhmE?+G71E2uedNis|x zOKAJ(rqo17mLP#GE?Z_Y@(Bl2NclvnGu}__ zZnuYTlghq40iC2Z_i%g#-qTyLk?ub0%G)UzcN#g1?Bo*WiQHPl7k&w&$D34UTw@o{ z%@xM37X+kyV;V9I^=*SGUq;CT@FCiHXEpcSEqIwmD!{17D=_5*zwiWI%Fg#h({`~x zvon=r6l?q2L6gQ9ChvTfQHx1(7y**-cpI~j0ucFF1@)BE?22y-cnSO651UTpje2G{ zV!3;S2a>ft_A9IbvIWSUw@Ka1DS5{{u3PwX)Fe9Am`AwdxMI}CxY!BGu;eux$L!tE zZm2+y>U`!d{T%JU$8uBNMz%#LJ-Whc6}g*bTlmIMx~a;Z5Kw zXcXaLKBnyUQHc`~gOAoWEFT6QdaYpGcWXk15&%*OfC2zknxJ~Z!Z1^-u!f~49DR#`-dxQe>wi( zGym0!$6vGgg9ZS|!Bw63kWbrCFg$3ARgjrQ9t1T7Ier<~f07nFBuYQ?L+s;kKm0)g z073{5?5{6DkUhjdj`{x(pMeCylX3kgK;z$i^*FNQL)sp}FPhl@9p~}yV*bQYgiO^B zO99LiLW;oy^CMC|`J41l+$+dsWqm{nr}#}e?12X_cM&~o=072^2mt`*M;s37|8htl z;aKr%|FGH@<6bp`2Tm8 zgULP-LRuUk^IKlUPgoDjeC&?)r$hh%OCF+uq7Si;9n&6SNfiF;rUw77PJjH?{vk

    hI`0){3h*T9WZ+T<@|cD_Ne3k ze)#lt#~`|ufl0UXUAjb~Fk>wk@D@L}Jt-p(`!bBq#!$Bl{q)gynN zQ2_w|OkBUKXR?Bj#2&V6hX(DtG{jp_9?~P6e?)}*8?^%)$C84p1|F#&1FQ@x5d?1u z!9(`;K1gN`zwxM%82=K8+Gju~9bIffV_pYwi+jqvx7CiVF*hwvd@ zd;))K4^oF8WXctKC^P`m|2KYSjOro&kM#@yNIk-<1^&kO53@YPixODf_d#R{g-p!C zkMQ4vf8+56A1SV?7!p}AME(ni;y#uKH|jTD*_`xWUEk1A<(dK^l|o35E5VQbO?o|u z045zHc_`RF<`{&-`$#aP`2TVq=|sAn-dGVtCki2*#rFtLkoZp>i+BGJwXN{~MP$XZ3POPN#7BryaXqSWBKH@JO1YcJOk9(7y;^fZ)4dgj+}x_#Y9vJHZs=&mUfq z$Lp(y;G}NIYKunaKkJ?Ut*jodm>{>V2Wh(-05Ea4wliaOF*h~0c5-8La&dGrcX6{e YcU6{$gDBOn3%~_1!~y`sKK;u5KYYNIGXMYp delta 27617 zcmZ6yQuX*;tL)zzmT?m7J!W9)eQ##(d5jNMTK z8eRg5pdDSgY7}SaFdt@2UYsvovM6h0I8LsawDy3-ypAL zer}9Mdcjv}ll5VUH^i>z=KwUh%63fVTEb^ia)np(4P+H>5|X=C!<*`MsMPfXnpC*8 zD1c{oKt*drga!O>V zDql78?}}Eg>WeFbEN8jeMMDc=fb+<6-4dxzah`0t0UAd8JkqC}0-Ux^N(j-oxlWxi zh@Bp3ju65Ja+i#9+{pm(&z?fA=`V$?F>Vz@rwh2#iYOP8TD<_R_(3>_*eQNWTwH78 z<+k79(4_QR6uQ4=f$iNB0kE2rTffVcN6<4n6)-ke82>klDRlG)|CP6v48lB+{|X%5 zk^c55I1tdEe;H)`PvBs&NdU4=sOETo?XxMYyX}dEh?VSs!djJrsW6s<(>F}4bPD0i zz|lJ{fPuFKi%)o^GaAq>9ruouq7qOLwJsZ9)?D-8S}9i2}bria?<

    Q11vU%I#u1|rr%4)iLc*ET zGy&@-ED690$V6Eyji=<9cPCT5IrC9;cO&~G4^=r!Vl6JDG8}g(|~YSe`jTl-mB2c?#zfZ@YO0w)0%2^Y051#b7$vpp#JE!8Qo$udo~fq zRpn*Vx1Tm!5h;>oPR+GJu2gh}LfI;Iz{|!)b)mnlv$FM}GX>ye6`NOui%0|wd^peEJf;#`%Y6xPmCP}}d80joIw(7FxdOwEKo8&8 zmS@)`^N;YF=qz9x2RGDzvE~GBltE|Z&PIMb!{X)Me|6M-X;6J=OWE8+N&ehIm|uHE zO}BvP-FFQyTvxU|CxJ^Y$Yhug%2mp`j)aw#(Ri0d0!ynwyOpJkWvyYy-8ZZ`mg&`Wk~ z-b-S8fnjbq|>)=XT)aue@5f)@>i z=ji-Ql3r~VzG`Qtw`gY)2bZN`R^LT>n=5X|$|CmWT-|pg5-^_NY2F+0_Wdq27BTVp zkQAGcWC_U6pbiTy|IBtq7(U8RR>DAMZ=XOs`2QphO3 z%BG#EpqgMpX(8>m=SWr$_q(bFnTfHNFU>u>2?C(kr4e*`3oQ>{BcE3H7S_;QUBwZg zCasF6)RQP3K(2|UXL=gI>hkq znEVXs6`cHx>E-?XEt+5Hf(X*;7n1+l5)8>KYM78bCNrmzUjh0|$6aAD$zv8O>aCT znO-=jdzcv`cunPLyA6FDD5dlbd0#><+jiSNbZI)v>drQH<#wqgHVre;MJL}Y;s0I; zGw=oyN&gVz10tn}5Ibdh3a7z9 zOqVFh$b>))e3l8O9o% zoACTg_2S7{G#j&J;d9$}=05+C=Q@AA|M%+yGw}Oedzc}%(o0knRe~*dq%xY5{hj^ z1ja#Hj6E!A;ViOTh%6nj`^)XGAKI(nt{kYagA+v9fkIx1fZei(3UZoHBEA%vS-#`Xe1jJqO8dkZY#U}(sklyzG9|kSzB0E+cXhs zj%D<)o#iV|-BE1n-K_&Iz(nmt}MOGERNBEqxZpU?L0#M0o zeb_bjBoYDATUrkvTrUN9mUbi4F#OO2P%5}pXCgdKBCz0$PgwmfH9p>1bcBju5UlMN zKJSh<2*9N2UvXA-=^t9B>0f-H?_Yl44^VQJ9Zz_d7IwQq%nDk09RTS=kww64*}1@D zz|_j!s!s%!q-ZeOj;?%UmxTwgn4XofV#kZ`_DlJgeBsY~h4@uU^X5F=1GLG@!V@?LxzmyALt9ni=wNhfuv{F{ zEs+TQ(|ktlY--L}`dgCoTYAfI;!lb^8(IDwTMtV|)_@z6i-+K8NLb@D94kUL zI?koKzgFP&403M}VB2ej#kKJrB_nH0$@{`{%48>h+;GLtmrz03_`K5)r||h7)gC+h{w#pu1p`PVQhoV$SJI zK8=XbsEV%;UPD3pN%C>kL~W3HZ;SgVOJ8N`^3?odoml7gKi`z+{n=2H{Ld{xzP%Q8 zMY%$STa`$JGevkW5Xw7IagoL-E+Zm$7-Zg%;|V~P=3kI}tkK1DbDw_ekyNi=03Ij= z+Z~*DcE^NwMrAt{XCb++gLoi6j8_z<1@g$Eb|on3-=3DQDEJ8q%>(-(toR2} zN}XXCmtz!z6%-qs;Ey=>2rQFC-qA(5hK}@(-qw_RWdIFnuv$w#c|E2ENeQl=L)-9w zA^-QNEc0gS)@VtR3jJ?lo^!d1Mu!9fazFqAqWDM5m?`q|=m1k(P2``oHFWP3u4vbd zwh$A_IJ!*?X;ms|)caMjx`sbA3R|7Fwxo_UcilZTBjGNnRB!%ZAgP}am^#!Wt-t}l z6F~Ym^>y26mR}+JIPa|YW;j{*b2|S1%ztwN=Z*~=aYYXiHxKIr~T#Im~{t zr;g+Id073p<8Sx=znm$<2}kZlxxFc)dteY^wPJLfCjd@*+gemz;pQ|-uH{wRA>y;n zZj!%}WK+IHNr#_URfkiaGR7q%&ML|&T}?C4$Rr%BINR)$Sz$M+ahPDw(68}Z<;GX# zJT++3-M~L7D8>u$2y*v$+8T^(KZ*s8&tJ`Lwb}ew!_)kSs4m%%y?2=4--?6B)ZS`? z7YfrVwg5lK!z<%5)3DwAb@XWN4~z$v?AwX%jo2<3aK*QHP1ho8o@ssNwzRa}d3g&v zWdRaxa`HLx2&Ep~7>=K?-94;?PgOqc`45&7zPnVZcJglXo&$C3sE*Q|zwMlR(81kj zSKKR!+6%e=WKu-Mo>;G8=GaY}V6_iKPZ8>YF9Fa^#eaES)~)(tIS0yxXcVB*3!ei;XrkV*)Hk}Ke`sBWXOU&uc$;+C#L1`{^ z9Rn~pBzdXkjA?3lfIH*h8-?{=d*0^9(R_-kC=ui~%e)pdbI=}DypN6sr+buHFUF$69_@9gd9OPEf)liqXr>7u zu7Q0oi#CjUw(TuGnCY?X+<2Eq8&GEsssLzrrzpIV$*${(Gxs}*8oHId=SA-yBlfr;1174rm|^o=E`c82%WcE-b_2V^B~V|q@vAOKZI z@V(M5V0j6cEh84|8JyQt1eNa*Xy+h5(he}JbL7%}rH5p)r?BPjF-FOb87ANrdx!Gm z-D;ln?1*cvoxllB^i|zx4eI&#R`|%876!t1yxj-D1Eu<1Yidr!JvQBr_@XU8;3mV5 z%n-1IqJZ80-j?z8r^deRnr+pV5I}Mc73n&CPN?dA5Koc;e$|Q7x4^@~Uu%ni<4Ny? zQxa3YK3@sXn{eg}mH<(>vhFf+2^m`N*|ILD%gaD9oJq(%lB(PRSE{rcSPuO=^U7N} z#?3W)Z`(dFjtADDgQq{2@9xanki1XbjVb2arbU-8 zBFzLR4X%h~bgBlQYQiI|GdMS*k-L@vnd*j6U^o{HP|=_u%1Y{PXXeDY-uUq9H2+X= zGV2qr*TUr%w~ciKx+tAfog~z#%isFls87So0Qr+R26gd5r z?qMtO5c8-~cN3vCo^d+lVu-qhrby3q$R5lM2ZM8h-Y?mSCJcB!cL1uQip_8B6vZu5 zxmltY$;@D;o9j`kGAQO8aKIwaQCDtHkMD>tf%b@)pK1#|@6#KXP1*()afwg5_iE)Vd z*|5kHesr=N#Z+fMe}EgPzH1a}?cyY4{>d?&$60*siz}WGDl=OOPad5I<~05L97{jc z7IPGJzLkkZIkPC8eyZg|ET;TE#vcK{4bgofLvY|F+ZBxBHZv5&4ChOcPyJBVtE{nC z7M0arvR*vN3hOtHqoGVn7gM&X!3<+{Zb=4f1Jj z_l9r;{}n%e(b=?=CO?6NnUR{YL@7(T#aIf_kB$0PdvCij!Tho$Rz#t%3Ya-nm5amtpu@B0QmfO2E6k0Q|crt?`JoHb1nfzyH7aDA!*5E-oq% z&^0v>5Yd08PV(%C09Ze?lQu!hp3P0VVOpf2DO)iT?+8vDBzWn>4Wz_-CZbC-&*a!^ z(%ZRYu{xa?nbm$J-9?Ss1QU97W$6W~4IBUV*P^zzwp9&V$E%yZzPH!C?N6FFfQ1KV zjv3MsSpIMs z)D34I{vMl8(MP6~TlMe*ID7aQRBy^0xcb=Tqc*WzCA->LZPnP55XL}EVE85SEi~( zhB^E6u6t&q_6r&+cyw(kSeKTR4V*iES+aw=59?JvTmz~*c?j4eo1a}$1NVY;RDUN4 z+csNj2dX=}0k%1eLEl3z)HvK8%6R`!`RJ6(rU`oi0JS;y3>5*{pLqyIeA~TuiT=K! zJ7C}tubKYtRn136rl@wJ6T)vK%YNs&J*6#`rpkp|P;f{cOOB!r^Q7KEEp*7jJu}c= zpo})aKB#%O`RLWz6QCF9(mAxv42F-hz2x9@>~1Zpa$g=RU(MlUuXF5j6WuLulM9>8 zxaxTZWJeNQ)mpcqSk$_LFTelsqO-nWwv8c>7JjtN@=NlvtVlA-Uc^&$TXUm$b{eU? zkUrntl_)OXBp8taaP6VSjJz-{+jT_A!(TWbF4gJkyHA!P+b2MsbUDCs{GR(&yd1ly zM3%*V*QdOMUGXUJI0X&S&9&f9B!8dXQp)2 zt-S_9H;)o;%h;nFrQ3YWly(ZQBIg08DXNnsm3L0fW2&@J8S5I=!dIrBlafRZ74MN5 zFkHRa+XSr|?NZU|aS@ZwU@I@4MIkBn5}qp@)*$NnG~qzeMvVb_7l}|Z#3ZX=^Xy#` zDTE{rX3d1EjU6FZxYcaO+H-bGG!)F`}_nDBnK;B*kpY)*EC@k zHjhN(#)_@TQGicG)+VjALrt3jFi6im8;wIbaeOwUlcZ_!6t%U9wrbZ$P?dL^uc7y# zRB+DSI!O7?u-rvS7$!a3r((HM8;cfG7g5bdu)?2SHV<@@z=QgVsUgy%)Hq`cc*zgQ zg4-Ub&SO0VE{_*02}Dz?w2&2Rr+rrC9N0}a$_v{nTJ3PrVkvdqw6&1{93cccuBLsC|uYiSFV4;+Gdw6^D>xzUe4m{KiD+2W$L z7%;SNvn<0??w`|ZT-YKpp$k(@s8e-8EUg#?xogs{vf*neMe6JrI@{qGjA^5o`9nud zr>nhtqoR80&>85a9-a=ZW>WNoE$z*ns|sfbN2|}hHd*CMPtU;snSmltbso8In(y3z zq9r2!k|pHK(cK`Rdjj2c-+2Keait2EY9ADPCHH5) zk1f(14|Ual`1Yf3K1O6_L2ISY{e@2$A2SNY>yx3**V^3GD^<~cnx}j}IKVt^Lvx+L zk$r>@x7GVY28tg6njZ>jZx8wt47cVXL!LEX>_6R8Z}$OdcPEZgt_4$XF(`TJKTpmzK63m3LU6B%Q~O8Xa=Z#uV%eT3 zbVoYRDbj_H!(aAkx5Sy}L;%SXNh9>zW?FhyvkHCsy`rwkg`#2M!(~|$@+OYK6?X-- zyXEd@{qQvhEauszlHF!YTmJG@b$yf;*=I4emTouLxT0B4`oS~X-6$>$9e*6UUCLg( zM)k5^8hHeOh^_3Ey6mh_XOv=6Jk)gZ?EFBhi%^Kzjq9c9wsWl0&De_SR1$cG( zXzQ-g^6=(r4O;2+5tc0FlxU^!+}qkw(Tm6WH4n52D|e4L4jZ8pN$|LYD!)iV&3EXu z0<>|$(}3$W(X1ARq=ip1|7t!VLDdf^!LrBYM-C`J!D3GEZ=oJ+S;(TnGi)DD9&B-YDQrx{i@pJ}OS-lU-USC<>AzS@;|fzR#KD zi%ssUFPS;ZE|V1b?ri|0GQbo)%PosRn?W@+**6KIXs`42%j&Vqfa~ zR6Y~`6*=rs|0ydx*8EZ!TPm4iO9Q|YJ~@H`cv&TROurld*!wuh%;BR5_R>0dE0zf^ zO2YirIN5)Ap&6L>BhCGS2w0}E!hdXIbttyk3f6N6$Yl2bX zIZxp!0933uCwPM|NGVS6S}M}J32NI?tjF9gv~I-&7UM(?p_*=nMK$`0OxOawH>bk@ zy9J$(*bZ%~8gqSQPZt-*a(P#qBN801LrO5%f)=YQqFhl6#_M=>a24$>bX2$1FdPC8(Y)>(SVas>HV z*S^Ef8h$qm{?f|UDYScjTup8|`W|=yD8{B1JSNN3wS@eFK9efGrh13vZw)@u@z6tb zfmD{1xJ3t8NioBSy!0E_8LG=|$)0ib0us*2-{maU2GFQJ2hGZ36W!aEdO>Kpc*5kv z3JR-p##>Q8-LGbR$<1(46^A)ygfyVlIBRO;!-A&(jjSmAq*#*3Crf4WeLzaMpRQk2FjzxnPACA36e2YE0+UY>i77 zXrnXPLIGo4mi0tYPr?*Q_o=bc9E^5@Pcv7U`^lXqFhwtyG*VAJBl8Fo1vvCGvzkNA z*yY{%x8l{*qM~Q{q~|~6csVzKtHLx>k<&_9^2v-%Uff9n+@&&_<*W*`WOVFYZeX%1 zU4|--4vW8MNvw@SbHK9?1Y`H3i7WnT1(CYSMFXS@`C?ciQewEEt=<5PJ(c z(pE!MRu+`=OO7VQCAPrvq=&eWo7EL#tHDoVZn)IDcbcE*i5r#z7=?@dcyjvXe1*w{ zWY0M^);V!nj;7eB3jneh-jqkmu;x`rw=&_iK7FYDd)u`yK2=`1K(+QHK- zu{&Nl2oiGr_$XaeBonvP7EM}ph+M~5k+cEhY#$8WuthMGf3$*zcZbfA;GaS*exQ}d znJt*F2vfK(p<8_}c+!j8p?V1Ynr{4KfK?U9P+0a1-<|LUD0W+`lL4Jb0WG9m zpdfNZDU=5W2=r9u`?v<~C9k%QLO^UgCj27t*6K<}SUfI#1DiBQt?<5rs1mGO zZfOj3vaj_?x}2tktSn#CC9gDe!lZ{F!f>G1E8Qs8Y;E^mZ0c;IAE<7d#e9yYY5Pc6 zM)|d@a)+$Lf!5+`U!eydzP`Z4WkN&39I3?zTU-zUz_J5+bZj>rj1WkJAPNw$$d9Y+ zoziGwuCXBHH$TJkM*TY5bX44SPOe!Y!uw72g33p}b_rq4-RPGN_O|(H#Z1sfuP&M4 z%|HDZg9QJUq@kWVP`QTr{IVSFU#bD&uednHx7E!{C>7A*GQrtmAKW2aw|$tyTbbG) z-1tKS0P7E#3)9L1Vg%p1>j-h9Q6KDMB^rD`X2% zRc4>K)-m}759)RZWS=Ux7!96dgU-Bv+OG3AMh#C5rh`if@sIAO^_9kNbZIEeHoQV> zgl&5-{Igzz9gvN*6PA7E>pz5*cx=Rpr>^pah~t9{s*-3a+J zK!ERJV^XF(1<5c z@BXo@-!{C{D-q6sBID5Iy78xXlmWyzKU<6eFxvM)dZFe8yH{7b#}eXs#C@|_+WLj$ zIj`No_eS_@g!ViDsxtblVgdJA6?aP;AD=AAtslBm+y z)jJd)VZ#^ep0D0Q(*&Js;4oa{vPN=X8`#?%WTg9uw{w%E^=l^P!T-U1Ul}BS>98+>^K1aJ&qt;g73y{ zB>b-Z${MKWg__U#ONKLUgJgf`e47p02eRmc?_3L-e!l1}k$E4oY(Hd;kV@e%kPJY` z;hWUTKa}T+Q22sW4GAzuQRKa0vtP15)Xv*D`-4zJK6*F1<;>`xxX1Mh2(BJc>~{OPFl+CXLggL<;FPjm?|4+1LIMwTsH1J)fup?dA_>zHlwv_mvfk^5_Q zwp<&T^7TAlcpJG9$xXlqNPC2_v-&Ag8L>poHD+DOrG93#4u|=+-J*aIOPyuw=?Hu` zyx3nRRtc$?naUXXt)Syh!ByJyE?9VVKH9IU`^AYhp{$*qe^;X(NV5=+S{-_m_AjtW$0C+2UW& z|8-ZpgK|+L|0I4I^nWzJA}~HccUuFAf2bL}8fDOq8O#X;EoW3gRR*_^4LYn5E7>TW zr!=OKQ9>-ek`DDeox%DO_s>>FKL=xYgP;8VC*z+Lz@kUJSqSIjDxzmr7O(5vOrPoZ z$GjsWaBmQy`Ma~G1fvC6FQc)B1nsHW&Twz+KaF549S+HiJ*{;({OtqaJ4lXr(Mm(g zU?M9)wnbSpw=+^5f_#y3;!JpN9E})%c~=^feAgPHd{O#v(UrhVMo|IrQ_U{gTn6DQ zx(^3(9+P7Rj$@uRqV1aDXjxma&m;TE%u;)SUtN0dCqlc?r>7;tW36*zv#+Ag6ED!Y z{1Ip=_LN_q+iKO;U?&AQ)TGbWN(1lF8N!?++K;}jfU;NDNbulv(9cRKcB9P3&RO&1 zxZPK3WRtY6Fkuk&5TDJ$8F&5LzY_|n(O07vDPV)o!QW!-&Ym5UV;_~ma#F;aiLY-M zlTrI}3RmJK0iGpq!be7&eVLxwW2cVDBZt7)UesQS@8)T?V^jp}ECw)p*QGgBgqL@n zZ4Q%dcd#!{$Q+byEXJzI&0H`hOTq{Y zpNX4=u`PtHx7o)Z!Gaqe(Spz4vp={DCtwUGNQ4uhddm)R!t{@Ns$D+;)o>55v0T?0 zfmm41K)?Rm%f!5|@ zM&4njXQ?LAB28e2$``M{@B>{x|GS3x^CWaYA2$ZLYK>W8>mPA=rl!63^$h?m|J)9lJ6bwq0{AR|X7(tx? zW=>cKDK?xuUJB(rcrm278Vjuemw({Prt||Y5JCYAvC4g>&tf9zAXJP)qZT^lwcZUe z)tdm|cAsY#LIR!~gl95R7Q7}F$DoyM%i$5>+x<#*zG!EQg-FUmvCG_UL#>Uv2yv?8E zTcbL*e*Shv6ofJALwP)=57MP`vDn39mD|pQ3o zP*?6R;o@W8-r(S0;NxxZ^Xzo^T6sBDT6&qu30K~+*f*>tt*D>^@Rg_KN+~#~wU+El zm(tUlJ**~U>y6Cw1MBCk5+p@_PL<9kIOb2IduPwv=XP=D_-sAhS`B|a>i3EEl#qrD z^M|4E^MW4*FPAgdaK9t(lBxY0C z2ZIghQ#h&52iLLvm9|d$B`o%onL|FG7=^Nb^SxtUc8L1mKzZx*vK;OTJF@mUMpa>P zKVdEPrG+DaXug=e4qA4wE>V&R^5vGA0_yy8d1lGnBZXkVuS&)9bWFtHUunH`Y70wW zA}VVQ+li<>vN??W-Q;K~E41_^S#;g&=2#dv%jKW!4^ph)8)9QK_!cxw^Hb zo&A#2WM`i@T`|e6HIrpK*p43#^SCO@**x2Q(1yU809S3$yHtVsff7`%c z))^krYk&Tx$zb+U*B5pL?h|k8HTxHd)?|(_R&{I1PwX|ANmahH2MqJ*J9%-XG^42c z)Hk97TDv*IFhe8W7BZcdv)xp=91`WQHMI8Um#i&k=_YXSGH%bu9o6~b_M?XOTWm13 z`Dr|k&1uFOWk<`rnxZof%4iS{WYDRxXnq0(RkU|h8}xN_G#@DWN{-w+rAG_I^$Erk zE##};W1PO!Ij(wh7Nz4dtk_t~jvV7-bJY0&w04(iQ5RGtMZ7pa3_X7*ww=KKs&f30 zQ%WDsTyDi1~ z`t#xEtlJtL+sME;b5<`$f}OGSYB}?piB?-3tt>QaR~aZ%HAn`R|B>NtVYc;Vk>$3UJg&XN>|*;7$SpxW!dir&RiGt6V?t<2y6oC(T@ z)|4OcWE`1`+A&mDQ|QNbDTStfEcVV_MuY6>@g8dY$7bvQeVYKk5}|OxFT;G}USI z8lxt>UF_i6u{iNIQfRv)=7MP zm6hRP6h>7(mqvs8b z6W=cYzdc6#`K$v&660?-%~VTOgI-G^cvzs7O7J;{SmNb-o-FnBIaiei3`~eNh)X$m z%!#^mZ5!_0y=JTn45(KIhCAm+CgBKiD+1G=6~@jMnbD!p+yLNt`e>W{OPc(4P7Ky%U$Do`ox0v>r3h zS+4%i1i`uqMYZ{xjl-#zIytc+B z9x@m;d84X%qY^pJaV_m@+P2N>*@#8PCG+8=M}vzbPAi`li;o+CZ#Pr;+aZ3fGmxf# z%e)SO30JFJ_-$Mv%7UnrhjjJ1Ay9MVZODBaVq6@cOTDFABcD^7l9;bYP z*>qc7OTXZzj#{$->0w6UcJ{he-z1f-ZtM&XV6BU7#jzj!akN%p(o^G(*`FE;Zyvtz zNE@O>FWPoo!DX&odD=Yz7?v|fGuS%r8pa6l$|oM>=aF?P?qZ!ZebFr3;`1zkf7~7s-EmFzxXaeGP)B*G4jv=i{i+;-*rXw<_fQ<)#sUv ztpeHN^9m|$b=r3RP?%{(XT)l++qZVx)cI|Zs;v#7=@u^3P3d6}7y;0rtHVeyEuF`7 zjwz<$Cw&=Ap+Ab!pO0PIa)=s>?H6v9MDoVTt>Nwt#X7|&Q617>I%m}<8$_^O7oapb1jxFhHx z2Iw%yN{x$n-_)Go1$mnncYU+I=S+9DVE@g|PjiqVwE19F-OrFh+CwPsL3i~+mhH|H z{_O_~x1rZN&HQ2W&Y`VyUW|0Iglqq^+0Pf}5l`bL{isl>xbYoV*+Rq>KRL#P zHQWKbN3Hmhe0^zMs@kQL4L!w$gkoWQI#~;+i-yT`?`E1eY!yq7a--HJI)~l!>Sugm z?;a;$*$2$Lj`&@2Dw z*Q2*boZ(sBiFl`S?*M-Ha2x>b(-3{)RS^7EqZNi&sTJM=L!hKg+O-pbNb{aZh*(`Y z8lhgwOgunw@|dVc@M=1hkWhJlj11@ksQuzbCmc1UAs8Q5^TR7pdXVNTI#7iXs54>B zS9`$rTT7Ui&Yu5#00%crEig=uP@EiQl$H30$LN#iH35F|zt+p>Jtt3a19lhy0~g~R zYJRx;%J=$${ghDy4M&jgMS20HN8|5N(FI4AFah<&hyzHKTi%6jeKp5U0D$9o&~T4W zH!s55_0`q&CN0uAPkF(@Ff+dZ%559U;`;XDa{T}oMfFn2<+7*B$%K5DN|XRyZG_g-pU3t>0$+86t;>q zYwn~|&fLyO3ZRJ*-e$x0raJ9J zH~-=j+j0qSIgd2RpD~uj89H;+QlQ2GdHA=8!mPjm@p` z8WMK81yz^El4ebd#Hy1%!updrcogf9_wbN<5-44bFD-653>h1VjNidx#qnuJ&$*({ zCI2|HFThyb8&ua$v|=&m zjvIIRuoJTWFac}>U0qa$pPX}^k62rFIpmuZJ0x^dq)lVd*@+*1ro$M7>L4H0YptbY zaIp&vWIN)}Md*yG%_|^h-Klacr;5#bl#m7F(IQ3UsbVYdRoRt8x3whdjXP1J_72`C;q9|mQRXIHgY6vr1e|pXjl!O>%*Y~1nfY?@%PmDE79_u`aZS#k z@Z~d2i=c9Lkom1cZPv+bb1ciO=aWn&MwUiknp9-qtWDC;FyXd2yQmNKPP7Gd=ou`m z-m-~t$pOEML(6Nx)TKrvsLPgWN^OfJY8OXzlxZ8I_;z$qqqj#XQt>!R7E6>(xzOVN zsh}2;OE%>MTkU-QDOl2Vi*hN>ZP8@`=M#2WQ(Uyg$|(2ZsCDH;do9$A0}k=pjJIP9 zp2OZzF%`$e8)6Jr#n*Ai$S0gv}Ez z-KIxK@Xcd*ng6hR7VEO=o`0sI^B^ZIFdK3I z;XSX>KNLbpiX4^IY|b{5SJDESwEgsG!y7W3&$;`TZ+9%y?nUieq?Z-n<>rg&gQ#S9 z=kL-zj+5t{&#p(-DSValSkbM~HvmY(7_=Suo;dQ~MzvUfb`Ln!Q2>=~gA0Jk)^O;% zsMepFI@E8?sV_c}>31hQ8k?LDpHT_ksg4z>=7CRrb_fu@Z;ZB?g#3I7?8iXbQ!};?Ojq7z#znR58n1$^TDdR~;7B7VYWo?i2*1LmDNd zyGwfL?jA~zk{Cd`Q@R`JkS=MEMidwX0TJX4KJUHEz3-2AzHh#n^X>IpYwa~_uYGo$ zwWYP%6)HxC*G>Jtlq$`%NrT3KbO1A`3fM}lh4e$Po~QinT(I3WIC@p^(=TE zzr`DkZ>Nzlwvi|0L^|jTnUF7=LuFClq1r1XRvCKs0imxFJQu)O2OV?K&U##=ypkH! zW-}f~8FZtNN`cTPrzH9gC$Yj*bn9xu{&C*~1?A(2Hfg1PFi3u*p-w5#l@+|)M(Tgv zNIk?AF3=e$+Uc06Kr3jdU=4=`Xmnwse3div^mICfu9;`<3xYB?U5ULQ+}h1v`2?nW zSO*cfIi)L8;z$)Jnd=Dm9h4hdZm4QbWIae*q={iIgD7;6NZF*OK%w1|Am9B<$dbJ1 z199CGFgJgF)JWCz53i?>-wV}$D_^`oAKu!-h}I^q{-7-o>v3a;#*5r*W5^x+z4dEb z{h}6N@$~y~UHjWAknc%VcR26UViy~-w?(x!lL1Vh(|h;wfP<)pE^oN?;S6RLeX-v$ za5@*A;M$YxFHQ+Z9^rd`6x2!kh-&EZF-1#x!|>R)E`&SCs22(4NRr_?O(;|`zuX6i zZP}qS62BI;AK#TJcV;LO=ltHEt-Vcb9ES+(71V~bJY&5v3r3K)4-leY@Z|LH^5jzo zy^IZ>;Pv#X24)N{(sO-0+Evq;T_W;{Xbb6b^ZQ!g*>NDcKODf&G}SXR;&8!HlQpt2 zezJB^Z9a)~jeo_#6zUi`M&WUiUh!HdWAl2flJ7Bgu|R>7&{ zs7}6d3(=c)^9gDIE=U)rK9$P5ma^c{vGdWoRSEm;qA?DWy!pmjse58_#;pxSyjYIZ ziQ-Py*9IoqS4E0jl0@phi2?nMC3q-@t0<`lYpC?>fAk&Kyk)wcxRf|C2qFLc5X*6{S;hLsuwkmwF0?*g+L)@oM&a#eBf7^Kty0JfI8M< zv~G*>yS~`vI?sM~{fXdDvGd70e^;t*>6-x3HW%G~FDZYW{!1wl8es41JY;iR3-i}> z#KEMJ80OsGs=hy%28$Y{cxbm?qZssCU=%p#I|EeXdoByW{cR+naf#N&%gzKt{hwMe zsBQbO!F^9}P@WN;IupdbzMdut3>h->Hr$HYie$=;*<${jjdR*C24s|On3M(^B0Q$| zHp&b5P$TTXZ?{;Wy$MtveM+cp!5G6#@6c*Pddb`nsWicI4WClttmm+!XU?$UngyRN z%0I$&dmbDRn%N*fR#$bFo6O-8zR2tjFyzRY@ROc+Ix8bupvu{KE<1Ixkl`Ab&ou^a znQpf^bGA{Zr9!C3e4Im791PkI(*~}xcsi;cSQ7!Jn6JYfqJe8qM{92=(c&JzFGPg--@1FD#mQkSFrxKySaAFVp9&JkU8a0{f z*9OkzZjW@Sa+o*wCdY;?Y1Mtr{QUV!&}jM;z2k%L_ZAs<#nBe* z>d)SkP6yr^{Ra7dW-5?UET5)sy#+3+Yv2b8 z8-l7XXNr(<-{Q>-_?_(Dlw#x;vus_~POlQ{hTiTg0fE)DHE>%}0#w|L9$)RtQ- zn}LVQPX+mi%HDN+>MuP(VtMriG_G7ZxLq>Nf;A_v6FiE!YD);8`6SY9Uw67up{sQP zJd{R4pTQR`V#NI@2LBXjM#dGWkbd zKJ@%xah8{%0i$=}Yn(o(d0l3C;kA{AMLi&~rTH7~NRE%Lelg?&v}YtQt(A+)dOEaB zJdM#5t4@ofS36+WkPovdJeqfJ`lPF=CnZ-NO#zSJqga0*YfZ(}N3YR?JHOp8pZ2V@wB`-=2$h)RjJ1Q)+jdNo6$1yKJ}rGC*W{E-!=l|6!~5>9?*bMF zmsXZmqKTvy*HiVM!6&!*AhDGllZL6IIw0w@;AV>pBwJ^R+Z|%iQBT()3Nsl9Qpfls z7a0LRZ@hNOr~W3er6lph*Xb0um|@OfA4fFlnuHC&Djczx$778y?Mphk{e2w#eanqIC(kc3x@BhTyd%olRNa|7LbqQLZKJxPbYfO6 zfO*b9!e<&q$qBhyq&mnO?`hkP>km-!gq@p!mWAP#L<<3Z4$jiupD`g8V@a857 zC%e_A#Ie+Q`KzUyJoSC0K10LPXacv6z(;vmRW%atgreq?I%N|j4TkeETHR7Rk%u$! zffnO(-}WT6FGMQug0rspJvX16mkS_iB5z>Zxx8(o47G*3EZrV11uYgB^~C#TiQpH1 z-LB5X|GFku94_D{dThoK!!eT9&V{1XqQ0x9uCz%Ns3lP0xoKo(W7+{U#SX{G-`8r1 zGv@QfA|df_&k4L#D^TNE?E(c}A{*`)391ohhid><{e4mCBFbU*R#mA=7pePYi*AXq zw7O!qRuo~b+!BPzlHXUuLeJS0P%Auq!8p-eo}3f~21WrJ28QSfAIn0Q-)~~vizeV0)M>$9dJubHOp0c)cBPz z%=)BMdx%r1H{6EiDO;vyjfP?uTTCWiq6S*FCYTa&Lkld5xS<1v2O_GBT;)a0UM0ko z07U4whwx~`qtl8_<19UfxF}F5Px0T_v3_l?%rtE$?b|%KirnlFc1m?*ENf3;%X7J# z+o9C>xphik;H9jx$|fEqrz>ik_NF}JI3|0X9FtBSwOqP*Xz)UbGI9v(sKDBwKFs8! zd5i77XlIa+wuR14(pSw|(y@2yW{VU}IQ zV?(kcJMKQ$tRS!*BqJHMz~oTWQz^3V%R3odu9Tt$2sjS1E;&}rC+8juXY;z6O(&e< zqmGX{nitzG7H0!Qe1zM3!2*2bXKzfgOJea3#%}Bb9hHN=2#fT_kKObp(e^rhSkl&) z;rKY~et`BVdRu$IdDgyH1f9sXzD`}kT@yxB?dtt$wVN{WxIb;{hIQ&Z`;5JpS zl8RAa#9!Mh_VX?iLq)9Td_jVI(1&xifpF43MbxTl7G-$_w&WtT71k2jsrX$@^oXM! zk=TLCjx1iOcRn&Hq*EMUtrw$RFq^kU$-=qvO*MJF%rk+JSzc>~KxI`)1diNC$|wvIcx=@L&6y{XE|e0NjYo;;?Q^YU6o}-p2uqPJ?$?{ zmzY=R?RhcX!MaU|`4;Sk$qYVtCm@p(5fWo;eZ>%1w7KA|H3-0b-{S8hP2J#b$x z$eM`1T+!m&!7NvGcD-Qw%(~5n>9oy4SPjoDvD`$0TqY`f7D494jNN#&)9fe^g zG!t&HHI0gDP~gF0-f1_d*1WDw2S(E}_#sq%RZ;q8^O$e%+&_ib!(6MUd})7a>fMha zKZf3Uhhyf6jJHlcMZ<{1#|i-Zp(m9QDVRy}g<(N^S!UTP8P?u9gO&v}Q!O(X(ZFgQS_8!x;Q>!->Pg&YbCws0H$hcY<66 z+M?2ECcpOgvoUZqKya=|P56-nskYju zA`ArISK4+Vo1O-23b2ZC89h7v4tu;59bi8s;MTe@85nShS9qLV5B9jsej(|#p?AfM z2FyIISnKhG53%QWxwH(cL5#H?B9m^wqHF;@S$iS;dXxR6vTgm0!-pO>c#=GlgiK&> zsEVM6WZ!II*%?f+Giu!d!$8YM>yBkR;79KTo8|B7RJ2gKulD3HSwy}C8Gn_{UfHdA zabwP)`yE_E=$dG!f^O%GA!IX|>*{`*9gyQ)682P>+}BG(-p$U`e9KJpXAH;!*+!m> zx6#Zm$CJxv!gxszm~&k?y_+i=@e@p7?T}tbFJOJZ1-9_f5l<1F64y9# zPSsWZ$mcE%^DIkvF2!sTU}iOZ`G|3nR=qp?@zk}*({LvylUYQ2<*;Vfyj%5*Wip_z z?nY5lnVrkF#KEbC^-&;h7gz~#)qe_~tbMNQ^~@9*OYYe@F#AHgnW%;huLXkvkBMg4 zi*zjj(LHNWDOCT~L$5-nLBYy<4o3OgE0|v|&#%qssAJV9Qjgq~*fo(cFT zjPmWR+{4p2t@i#Gvlb17biX0~;Ig$w8neEs-Q8>F1V6hKc zu-c@1<$%=c6qAchHZjRJ@mQ4JsMHn}f!Rk&V{{{3=?MOxl5|;Rb88mTWbc69Eg70! zk5~IV=~hQQr>KwdU(&2{w3&E6cXz!?vD^jNtMICp(`JJ&`|*QiX|hqILr%je61Gu` z__M9CE^28Vw)Vkq_CJ)JfGH-XPCFIF;n+mheW^Uqv0IaezHZY>MHNQ{Cu323I;7C( zQ4#f+T_3PFh1=_lX0CDyr+@amGUgN09b5ySRH=08x)@P*#akn4q7DH?2{!rYxTzDE zbi(Up@$UIMIOt~!1DZf8RY&V#Tw)(~Rq=vJC zk~(LNDrZdtH)jp^Q&neAOf^n!PTp-6rVo?yO6<%{N*wZvBXaXf6C*n;EaOi`#*`X& z;U2uz4G+nBnYjFvYZ4-h0v2@FM+Fp(6L;#P!FY9YlU#-}iyZ=Z{f5N&a_3D8r5I>36uR)Ap2?aZ=H%2C zss9ko&07m6ulZiPWB-J7WRqHiQuzA8L)(32K>+}UKY11JZ|f@n!Eg`mL8TsBf6^@8 z7pqI4QkFuJZe2pAD?~tfo{#D^Vus-UWEH(9=%`TQzR$`P3c^ZP0h&7GeJe0&4AIY1 zwP2=!thy9NqSoT#V??eO9~EQhd5#>XTvd0EWFY&U!bLI;)&rAH$2}`0>}&hN9kRg$J{J9yZCKP04v? zGWEf6&5Wc>cF1>L1X!iNPOjF-%&qTY*DHpTt@)Fw6ft7PO7cKCc$Q~+@<$cPsAMsfiiWoZ3&9av##*PtU>XDePNoKJRo zIjZ9)C74WGv_OKB^5y4`fyB13)jJr{@C!g6uDC{I-FJ?6m>o58d=JegoJvXn*4**oCoEHZMvw8-Vj}y8wYP-ct7E z{&gzDlwYC6e^D_M`}Rk$^o<#kpMx2Qy@^cIn*9;P9xLc8g5 znuU!%j&TdZog&r{N+mqYu!d|xjy1e9`L?Rti*Hg9&uFDxOiG*~Tj*fhE^zhj{Z3ET z+EPSAx$?5;Ck&R{cC%@#8oMO?M%i&WaC~#;ZTxe7Np=HexW~@7T^I?<123kqBdQ&Tc<~B|Chn`(eYdFmvQl_CudJMJ zRuy$+szQ=#546|P#^VDj7^a!Op+vlnZ+}GtW*wye2H0u{Xzk|2J(&vOn0ZMKM7ks^ zrLcLNZfmRkveNgB>3no>ER9C%)9v-11f#R4EV4i}NCpOvcTZGRfRPOF znn<-z7V*SWRx+NDJAAs)z*}cw0&Y@z?2b!t*$;%X+Ef`e8LqivP6|v5o5u~%5X5{E z?ejMLDBzfs=X}!I;#r&PiXo|1R6rExgwFEv26&{YZ&MYY zDHZ_^MSu-GKi-n}d_|j>3=?E3f{^o^MeX1wYdEo?YAlo11@;tOSmxW~Hg?*h3D44x z1L%cUU4D*IBBB1vQ6Hlbh`E@ivJiUW+9)_9)LPF`;%-3|pHSsi&jhp0V*E=E%(fpz zra1_$)q~xsM+clt>PUyN% z)WPPyNPW~xv4lVT^U7xI>I*B?l#sZOq6Twa(h_k49f&Sg{#l*cwRoBCz=83Q-}f6J_elmw=I57?%+Ca(JpQbo%o3We^ai3*INFP@&1hof%_JV# zO3YruB4t`GIe5N>Bn|_98hD$2W#WG!LwdqWu859=FPZAMKhe5BA?kIpU+uI3%L^8P zhgad2{mc+7+L?r@>E2?S>-ecGI3V0YL6Akj>~->!#~7zg!buq9FL#H%fU|POzC-y= z8jm(SkbA{&vuA!Xkzk9l6-V`UV>^7Rn=wCDGkS`}y-%83n@l?rY2a94vNx8LR6;({8ipvdv9D zq%|h<&pDolyQ{=f;wm=cRla}eLB_OlO>b=1`jbBY!})3%f4ox?m7{6dYL_y1LJuE4 zvUREe0&`lsJirX!04bZR<7R`)*er*?zozjWefZjUTP}O)xV^=r7)sy*s7 zIo&9iH0sNN9A2mx|A%~RPOK`A{46=)XjV+>$U_}4$BwgwjnW~F%9hZNJ&GXAoxB`W zITD6A6w1dL)$<%U!plEm`X>~Bo-TzjSL2GjdZa?%$HUo5TBp6AhFgBoq=xcj*99b^ z?8WJV-i$linV;^JpYCLcTRg|du|AoYVjPj7Xgvqp+Yc;}QtHquP$ zCWJLE$)r4)X$%&<7XeqS@QlP$@`gT1;wK4zub9Ayk>s1>z#LZBvWh<2{|CK%Tp1_-NOVb4M;t?4-NZeWq=tZVQ z@Cyq=kvnZKB#)1=QVxR<4zse!hK*N#cFttvMOmcH6VD*jGy+R!%hytOp(j3W{Re~An(Lcnf9su2xy@gI0B+~GsBMZ0k) z+rL2Pit($k*zN)^l%HbB3@0Tv{{v^@w7W;A_DK0&0E0239E~|a(zg^dTmN-Dw2N#@ zF2hL@OJ*bPWHY^3;t%>F$<6i}%!?VtYFbJL8kK+ob+vOk{&xEV*g8i%Au;h%tDG z8%s_Zs_9^rMx6;dnDC^|>^nT&Z2ef7?%wCFkkax#*;m|wj^Gp%z+ay1KHypRbDR8Z zsbpoCxu@y^b9kvWVM7PqlWvPh?Ze?)AGadmf}`Cn;LBbk9S)y&&J@6ldg*c2QZK&j zUTYw6K=4W;WY-&sw90bwDSI(s%tXK6ob*HzH{q??bG1QMW3}2R`TaTZ2Lv)q)u>Avww7uLV08 zxJD3ydsNpxE?pc&572#9A3vA}Z8|Pt9Yrgb^T@KpJD%wB*JGbcjhtxL_m~(@X(&x! zo8HdgT7>7+NQClWaBQ|RE20E%B1sbkh;K}iL{Gb)qnI1nKO1;7-XU%bAB;+SfKSB_ z*0%e6LR@RL)AJ>_>v>YhA_^`(dVg<3zho5n2&YaaELx^U>NSuEr^qR0>XkF$TWfX` zj7uiOw^>)NM^~=rPG|5D-zoIk0?!pS*)qK|l=QNa8yITv@|B$?GU zW5%EN0&g1=A?N#;14yszNKc|}SFe{%V(s(L?6-3+F_-JP2YVpzBM>O7gRlbJ8%R9tO9^2(lQYurM%I5DMn|KD<8Dkrk@HrV0>3 zco-e0dq$wjH!_tTij!_axfF$mf%$+635J9q>M%Ue^9i#04Nr0S2anD6nQBSWkb$!l zsIrd{`2QQEb87r|kiH8wG>3&FfdAj5&i98{K-j}TYW4d<{qqe2g9CFUet)C%LQ|Yg zPFWgqcmNO@_xEuZ5GsPdaWF82!I0c-|KOCwmF1*gXlSx4NvkUS!{Pq;i+j*tk|F)V zp9=@(Ry+gp+sV}mBJ=@5=U))?zYq{*Z^hsLfmpekTG#`ik^cfh^2`0X?tzDpl@#*F ze++W3mUW1QWdVd>9`d38ixTqP5KIXR${VBvp$}p{q=!111BxCN?wp0)zu0NQDiX70MP6H4(~u*9;-%Q-hd}4GLt6_!~qsj0EbYCID>= zAwCoab+7{z4h7{m4xN{lEVkO zhnoG)s(bGckmgAZMbAbAQX64>NDGy07mBu<8*gAVEwU5_bfk1i5Sd1}Z9Hg1SebToY=|5egHi1HmvoRP665 zMF<^q!}(|oVg)b&AG+{;JpN~15C#TP`MYY^9r6#Ab~bq@=-@X;vF=j6#L(T z4<>0J3jTYJ1~L`N4<)$89U_<+#6C?3CH&LOz3^qP-^{3I=^rwK8uUUjBk}vKY}`05 z6f@=NduGQ$za<$a>A)4JGKGQ4$P{Fckez3Fw8j=zsk2;X@586d \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Djava.net.preferIPv4Stack=true" + # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" @@ -30,6 +48,7 @@ die ( ) { cygwin=false msys=false darwin=false +nonstop=false case "`uname`" in CYGWIN* ) cygwin=true @@ -40,26 +59,11 @@ case "`uname`" in MINGW* ) msys=true ;; + NONSTOP* ) + nonstop=true + ;; esac -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. @@ -85,7 +89,7 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then MAX_FD_LIMIT=`ulimit -H -n` if [ $? -eq 0 ] ; then if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then @@ -150,11 +154,19 @@ if $cygwin ; then esac fi -# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules -function splitJvmOpts() { - JVM_OPTS=("$@") +# Escape application args +save ( ) { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " } -eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS -JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi -exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index b5adeb2fde6e8..156038a96083a 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -8,14 +8,14 @@ @rem Set local scope for the variables with windows NT shell if "%OS%"=="Windows_NT" setlocal -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS=-Xmx2g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m - set DIRNAME=%~dp0 if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS=-Xmx2g -XX:ReservedCodeCacheSize=512m -Djava.net.preferIPv4Stack=true + @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome @@ -49,7 +49,6 @@ goto fail @rem Get command-line arguments, handling Windows variants if not "%OS%" == "Windows_NT" goto win9xME_args -if "%@eval[2+2]" == "4" goto 4NT_args :win9xME_args @rem Slurp the command line arguments. @@ -60,11 +59,6 @@ set _SKIP=2 if "x%~1" == "x" goto execute set CMD_LINE_ARGS=%* -goto execute - -:4NT_args -@rem Get arguments from the 4NT Shell from JP Software -set CMD_LINE_ARGS=%$ :execute @rem Setup the command line diff --git a/graphx/build.gradle b/graphx/build.gradle index f7393440f8c8c..3dfe6288223c2 100644 --- a/graphx/build.gradle +++ b/graphx/build.gradle @@ -22,7 +22,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-mllib-local_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' + compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version compile group: 'com.google.guava', name: 'guava', version: guavaVersion compile group: 'com.github.fommil.netlib', name: 'core', version: '1.1.2' compile group: 'net.sourceforge.f2j', name: 'arpack_combined_all', version: '0.1' diff --git a/repl/build.gradle b/repl/build.gradle index ea40dc5577b94..8e4bde82ea11c 100644 --- a/repl/build.gradle +++ b/repl/build.gradle @@ -22,7 +22,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' + compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion compile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion compile group: 'jline', name: 'jline', version: jlineVersion diff --git a/sql/catalyst/build.gradle b/sql/catalyst/build.gradle index 27d81efd95bc3..9f2b52ac4ca2d 100644 --- a/sql/catalyst/build.gradle +++ b/sql/catalyst/build.gradle @@ -25,7 +25,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion - compile group: 'org.scala-lang.modules', name: 'scala-parser-combinators_' + scalaBinaryVersion, version: '1.0.4' + // compile group: 'org.scala-lang.modules', name: 'scala-parser-combinators_' + scalaBinaryVersion, version: '1.0.4' compile group: 'org.codehaus.janino', name: 'janino', version: '3.0.7' compile group: 'org.codehaus.janino', name: 'commons-compiler', version: '3.0.7' compile group: 'org.antlr', name: 'antlr4-runtime', version: antlrVersion diff --git a/sql/core/build.gradle b/sql/core/build.gradle index e83a8b2697419..04032503641c4 100644 --- a/sql/core/build.gradle +++ b/sql/core/build.gradle @@ -23,19 +23,24 @@ dependencies { compile project(subprojectBase + 'snappy-spark-sketch_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile group: 'com.univocity', name: 'univocity-parsers', version: '2.2.1' + compile group: 'com.univocity', name: 'univocity-parsers', version: '2.2.3' compile group: 'org.apache.parquet', name: 'parquet-column', version: parquetVersion compile group: 'org.apache.parquet', name: 'parquet-hadoop', version: parquetVersion compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: fasterXmlVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion, configuration: 'testOutput') testCompile group: 'com.h2database', name: 'h2', version: '1.4.183' testCompile group: 'mysql', name: 'mysql-connector-java', version: '5.1.38' testCompile group: 'org.postgresql', name: 'postgresql', version: '9.4.1207.jre7' - testCompile group: 'org.apache.parquet', name: 'parquet-avro', version: parquetVersion testCompile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' + testCompile(group: 'org.apache.parquet', name: 'parquet-avro', version: parquetVersion) { + exclude(group: 'it.unimi.dsi', module: 'fastutil') + } + testCompile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: '4.4' + // different avro version from parent (1.7.7) since parquet-avro depends on 1.8.x // which is used by ParquetAvroCompatibilitySuite that uses AvroParquetWriter testCompile group: 'org.apache.avro', name: 'avro', version: '1.8.1' diff --git a/sql/hive/build.gradle b/sql/hive/build.gradle index bed3e2da88f83..1947b53bc47c3 100644 --- a/sql/hive/build.gradle +++ b/sql/hive/build.gradle @@ -22,9 +22,13 @@ dependencies { compile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - compile group: 'com.twitter', name: 'parquet-hadoop-bundle', version: hiveParquetVersion + // compile group: 'com.twitter', name: 'parquet-hadoop-bundle', version: hiveParquetVersion compile group: 'org.apache.derby', name: 'derby', version: derbyVersion + compile group: 'org.datanucleus', name: 'datanucleus-core', version: '3.2.15' + compile group: 'org.datanucleus', name: 'datanucleus-api-jdo', version: '3.2.8' + compile group: 'org.datanucleus', name: 'datanucleus-rdbms', version: '3.2.13' compile(group: 'org.spark-project.hive', name: 'hive-exec', version: hiveVersion) { + exclude(group: 'org.datanucleus', module: 'datanucleus-core') exclude(group: 'org.spark-project.hive', module: 'hive-metastore') exclude(group: 'org.spark-project.hive', module: 'hive-shims') exclude(group: 'org.spark-project.hive', module: 'hive-ant') @@ -50,6 +54,9 @@ dependencies { exclude(group: 'org.json', module: 'json') } compile(group: 'org.spark-project.hive', name: 'hive-metastore', version: hiveVersion) { + exclude(group: 'org.datanucleus', module: 'datanucleus-core') + exclude(group: 'org.datanucleus', module: 'datanucleus-api-jdo') + exclude(group: 'org.datanucleus', module: 'datanucleus-rdbms') exclude(group: 'org.spark-project.hive', module: 'hive-serde') exclude(group: 'org.spark-project.hive', module: 'hive-shims') exclude(group: 'org.apache.thrift', module: 'libfb303') @@ -96,12 +103,11 @@ dependencies { exclude(group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm') } compile group: 'org.apache.httpcomponents', name: 'httpclient', version: httpClientVersion - compile group: 'org.codehaus.jackson', name: 'jackson-mapper-asl', version: '1.9.13' + compile group: 'org.codehaus.jackson', name: 'jackson-mapper-asl', version: jackson1Version compile group: 'commons-codec', name: 'commons-codec', version: commonsCodecVersion - compile group: 'joda-time', name: 'joda-time', version: '2.9.4' - compile group: 'org.jodd', name: 'jodd-core', version: '3.5.2' + compile group: 'joda-time', name: 'joda-time', version: '2.9.9' + compile group: 'org.jodd', name: 'jodd-core', version: '3.9.1' compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version - compile group: 'org.datanucleus', name: 'datanucleus-core', version: '3.2.10' compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) { exclude(group: 'org.slf4j', module: 'slf4j-api') } From ddc54ad4b996a5b283a614672555549f8af43d41 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 24 Oct 2017 18:55:17 +0530 Subject: [PATCH 110/142] [SNAP-2061] fix scalastyle errors, add test - fix scalastyle errors in SQLContext - moved the Dataset/DataFrame nested POJO tests to JavaDatasetSuite from SQLContextSuite - added test for Dataset.as(Encoder) for nested POJO in the same Conflicts: sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala --- .../org/apache/spark/sql/SQLContext.scala | 22 +- .../apache/spark/sql/JavaDatasetSuite.java | 280 ++++++++++++++++++ .../apache/spark/sql/SQLContextSuite.scala | 112 +------ 3 files changed, 292 insertions(+), 122 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index b73de9724d48f..036e90f7d1aa1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -18,11 +18,11 @@ package org.apache.spark.sql import java.beans.{BeanInfo, Introspector, PropertyDescriptor} -import java.lang.reflect.Method import java.util.Properties import scala.collection.immutable import scala.reflect.runtime.universe.TypeTag + import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.annotation.{DeveloperApi, Experimental, InterfaceStability} import org.apache.spark.api.java.{JavaRDD, JavaSparkContext} @@ -32,7 +32,7 @@ import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.command.ShowTablesCommand -import org.apache.spark.sql.internal.{SQLConf, SessionState, SharedState} +import org.apache.spark.sql.internal.{SessionState, SharedState, SQLConf} import org.apache.spark.sql.sources.BaseRelation import org.apache.spark.sql.streaming.{DataStreamReader, StreamingQueryManager} import org.apache.spark.sql.types._ @@ -1129,21 +1129,21 @@ object SQLContext { } } - def getExtractors( beanInfo: BeanInfo, - attrs: Seq[AttributeReference]): Array[(PropertyDescriptor, Any => Any)] = { - val methodsToConverts = beanInfo.getPropertyDescriptors. - filterNot(_.getName == "class").zip(attrs) - methodsToConverts.map { case (desc, attr) => + def getExtractors( + beanInfo: BeanInfo, + attrs: Seq[AttributeReference]): Array[(PropertyDescriptor, Any => Any)] = { + val methodsToConverts = beanInfo.getPropertyDescriptors. + filterNot(_.getName == "class").zip(attrs) + methodsToConverts.map { case (desc, attr) => attr.dataType match { - case strct: StructType => { + case struct: StructType => val extractors = getExtractors(Introspector.getBeanInfo(desc.getPropertyType), - strct.toAttributes) + struct.toAttributes) (desc, (x: Any) => { - val arr = Array.tabulate[Any](strct.length)(i => + val arr = Array.tabulate[Any](struct.length)(i => extractors(i)._2(extractors(i)._1.getReadMethod.invoke(x))) new GenericInternalRow(arr) }) - } case _ => (desc, CatalystTypeConverters.createToCatalystConverter(attr.dataType)) } } diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java index c132cab1b38cf..ae07ac2fae942 100644 --- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java +++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java @@ -34,6 +34,7 @@ import org.junit.*; import org.junit.rules.ExpectedException; +import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.*; import org.apache.spark.sql.*; @@ -793,6 +794,208 @@ public int hashCode() { } } + public static class NestedBean implements Serializable { + + private int id; + private String name; + private long longField; + private short shortField; + private byte byteField; + private double doubleField; + private float floatField; + private boolean booleanField; + private byte[] binaryField; + private Date date; + private Timestamp timestamp; + private Address address; + + public NestedBean(int id, String name, long longValue, short shortValue, byte byteValue, + double doubleValue, float floatValue, boolean booleanValue, byte[] binaryValue, + Date date, Timestamp timestamp, Address address) { + this.id = id; + this.name = name; + this.longField = longValue; + this.shortField = shortValue; + this.byteField = byteValue; + this.doubleField = doubleValue; + this.floatField = floatValue; + this.booleanField = booleanValue; + this.binaryField = binaryValue; + this.date = date; + this.timestamp = timestamp; + this.address = address; + } + + public NestedBean() { + this(0, null, 0, (short)0, (byte)0, 0d, 0f, false, null, null, null, null); + } + + public String getName() { + return name; + } + + public int getId() { + return id; + } + + public long getLongField() { + return longField; + } + + public short getShortField() { + return shortField; + } + + public byte getByteField() { + return byteField; + } + + public double getDoubleField() { + return doubleField; + } + + public float getFloatField() { + return floatField; + } + + public boolean getBooleanField() { + return booleanField; + } + + public byte[] getBinaryField() { + return binaryField; + } + + public Date getDate() { + return date; + } + + public Timestamp getTimestamp() { + return timestamp; + } + + public Address getAddress() { + return address; + } + + public void setName(String name) { + this.name = name; + } + + public void setId(int id) { + this.id = id; + } + + public void setLongField(long longValue) { + this.longField = longValue; + } + + public void setShortField(short shortValue) { + this.shortField = shortValue; + } + + public void setByteField(byte byteValue) { + this.byteField = byteValue; + } + + public void setDoubleField(double doubleValue) { + this.doubleField = doubleValue; + } + + public void setFloatField(float floatValue) { + this.floatField = floatValue; + } + + public void setBooleanField(boolean booleanValue) { + this.booleanField = booleanValue; + } + + public void setBinaryField(byte[] binaryValue) { + this.binaryField = binaryValue; + } + + public void setDate(Date date) { + this.date = date; + } + + public void setTimestamp(Timestamp timestamp) { + this.timestamp = timestamp; + } + + public void setAddress(Address address) { + this.address = address; + } + } + + public static class Address implements Serializable { + + private String street; + private int zip; + + public Address(String street, int zip) { + this.street = street; + this.zip = zip; + } + + public Address() { + this(null, -1); + } + + public String getStreet() { + return this.street; + } + + public int getZip() { + return this.zip; + } + + public void setStreet(String street) { + this.street = street; + } + + public void setZip(int zip) { + this.zip = zip; + } + } + + private void checkNestedBeansResult(List rows) { + Set keys = new HashSet<>(100); + for (int k = 1; k <= 100; k++) { + keys.add(k); + } + for (Row row : rows) { + int k = row.getAs("id"); + Assert.assertTrue(keys.remove(k)); + Assert.assertEquals("String field match not as expected", + "name_" + k, row.getAs("name")); + Assert.assertEquals("Long field match not as expected", + (long)k, row.getAs("longField").longValue()); + Assert.assertEquals("Short field match not as expected", + (short)k, row.getAs("shortField").shortValue()); + Assert.assertEquals("Byte field match not as expected", + (byte)k, row.getAs("byteField").byteValue()); + Assert.assertEquals("Double field match not as expected", + k * 86.7543d, row.getAs("doubleField"), 0.0); + Assert.assertEquals("Float field match not as expected", + k * 7.31f, row.getAs("floatField"), 0.0f); + Assert.assertTrue("Boolean field match not as expected", + row.getAs("booleanField")); + byte[] bytesValue = new byte[k]; + Arrays.fill(bytesValue, (byte)k); + Assert.assertTrue(Arrays.equals(bytesValue, row.getAs("binaryField"))); + Assert.assertEquals("Date field match not as expected", + new Date(7836L * k * 1000L).toString(), row.getAs("date").toString()); + Assert.assertEquals("TimeStamp field match not as expected", + new Timestamp(7896L * k * 1000L), row.getAs("timestamp")); + Row addressStruct = row.getAs("address"); + Assert.assertEquals("Address.street field match not as expected", + "12320 sw horizon," + k, addressStruct.getAs("street")); + Assert.assertEquals("Address.zip field match not as expected", + 97007 * k, addressStruct.getAs("zip").intValue()); + } + assert (keys.isEmpty()); + } + @Rule public transient ExpectedException nullabilityCheck = ExpectedException.none(); @@ -1534,4 +1737,81 @@ public int hashCode() { return Objects.hashCode(arrayList, linkedList, list); } } + + // see SNAP-2061 + @Test + public void testNestedBeanInDataFrameFromRDD() { + List beanCollection = new ArrayList<>(100); + for (int k = 1; k <= 100; k++) { + byte[] bytesValue = new byte[k]; + Arrays.fill(bytesValue, (byte)k); + beanCollection.add(new NestedBean(k, "name_" + k, (long)k, (short)k, + (byte)k, (double)k * 86.7543d, (float)k * 7.31f, true, + bytesValue, new Date(7836L * k * 1000L), new Timestamp(7896L * k * 1000L), + new Address("12320 sw horizon," + k, 97007 * k))); + } + + JavaRDD beanRDD = jsc.parallelize(beanCollection); + Dataset df = spark.createDataFrame(beanRDD, NestedBean.class); + checkNestedBeansResult(df.collectAsList()); + } + + // see SNAP-2061 + @Test + public void testNestedBeanInDatasetFromRDD() { + List beansCollection = new ArrayList<>(100); + for (int k = 1; k <= 100; k++) { + byte[] bytesValue = new byte[k]; + Arrays.fill(bytesValue, (byte)k); + beansCollection.add(new NestedBean(k, "name_" + k, (long)k, (short)k, + (byte)k, (double)k * 86.7543d, (float)k * 7.31f, true, + bytesValue, new Date(7836L * k * 1000L), new Timestamp(7896L * k * 1000L), + new Address("12320 sw horizon," + k, 97007 * k))); + } + + Encoder encoder = Encoders.bean(NestedBean.class); + Dataset beansDataset = spark.createDataset(beansCollection, encoder); + checkNestedBeansResult(beansDataset.toDF().collectAsList()); + + beansDataset.createOrReplaceTempView("tempPersonsTable"); + List rows = spark.sql("select * from tempPersonsTable").collectAsList(); + checkNestedBeansResult(rows); + + // test Dataset.as[Person] + JavaRDD beansRDD = jsc.parallelize(rows); + Dataset beansDF = spark.createDataFrame(beansRDD, beansDataset.schema()); + List results = beansDF.as(encoder).collectAsList(); + Set keys = new HashSet<>(100); + for (int k = 1; k <= 100; k++) { + keys.add(k); + } + for (NestedBean bean : results) { + int k = bean.getId(); + Assert.assertTrue(keys.remove(k)); + Assert.assertEquals("String field match not as expected", "name_" + k, bean.getName()); + Assert.assertEquals("Long field match not as expected", k, bean.getLongField()); + Assert.assertEquals("Short field match not as expected", (short)k, bean.getShortField()); + Assert.assertEquals("Byte field match not as expected", (byte)k, bean.getByteField()); + Assert.assertEquals("Double field match not as expected", + k * 86.7543d, bean.getDoubleField(), 0.0); + Assert.assertEquals("Float field match not as expected", + k * 7.31f, bean.getFloatField(), 0.0f); + Assert.assertTrue("Boolean field match not as expected", bean.getBooleanField()); + byte[] bytesValue = new byte[k]; + Arrays.fill(bytesValue, (byte)k); + Assert.assertTrue(Arrays.equals(bytesValue, bean.getBinaryField())); + Assert.assertEquals("Date field match not as expected", + new Date(7836L * k * 1000L).toString(), bean.getDate().toString()); + Assert.assertEquals("TimeStamp field match not as expected", + new Timestamp(7896L * k * 1000L), bean.getTimestamp()); + Address address = bean.getAddress(); + Assert.assertEquals("Address.street field match not as expected", + "12320 sw horizon," + k, address.getStreet()); + Assert.assertEquals("Address.zip field match not as expected", + 97007 * k, address.getZip()); + } + assert (keys.isEmpty()); + + spark.catalog().dropTempView("tempPersonsTable"); + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala index 2efa9669fcfd1..3e828449ff0be 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala @@ -17,8 +17,6 @@ package org.apache.spark.sql -import java.sql.{Date, Timestamp} - import org.apache.spark.{SharedSparkContext, SparkFunSuite} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -143,115 +141,7 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext { sqlContext.dropTempTable("tables") } } - - test("Bug SNAP-2061 Nested POJO object not handled when creating DataFrame from RDD") { - val sqlContext = SQLContext.getOrCreate(sc) - val personsCollection = for (k <- 1 until 100) yield { - new Person(k, "name_" + k, k.toLong, k.toShort, - k.toByte, k.toDouble *86.7543d, k.toFloat *7.31f, - true, Array.fill[Byte](k)(k.toByte), - new java.sql.Date(7836*k*1000), new Timestamp(7896*k*1000), - new Address("12320 sw horizon," + k, 97007*k)) - } - - val personsRDD = sc.parallelize(personsCollection) - val df = sqlContext.createDataFrame(personsRDD, classOf[Person]) - val rows = df.collect() - val keys = scala.collection.mutable.Set[Int]() - for(i <- 1 until 100) keys.add(i) - for(row <- rows) { - assert(keys.remove(row.getAs[Int]("id"))) - val k = row.getAs[Int]("id") - assert("name_" + k == row.getAs[String]("name"), "String field match not as expected") - assert(k.toLong == row.getAs[Long]("longField"), "Long field match not as expected") - assert(k.toShort == row.getAs[Short]("shortField"), "Short field match not as expected") - assert(k.toByte == row.getAs[Byte]("byteField"), "Byte field match not as expected") - assert(k*86.7543d == row.getAs[Double]("doubleField"), "Double field match not as expected") - assert(k*7.31f == row.getAs[Float]("floatField"), "Float field match not as expected") - assert(true == row.getAs[Boolean]("booleanField"), "Boolean field match not as expected") - assertResult(Array.fill[Byte](k)(k.toByte).seq) {row.getAs[Array[Byte]]("binaryField").toSeq} - assert(new java.sql.Date(7836*k*1000).toString == row.getAs[Date]("datee").toString, - "Date field match not as expected") - assert(new Timestamp(7896*k*1000).toString == row.getAs[Timestamp]("timeeStamp").toString, - "TimeStamp field match not as expected") - val addressStruct = row.getAs[Row]("address") - assert("12320 sw horizon," + k == addressStruct.getAs[String]("street"), - "struct field match not as expected") - assert(97007*k == addressStruct.getAs[Int]("zip"), "struct field match not as expected") - } - assert(keys.isEmpty) - } - - test("Bug SNAP-2061 Nested POJO object not handled when creating DataSet from RDD") { - val sqlContext = SQLContext.getOrCreate(sc) - val spark = sqlContext.sparkSession - - val personsCollection = for (k <- 1 until 100) yield { - new Person(k, "name_" + k, k.toLong, k.toShort, - k.toByte, k.toDouble *86.7543d, k.toFloat *7.31f, - true, Array.fill[Byte](k)(k.toByte), - new java.sql.Date(7836*k*1000), new Timestamp(7896*k*1000), - new Address("12320 sw horizon," + k, 97007*k)) - } - - - - val personsDataSet = spark.createDataset(personsCollection)(Encoders.bean(classOf[Person])) - - var rows = personsDataSet.toDF().collect() - val keys = scala.collection.mutable.Set[Int]() - for(i <- 1 until 100) keys.add(i) - for(row <- rows) { - assert(keys.remove(row.getAs[Int]("id"))) - val k = row.getAs[Int]("id") - assert("name_" + k == row.getAs[String]("name"), "String field match not as expected") - assert(k.toLong == row.getAs[Long]("longField"), "Long field match not as expected") - assert(k.toShort == row.getAs[Short]("shortField"), "Short field match not as expected") - assert(k.toByte == row.getAs[Byte]("byteField"), "Byte field match not as expected") - assert(k*86.7543d == row.getAs[Double]("doubleField"), "Double field match not as expected") - assert(k*7.31f == row.getAs[Float]("floatField"), "Float field match not as expected") - assert(true == row.getAs[Boolean]("booleanField"), "Boolean field match not as expected") - assertResult(Array.fill[Byte](k)(k.toByte).seq) {row.getAs[Array[Byte]]("binaryField").toSeq} - assert(new java.sql.Date(7836*k*1000).toString == row.getAs[Date]("datee").toString, - "Date field match not as expected") - assert(new Timestamp(7896*k*1000).toString == row.getAs[Timestamp]("timeeStamp").toString, - "TimeStamp field match not as expected") - val addressStruct = row.getAs[Row]("address") - assert("12320 sw horizon," + k == addressStruct.getAs[String]("street"), - "struct field match not as expected") - assert(97007*k == addressStruct.getAs[Int]("zip"), "struct field match not as expected") - } - assert(keys.isEmpty) - personsDataSet.createOrReplaceTempView("tempPersonsTable") - rows = spark.sql("select * from tempPersonsTable").collect() - for(i <- 1 until 100) keys.add(i) - for(row <- rows) { - assert(keys.remove(row.getAs[Int]("id"))) - val k = row.getAs[Int]("id") - assert("name_" + k == row.getAs[String]("name"), "String field match not as expected") - assert(k.toLong == row.getAs[Long]("longField"), "Long field match not as expected") - assert(k.toShort == row.getAs[Short]("shortField"), "Short field match not as expected") - assert(k.toByte == row.getAs[Byte]("byteField"), "Byte field match not as expected") - assert(k*86.7543d == row.getAs[Double]("doubleField"), "Double field match not as expected") - assert(k*7.31f == row.getAs[Float]("floatField"), "Float field match not as expected") - assert(true == row.getAs[Boolean]("booleanField"), "Boolean field match not as expected") - assertResult(Array.fill[Byte](k)(k.toByte).seq) {row.getAs[Array[Byte]]("binaryField").toSeq} - assert(new java.sql.Date(7836*k*1000).toString == row.getAs[Date]("datee").toString, - "Date field match not as expected") - assert(new Timestamp(7896*k*1000).toString == row.getAs[Timestamp]("timeeStamp").toString, - "TimeStamp field match not as expected") - val addressStruct = row.getAs[Row]("address") - assert("12320 sw horizon," + k == addressStruct.getAs[String]("street"), - "struct field match not as expected") - assert(97007*k == addressStruct.getAs[Int]("zip"), "struct field match not as expected") - } - assert(keys.isEmpty) - sqlContext.dropTempTable("tempPersonsTable") - } - - } - class Person(var id: Int, var name: String, var longField: Long, var shortField: Short, var byteField: Byte, var doubleField: Double, var floatField: Float, var booleanField: Boolean, var binaryField: Array[Byte], @@ -293,4 +183,4 @@ class Address(var street: String, var zip: Int) extends java.io.Serializable { def getZip: Int = this.zip def setStreet(street: String): Unit = {this.street = street} def setZip(zip: Int): Unit = {this.zip = zip} -} +} \ No newline at end of file From 2e01d6ebaa07338a5a262f11f93630cfd7de84c4 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 5 Dec 2017 00:50:29 +0530 Subject: [PATCH 111/142] [SNAPPYDATA] add missing jersey-hk2 dependency required after the upgrade to jersey 2.26 that does not include it automatically (used by Executors tab in the GUI) guard debug logs with "debugEnabled()" Conflicts: core/build.gradle core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java --- core/build.gradle | 3 ++- .../spark/memory/TaskMemoryManager.java | 25 +++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index a4973db71a839..c51c606aa4b93 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -116,7 +116,7 @@ dependencies { compile group: 'com.ning', name: 'compress-lzf', version: lzfVersion compile group: 'commons-net', name: 'commons-net', version: '3.6' compile group: 'org.json4s', name: 'json4s-jackson_' + scalaBinaryVersion, version: '3.2.11' - compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: jerseyVersion + compile group: 'org.gla>>>ssfish.jersey.core', name: 'jersey-client', version: jerseyVersion compile group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion compile group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet', version: jerseyVersion @@ -124,6 +124,7 @@ dependencies { compile(group: 'org.apache.mesos', name: 'mesos', version: '0.21.1', classifier: 'shaded-protobuf') { exclude(group: 'com.google.protobuf', module: 'protobuf-java') } + compile group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: jerseyVersion compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion compile(group: 'com.clearspring.analytics', name: 'stream', version: '2.8.0') { exclude(group: 'it.unimi.dsi', module: 'fastutil') diff --git a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java index 632d718062212..dd78f691b835e 100644 --- a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java +++ b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java @@ -203,8 +203,10 @@ public long acquireExecutionMemory(long required, MemoryConsumer consumer) { try { long released = consumer.spill(required - got, consumer); if (released > 0) { - logger.debug("Task {} released {} from itself ({})", taskAttemptId, - Utils.bytesToString(released), consumer); + if (logger.isDebugEnabled()) { + logger.debug("Task {} released {} from itself ({})", taskAttemptId, + Utils.bytesToString(released), consumer); + } got += memoryManager.acquireExecutionMemory(required - got, taskAttemptId, mode); } } catch (ClosedByInterruptException e) { @@ -219,7 +221,10 @@ public long acquireExecutionMemory(long required, MemoryConsumer consumer) { } consumers.add(consumer); - logger.debug("Task {} acquired {} for {}", taskAttemptId, Utils.bytesToString(got), consumer); + if (logger.isDebugEnabled()) { + logger.debug("Task {} acquired {} for {}", taskAttemptId, + Utils.bytesToString(got), consumer); + } return got; } } @@ -228,7 +233,10 @@ public long acquireExecutionMemory(long required, MemoryConsumer consumer) { * Release N bytes of execution memory for a MemoryConsumer. */ public void releaseExecutionMemory(long size, MemoryConsumer consumer) { - logger.debug("Task {} release {} from {}", taskAttemptId, Utils.bytesToString(size), consumer); + if (logger.isDebugEnabled()) { + logger.debug("Task {} release {} from {}", taskAttemptId, + Utils.bytesToString(size), consumer); + } memoryManager.releaseExecutionMemory(size, taskAttemptId, consumer.getMode()); } @@ -424,15 +432,18 @@ public long cleanUpAllAllocatedMemory() { for (MemoryConsumer c: consumers) { if (c != null && c.getUsed() > 0) { // In case of failed task, it's normal to see leaked memory - logger.debug("unreleased " + Utils.bytesToString(c.getUsed()) + " memory from " + c); + if (logger.isDebugEnabled()) { + logger.debug("unreleased " + Utils.bytesToString(c.getUsed()) + " memory from " + c); + } } } consumers.clear(); for (MemoryBlock page : pageTable) { if (page != null) { - logger.debug("unreleased page: " + page + " in task " + taskAttemptId); - page.pageNumber = MemoryBlock.FREED_IN_TMM_PAGE_NUMBER; + if (logger.isDebugEnabled()) { + logger.debug("unreleased page: " + page + " in task " + taskAttemptId); + } memoryManager.tungstenMemoryAllocator().free(page); } } From ab61be299646bf26c88eb9afe2a8cd460fe59a05 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 26 Dec 2017 12:17:56 +0530 Subject: [PATCH 112/142] [SNAPPYDATA][SNAP-2120] make codegen cache size configurable (#87) - use "spark.sql.codegen.cacheSize" to set codegenerator cache size else default to 1000 - also added explicit returns in MemoryPool else it does boxing/unboxing inside the sync block that also shows up in perf analysis (can be seen via decompiler too) - avoid NPE for "Stages" tab of a standby lead Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala --- core/src/main/scala/org/apache/spark/SparkContext.scala | 1 + core/src/main/scala/org/apache/spark/memory/MemoryPool.scala | 4 ++-- .../sql/catalyst/expressions/codegen/CodeGenerator.scala | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index f6dc1b22a9482..5717d991daa95 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1770,6 +1770,7 @@ class SparkContext(config: SparkConf) extends Logging { @DeveloperApi def getAllPools: Seq[Schedulable] = { assertNotStopped() + if (taskScheduler eq null) return Seq.empty // TODO(xiajunluan): We should take nested pools into account taskScheduler.rootPool.schedulableQueue.asScala.toSeq } diff --git a/core/src/main/scala/org/apache/spark/memory/MemoryPool.scala b/core/src/main/scala/org/apache/spark/memory/MemoryPool.scala index 1b9edf9c43bda..32c25f47225d9 100644 --- a/core/src/main/scala/org/apache/spark/memory/MemoryPool.scala +++ b/core/src/main/scala/org/apache/spark/memory/MemoryPool.scala @@ -36,14 +36,14 @@ private[memory] abstract class MemoryPool(lock: Object) { * Returns the current size of the pool, in bytes. */ final def poolSize: Long = lock.synchronized { - _poolSize + return _poolSize } /** * Returns the amount of free memory in the pool, in bytes. */ final def memoryFree: Long = lock.synchronized { - _poolSize - memoryUsed + return _poolSize - memoryUsed } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 5f2bc633812ba..903bfe580e76f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1504,7 +1504,7 @@ object CodeGenerator extends Logging { * weak keys/values and thus does not respond to memory pressure. */ private val cache = CacheBuilder.newBuilder() - .maximumSize(300) + .maximumSize(1000) .build( new CacheLoader[CodeAndComment, (GeneratedClass, Int)]() { override def load(code: CodeAndComment): (GeneratedClass, Int) = { @@ -1518,4 +1518,5 @@ object CodeGenerator extends Logging { result } }) + } } From 754161962527e9a4031870fc8c5cd5e097105a64 Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Thu, 28 Dec 2017 11:59:05 +0530 Subject: [PATCH 113/142] Snap 2084 (#86) If SnappyUMM is found in classpath , SparkEnv will assign the memory manager to SnappyUMM.If user has explicitly set the memory manager that will take precedence. Conflicts: core/src/main/scala/org/apache/spark/SparkEnv.scala --- .../scala/org/apache/spark/SparkEnv.scala | 3 +- .../org/apache/spark/SparkSnappyUtils.scala | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 core/src/main/scala/org/apache/spark/SparkSnappyUtils.scala diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index d7c1568ad58d7..e2f4d4606543f 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -370,12 +370,13 @@ object SparkEnv extends Logging { val useLegacyMemoryManager = conf.getBoolean("spark.memory.useLegacyMode", false) val memoryManager: MemoryManager = + SparkSnappyUtils.loadSnappyManager(conf, numUsableCores).getOrElse { if (useLegacyMemoryManager) { new StaticMemoryManager(conf, numUsableCores) } else { UnifiedMemoryManager(conf, numUsableCores) } - + } val blockManagerPort = if (isDriver) { conf.get(DRIVER_BLOCK_MANAGER_PORT) } else { diff --git a/core/src/main/scala/org/apache/spark/SparkSnappyUtils.scala b/core/src/main/scala/org/apache/spark/SparkSnappyUtils.scala new file mode 100644 index 0000000000000..0d6bc27147b4e --- /dev/null +++ b/core/src/main/scala/org/apache/spark/SparkSnappyUtils.scala @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package org.apache.spark + +import org.apache.spark.memory.MemoryManager +import org.apache.spark.util.Utils + + +object SparkSnappyUtils { + + val SNAPPY_UNIFIED_MEMORY_MANAGER_CLASS = "org.apache.spark.memory.SnappyUnifiedMemoryManager" + + def loadSnappyManager(conf: SparkConf, numUsableCores: Int): Option[MemoryManager] = { + try { + Some(Utils.classForName(SNAPPY_UNIFIED_MEMORY_MANAGER_CLASS) + .getConstructor(classOf[SparkConf], classOf[Int]) + .newInstance(conf, Int.box(numUsableCores)) + .asInstanceOf[MemoryManager]) + } catch { + case ex: ClassNotFoundException => None + } + } + +} From 6f1b86065be6079102229d099268f56ce6acbe45 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 9 Jan 2018 03:14:38 +0530 Subject: [PATCH 114/142] [SNAPPYDATA] some optimizations to ExecutionMemoryPool - avoid multiple lookups into the map in ExecutionMemoryPool.releaseMemory - avoid an unnecessary boxing/unboxing by adding explicit return from lock.synchronized blocks --- .../spark/memory/ExecutionMemoryPool.scala | 37 ++++++++++++------- .../sql/execution/WholeStageCodegenExec.scala | 4 +- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala b/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala index f1915857ea43a..ab783554f0b2c 100644 --- a/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala +++ b/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala @@ -56,14 +56,14 @@ private[memory] class ExecutionMemoryPool( private val memoryForTask = new mutable.HashMap[Long, Long]() override def memoryUsed: Long = lock.synchronized { - memoryForTask.values.sum + return memoryForTask.values.sum } /** * Returns the memory consumption, in bytes, for the given task. */ def getMemoryUsageForTask(taskAttemptId: Long): Long = lock.synchronized { - memoryForTask.getOrElse(taskAttemptId, 0L) + return memoryForTask.getOrElse(taskAttemptId, 0L) } /** @@ -99,10 +99,12 @@ private[memory] class ExecutionMemoryPool( // Add this task to the taskMemory map just so we can keep an accurate count of the number // of active tasks, to let other tasks ramp down their memory in calls to `acquireMemory` - if (!memoryForTask.contains(taskAttemptId)) { - memoryForTask(taskAttemptId) = 0L - // This will later cause waiting tasks to wake up and check numTasks again - lock.notifyAll() + var curMem = memoryForTask.get(taskAttemptId) match { + case Some(m) => m + case _ => memoryForTask(taskAttemptId) = 0L + // This will later cause waiting tasks to wake up and check numTasks again + lock.notifyAll() + 0L } // Keep looping until we're either sure that we don't want to grant this request (because this @@ -111,7 +113,6 @@ private[memory] class ExecutionMemoryPool( // TODO: simplify this to limit each task to its own slot while (true) { val numActiveTasks = memoryForTask.keys.size - val curMem = memoryForTask(taskAttemptId) // In every iteration of this loop, we should first try to reclaim any borrowed execution // space from storage. This is necessary because of the potential race condition where new @@ -138,30 +139,38 @@ private[memory] class ExecutionMemoryPool( if (toGrant < numBytes && curMem + toGrant < minMemoryPerTask) { logInfo(s"TID $taskAttemptId waiting for at least 1/2N of $poolName pool to be free") lock.wait() + curMem = memoryForTask(taskAttemptId) } else { memoryForTask(taskAttemptId) += toGrant return toGrant } } - 0L // Never reached + return 0L // Never reached } /** * Release `numBytes` of memory acquired by the given task. */ def releaseMemory(numBytes: Long, taskAttemptId: Long): Unit = lock.synchronized { - val curMem = memoryForTask.getOrElse(taskAttemptId, 0L) + val curMemOpt = memoryForTask.get(taskAttemptId) + var curMem = curMemOpt match { + case Some(m) => m + case _ => 0L + } var memoryToFree = if (curMem < numBytes) { + val mem = curMem logWarning( - s"Internal error: release called on $numBytes bytes but task only has $curMem bytes " + + s"Internal error: release called on $numBytes bytes but task only has $mem bytes " + s"of memory from the $poolName pool") curMem } else { numBytes } - if (memoryForTask.contains(taskAttemptId)) { - memoryForTask(taskAttemptId) -= memoryToFree - if (memoryForTask(taskAttemptId) <= 0) { + if (curMemOpt.isDefined) { + curMem -= memoryToFree + if (curMem > 0) { + memoryForTask(taskAttemptId) = curMem + } else { memoryForTask.remove(taskAttemptId) } } @@ -175,7 +184,7 @@ private[memory] class ExecutionMemoryPool( def releaseAllMemoryForTask(taskAttemptId: Long): Long = lock.synchronized { val numBytesToFree = getMemoryUsageForTask(taskAttemptId) releaseMemory(numBytesToFree, taskAttemptId) - numBytesToFree + return numBytesToFree } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala index e8405c798c4ab..c61c770b2fb42 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala @@ -784,7 +784,7 @@ case class WholeStageCodegenRDD(@transient sc: SparkContext, var source: CodeAnd output.writeInt(source.hashCode()) output.writeString(source.body) val comment = source.comment - output.writeVarInt(comment.size, true) + output.writeInt(comment.size) for ((k, v) <- comment) { output.writeString(k) output.writeString(v) @@ -812,7 +812,7 @@ case class WholeStageCodegenRDD(@transient sc: SparkContext, var source: CodeAnd val hash = input.readInt() val body = input.readString() - var commentSize = input.readVarInt(true) + var commentSize = input.readInt() val comment = new scala.collection.mutable.HashMap[String, String]() while (commentSize > 0) { val k = input.readString() From c03599e0027c552da987f5358b02c378b7874a5f Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 30 Jan 2018 15:17:42 +0530 Subject: [PATCH 115/142] [SNAPPYDATA] fixing all failures in snappy-spark test suite Three broad categories of issues fixed: - handling of double values in JSON conversion layer of the metrics; upstream spark has all metrics as Long but snappy-spark has the timings one as double to give more accurate results - library version differences between Spark's maven poms and SnappyData's gradle builds; these are as such not product issues but this checkin changes some versions to be matching to maven builds to be fully upstream compatible - path differences in test resource files/jars when run using gradle rather than using maven Other fixes and changes: - the optimized Decimal.equals gave incorrect result in case the scale of the two is different; this followed the Java BigDecimal convention of returning false if the scale is different but that is incorrect as per Spark's conventions; this should normally not happen from catalyst layer but can happen in RDD operations - correct accumulator result in Task to be empty rather than null when nothing present - override the extended two argument DStream.initialize in MapWithStateDStream.initialize - correct the UI path for Spark cache to be "/Spark Cache/" rather than "/storage/" - avoid sending the whole child plan across in DeserializeToObjectExec to executors when only the output is required (also see SNAP-1840 caused due to this) - rounding of some of the time statistics (that are accumulated as double) in Spark metrics - SparkListenerSuite local metrics tests frequently failed due to deserialization time being zero (despite above change); the reason being the optimizations in snappy-spark that allow it to run much quicker and not registering even with System.nanoTime(); now extended the closure to force a 1 milliseond sleep in its readExternal method - use spark.serializer consistently for data only and spark.closureSerializer for others (for the case the two are different) - don't allow inline message size to exceed spark.rpc.message.maxSize - revert default spark.locality.wait to be 3s in Spark (will be set at snappydata layer if required) - make SparkEnv.taskLogger to be serializable if required (extend Spark's Logging trait) - account for task decompression time in the deserialization time too The full spark test suite can be run either by: - ./dev/snappy-build.sh && ./dev/snappy-build.sh test (or equivalent) - ./gradlew check - from SnappyData: - ./gradlew snappy-spark:check, OR - ./gradlew precheckin -Pspark (for full test suite run including snappydata suite) For SnappyData product builds, one of the last two ways from SnappyData should be used Conflicts: core/src/main/scala/org/apache/spark/SparkContext.scala core/src/main/scala/org/apache/spark/SparkEnv.scala core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala core/src/main/scala/org/apache/spark/executor/Executor.scala core/src/main/scala/org/apache/spark/scheduler/Task.scala core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala core/src/main/scala/org/apache/spark/util/JsonProtocol.scala core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala --- build.gradle | 13 +++-- common/network-common/build.gradle | 2 +- common/network-shuffle/build.gradle | 2 +- common/network-yarn/build.gradle | 2 +- common/unsafe/pom.xml | 4 ++ core/build.gradle | 4 +- .../scala/org/apache/spark/SparkEnv.scala | 28 +++++++++- .../org/apache/spark/executor/Executor.scala | 34 ++++++------- .../spark/executor/ShuffleReadMetrics.scala | 2 +- .../apache/spark/executor/TaskMetrics.scala | 10 ++-- .../apache/spark/scheduler/DAGScheduler.scala | 12 ++++- .../org/apache/spark/scheduler/Task.scala | 51 +++++++++++-------- .../apache/spark/scheduler/TaskResult.scala | 2 +- .../spark/scheduler/TaskResultGetter.scala | 5 +- .../spark/scheduler/TaskSetManager.scala | 6 +-- .../storage/ShuffleBlockFetcherIterator.scala | 1 - .../org/apache/spark/util/JsonProtocol.scala | 13 ++--- .../spark/deploy/SparkSubmitSuite.scala | 5 +- .../scheduler/SchedulerIntegrationSuite.scala | 7 ++- .../spark/scheduler/SparkListenerSuite.scala | 22 +++++--- .../scheduler/TaskResultGetterSuite.scala | 4 +- .../org/apache/spark/ui/UISeleniumSuite.scala | 12 ++--- .../apache/spark/util/JsonProtocolSuite.scala | 12 ++--- dev/snappy-build.sh | 9 ++++ .../SparkSubmitCommandBuilderSuite.java | 6 ++- .../spark/ml/r/RWrapperUtilsSuite.scala | 7 ++- .../spark/repl/SingletonReplSuite.scala | 11 ++-- sql/catalyst/build.gradle | 4 +- .../org/apache/spark/sql/types/Decimal.scala | 22 +++----- .../catalyst/expressions/OrderingSuite.scala | 4 +- sql/core/build.gradle | 2 +- .../exchange/EnsureRequirements.scala | 4 +- .../apache/spark/sql/execution/objects.scala | 3 +- .../apache/spark/sql/JavaDatasetSuite.java | 2 +- .../apache/spark/sql/CachedTableSuite.scala | 3 +- .../org/apache/spark/sql/QueryTest.scala | 2 +- .../spark/sql/StatisticsCollectionSuite.scala | 4 +- .../spark/sql/execution/PlannerSuite.scala | 6 ++- .../HiveThriftServer2Suites.scala | 14 +++-- .../hive/execution/HiveComparisonTest.scala | 7 ++- .../dstream/MapWithStateDStream.scala | 4 +- .../streaming/StreamingContextSuite.scala | 5 ++ 42 files changed, 230 insertions(+), 142 deletions(-) create mode 100755 dev/snappy-build.sh diff --git a/build.gradle b/build.gradle index 3f5af8c43bef2..2ea2f044646a8 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,7 @@ allprojects { scalaBinaryVersion = '2.11' scalaVersion = scalaBinaryVersion + '.8' hadoopVersion = '2.7.3' - protobufVersion = '2.6.1' + protobufVersion = '2.5.0' jerseyVersion = '2.26' sunJerseyVersion = '1.19.4' jettyVersion = '9.2.22.v20170606' @@ -65,7 +65,7 @@ allprojects { javaxServletVersion = '3.1.0' guavaVersion = '14.0.1' hiveVersion = '1.2.1.spark2' - chillVersion = '0.8.1' + chillVersion = '0.8.4' kryoVersion = '4.0.1' nettyVersion = '3.10.6.Final' nettyAllVersion = '4.0.51.Final' @@ -73,13 +73,15 @@ allprojects { httpClientVersion = '4.5.3' httpCoreVersion = '4.4.7' jackson1Version = '1.9.13' - jacksonVersion = '2.9.1' + jacksonVersion = '2.6.7' + jacksonBindVersion = '2.6.7.1' snappyJavaVersion = '1.1.4' lz4Version = '1.4.0' lzfVersion = '1.0.4' parquetVersion = '1.8.2' // hiveParquetVersion = '1.6.0' metricsVersion = '3.2.5' + janinoVersion = '3.0.8' thriftVersion = '0.9.3' antlrVersion = '4.5.3' jpamVersion = '1.1' @@ -313,10 +315,11 @@ subprojects { 'spark.test.home': snappyProductDir, 'spark.project.home': "${project(sparkProjectRoot).projectDir}", 'spark.testing': '1', + 'spark.master.rest.enabled': 'false', 'spark.ui.enabled': 'false', 'spark.ui.showConsoleProgress': 'false', - 'spark.driver.allowMultipleContexts': 'true', - 'spark.unsafe.exceptionOnMemoryLeak': 'true' + 'spark.unsafe.exceptionOnMemoryLeak': 'true', + 'spark.memory.debugFill': 'true' testLogging.exceptionFormat = 'full' diff --git a/common/network-common/build.gradle b/common/network-common/build.gradle index 52dd2ec59b72e..4f81762413192 100644 --- a/common/network-common/build.gradle +++ b/common/network-common/build.gradle @@ -24,7 +24,7 @@ dependencies { compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version compile group: 'com.google.guava', name: 'guava', version: guavaVersion compile group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: '1.8' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion compile group: 'org.apache.commons', name: 'commons-lang3', version: commonsLang3Version } diff --git a/common/network-shuffle/build.gradle b/common/network-shuffle/build.gradle index de3cdc886be68..76e3bc7c281f4 100644 --- a/common/network-shuffle/build.gradle +++ b/common/network-shuffle/build.gradle @@ -22,7 +22,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) compile group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: '1.8' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion compile group: 'com.google.guava', name: 'guava', version: guavaVersion compile(group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion) { diff --git a/common/network-yarn/build.gradle b/common/network-yarn/build.gradle index c04629191cc78..61cb12776bd4d 100644 --- a/common/network-yarn/build.gradle +++ b/common/network-yarn/build.gradle @@ -49,7 +49,7 @@ dependencies { runtimeJar project(subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion) runtimeJar project(subprojectBase + 'snappy-spark-network-shuffle_' + scalaBinaryVersion) runtimeJar group: 'io.netty', name: 'netty-all', version: nettyAllVersion - runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion */ } diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 5613f5e81384c..715be536d6c9e 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -65,6 +65,10 @@ com.google.guava guava + + log4j + log4j + diff --git a/core/build.gradle b/core/build.gradle index c51c606aa4b93..aadc5e2e1a213 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -145,7 +145,7 @@ dependencies { exclude(group: 'org.slf4j', module: 'slf4j-api') exclude(group: 'org.slf4j', module: 'slf4j-log4j12') } - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion compile(group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_' + scalaBinaryVersion, version: jacksonVersion) { exclude(group: 'com.google.guava', module: 'guava') } @@ -164,7 +164,7 @@ dependencies { testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-htmlunit-driver', version: seleniumVersion) { exclude(group: 'com.google.guava', module: 'guava') } - testCompile group: 'xml-apis', name: 'xml-apis', version: '1.0.b2' + testCompile group: 'xml-apis', name: 'xml-apis', version: '1.4.01' testCompile group: 'org.hamcrest', name: 'hamcrest-core', version: '1.3' testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '1.3' testCompile(group: 'org.apache.curator', name: 'curator-test', version: curatorVersion) { diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index e2f4d4606543f..5aaa63b5a8d17 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -94,6 +94,9 @@ class SparkEnv ( private[spark] var isStopped = false private val pythonWorkers = mutable.HashMap[(String, Map[String, String]), PythonWorkerFactory]() + // This logger is used to do task related logging across multiple classes + private[spark] val taskLogger = new NamedLogger("org.apache.spark.Task") + // A general, soft-reference map for metadata needed during HadoopRDD split computation // (e.g., HadoopFileRDD uses this to cache JobConfs and InputFormats). private[spark] val hadoopJobMetadata = new MapMaker().softValues().makeMap[String, Any]() @@ -368,7 +371,7 @@ object SparkEnv extends Logging { shortShuffleMgrNames.getOrElse(shuffleMgrName.toLowerCase(Locale.ROOT), shuffleMgrName) val shuffleManager = instantiateClass[ShuffleManager](shuffleMgrClass) - val useLegacyMemoryManager = conf.getBoolean("spark.memory.useLegacyMode", false) + val useLegacyMemoryManager = conf.getBoolean("spark.memory.useLegacyMode", defaultValue = false) val memoryManager: MemoryManager = SparkSnappyUtils.loadSnappyManager(conf, numUsableCores).getOrElse { if (useLegacyMemoryManager) { @@ -497,3 +500,26 @@ object SparkEnv extends Logging { "Classpath Entries" -> classPaths) } } + +private[spark] class NamedLogger(override val logName: String) extends Logging with Serializable { + + override def logInfo(msg: => String): Unit = super.logInfo(msg) + + override def logDebug(msg: => String): Unit = super.logDebug(msg) + + override def logTrace(msg: => String): Unit = super.logTrace(msg) + + override def logWarning(msg: => String): Unit = super.logWarning(msg) + + override def logError(msg: => String): Unit = super.logError(msg) + + override def logInfo(msg: => String, t: Throwable): Unit = super.logInfo(msg, t) + + override def logDebug(msg: => String, t: Throwable): Unit = super.logDebug(msg, t) + + override def logTrace(msg: => String, t: Throwable): Unit = super.logTrace(msg, t) + + override def logWarning(msg: => String, t: Throwable): Unit = super.logWarning(msg, t) + + override def logError(msg: => String, t: Throwable): Unit = super.logError(msg, t) +} diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 0a9f572d698a5..ae33b9765ebab 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -251,7 +251,8 @@ private[spark] class Executor( val attemptNumber: Int, taskName: String, serializedTask: ByteBuffer, - taskDataBytes: Array[Byte]) + taskDataBytes: Array[Byte], + taskDecompressTime: Long) extends Runnable { val taskId = taskDescription.taskId @@ -410,16 +411,15 @@ private[spark] class Executor( // Deserialization happens in two parts: first, we deserialize a Task object, which // includes the Partition. Second, Task.run() deserializes the RDD and function to be run. - task.metrics.setExecutorDeserializeTime(math.max( - taskStart - deserializeStartTime + task.executorDeserializeTime, 0L) / 1000000.0) - task.metrics.setExecutorDeserializeCpuTime(math.max( - taskStartCpu - deserializeStartCpuTime + task.executorDeserializeCpuTime, 0L) / - 1000000.0) + task.metrics.setExecutorDeserializeTime(math.max(taskStart - deserializeStartTime + + task.executorDeserializeTime + taskDecompressTime, 0L).toDouble / 1000000.0) + task.metrics.setExecutorDeserializeCpuTime(math.max(taskStartCpu - + deserializeStartCpuTime + task.executorDeserializeCpuTime, 0L).toDouble / 1000000.0) // We need to subtract Task.run()'s deserialization time to avoid double-counting - task.metrics.setExecutorRunTime(math.max( - taskFinish - taskStart - task.executorDeserializeTime, 0L) / 1000000.0) - task.metrics.setExecutorCpuTime(math.max( - taskFinishCpu - taskStartCpu - task.executorDeserializeCpuTime, 0L) / 1000000.0) + task.metrics.setExecutorRunTime(math.max(taskFinish - taskStart - + task.executorDeserializeTime, 0L).toDouble / 1000000.0) + task.metrics.setExecutorCpuTime(math.max(taskFinishCpu - taskStartCpu - + task.executorDeserializeCpuTime, 0L).toDouble / 1000000.0) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) // Expose task metrics using the Dropwizard metrics system. @@ -467,8 +467,9 @@ private[spark] class Executor( val accumUpdates = task.collectAccumulatorUpdates() val directResult = new DirectTaskResult(value, accumUpdates, Some(task.metrics.resultSerializationTimeMetric)) - val serializedDirectResult = ser.serialize(directResult) - val resultSize = serializedDirectResult.limit() + val taskSer = env.serializer.newInstance() + val serializedDirectResult = taskSer.serialize(directResult) + val resultSize = serializedDirectResult.limit // directSend = sending directly back to the driver val serializedResult: ByteBuffer = { @@ -476,7 +477,7 @@ private[spark] class Executor( logWarning(s"Finished $taskName (TID $taskId). Result is larger than maxResultSize " + s"(${Utils.bytesToString(resultSize)} > ${Utils.bytesToString(maxResultSize)}), " + s"dropping it.") - ser.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize)) + taskSer.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize)) } else if (resultSize > maxDirectResultSize) { val blockId = TaskResultBlockId(taskId) env.blockManager.putBytes( @@ -485,7 +486,7 @@ private[spark] class Executor( StorageLevel.MEMORY_AND_DISK_SER) logInfo( s"Finished $taskName (TID $taskId). $resultSize bytes result sent via BlockManager)") - ser.serialize(new IndirectTaskResult[Any](blockId, resultSize)) + taskSer.serialize(new IndirectTaskResult[Any](blockId, resultSize)) } else { logInfo(s"Finished $taskName (TID $taskId). $resultSize bytes result sent to driver") serializedDirectResult @@ -532,7 +533,6 @@ private[spark] class Executor( logError(s"Store closed exception in $taskName (TID $taskId)", t) setTaskFinishedAndClearInterruptStatus() val reason = new ExecutorLostFailure(executorId, false, Some(t.getMessage)) - val ser = env.closureSerializer.newInstance() execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) case t: Throwable if isStoreException(t) => @@ -557,12 +557,12 @@ private[spark] class Executor( val accums: Seq[AccumulatorV2[_, _]] = if (task != null) { task.metrics.setExecutorRunTime( - math.max(System.nanoTime() - taskStart, 0L) / 1000000.0) + math.max(System.nanoTime() - taskStart, 0L).toDouble / 1000000.0) val taskEndCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { threadMXBean.getCurrentThreadCpuTime } else 0L task.metrics.setExecutorCpuTime( - math.max(taskEndCpu - taskStartCpu, 0L) / 1000000.0) + math.max(taskEndCpu - taskStartCpu, 0L).toDouble / 1000000.0) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) task.collectAccumulatorUpdates(taskFailed = true) } else { diff --git a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala index 6b49c8cbeb629..ca27ef1ca145d 100644 --- a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala @@ -70,7 +70,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable with KryoSeriali * blocking on shuffle input data. For instance if block B is being fetched while the task is * still not finished processing block A, it is not considered to be blocking on block B. */ - def fetchWaitTime: Long = _fetchWaitTime.sum.toLong + def fetchWaitTime: Long = math.round(_fetchWaitTime.sum) /** * Total number of records read from the shuffle by this task. diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala index e126d9b495d65..4b45b0bf996b1 100644 --- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala @@ -62,23 +62,23 @@ class TaskMetrics private[spark] () extends Serializable with KryoSerializable { /** * Time taken on the executor to deserialize this task. */ - def executorDeserializeTime: Long = _executorDeserializeTime.sum.toLong + def executorDeserializeTime: Long = math.round(_executorDeserializeTime.sum) /** * CPU Time taken on the executor to deserialize this task in nanoseconds. */ - def executorDeserializeCpuTime: Long = _executorDeserializeCpuTime.sum.toLong + def executorDeserializeCpuTime: Long = math.round(_executorDeserializeCpuTime.sum) /** * Time the executor spends actually running the task (including fetching shuffle data). */ - def executorRunTime: Long = _executorRunTime.sum.toLong + def executorRunTime: Long = math.round(_executorRunTime.sum) /** * CPU Time the executor spends actually running the task * (including fetching shuffle data) in nanoseconds. */ - def executorCpuTime: Long = _executorCpuTime.sum.toLong + def executorCpuTime: Long = math.round(_executorCpuTime.sum) /** * The number of bytes this task transmitted back to the driver as the TaskResult. @@ -93,7 +93,7 @@ class TaskMetrics private[spark] () extends Serializable with KryoSerializable { /** * Amount of time spent serializing the task result. */ - def resultSerializationTime: Long = _resultSerializationTime.sum.toLong + def resultSerializationTime: Long = math.round(_resultSerializationTime.sum) /** * The number of in-memory bytes spilled by this task. diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index adcfb4eaf932d..c12ce76d8ea98 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -183,6 +183,8 @@ class DAGScheduler( // This is only safe because DAGScheduler runs in a single thread. private val closureSerializer = SparkEnv.get.closureSerializer.newInstance() + private lazy val maxRpcMessageSize = RpcUtils.maxMessageSizeBytes(sc.conf) + /** If enabled, FetchFailed will not cause stage retry, in order to surface the problem. */ private val disallowStageRetryForTest = sc.getConf.getBoolean("spark.test.noStageRetry", false) @@ -1039,7 +1041,9 @@ class DAGScheduler( // use direct byte shipping for small size or if number of partitions is small val taskBytesLen = taskBinaryBytes.length if (taskBytesLen <= DAGScheduler.TASK_INLINE_LIMIT || - partitionsToCompute.length <= DAGScheduler.TASK_INLINE_PARTITION_LIMIT) { + partitionsToCompute.length == 1 || + (taskBytesLen < math.min(maxRpcMessageSize, DAGScheduler.TASK_INLINE_UPPER_LIMIT) && + partitionsToCompute.length <= DAGScheduler.TASK_INLINE_PARTITION_LIMIT)) { if (stage.taskData.uncompressedLen > 0) { taskData = stage.taskData } else { @@ -1864,7 +1868,11 @@ private[spark] object DAGScheduler { // The maximum size of uncompressed common task bytes (rdd, closure) // that will be shipped with the task else will be broadcast separately. - val TASK_INLINE_LIMIT = 100 * 1024 + val TASK_INLINE_LIMIT: Int = 128 * 1024 + + // Maximum size beyond which common task bytes will always be broadcast even if number + // of partitions is smaller than TASK_INLINE_PARTITION_LIMIT (except if it is 1) + val TASK_INLINE_UPPER_LIMIT: Int = 4 * 1024 * 1024 // The maximum number of partitions below which common task bytes will be // shipped with the task else will be broadcast separately. diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index ac0586cb6066d..b152b977db6f6 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -103,9 +103,9 @@ private[spark] abstract class Task[T]( * @return the result of the task along with updates of Accumulators. */ final def run( - taskAttemptId: Long, - attemptNumber: Int, - metricsSystem: MetricsSystem): T = { + taskAttemptId: Long, + attemptNumber: Int, + metricsSystem: MetricsSystem): T = { SparkEnv.get.blockManager.registerTask(taskAttemptId) context = new TaskContextImpl( stageId, @@ -304,27 +304,33 @@ private[spark] object Task { * Serialize a task and the current app dependencies (files and JARs added to the SparkContext) */ def serializeWithDependencies( - task: Task[_], - currentFiles: mutable.Map[String, Long], - currentJars: mutable.Map[String, Long], - serializer: SerializerInstance) - : ByteBuffer = { + task: Task[_], + currentFiles: mutable.Map[String, Long], + currentJars: mutable.Map[String, Long], + serializer: SerializerInstance) + : ByteBuffer = { val out = new ByteBufferOutputStream(4096) val dataOut = new DataOutputStream(out) // Write currentFiles - dataOut.writeInt(currentFiles.size) - for ((name, timestamp) <- currentFiles) { - dataOut.writeUTF(name) - dataOut.writeLong(timestamp) + val numFiles = currentFiles.size + dataOut.writeInt(numFiles) + if (numFiles > 0) { + for ((name, timestamp) <- currentFiles) { + dataOut.writeUTF(name) + dataOut.writeLong(timestamp) + } } // Write currentJars - dataOut.writeInt(currentJars.size) - for ((name, timestamp) <- currentJars) { - dataOut.writeUTF(name) - dataOut.writeLong(timestamp) + val numJars = currentJars.size + dataOut.writeInt(numJars) + if (numJars > 0) { + for ((name, timestamp) <- currentJars) { + dataOut.writeUTF(name) + dataOut.writeLong(timestamp) + } } // Write the task properties separately so it is available before full task deserialization. @@ -345,10 +351,10 @@ private[spark] object Task { * and return the task itself as a serialized ByteBuffer. The caller can then update its * ClassLoaders and deserialize the task. * - * @return (taskFiles, taskJars, taskBytes) + * @return (taskFiles, taskJars, taskProps, taskBytes) */ def deserializeWithDependencies(serializedTask: ByteBuffer) - : (HashMap[String, Long], HashMap[String, Long], Properties, ByteBuffer) = { + : (HashMap[String, Long], HashMap[String, Long], Properties, ByteBuffer) = { val in = new ByteBufferInputStream(serializedTask) val dataIn = new DataInputStream(in) @@ -387,14 +393,15 @@ private[spark] final class TaskData private(var compressedBytes: Array[Byte], @transient private var decompressed: Array[Byte] = _ /** decompress the common task data if present */ - def decompress(env: SparkEnv = SparkEnv.get): Array[Byte] = { + def decompress(env: SparkEnv = SparkEnv.get): (Array[Byte], Long) = { if (uncompressedLen > 0) { if (decompressed eq null) { + val startDecompression = System.nanoTime() decompressed = env.createCompressionCodec.decompress(compressedBytes, 0, compressedBytes.length, uncompressedLen) - } - decompressed - } else TaskData.EMPTY_BYTES + decompressed -> math.max(System.nanoTime() - startDecompression, 0L) + } else decompressed -> 0L + } else TaskData.EMPTY_BYTES -> 0L } override def hashCode(): Int = java.util.Arrays.hashCode(compressedBytes) diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala index 5b564366439a8..bb4565f91d919 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala @@ -100,7 +100,7 @@ private[spark] class DirectTaskResult[T]( var numUpdates = input.readVarInt(true) val hasTimeMetric = input.readBoolean() if (numUpdates == 0 && !hasTimeMetric) { - accumUpdates = null + accumUpdates = Seq.empty } else { val _accumUpdates = new ArrayBuffer[AccumulatorV2[_, _]]( if (hasTimeMetric) numUpdates + 1 else numUpdates) diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala index fa5f6592ba92d..995cbd9a4bde9 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala @@ -61,7 +61,8 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul getTaskResultExecutor.execute(new Runnable { override def run(): Unit = Utils.logUncaughtExceptions { try { - val (result, size) = serializer.get().deserialize[TaskResult[_]](serializedData) match { + val resultSerializer = taskResultSerializer.get() + val (result, size) = resultSerializer.deserialize[TaskResult[_]](serializedData) match { case directResult: DirectTaskResult[_] => if (!taskSetManager.canFetchMoreResults(serializedData.limit())) { return @@ -88,7 +89,7 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul taskSetManager, tid, TaskState.FINISHED, TaskResultLost) return } - val deserializedResult = serializer.get().deserialize[DirectTaskResult[_]]( + val deserializedResult = resultSerializer.deserialize[DirectTaskResult[_]]( serializedTaskResult.get.toByteBuffer) // force deserialization of referenced value deserializedResult.value() diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index f65b8a7292cc2..e44a4053ca04b 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -403,7 +403,7 @@ private[spark] class TaskSetManager( * @return An option containing (task index within the task set, locality, is speculative?) */ private def dequeueTask(execId: String, host: String, maxLocality: TaskLocality.Value) - : Option[(Int, TaskLocality.Value, Boolean)] = + : Option[(Int, TaskLocality.Value, Boolean)] = { for (index <- dequeueTaskFromList(execId, host, getPendingTasksForExecutor(execId))) { return Some((index, TaskLocality.PROCESS_LOCAL, false)) @@ -1005,7 +1005,7 @@ private[spark] class TaskSetManager( private def getLocalityWait(level: TaskLocality.TaskLocality): Long = { val defaultWait = conf.get(config.LOCALITY_WAIT) - // val defaultWait = conf.get("spark.locality.wait", "10s") + // val defaultWait = conf.get("spark.locality.wait", "3s") val localityWaitKey = level match { case TaskLocality.PROCESS_LOCAL => "spark.locality.wait.process" case TaskLocality.NODE_LOCAL => "spark.locality.wait.node" @@ -1063,5 +1063,5 @@ private[spark] class TaskSetManager( private[spark] object TaskSetManager { // The user will be warned if any stages contain a task that has a serialized size greater than // this. - val TASK_SIZE_TO_WARN_KB = 128 + val TASK_SIZE_TO_WARN_KB = 512 } diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala index 5df11ddcaad06..20bec146edc79 100644 --- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala +++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala @@ -380,7 +380,6 @@ final class ShuffleBlockFetcherIterator( } numBlocksProcessed += 1 - var result: FetchResult = null var input: InputStream = null // Take the next fetched result and try to decompress it to detect data corruption, diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala index ea811f45086c2..368084cc61c3a 100644 --- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala +++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala @@ -786,6 +786,7 @@ private[spark] object JsonProtocol { if (name.exists(_.startsWith(InternalAccumulator.METRICS_PREFIX))) { value match { case JInt(v) => v.toLong + case JDouble(v) => v case JArray(v) => v.map { blockJson => val id = BlockId((blockJson \ "Block ID").extract[String]) @@ -805,19 +806,19 @@ private[spark] object JsonProtocol { if (json == JNothing) { return metrics } - metrics.setExecutorDeserializeTime((json \ "Executor Deserialize Time").extract[Long]) + metrics.setExecutorDeserializeTime((json \ "Executor Deserialize Time").extract[Double]) metrics.setExecutorDeserializeCpuTime((json \ "Executor Deserialize CPU Time") match { case JNothing => 0 - case x => x.extract[Long] + case x => x.extract[Double] }) - metrics.setExecutorRunTime((json \ "Executor Run Time").extract[Long]) + metrics.setExecutorRunTime((json \ "Executor Run Time").extract[Double]) metrics.setExecutorCpuTime((json \ "Executor CPU Time") match { case JNothing => 0 - case x => x.extract[Long] + case x => x.extract[Double] }) metrics.setResultSize((json \ "Result Size").extract[Long]) metrics.setJvmGCTime((json \ "JVM GC Time").extract[Long]) - metrics.setResultSerializationTime((json \ "Result Serialization Time").extract[Long]) + metrics.setResultSerializationTime((json \ "Result Serialization Time").extract[Double]) metrics.incMemoryBytesSpilled((json \ "Memory Bytes Spilled").extract[Long]) metrics.incDiskBytesSpilled((json \ "Disk Bytes Spilled").extract[Long]) @@ -831,7 +832,7 @@ private[spark] object JsonProtocol { .foreach { v => readMetrics.incRemoteBytesReadToDisk(v.extract[Long])} readMetrics.incLocalBytesRead( Utils.jsonOption(readJson \ "Local Bytes Read").map(_.extract[Long]).getOrElse(0L)) - readMetrics.incFetchWaitTime((readJson \ "Fetch Wait Time").extract[Long]) + readMetrics.incFetchWaitTime((readJson \ "Fetch Wait Time").extract[Double]) readMetrics.incRecordsRead( Utils.jsonOption(readJson \ "Total Records Read").map(_.extract[Long]).getOrElse(0L)) metrics.mergeShuffleReadMetrics() diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 27dd435332348..8d488aed8749f 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -1054,10 +1054,11 @@ object SparkSubmitSuite extends SparkFunSuite with TimeLimits { def runSparkSubmit(args: Seq[String], root: String = ".."): Unit = { val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!")) val sparkSubmitFile = if (Utils.isWindows) { - new File(s"$root\\bin\\spark-submit.cmd") + new File(s"$sparkHome\\bin\\spark-submit.cmd") } else { - new File(s"$root/bin/spark-submit") + new File(s"$sparkHome/bin/spark-submit") } + val process = Utils.executeCommand( Seq(sparkSubmitFile.getCanonicalPath) ++ args, new File(sparkHome), diff --git a/core/src/test/scala/org/apache/spark/scheduler/SchedulerIntegrationSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/SchedulerIntegrationSuite.scala index 75ea409e16b4b..0be51478e9476 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/SchedulerIntegrationSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/SchedulerIntegrationSuite.scala @@ -306,9 +306,7 @@ private[spark] abstract class MockBackend( * updates some internal state for this mock. */ def taskSuccess(task: TaskDescription, result: Any): Unit = { - val ser = env.serializer.newInstance() - val resultBytes = ser.serialize(result) - val directResult = new DirectTaskResult(resultBytes, Seq()) // no accumulator updates + val directResult = new DirectTaskResult(result, Seq()) // no accumulator updates taskUpdate(task, TaskState.FINISHED, directResult) } @@ -325,7 +323,8 @@ private[spark] abstract class MockBackend( } def taskUpdate(task: TaskDescription, state: TaskState, result: Any): Unit = { - val ser = env.serializer.newInstance() + val ser = if (state == TaskState.FINISHED) env.serializer.newInstance() + else env.closureSerializer.newInstance() val resultBytes = ser.serialize(result) // statusUpdate is safe to call from multiple threads, its protected inside taskScheduler taskScheduler.statusUpdate(task.taskId, state, resultBytes) diff --git a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala index da6ecb82c7e42..02ec653777dcf 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.scheduler +import java.io.{ObjectInput, ObjectOutput} import java.util.concurrent.Semaphore import scala.collection.JavaConverters._ @@ -295,13 +296,7 @@ class SparkListenerSuite extends SparkFunSuite with LocalSparkContext with Match sc.addSparkListener(listener) sc.addSparkListener(new StatsReportListener) // just to make sure some of the tasks take a noticeable amount of time - val w = { i: Int => - if (i == 0) { - Thread.sleep(100) - } - i - } - + val w = new WaitForTask val numSlices = 16 val d = sc.parallelize(0 to 10000, numSlices).map(w) d.count() @@ -583,3 +578,16 @@ private class FirehoseListenerThatAcceptsSparkConf(conf: SparkConf) extends Spar case _ => } } + +class WaitForTask extends (Int => Int) with java.io.Externalizable { + override def apply(i: Int): Int = { + if (i == 0) { + Thread.sleep(100) + } + i + } + + override def writeExternal(out: ObjectOutput): Unit = {} + + override def readExternal(in: ObjectInput): Unit = Thread.sleep(1) +} diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala index 1bddba8f6c82b..9b9eb500a35e2 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala @@ -56,7 +56,7 @@ private class ResultDeletingTaskResultGetter(sparkEnv: SparkEnv, scheduler: Task if (!removedResult) { // Only remove the result once, since we'd like to test the case where the task eventually // succeeds. - serializer.get().deserialize[TaskResult[_]](serializedData) match { + taskResultSerializer.get().deserialize[TaskResult[_]](serializedData) match { case IndirectTaskResult(blockId, size) => sparkEnv.blockManager.master.removeBlock(blockId) // removeBlock is asynchronous. Need to wait it's removed successfully @@ -97,7 +97,7 @@ private class MyTaskResultGetter(env: SparkEnv, scheduler: TaskSchedulerImpl) override def enqueueSuccessfulTask(tsm: TaskSetManager, tid: Long, data: ByteBuffer): Unit = { // work on a copy since the super class still needs to use the buffer val newBuffer = data.duplicate() - _taskResults += env.closureSerializer.newInstance().deserialize[DirectTaskResult[_]](newBuffer) + _taskResults += env.serializer.newInstance().deserialize[DirectTaskResult[_]](newBuffer) super.enqueueSuccessfulTask(tsm, tid, data) } } diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 326546787ab6c..5414d37a8de3a 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -119,12 +119,12 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B val rdd = sc.parallelize(Seq(1, 2, 3)) rdd.persist(StorageLevels.DISK_ONLY).count() eventually(timeout(5 seconds), interval(50 milliseconds)) { - goToUi(ui, "/storage") + goToUi(ui, "/Spark Cache") val tableRowText = findAll(cssSelector("#storage-by-rdd-table td")).map(_.text).toSeq tableRowText should contain (StorageLevels.DISK_ONLY.description) } eventually(timeout(5 seconds), interval(50 milliseconds)) { - goToUi(ui, "/storage/rdd/?id=0") + goToUi(ui, "/Spark Cache/rdd/?id=0") val tableRowText = findAll(cssSelector("#rdd-storage-by-block-table td")).map(_.text).toSeq tableRowText should contain (StorageLevels.DISK_ONLY.description) } @@ -138,12 +138,12 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B rdd.unpersist() rdd.persist(StorageLevels.MEMORY_ONLY).count() eventually(timeout(5 seconds), interval(50 milliseconds)) { - goToUi(ui, "/storage") + goToUi(ui, "/Spark Cache") val tableRowText = findAll(cssSelector("#storage-by-rdd-table td")).map(_.text).toSeq tableRowText should contain (StorageLevels.MEMORY_ONLY.description) } eventually(timeout(5 seconds), interval(50 milliseconds)) { - goToUi(ui, "/storage/rdd/?id=0") + goToUi(ui, "/Spark Cache/rdd/?id=0") val tableRowText = findAll(cssSelector("#rdd-storage-by-block-table td")).map(_.text).toSeq tableRowText should contain (StorageLevels.MEMORY_ONLY.description) } @@ -474,7 +474,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B goToUi(sc, "") find(cssSelector("""ul li a[href*="jobs"]""")) should not be(None) find(cssSelector("""ul li a[href*="stages"]""")) should not be(None) - find(cssSelector("""ul li a[href*="storage"]""")) should not be(None) + find(cssSelector("""ul li a[href*="Spark Cache"]""")) should not be(None) find(cssSelector("""ul li a[href*="environment"]""")) should not be(None) find(cssSelector("""ul li a[href*="foo"]""")) should not be(None) } @@ -488,7 +488,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B goToUi(sc, "") find(cssSelector("""ul li a[href*="jobs"]""")) should not be(None) find(cssSelector("""ul li a[href*="stages"]""")) should not be(None) - find(cssSelector("""ul li a[href*="storage"]""")) should not be(None) + find(cssSelector("""ul li a[href*="Spark Cache"]""")) should not be(None) find(cssSelector("""ul li a[href*="environment"]""")) should not be(None) find(cssSelector("""ul li a[href*="foo"]""")) should be(None) } diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index 4abbb8e7894f5..5071b625979a2 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -1811,14 +1811,14 @@ private[spark] object JsonProtocolSuite extends Assertions { | { | "ID": 0, | "Name": "$EXECUTOR_DESERIALIZE_TIME", - | "Update": 300, + | "Update": 300.0, | "Internal": true, | "Count Failed Values": true | }, | { | "ID": 1, | "Name": "$EXECUTOR_DESERIALIZE_CPU_TIME", - | "Update": 300, + | "Update": 300.0, | "Internal": true, | "Count Failed Values": true | }, @@ -1826,14 +1826,14 @@ private[spark] object JsonProtocolSuite extends Assertions { | { | "ID": 2, | "Name": "$EXECUTOR_RUN_TIME", - | "Update": 400, + | "Update": 400.0, | "Internal": true, | "Count Failed Values": true | }, | { | "ID": 3, | "Name": "$EXECUTOR_CPU_TIME", - | "Update": 400, + | "Update": 400.0, | "Internal": true, | "Count Failed Values": true | }, @@ -1854,7 +1854,7 @@ private[spark] object JsonProtocolSuite extends Assertions { | { | "ID": 6, | "Name": "$RESULT_SERIALIZATION_TIME", - | "Update": 700, + | "Update": 700.0, | "Internal": true, | "Count Failed Values": true | }, @@ -1938,7 +1938,7 @@ private[spark] object JsonProtocolSuite extends Assertions { | { | "ID": 16, | "Name": "${shuffleRead.FETCH_WAIT_TIME}", - | "Update": 0, + | "Update": 0.0, | "Internal": true, | "Count Failed Values": true | }, diff --git a/dev/snappy-build.sh b/dev/snappy-build.sh new file mode 100755 index 0000000000000..f3a581de8a5de --- /dev/null +++ b/dev/snappy-build.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +flags="-Pyarn -Phive-thriftserver -Phadoop-2.7 -Dhadoop.version=2.7.3" + +if [ -z "$1" ]; then + ./build/mvn $flags -DskipTests package +else + ./build/mvn $flags "$@" +fi diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index 2e050f8413074..376c013be1cc8 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -236,7 +236,11 @@ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) th launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver"); launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native"); } else { - launcher.childEnv.put("SPARK_CONF_DIR", System.getProperty("spark.test.home") + String projectHome = System.getProperty("spark.project.home"); + if (projectHome == null) { + projectHome = System.getProperty("spark.test.home"); + } + launcher.childEnv.put("SPARK_CONF_DIR", projectHome + "/launcher/src/test/resources"); } diff --git a/mllib/src/test/scala/org/apache/spark/ml/r/RWrapperUtilsSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/r/RWrapperUtilsSuite.scala index 27b03918d951e..a3c3c34533c6d 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/r/RWrapperUtilsSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/r/RWrapperUtilsSuite.scala @@ -25,8 +25,11 @@ class RWrapperUtilsSuite extends SparkFunSuite with MLlibTestSparkContext { test("avoid libsvm data column name conflicting") { val rFormula = new RFormula().setFormula("label ~ features") - val data = spark.read.format("libsvm").load("../data/mllib/sample_libsvm_data.txt") - + val dataDir = sys.props.get("spark.project.home") match { + case Some(h) => h + case None => ".." + } + val data = spark.read.format("libsvm").load(s"$dataDir/data/mllib/sample_libsvm_data.txt") // if not checking column name, then IllegalArgumentException intercept[IllegalArgumentException] { rFormula.fit(data) diff --git a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala index ec3d790255ad3..0ec441cf4f45a 100644 --- a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala +++ b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala @@ -369,11 +369,15 @@ class SingletonReplSuite extends SparkFunSuite { } test("should clone and clean line object in ClosureCleaner") { - val output = runInterpreter( + val projectDir = sys.props.get("spark.project.home") match { + case Some(h) => s"$h/repl" + case None => "." + } + val command = """ |import org.apache.spark.rdd.RDD | - |val lines = sc.textFile("pom.xml") + |val lines = sc.textFile("$$projectDir/pom.xml") |case class Data(s: String) |val dataRDD = lines.map(line => Data(line.take(3))) |dataRDD.cache.count @@ -390,7 +394,8 @@ class SingletonReplSuite extends SparkFunSuite { |val deviation = math.abs(cacheSize2 - cacheSize1).toDouble / cacheSize1 |assert(deviation < 0.2, | s"deviation too large: $deviation, first size: $cacheSize1, second size: $cacheSize2") - """.stripMargin) + """.stripMargin.replace("$$projectDir", projectDir) + val output = runInterpreterInPasteMode("local-cluster[1,4,4096]", command) assertDoesNotContain("AssertionError", output) assertDoesNotContain("Exception", output) } diff --git a/sql/catalyst/build.gradle b/sql/catalyst/build.gradle index 9f2b52ac4ca2d..97cbbd9c11b14 100644 --- a/sql/catalyst/build.gradle +++ b/sql/catalyst/build.gradle @@ -26,8 +26,8 @@ dependencies { compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion // compile group: 'org.scala-lang.modules', name: 'scala-parser-combinators_' + scalaBinaryVersion, version: '1.0.4' - compile group: 'org.codehaus.janino', name: 'janino', version: '3.0.7' - compile group: 'org.codehaus.janino', name: 'commons-compiler', version: '3.0.7' + compile group: 'org.codehaus.janino', name: 'janino', version: janinoVersion + compile group: 'org.codehaus.janino', name: 'commons-compiler', version: janinoVersion compile group: 'org.antlr', name: 'antlr4-runtime', version: antlrVersion compile group: 'commons-codec', name: 'commons-codec', version: commonsCodecVersion antlr group: 'org.antlr', name: 'antlr4', version: antlrVersion diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala index 98f612e8c24ba..fb1b7b874d53a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.AnalysisException final class Decimal extends Ordered[Decimal] with Serializable { import org.apache.spark.sql.types.Decimal._ - private var decimalVal: BigDecimal = null + private var decimalVal: BigDecimal = _ private var longVal: Long = 0L private var _precision: Int = 1 private var _scale: Int = 0 @@ -346,21 +346,11 @@ final class Decimal extends Ordered[Decimal] with Serializable { override def hashCode(): Int = toBigDecimal.hashCode() def equals(other: Decimal): Boolean = { - if (other != null) { - val decimalVal = this.decimalVal - val otherDecimalVal = other.decimalVal - if (decimalVal eq null) { - if (otherDecimalVal eq null) { - if (_scale == other._scale) longVal == other.longVal - else toJavaBigDecimal.equals(other.toJavaBigDecimal) - } else { - toJavaBigDecimal.equals(otherDecimalVal.bigDecimal) - } - } else if (otherDecimalVal ne null) { - decimalVal.bigDecimal.equals(otherDecimalVal.bigDecimal) - } else { - decimalVal.bigDecimal.equals(other.toJavaBigDecimal) - } + if (other ne null) { + if (_scale == other._scale) { + if ((decimalVal eq null) && (other.decimalVal eq null)) longVal == other.longVal + else toJavaBigDecimal.equals(other.toJavaBigDecimal) + } else toJavaBigDecimal.compareTo(other.toJavaBigDecimal) == 0 } else false } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala index d0604b8eb7675..a1fad43430d54 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala @@ -134,8 +134,8 @@ class OrderingSuite extends SparkFunSuite with ExpressionEvalHelper { // this is passing prior to SPARK-16845, and it should also be passing after SPARK-16845 GenerateOrdering.generate(Array.fill(40)(sortOrder)) - // verify that we can support up to 5000 ordering comparisons, which should be sufficient - GenerateOrdering.generate(Array.fill(5000)(sortOrder)) + // verify that we can support up to 4000 ordering comparisons, which should be sufficient + GenerateOrdering.generate(Array.fill(4000)(sortOrder)) } test("SPARK-21344: BinaryType comparison does signed byte array comparison") { diff --git a/sql/core/build.gradle b/sql/core/build.gradle index 04032503641c4..061d935b18018 100644 --- a/sql/core/build.gradle +++ b/sql/core/build.gradle @@ -27,7 +27,7 @@ dependencies { compile group: 'org.apache.parquet', name: 'parquet-column', version: parquetVersion compile group: 'org.apache.parquet', name: 'parquet-hadoop', version: parquetVersion compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala index 6fb19a67401c8..a270b13810a45 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala @@ -200,10 +200,10 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { // number of partitions. Otherwise, we use maxChildrenNumPartitions. if (shufflesAllChildren) defaultNumPreShufflePartitions else maxChildrenNumPartitions } + children.zip(requiredChildDistributions).map { case (child, distribution) => - val targetPartitioning = createPartitioning(distribution, - numPartitions) + val targetPartitioning = createPartitioning(distribution, numPartitions) if (child.outputPartitioning.guarantees(targetPartitioning)) { child } else { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala index 03d1bbf2ab882..a58401130a14e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala @@ -86,8 +86,9 @@ case class DeserializeToObjectExec( } override protected def doExecute(): RDD[InternalRow] = { + val output = child.output child.execute().mapPartitionsWithIndexInternal { (index, iter) => - val projection = GenerateSafeProjection.generate(deserializer :: Nil, child.output) + val projection = GenerateSafeProjection.generate(deserializer :: Nil, output) projection.initialize(index) iter.map(projection) } diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java index ae07ac2fae942..4a26b4a6bb5b5 100644 --- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java +++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java @@ -982,7 +982,7 @@ private void checkNestedBeansResult(List rows) { row.getAs("booleanField")); byte[] bytesValue = new byte[k]; Arrays.fill(bytesValue, (byte)k); - Assert.assertTrue(Arrays.equals(bytesValue, row.getAs("binaryField"))); + Assert.assertTrue(Arrays.equals(bytesValue, (byte[])row.getAs("binaryField"))); Assert.assertEquals("Date field match not as expected", new Date(7836L * k * 1000L).toString(), row.getAs("date").toString()); Assert.assertEquals("TimeStamp field match not as expected", diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala index 669e5f2bf4e65..3f67fad0504b2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala @@ -522,7 +522,8 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with SharedSQLContext spark.catalog.cacheTable("t2") val query = sql("SELECT key, value, a, b FROM t1 t1 JOIN t2 t2 ON t1.key = t2.a") - verifyNumExchanges(query, 2) + // SNAP: expect 1 exchanges here instead of 2 due to changes for SNAP-1251 + verifyNumExchanges(query, 1) checkAnswer( query, testData.join(testData2, $"key" === $"a").select($"key", $"value", $"a", $"b")) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala index aaaf4c3acb667..09c3850707d01 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala @@ -294,7 +294,7 @@ object QueryTest { case null => null case d: java.math.BigDecimal => BigDecimal(d) case d: Decimal => d.toBigDecimal // to use BigDecimal.compareTo == 0 - case d: Double => math.floor(d * 1000.0 + 0.5) // round to three digits + case d: Double => math.floor(d * 1000.0 + 0.5) / 1000.0 // round to three digits // Convert array to Seq for easy equality check. case b: Array[_] => b.toSeq case r: Row => prepareRow(r) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala index b11e798532056..f2da3f5674fec 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala @@ -46,8 +46,8 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared } assert(sizes.size === 1, s"number of Join nodes is wrong:\n ${df.queryExecution}") - assert(sizes.head === BigInt(96), - s"expected exact size 96 for table 'test', got: ${sizes.head}") + assert(sizes.head === BigInt(97), + s"expected exact size 97 for table 'test', got: ${sizes.head}") } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala index f8b26f5b28cc7..6bf69077004f8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala @@ -216,7 +216,8 @@ class PlannerSuite extends SharedSQLContext { ).queryExecution.executedPlan.collect { case exchange: ShuffleExchangeExec => exchange }.length - assert(numExchanges === 5) + // SNAP: expect 3 exchanges here instead of 5 due to changes for SNAP-1251 + assert(numExchanges === 3) } { @@ -231,7 +232,8 @@ class PlannerSuite extends SharedSQLContext { ).queryExecution.executedPlan.collect { case exchange: ShuffleExchangeExec => exchange }.length - assert(numExchanges === 5) + // SNAP: expect 3 exchanges here instead of 5 due to changes for SNAP-1251 + assert(numExchanges === 3) } } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 3af4630076b1d..0159dc8f5af63 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -575,8 +575,11 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { test("SPARK-11595 ADD JAR with input path having URL scheme") { withJdbcStatement("test_udtf") { statement => try { - val jarPath = "../hive/src/test/resources/TestUDTF.jar" - val jarURL = s"file://${System.getProperty("user.dir")}/$jarPath" + val jarPath = sys.props.get("spark.project.home") match { + case Some(h) => s"$h/sql/hive/src/test/resources/TestUDTF.jar" + case _ => s"${System.getProperty("user.dir")}/../hive/src/test/resources/TestUDTF.jar" + } + val jarURL = s"file://$jarPath" Seq( s"ADD JAR $jarURL", @@ -642,8 +645,11 @@ class SingleSessionSuite extends HiveThriftJdbcTest { test("share the temporary functions across JDBC connections") { withMultipleConnectionJdbcStatement()( { statement => - val jarPath = "../hive/src/test/resources/TestUDTF.jar" - val jarURL = s"file://${System.getProperty("user.dir")}/$jarPath" + val jarPath = sys.props.get("spark.project.home") match { + case Some(h) => s"$h/sql/hive/src/test/resources/TestUDTF.jar" + case _ => s"${System.getProperty("user.dir")}/../hive/src/test/resources/TestUDTF.jar" + } + val jarURL = s"file://$jarPath" // Configurations and temporary functions added in this session should be visible to all // the other sessions. diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala index cee82cda4628a..457798d02351e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala @@ -98,8 +98,13 @@ abstract class HiveComparisonTest .map(name => new File(targetDir, s"$suiteName.$name")) /** The local directory with cached golden answer will be stored. */ - protected val answerCache = new File("src" + File.separator + "test" + + protected var answerCache = new File("src" + File.separator + "test" + File.separator + "resources" + File.separator + "golden") + sys.props.get("spark.project.home") match { + case Some(h) => answerCache = new File(h, "sql" + File.separator + "hive" + + File.separator + answerCache.getPath) + case None => + } if (!answerCache.exists) { answerCache.mkdir() } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/MapWithStateDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/MapWithStateDStream.scala index 9512db7d7d757..74dec504728d5 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/MapWithStateDStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/MapWithStateDStream.scala @@ -121,11 +121,11 @@ class InternalMapWithStateDStream[K: ClassTag, V: ClassTag, S: ClassTag, E: Clas override val mustCheckpoint = true /** Override the default checkpoint duration */ - override def initialize(time: Time): Unit = { + override def initialize(time: Time, skipInitialized: Boolean): Unit = { if (checkpointDuration == null) { checkpointDuration = slideDuration * DEFAULT_CHECKPOINT_DURATION_MULTIPLIER } - super.initialize(time) + super.initialize(time, skipInitialized) } /** Method that generates an RDD for the given time */ diff --git a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala index 52c8959351fe7..c1c5152d71e15 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala @@ -755,12 +755,17 @@ class StreamingContextSuite extends SparkFunSuite with BeforeAndAfter with TimeL ssc.start() require(ssc.getState() === StreamingContextState.ACTIVE) + /* SNAP: allowed in SnappyData testForException("no error on adding input after start", "start") { addInputStream(ssc) } testForException("no error on adding transformation after start", "start") { input.map { x => x * 2 } } testForException("no error on adding output operation after start", "start") { transformed.foreachRDD { rdd => rdd.collect() } } + */ + addInputStream(ssc) + input.map { x => x * 2 } + transformed.foreachRDD { rdd => rdd.collect() } ssc.stop() require(ssc.getState() === StreamingContextState.STOPPED) From b5136bf2f463e1cab8a0ab4d8310f0a728cecc4d Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 31 Jan 2018 13:28:16 +0530 Subject: [PATCH 116/142] [SNAPPYDATA] fixing one remaining failure in gradle runs --- .../sql/hive/thriftserver/HiveThriftServer2Suites.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 0159dc8f5af63..0e9ccf908933c 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -601,7 +601,10 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { assert(rs1.next()) assert(rs1.getString(1) === "Usage: N/A.") - val dataPath = "../hive/src/test/resources/data/files/kv1.txt" + val dataPath = sys.props.get("spark.project.home") match { + case Some(h) => s"$h/sql/hive/src/test/resources/data/files/kv1.txt" + case _ => "../hive/src/test/resources/data/files/kv1.txt" + } Seq( "CREATE TABLE test_udtf(key INT, value STRING)", From 6532714ecd4d977bce6c8765738089a15bad69df Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Mon, 12 Feb 2018 09:30:07 +0530 Subject: [PATCH 117/142] Preserve the preferred location in MapPartitionRDD. (#92) --- .../src/main/scala/org/apache/spark/rdd/MapPartitionsRDD.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/rdd/MapPartitionsRDD.scala b/core/src/main/scala/org/apache/spark/rdd/MapPartitionsRDD.scala index e4587c96eae1c..621b8d4a5e4e3 100644 --- a/core/src/main/scala/org/apache/spark/rdd/MapPartitionsRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/MapPartitionsRDD.scala @@ -34,6 +34,9 @@ private[spark] class MapPartitionsRDD[U: ClassTag, T: ClassTag]( override def getPartitions: Array[Partition] = firstParent[T].partitions + override def getPreferredLocations( + split: Partition): Seq[String] = firstParent[T].preferredLocations(split) + override def compute(split: Partition, context: TaskContext): Iterator[U] = f(context, split.index, firstParent[T].iterator(split, context)) From b35bd984857f5b8751d40151ebf8a65d506623bd Mon Sep 17 00:00:00 2001 From: Amogh Shetkar Date: Mon, 12 Feb 2018 12:50:41 +0530 Subject: [PATCH 118/142] * SnappyData Spark Version 2.1.1.2 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2ea2f044646a8..5ef63a4df3e33 100644 --- a/build.gradle +++ b/build.gradle @@ -47,7 +47,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.1.1.1' + version = '2.1.1.2' ext { productName = 'SnappyData' From 55aa6e3e6b6a33106991b949e823d509e4948d4e Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Fri, 16 Feb 2018 15:43:04 +0530 Subject: [PATCH 119/142] [SNAP-2218] honour timeout in netty RPC transfers (#93) use a future for enforcing timeout (2 x configured value) in netty RPC transfers after which the channel will be closed and fail Conflicts: core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala --- .../scala/org/apache/spark/rpc/RpcEnv.scala | 10 +++++++ .../apache/spark/rpc/netty/NettyRpcEnv.scala | 29 ++++++++++++++----- .../scala/org/apache/spark/util/Utils.scala | 4 ++- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala index 59fce6a4e731b..e55ba295aa0d5 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala @@ -151,6 +151,16 @@ private[spark] abstract class RpcEnv(conf: SparkConf) { * @param uri URI with location of the file. */ def openChannel(uri: String): ReadableByteChannel + + /** + * Open a channel to download a file from the given URI. If the URIs returned by the + * RpcEnvFileServer use the "spark" scheme, this method will be called by the Utils class to + * retrieve the files. + * + * @param uri URI with location of the file. + * @param readTimeoutMs timeout in reading in millisecond + */ + def openChannel(uri: String, readTimeoutMs: Long): ReadableByteChannel } /** diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala index 79639f2f38745..3a57a151c4db4 100644 --- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala +++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala @@ -24,7 +24,8 @@ import java.util.concurrent._ import java.util.concurrent.atomic.AtomicBoolean import javax.annotation.Nullable -import scala.concurrent.{Future, Promise} +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.concurrent.duration.Duration import scala.reflect.ClassTag import scala.util.{DynamicVariable, Failure, Success, Try} import scala.util.control.NonFatal @@ -351,6 +352,14 @@ private[netty] class NettyRpcEnv( source } + override def openChannel(uri: String, readTimeoutMs: Long): ReadableByteChannel = { + val source = openChannel(uri) + if (readTimeoutMs > 0) { + source.asInstanceOf[FileDownloadChannel].setTimeoutMs(readTimeoutMs) + } + source + } + private def downloadClient(host: String, port: Int): TransportClient = { if (fileDownloadFactory == null) synchronized { if (fileDownloadFactory == null) { @@ -378,6 +387,7 @@ private[netty] class NettyRpcEnv( private class FileDownloadChannel(source: Pipe.SourceChannel) extends ReadableByteChannel { @volatile private var error: Throwable = _ + private var timeoutMs: Long = _ def setError(e: Throwable): Unit = { // This setError callback is invoked by internal RPC threads in order to propagate remote @@ -392,14 +402,17 @@ private[netty] class NettyRpcEnv( error = e } + def setTimeoutMs(millis: Long): Unit = { + timeoutMs = millis + } + override def read(dst: ByteBuffer): Int = { - Try(source.read(dst)) match { - // See the documentation above in setError(): if an RPC error has occurred then setError() - // will be called to propagate the RPC error and then `source`'s corresponding - // Pipe.SinkChannel will be closed, unblocking this read. In that case, we want to propagate - // the remote RPC exception (and not any exceptions triggered by the pipe close, such as - // ChannelClosedException), hence this `error != null` check: - case _ if error != null => throw error + def readBuffer: Int = if (timeoutMs > 0) { + val context = ExecutionContext.fromExecutorService(clientConnectionExecutor) + val future = Future(source.read(dst))(context) + ThreadUtils.awaitResult(future, Duration(timeoutMs, TimeUnit.MILLISECONDS)) + } else source.read(dst) + Try(readBuffer) match { case Success(bytesRead) => bytesRead case Failure(readErr) => throw readErr } diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 56e159746b518..f80ebbad28ce0 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -714,7 +714,9 @@ private[spark] object Utils extends Logging { throw new IllegalStateException( "Cannot retrieve files with 'spark' scheme without an active SparkEnv.") } - val source = SparkEnv.get.rpcEnv.openChannel(url) + // wait for max double the configured time (connect + read time) + val timeoutMs = conf.getTimeAsSeconds("spark.files.fetchTimeout", "60s") * 2000L + val source = SparkEnv.get.rpcEnv.openChannel(url, timeoutMs) val is = Channels.newInputStream(source) downloadFile(url, is, targetFile, fileOverwrite) case "http" | "https" | "ftp" => From 5db16fe9a6f437db804276a936e43a6625b87b4e Mon Sep 17 00:00:00 2001 From: ymahajan Date: Mon, 12 Mar 2018 13:52:34 -0700 Subject: [PATCH 120/142] Adding commons crypto libraries --- build.gradle | 4 ++-- common/network-common/build.gradle | 17 +++++++++++++++++ core/build.gradle | 3 +++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 5ef63a4df3e33..939015659e66f 100644 --- a/build.gradle +++ b/build.gradle @@ -58,7 +58,7 @@ allprojects { protobufVersion = '2.5.0' jerseyVersion = '2.26' sunJerseyVersion = '1.19.4' - jettyVersion = '9.2.22.v20170606' + jettyVersion = '9.3.20.v20170531' log4jVersion = '1.2.17' slf4jVersion = '1.7.25' junitVersion = '4.12' @@ -96,7 +96,7 @@ allprojects { xbeanAsm5Version = '4.5' scalatestVersion = '2.2.6' pegdownVersion = '1.6.0' - + commonsCryptoVersion = '1.0.0' shadePackageName = 'org.spark_project' } diff --git a/common/network-common/build.gradle b/common/network-common/build.gradle index 4f81762413192..e028aefec5437 100644 --- a/common/network-common/build.gradle +++ b/common/network-common/build.gradle @@ -27,4 +27,21 @@ dependencies { compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion compile group: 'org.apache.commons', name: 'commons-lang3', version: commonsLang3Version + + testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') + compile group: 'org.apache.commons', name: 'commons-crypto', version: '1.0.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: '3.1.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-json', version: '3.1.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: '3.1.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-ganglia', version: '3.1.0' + compile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' + compile group: 'org.apache.commons', name: 'commons-crypto', version: '1.0.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: '3.1.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-json', version: '3.1.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: '3.1.0' + compile group: 'io.dropwizard.metrics', name: 'metrics-ganglia', version: '3.1.0' + compile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' + } diff --git a/core/build.gradle b/core/build.gradle index aadc5e2e1a213..90532a5c0a1dc 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -156,6 +156,9 @@ dependencies { } compile group: 'net.sf.py4j', name: 'py4j', version: '0.10.4' + testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') + // compile group: 'org.apache.commons', name: 'commons-crypto', version: commonsCryptoVersion + testCompile group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion, classifier: 'tests' testCompile "org.apache.derby:derby:${derbyVersion}" testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-java', version: seleniumVersion) { From 787deb8875d550bece580ec2a5cef6cda52652ae Mon Sep 17 00:00:00 2001 From: ymahajan Date: Mon, 12 Mar 2018 15:51:49 -0700 Subject: [PATCH 121/142] compilation issues --- .../spark/unsafe/array/ByteArrayMethods.java | 7 +- .../apache/spark/unsafe/types/UTF8String.java | 85 ++++--------------- core/build.gradle | 2 +- 3 files changed, 22 insertions(+), 72 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java index 776ff2881f64c..3bf8eb28a5ed7 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/array/ByteArrayMethods.java @@ -66,9 +66,9 @@ public static boolean arrayEquals(final Object leftBase, long leftOffset, long endOffset = leftOffset + length; // try to align at least one side if ((rightOffset & 0x7) != 0 && (leftOffset & 0x7) != 0) { // mod 8 - final long endOffset = Math.min(((leftOffset + 7) >>> 3) << 3, leftOffset + length); + final long alignedOffset = Math.min(((leftOffset + 7) >>> 3) << 3, endOffset); if (Platform.unaligned()) { - if (leftOffset <= (endOffset - 4)) { + if (leftOffset <= (alignedOffset - 4)) { if (Platform.getInt(leftBase, leftOffset) != Platform.getInt(rightBase, rightOffset)) { return false; @@ -77,7 +77,7 @@ public static boolean arrayEquals(final Object leftBase, long leftOffset, rightOffset += 4; } } - while (leftOffset < endOffset) { + while (leftOffset < alignedOffset) { if (Platform.getByte(leftBase, leftOffset) != Platform.getByte(rightBase, rightOffset)) { return false; @@ -86,7 +86,6 @@ public static boolean arrayEquals(final Object leftBase, long leftOffset, rightOffset++; } } - long endOffset = leftOffset + length; // for architectures that support unaligned accesses, chew it up 8 bytes at a time if (Platform.unaligned() || (((leftOffset & 0x7) == 0) && ((rightOffset & 0x7) == 0))) { endOffset -= 8; diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index 23d834a9efaad..93e2a6a14b254 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -1236,77 +1236,28 @@ public int compareTo(@Nonnull final UTF8String other) { return compare(other); } - /** Read integer in big-endian format */ - static int getIntBigEndian(final Object base, final long offset) { - return isLittleEndian ? Integer.reverseBytes(Platform.getInt(base, offset)) - : Platform.getInt(base, offset); - } - - /** Read long in big-endian format */ - static long getLongBigEndian(final Object base, final long offset) { - return isLittleEndian ? Long.reverseBytes(Platform.getLong(base, offset)) - : Platform.getLong(base, offset); - } - public final int compare(final UTF8String other) { - final Object rightBase = other.getBaseObject(); - long rightOffset = other.getBaseOffset(); - final Object leftBase = base; - long leftOffset = offset; - - final int len = Math.min(numBytes, other.numBytes); - - // for the case that compare will fail in first few bytes itself, the overhead - // of JNI call is too high - /* - // noinspection ConstantConditions - if (leftBase == null && rightBase == null && - len >= Native.MIN_JNI_SIZE && Native.isLoaded()) { - final int result = Native.compareString(leftOffset, rightOffset, len); - return result != 0 ? result : (numBytes - other.numBytes); - } - */ - - long endOffset = leftOffset + len; - // for architectures that support unaligned accesses, read 8 bytes at a time - if (Platform.unaligned() || (((leftOffset & 0x7) == 0) && ((rightOffset & 0x7) == 0))) { - endOffset -= 8; - while (leftOffset <= endOffset) { - // In UTF-8, the byte should be unsigned, so we should compare them as unsigned long. - final long ll = getLongBigEndian(leftBase, leftOffset); - final long rl = getLongBigEndian(rightBase, rightOffset); - final long res = ll - rl; - // If the sign of both values is same then "res" is with correct sign. - // If the sign of values is different then "res" has opposite sign. - // The XOR operations will revert the sign bit of res if sign of values is different. - // After that converting to signum is "(1 + ((v >> 63) << 1))" - // where (v >> 63) will flow the sign to give -1 or 0, and (1 + 2 times) - // of that will give -1 or 1 respectively. - if (res != 0) return (int)(1 + (((ll ^ rl ^ res) >> 63) << 1)); - leftOffset += 8; - rightOffset += 8; - } - endOffset += 4; - if (leftOffset <= endOffset) { - // In UTF-8, the byte should be unsigned, so we should compare them as unsigned int - // which is done by converting to unsigned longs. - // After that conversion to signed integer is "(1 + ((v >> 63) << 1))" as above. - final long res = (getIntBigEndian(leftBase, leftOffset) & 0xffffffffL) - - (getIntBigEndian(rightBase, rightOffset) & 0xffffffffL); - if (res != 0) return (int)(1 + ((res >> 63) << 1)); - leftOffset += 4; - rightOffset += 4; + int len = Math.min(numBytes, other.numBytes); + int wordMax = (len / 8) * 8; + long roffset = other.offset; + Object rbase = other.base; + for (int i = 0; i < wordMax; i += 8) { + long left = getLong(base, offset + i); + long right = getLong(rbase, roffset + i); + if (left != right) { + if (IS_LITTLE_ENDIAN) { + return Long.compareUnsigned(Long.reverseBytes(left), Long.reverseBytes(right)); + } else { + return Long.compareUnsigned(left, right); + } } - endOffset += 4; } - // finish the remaining bytes - while (leftOffset < endOffset) { + for (int i = wordMax; i < len; i++) { // In UTF-8, the byte should be unsigned, so we should compare them as unsigned int. - final int res = (Platform.getByte(leftBase, leftOffset) & 0xff) - - (Platform.getByte(rightBase, rightOffset) & 0xff); - if (res != 0) return res; - leftOffset++; - rightOffset++; + int res = (getByte(i) & 0xFF) - (Platform.getByte(rbase, roffset + i) & 0xFF); + if (res != 0) { + return res; + } } return numBytes - other.numBytes; } diff --git a/core/build.gradle b/core/build.gradle index 90532a5c0a1dc..341c65d7a5c7b 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -116,7 +116,7 @@ dependencies { compile group: 'com.ning', name: 'compress-lzf', version: lzfVersion compile group: 'commons-net', name: 'commons-net', version: '3.6' compile group: 'org.json4s', name: 'json4s-jackson_' + scalaBinaryVersion, version: '3.2.11' - compile group: 'org.gla>>>ssfish.jersey.core', name: 'jersey-client', version: jerseyVersion + compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: jerseyVersion compile group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion compile group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet', version: jerseyVersion From 39fb136755a2e71f7f469f68bb4db8db5659b14d Mon Sep 17 00:00:00 2001 From: ymahajan Date: Tue, 13 Mar 2018 11:45:55 -0700 Subject: [PATCH 122/142] fixing compilation errors --- .../plans/logical/basicLogicalOperators.scala | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index fbf1b6f8783f6..c538b22a08419 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -149,7 +149,7 @@ case class Filter(condition: Expression, child: LogicalPlan) override def maxRows: Option[Long] = child.maxRows - override lazy val statistics: Statistics = { + override lazy val stats: Statistics = { // Expected filtering by expressions based on some constants for now. def expectedFilterDivisor(cond: Expression): Int = cond match { case EqualTo(_, _) => 10 @@ -169,7 +169,7 @@ case class Filter(condition: Expression, child: LogicalPlan) case _ => 1 } - child.statistics.copy(sizeInBytes = child.statistics.sizeInBytes / + child.stats.copy(sizeInBytes = child.stats.sizeInBytes / expectedFilterDivisor(condition)) } @@ -385,16 +385,16 @@ case class Join( case _ => resolvedExceptNatural } - override lazy val statistics: Statistics = joinType match { + override lazy val stats: Statistics = joinType match { case LeftAnti | LeftSemi => // LeftSemi and LeftAnti won't ever be bigger than left - left.statistics.copy() + left.stats.copy() case _ if ExtractEquiJoinKeys.unapply(this).isDefined => - Statistics(sizeInBytes = children.map(_.statistics.sizeInBytes).sum) + Statistics(sizeInBytes = children.map(_.stats.sizeInBytes).sum) case _ => // make sure we don't propagate isBroadcastable in other joins, because // they could explode the size. - super.statistics.copy(isBroadcastable = false) + super.stats.copy(isBroadcastable = false) } } @@ -451,12 +451,12 @@ case class InsertIntoDir( storage: CatalogStorageFormat, provider: Option[String], child: LogicalPlan, - overwrite: Boolean = true, - ifNotExists: Boolean) + overwrite: Boolean = true) extends LogicalPlan { override def output: Seq[Attribute] = Seq.empty override lazy val resolved: Boolean = false + override def children: Seq[LogicalPlan] = child :: Nil } /** @@ -608,11 +608,11 @@ case class Aggregate( child.constraints.union(getAliasedConstraints(nonAgg)) } - override lazy val statistics: Statistics = { + override lazy val stats: Statistics = { if (groupingExpressions.isEmpty) { - super.statistics.copy(sizeInBytes = 1) + super.stats.copy(sizeInBytes = 1) } else { - val stats = super.statistics + val stats = super.stats stats.copy(sizeInBytes = stats.sizeInBytes / 2) } } From 1cebe77295ca884e7c2a660ffe93668369000f44 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Thu, 15 Mar 2018 23:27:17 -0700 Subject: [PATCH 123/142] compilation errors --- .../plans/logical/basicLogicalOperators.scala | 2 +- .../scala/org/apache/spark/sql/internal/SQLConf.scala | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index c538b22a08419..683ab6a9fe572 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -394,7 +394,7 @@ case class Join( case _ => // make sure we don't propagate isBroadcastable in other joins, because // they could explode the size. - super.stats.copy(isBroadcastable = false) + super.stats.copy(hints = HintInfo(broadcast = false)) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index f5ca04587932d..56ea6d8ed055a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -29,7 +29,6 @@ import org.apache.parquet.hadoop.ParquetOutputCommitter import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ import org.apache.spark.network.util.ByteUnit -import org.apache.spark.sql.catalyst.CatalystConf import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol import org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol import org.apache.spark.util.Utils @@ -636,7 +635,7 @@ object SQLConf { * * SQLConf is thread-safe (internally synchronized, so safe to be used in multiple threads). */ -private[sql] class SQLConf extends Serializable with CatalystConf with Logging { +private[sql] class SQLConf extends Serializable with Logging { import SQLConf._ /** Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap. */ @@ -790,7 +789,7 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging { def dataFramePivotMaxValues: Int = getConf(DATAFRAME_PIVOT_MAX_VALUES) - override def runSQLonFile: Boolean = getConf(RUN_SQL_ON_FILES) + def runSQLonFile: Boolean = getConf(RUN_SQL_ON_FILES) def enableTwoLevelAggMap: Boolean = getConf(ENABLE_TWOLEVEL_AGG_MAP) @@ -802,11 +801,11 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging { def ignoreCorruptFiles: Boolean = getConf(IGNORE_CORRUPT_FILES) - override def orderByOrdinal: Boolean = getConf(ORDER_BY_ORDINAL) + def orderByOrdinal: Boolean = getConf(ORDER_BY_ORDINAL) - override def groupByOrdinal: Boolean = getConf(GROUP_BY_ORDINAL) + def groupByOrdinal: Boolean = getConf(GROUP_BY_ORDINAL) - override def crossJoinEnabled: Boolean = getConf(SQLConf.CROSS_JOINS_ENABLED) + def crossJoinEnabled: Boolean = getConf(SQLConf.CROSS_JOINS_ENABLED) def ndvMaxError: Double = getConf(NDV_MAX_ERROR) /** ********************** SQLConf functionality methods ************ */ From 7940bbb0796493c32160f3a3218d42fa34338188 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Sun, 18 Mar 2018 19:49:42 -0700 Subject: [PATCH 124/142] compilation issues --- .../apache/spark/launcher/SparkLauncher.java | 6 +- .../apache/spark/sql/internal/SQLConf.scala | 1962 ++++++++--------- 2 files changed, 982 insertions(+), 986 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java index ca7b99a8fd6f6..900db16a8e797 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java @@ -188,6 +188,7 @@ public SparkLauncher directory(File dir) { */ public SparkLauncher redirectError() { redirectErrorStream = true; + return this; } public SparkLauncher setConf(String key, String value) { @@ -262,11 +263,6 @@ public SparkLauncher setPropertiesFile(String path) { return super.setPropertiesFile(path); } - @Override - public SparkLauncher setConf(String key, String value) { - return super.setConf(key, value); - } - @Override public SparkLauncher setAppName(String appName) { return super.setAppName(appName); diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 56ea6d8ed055a..2940302f27389 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1,981 +1,981 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.internal - -import java.util.{NoSuchElementException, Properties} -import java.util.concurrent.TimeUnit - -import scala.collection.JavaConverters._ -import scala.collection.immutable - -import org.apache.hadoop.fs.Path -import org.apache.parquet.hadoop.ParquetOutputCommitter - -import org.apache.spark.internal.Logging -import org.apache.spark.internal.config._ -import org.apache.spark.network.util.ByteUnit -import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol -import org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol -import org.apache.spark.util.Utils - -//////////////////////////////////////////////////////////////////////////////////////////////////// -// This file defines the configuration options for Spark SQL. -//////////////////////////////////////////////////////////////////////////////////////////////////// - - -object SQLConf { - - private val sqlConfEntries = java.util.Collections.synchronizedMap( - new java.util.HashMap[String, ConfigEntry[_]]()) - - private[sql] def register(entry: ConfigEntry[_]): Unit = sqlConfEntries.synchronized { - require(!sqlConfEntries.containsKey(entry.key), - s"Duplicate SQLConfigEntry. ${entry.key} has been registered") - sqlConfEntries.put(entry.key, entry) - } - - private[sql] object SQLConfigBuilder { - - def apply(key: String): ConfigBuilder = new ConfigBuilder(key).onCreate(register) - - } - - - val OPTIMIZER_MAX_ITERATIONS = SQLConfigBuilder("spark.sql.optimizer.maxIterations") - .internal() - .doc("The max number of iterations the optimizer and analyzer runs.") - .intConf - .createWithDefault(100) - - val OPTIMIZER_INSET_CONVERSION_THRESHOLD = - SQLConfigBuilder("spark.sql.optimizer.inSetConversionThreshold") - .internal() - .doc("The threshold of set size for InSet conversion.") - .intConf - .createWithDefault(10) - - val COMPRESS_CACHED = SQLConfigBuilder("spark.sql.inMemoryColumnarStorage.compressed") - .internal() - .doc("When set to true Spark SQL will automatically select a compression codec for each " + - "column based on statistics of the data.") - .booleanConf - .createWithDefault(true) - - val COLUMN_BATCH_SIZE = SQLConfigBuilder("spark.sql.inMemoryColumnarStorage.batchSize") - .internal() - .doc("Controls the size of batches for columnar caching. Larger batch sizes can improve " + - "memory utilization and compression, but risk OOMs when caching data.") - .intConf - .createWithDefault(10000) - - val IN_MEMORY_PARTITION_PRUNING = - SQLConfigBuilder("spark.sql.inMemoryColumnarStorage.partitionPruning") - .internal() - .doc("When true, enable partition pruning for in-memory columnar tables.") - .booleanConf - .createWithDefault(true) - - val PREFER_SORTMERGEJOIN = SQLConfigBuilder("spark.sql.join.preferSortMergeJoin") - .internal() - .doc("When true, prefer sort merge join over shuffle hash join.") - .booleanConf - .createWithDefault(true) - - val RADIX_SORT_ENABLED = SQLConfigBuilder("spark.sql.sort.enableRadixSort") - .internal() - .doc("When true, enable use of radix sort when possible. Radix sort is much faster but " + - "requires additional memory to be reserved up-front. The memory overhead may be " + - "significant when sorting very small rows (up to 50% more in this case).") - .booleanConf - .createWithDefault(true) - - val AUTO_BROADCASTJOIN_THRESHOLD = SQLConfigBuilder("spark.sql.autoBroadcastJoinThreshold") - .doc("Configures the maximum size in bytes for a table that will be broadcast to all worker " + - "nodes when performing a join. By setting this value to -1 broadcasting can be disabled. " + - "Note that currently statistics are only supported for Hive Metastore tables where the " + - "command ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan has been " + - "run, and file-based data source tables where the statistics are computed directly on " + - "the files of data.") - .longConf - .createWithDefault(10L * 1024 * 1024) - - val LIMIT_SCALE_UP_FACTOR = SQLConfigBuilder("spark.sql.limit.scaleUpFactor") - .internal() - .doc("Minimal increase rate in number of partitions between attempts when executing a take " + - "on a query. Higher values lead to more partitions read. Lower values might lead to " + - "longer execution times as more jobs will be run") - .intConf - .createWithDefault(4) - - val ENABLE_FALL_BACK_TO_HDFS_FOR_STATS = - SQLConfigBuilder("spark.sql.statistics.fallBackToHdfs") - .doc("If the table statistics are not available from table metadata enable fall back to hdfs." + - " This is useful in determining if a table is small enough to use auto broadcast joins.") - .booleanConf - .createWithDefault(false) - - val DEFAULT_SIZE_IN_BYTES = SQLConfigBuilder("spark.sql.defaultSizeInBytes") - .internal() - .doc("The default table size used in query planning. By default, it is set to Long.MaxValue " + - "which is larger than `spark.sql.autoBroadcastJoinThreshold` to be more conservative. " + - "That is to say by default the optimizer will not choose to broadcast a table unless it " + - "knows for sure its size is small enough.") - .longConf - .createWithDefault(Long.MaxValue) - - val SHUFFLE_PARTITIONS = SQLConfigBuilder("spark.sql.shuffle.partitions") - .doc("The default number of partitions to use when shuffling data for joins or aggregations.") - .intConf - .createWithDefault(200) - - val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE = - SQLConfigBuilder("spark.sql.adaptive.shuffle.targetPostShuffleInputSize") - .doc("The target post-shuffle input size in bytes of a task.") - .bytesConf(ByteUnit.BYTE) - .createWithDefault(64 * 1024 * 1024) - - val ADAPTIVE_EXECUTION_ENABLED = SQLConfigBuilder("spark.sql.adaptive.enabled") - .doc("When true, enable adaptive query execution.") - .booleanConf - .createWithDefault(false) - - val SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS = - SQLConfigBuilder("spark.sql.adaptive.minNumPostShufflePartitions") - .internal() - .doc("The advisory minimal number of post-shuffle partitions provided to " + - "ExchangeCoordinator. This setting is used in our test to make sure we " + - "have enough parallelism to expose issues that will not be exposed with a " + - "single partition. When the value is a non-positive value, this setting will " + - "not be provided to ExchangeCoordinator.") - .intConf - .createWithDefault(-1) - - val SUBEXPRESSION_ELIMINATION_ENABLED = - SQLConfigBuilder("spark.sql.subexpressionElimination.enabled") - .internal() - .doc("When true, common subexpressions will be eliminated.") - .booleanConf - .createWithDefault(true) - - val CASE_SENSITIVE = SQLConfigBuilder("spark.sql.caseSensitive") - .internal() - .doc("Whether the query analyzer should be case sensitive or not. " + - "Default to case insensitive. It is highly discouraged to turn on case sensitive mode.") - .booleanConf - .createWithDefault(false) - - val PARQUET_SCHEMA_MERGING_ENABLED = SQLConfigBuilder("spark.sql.parquet.mergeSchema") - .doc("When true, the Parquet data source merges schemas collected from all data files, " + - "otherwise the schema is picked from the summary file or a random data file " + - "if no summary file is available.") - .booleanConf - .createWithDefault(false) - - val PARQUET_SCHEMA_RESPECT_SUMMARIES = SQLConfigBuilder("spark.sql.parquet.respectSummaryFiles") - .doc("When true, we make assumption that all part-files of Parquet are consistent with " + - "summary files and we will ignore them when merging schema. Otherwise, if this is " + - "false, which is the default, we will merge all part-files. This should be considered " + - "as expert-only option, and shouldn't be enabled before knowing what it means exactly.") - .booleanConf - .createWithDefault(false) - - val PARQUET_BINARY_AS_STRING = SQLConfigBuilder("spark.sql.parquet.binaryAsString") - .doc("Some other Parquet-producing systems, in particular Impala and older versions of " + - "Spark SQL, do not differentiate between binary data and strings when writing out the " + - "Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide " + - "compatibility with these systems.") - .booleanConf - .createWithDefault(false) - - val PARQUET_INT96_AS_TIMESTAMP = SQLConfigBuilder("spark.sql.parquet.int96AsTimestamp") - .doc("Some Parquet-producing systems, in particular Impala, store Timestamp into INT96. " + - "Spark would also store Timestamp as INT96 because we need to avoid precision lost of the " + - "nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to " + - "provide compatibility with these systems.") - .booleanConf - .createWithDefault(true) - - val PARQUET_CACHE_METADATA = SQLConfigBuilder("spark.sql.parquet.cacheMetadata") - .doc("Turns on caching of Parquet schema metadata. Can speed up querying of static data.") - .booleanConf - .createWithDefault(true) - - val PARQUET_COMPRESSION = SQLConfigBuilder("spark.sql.parquet.compression.codec") - .doc("Sets the compression codec use when writing Parquet files. Acceptable values include: " + - "uncompressed, snappy, gzip, lzo.") - .stringConf - .transform(_.toLowerCase()) - .checkValues(Set("uncompressed", "snappy", "gzip", "lzo")) - .createWithDefault("snappy") - - val PARQUET_FILTER_PUSHDOWN_ENABLED = SQLConfigBuilder("spark.sql.parquet.filterPushdown") - .doc("Enables Parquet filter push-down optimization when set to true.") - .booleanConf - .createWithDefault(true) - - val PARQUET_WRITE_LEGACY_FORMAT = SQLConfigBuilder("spark.sql.parquet.writeLegacyFormat") - .doc("Whether to follow Parquet's format specification when converting Parquet schema to " + - "Spark SQL schema and vice versa.") - .booleanConf - .createWithDefault(false) - - val PARQUET_OUTPUT_COMMITTER_CLASS = SQLConfigBuilder("spark.sql.parquet.output.committer.class") - .doc("The output committer class used by Parquet. The specified class needs to be a " + - "subclass of org.apache.hadoop.mapreduce.OutputCommitter. Typically, it's also a subclass " + - "of org.apache.parquet.hadoop.ParquetOutputCommitter.") - .internal() - .stringConf - .createWithDefault(classOf[ParquetOutputCommitter].getName) - - val PARQUET_VECTORIZED_READER_ENABLED = - SQLConfigBuilder("spark.sql.parquet.enableVectorizedReader") - .doc("Enables vectorized parquet decoding.") - .booleanConf - .createWithDefault(true) - - val ORC_FILTER_PUSHDOWN_ENABLED = SQLConfigBuilder("spark.sql.orc.filterPushdown") - .doc("When true, enable filter pushdown for ORC files.") - .booleanConf - .createWithDefault(false) - - val HIVE_VERIFY_PARTITION_PATH = SQLConfigBuilder("spark.sql.hive.verifyPartitionPath") - .doc("When true, check all the partition paths under the table\'s root directory " + - "when reading data stored in HDFS.") - .booleanConf - .createWithDefault(false) - - val HIVE_METASTORE_PARTITION_PRUNING = - SQLConfigBuilder("spark.sql.hive.metastorePartitionPruning") - .doc("When true, some predicates will be pushed down into the Hive metastore so that " + - "unmatching partitions can be eliminated earlier. This only affects Hive tables " + - "not converted to filesource relations (see HiveUtils.CONVERT_METASTORE_PARQUET and " + - "HiveUtils.CONVERT_METASTORE_ORC for more information).") - .booleanConf - .createWithDefault(true) - - val HIVE_MANAGE_FILESOURCE_PARTITIONS = - SQLConfigBuilder("spark.sql.hive.manageFilesourcePartitions") - .doc("When true, enable metastore partition management for file source tables as well. " + - "This includes both datasource and converted Hive tables. When partition managment " + - "is enabled, datasource tables store partition in the Hive metastore, and use the " + - "metastore to prune partitions during query planning.") - .booleanConf - .createWithDefault(true) - - val HIVE_FILESOURCE_PARTITION_FILE_CACHE_SIZE = - SQLConfigBuilder("spark.sql.hive.filesourcePartitionFileCacheSize") - .doc("When nonzero, enable caching of partition file metadata in memory. All tables share " + - "a cache that can use up to specified num bytes for file metadata. This conf only " + - "has an effect when hive filesource partition management is enabled.") - .longConf - .createWithDefault(250 * 1024 * 1024) - - val OPTIMIZER_METADATA_ONLY = SQLConfigBuilder("spark.sql.optimizer.metadataOnly") - .doc("When true, enable the metadata-only query optimization that use the table's metadata " + - "to produce the partition columns instead of table scans. It applies when all the columns " + - "scanned are partition columns and the query has an aggregate operator that satisfies " + - "distinct semantics.") - .booleanConf - .createWithDefault(true) - - val COLUMN_NAME_OF_CORRUPT_RECORD = SQLConfigBuilder("spark.sql.columnNameOfCorruptRecord") - .doc("The name of internal column for storing raw/un-parsed JSON records that fail to parse.") - .stringConf - .createWithDefault("_corrupt_record") - - val BROADCAST_TIMEOUT = SQLConfigBuilder("spark.sql.broadcastTimeout") - .doc("Timeout in seconds for the broadcast wait time in broadcast joins.") - .intConf - .createWithDefault(5 * 60) - - // This is only used for the thriftserver - val THRIFTSERVER_POOL = SQLConfigBuilder("spark.sql.thriftserver.scheduler.pool") - .doc("Set a Fair Scheduler pool for a JDBC client session.") - .stringConf - .createOptional - - val THRIFTSERVER_UI_STATEMENT_LIMIT = - SQLConfigBuilder("spark.sql.thriftserver.ui.retainedStatements") - .doc("The number of SQL statements kept in the JDBC/ODBC web UI history.") - .intConf - .createWithDefault(200) - - val THRIFTSERVER_UI_SESSION_LIMIT = SQLConfigBuilder("spark.sql.thriftserver.ui.retainedSessions") - .doc("The number of SQL client sessions kept in the JDBC/ODBC web UI history.") - .intConf - .createWithDefault(200) - - // This is used to set the default data source - val DEFAULT_DATA_SOURCE_NAME = SQLConfigBuilder("spark.sql.sources.default") - .doc("The default data source to use in input/output.") - .stringConf - .createWithDefault("parquet") - - val CONVERT_CTAS = SQLConfigBuilder("spark.sql.hive.convertCTAS") - .internal() - .doc("When true, a table created by a Hive CTAS statement (no USING clause) " + - "without specifying any storage property will be converted to a data source table, " + - "using the data source set by spark.sql.sources.default.") - .booleanConf - .createWithDefault(false) - - val GATHER_FASTSTAT = SQLConfigBuilder("spark.sql.hive.gatherFastStats") - .internal() - .doc("When true, fast stats (number of files and total size of all files) will be gathered" + - " in parallel while repairing table partitions to avoid the sequential listing in Hive" + - " metastore.") - .booleanConf - .createWithDefault(true) - - val PARTITION_COLUMN_TYPE_INFERENCE = - SQLConfigBuilder("spark.sql.sources.partitionColumnTypeInference.enabled") - .doc("When true, automatically infer the data types for partitioned columns.") - .booleanConf - .createWithDefault(true) - - val BUCKETING_ENABLED = SQLConfigBuilder("spark.sql.sources.bucketing.enabled") - .doc("When false, we will treat bucketed table as normal table") - .booleanConf - .createWithDefault(true) - - val CROSS_JOINS_ENABLED = SQLConfigBuilder("spark.sql.crossJoin.enabled") - .doc("When false, we will throw an error if a query contains a cartesian product without " + - "explicit CROSS JOIN syntax.") - .booleanConf - .createWithDefault(false) - - val ORDER_BY_ORDINAL = SQLConfigBuilder("spark.sql.orderByOrdinal") - .doc("When true, the ordinal numbers are treated as the position in the select list. " + - "When false, the ordinal numbers in order/sort by clause are ignored.") - .booleanConf - .createWithDefault(true) - - val GROUP_BY_ORDINAL = SQLConfigBuilder("spark.sql.groupByOrdinal") - .doc("When true, the ordinal numbers in group by clauses are treated as the position " + - "in the select list. When false, the ordinal numbers are ignored.") - .booleanConf - .createWithDefault(true) - - // The output committer class used by data sources. The specified class needs to be a - // subclass of org.apache.hadoop.mapreduce.OutputCommitter. - val OUTPUT_COMMITTER_CLASS = - SQLConfigBuilder("spark.sql.sources.outputCommitterClass").internal().stringConf.createOptional - - val FILE_COMMIT_PROTOCOL_CLASS = - SQLConfigBuilder("spark.sql.sources.commitProtocolClass") - .internal() - .stringConf - .createWithDefault(classOf[SQLHadoopMapReduceCommitProtocol].getName) - - val PARALLEL_PARTITION_DISCOVERY_THRESHOLD = - SQLConfigBuilder("spark.sql.sources.parallelPartitionDiscovery.threshold") - .doc("The maximum number of files allowed for listing files at driver side. If the number " + - "of detected files exceeds this value during partition discovery, it tries to list the " + - "files with another Spark distributed job. This applies to Parquet, ORC, CSV, JSON and " + - "LibSVM data sources.") - .intConf - .createWithDefault(32) - - // Whether to automatically resolve ambiguity in join conditions for self-joins. - // See SPARK-6231. - val DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY = - SQLConfigBuilder("spark.sql.selfJoinAutoResolveAmbiguity") - .internal() - .booleanConf - .createWithDefault(true) - - // Whether to retain group by columns or not in GroupedData.agg. - val DATAFRAME_RETAIN_GROUP_COLUMNS = SQLConfigBuilder("spark.sql.retainGroupColumns") - .internal() - .booleanConf - .createWithDefault(true) - - val DATAFRAME_PIVOT_MAX_VALUES = SQLConfigBuilder("spark.sql.pivotMaxValues") - .doc("When doing a pivot without specifying values for the pivot column this is the maximum " + - "number of (distinct) values that will be collected without error.") - .intConf - .createWithDefault(10000) - - val RUN_SQL_ON_FILES = SQLConfigBuilder("spark.sql.runSQLOnFiles") - .internal() - .doc("When true, we could use `datasource`.`path` as table in SQL query.") - .booleanConf - .createWithDefault(true) - - val WHOLESTAGE_CODEGEN_ENABLED = SQLConfigBuilder("spark.sql.codegen.wholeStage") - .internal() - .doc("When true, the whole stage (of multiple operators) will be compiled into single java" + - " method.") - .booleanConf - .createWithDefault(true) - - val WHOLESTAGE_MAX_NUM_FIELDS = SQLConfigBuilder("spark.sql.codegen.maxFields") - .internal() - .doc("The maximum number of fields (including nested fields) that will be supported before" + - " deactivating whole-stage codegen.") - .intConf - .createWithDefault(100) - - val WHOLESTAGE_FALLBACK = SQLConfigBuilder("spark.sql.codegen.fallback") - .internal() - .doc("When true, whole stage codegen could be temporary disabled for the part of query that" + - " fail to compile generated code") - .booleanConf - .createWithDefault(true) - - val MAX_CASES_BRANCHES = SQLConfigBuilder("spark.sql.codegen.maxCaseBranches") - .internal() - .doc("The maximum number of switches supported with codegen.") - .intConf - .createWithDefault(20) - - val FILES_MAX_PARTITION_BYTES = SQLConfigBuilder("spark.sql.files.maxPartitionBytes") - .doc("The maximum number of bytes to pack into a single partition when reading files.") - .longConf - .createWithDefault(128 * 1024 * 1024) // parquet.block.size - - val FILES_OPEN_COST_IN_BYTES = SQLConfigBuilder("spark.sql.files.openCostInBytes") - .internal() - .doc("The estimated cost to open a file, measured by the number of bytes could be scanned in" + - " the same time. This is used when putting multiple files into a partition. It's better to" + - " over estimated, then the partitions with small files will be faster than partitions with" + - " bigger files (which is scheduled first).") - .longConf - .createWithDefault(4 * 1024 * 1024) - - val EXCHANGE_REUSE_ENABLED = SQLConfigBuilder("spark.sql.exchange.reuse") - .internal() - .doc("When true, the planner will try to find out duplicated exchanges and re-use them.") - .booleanConf - .createWithDefault(true) - - val STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT = - SQLConfigBuilder("spark.sql.streaming.stateStore.minDeltasForSnapshot") - .internal() - .doc("Minimum number of state store delta files that needs to be generated before they " + - "consolidated into snapshots.") - .intConf - .createWithDefault(10) - - val CHECKPOINT_LOCATION = SQLConfigBuilder("spark.sql.streaming.checkpointLocation") - .doc("The default location for storing checkpoint data for streaming queries.") - .stringConf - .createOptional - - val MIN_BATCHES_TO_RETAIN = SQLConfigBuilder("spark.sql.streaming.minBatchesToRetain") - .internal() - .doc("The minimum number of batches that must be retained and made recoverable.") - .intConf - .createWithDefault(100) - - val UNSUPPORTED_OPERATION_CHECK_ENABLED = - SQLConfigBuilder("spark.sql.streaming.unsupportedOperationCheck") - .internal() - .doc("When true, the logical plan for streaming query will be checked for unsupported" + - " operations.") - .booleanConf - .createWithDefault(true) - - val VARIABLE_SUBSTITUTE_ENABLED = - SQLConfigBuilder("spark.sql.variable.substitute") - .doc("This enables substitution using syntax like ${var} ${system:var} and ${env:var}.") - .booleanConf - .createWithDefault(true) - - val VARIABLE_SUBSTITUTE_DEPTH = - SQLConfigBuilder("spark.sql.variable.substitute.depth") - .internal() - .doc("Deprecated: The maximum replacements the substitution engine will do.") - .intConf - .createWithDefault(40) - - val VECTORIZED_AGG_MAP_MAX_COLUMNS = - SQLConfigBuilder("spark.sql.codegen.aggregate.map.columns.max") - .internal() - .doc("Sets the maximum width of schema (aggregate keys + values) for " + - "which aggregate with" + - "keys uses an in-memory columnar map to speed up execution. " + - "Setting this to 0 effectively" + - "disables the columnar map") - .intConf - .createWithDefault(3) - - val ENABLE_TWOLEVEL_AGG_MAP = - SQLConfigBuilder("spark.sql.codegen.aggregate.map.twolevel.enable") - .internal() - .doc("Enable two-level aggregate hash map. When enabled, records will first be " + - "inserted/looked-up at a 1st-level, small, fast map, and then fallback to a " + - "2nd-level, larger, slower map when 1st level is full or keys cannot be found. " + - "When disabled, records go directly to the 2nd level. Defaults to true.") - .booleanConf - .createWithDefault(true) - - val STREAMING_FILE_COMMIT_PROTOCOL_CLASS = - SQLConfigBuilder("spark.sql.streaming.commitProtocolClass") - .internal() - .stringConf - .createWithDefault(classOf[ManifestFileCommitProtocol].getName) - - val FILE_SINK_LOG_DELETION = SQLConfigBuilder("spark.sql.streaming.fileSink.log.deletion") - .internal() - .doc("Whether to delete the expired log files in file stream sink.") - .booleanConf - .createWithDefault(true) - - val FILE_SINK_LOG_COMPACT_INTERVAL = - SQLConfigBuilder("spark.sql.streaming.fileSink.log.compactInterval") - .internal() - .doc("Number of log files after which all the previous files " + - "are compacted into the next log file.") - .intConf - .createWithDefault(10) - - val FILE_SINK_LOG_CLEANUP_DELAY = - SQLConfigBuilder("spark.sql.streaming.fileSink.log.cleanupDelay") - .internal() - .doc("How long that a file is guaranteed to be visible for all readers.") - .timeConf(TimeUnit.MILLISECONDS) - .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes - - val FILE_SOURCE_LOG_DELETION = SQLConfigBuilder("spark.sql.streaming.fileSource.log.deletion") - .internal() - .doc("Whether to delete the expired log files in file stream source.") - .booleanConf - .createWithDefault(true) - - val FILE_SOURCE_LOG_COMPACT_INTERVAL = - SQLConfigBuilder("spark.sql.streaming.fileSource.log.compactInterval") - .internal() - .doc("Number of log files after which all the previous files " + - "are compacted into the next log file.") - .intConf - .createWithDefault(10) - - val FILE_SOURCE_LOG_CLEANUP_DELAY = - SQLConfigBuilder("spark.sql.streaming.fileSource.log.cleanupDelay") - .internal() - .doc("How long in milliseconds a file is guaranteed to be visible for all readers.") - .timeConf(TimeUnit.MILLISECONDS) - .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes - - val STREAMING_SCHEMA_INFERENCE = - SQLConfigBuilder("spark.sql.streaming.schemaInference") - .internal() - .doc("Whether file-based streaming sources will infer its own schema") - .booleanConf - .createWithDefault(false) - - val STREAMING_POLLING_DELAY = - SQLConfigBuilder("spark.sql.streaming.pollingDelay") - .internal() - .doc("How long to delay polling new data when no data is available") - .timeConf(TimeUnit.MILLISECONDS) - .createWithDefault(10L) - - val STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL = - SQLConfigBuilder("spark.sql.streaming.noDataProgressEventInterval") - .internal() - .doc("How long to wait between two progress events when there is no data") - .timeConf(TimeUnit.MILLISECONDS) - .createWithDefault(10000L) - - val STREAMING_METRICS_ENABLED = - SQLConfigBuilder("spark.sql.streaming.metricsEnabled") - .doc("Whether Dropwizard/Codahale metrics will be reported for active streaming queries.") - .booleanConf - .createWithDefault(false) - - val STREAMING_PROGRESS_RETENTION = - SQLConfigBuilder("spark.sql.streaming.numRecentProgressUpdates") - .doc("The number of progress updates to retain for a streaming query") - .intConf - .createWithDefault(100) - - val NDV_MAX_ERROR = - SQLConfigBuilder("spark.sql.statistics.ndv.maxError") - .internal() - .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " + - "column level statistics.") - .doubleConf - .createWithDefault(0.05) - - val IGNORE_CORRUPT_FILES = SQLConfigBuilder("spark.sql.files.ignoreCorruptFiles") - .doc("Whether to ignore corrupt files. If true, the Spark jobs will continue to run when " + - "encountering corrupted or non-existing and contents that have been read will still be " + - "returned.") - .booleanConf - .createWithDefault(false) - - object Deprecated { - val MAPRED_REDUCE_TASKS = "mapred.reduce.tasks" - } -} - -/** - * A class that enables the setting and getting of mutable config parameters/hints. - * - * In the presence of a SQLContext, these can be set and queried by passing SET commands - * into Spark SQL's query functions (i.e. sql()). Otherwise, users of this class can - * modify the hints by programmatically calling the setters and getters of this class. - * - * SQLConf is thread-safe (internally synchronized, so safe to be used in multiple threads). - */ -private[sql] class SQLConf extends Serializable with Logging { - import SQLConf._ - - /** Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap. */ - @transient protected[spark] val settings = java.util.Collections.synchronizedMap( - new java.util.HashMap[String, String]()) - - @transient private val reader = new ConfigReader(settings) - - /** ************************ Spark SQL Params/Hints ******************* */ - - def optimizerMaxIterations: Int = getConf(OPTIMIZER_MAX_ITERATIONS) - - def optimizerInSetConversionThreshold: Int = getConf(OPTIMIZER_INSET_CONVERSION_THRESHOLD) - - def stateStoreMinDeltasForSnapshot: Int = getConf(STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT) - - def checkpointLocation: Option[String] = getConf(CHECKPOINT_LOCATION) - - def isUnsupportedOperationCheckEnabled: Boolean = getConf(UNSUPPORTED_OPERATION_CHECK_ENABLED) - - def streamingFileCommitProtocolClass: String = getConf(STREAMING_FILE_COMMIT_PROTOCOL_CLASS) - - def fileSinkLogDeletion: Boolean = getConf(FILE_SINK_LOG_DELETION) - - def fileSinkLogCompactInterval: Int = getConf(FILE_SINK_LOG_COMPACT_INTERVAL) - - def fileSinkLogCleanupDelay: Long = getConf(FILE_SINK_LOG_CLEANUP_DELAY) - - def fileSourceLogDeletion: Boolean = getConf(FILE_SOURCE_LOG_DELETION) - - def fileSourceLogCompactInterval: Int = getConf(FILE_SOURCE_LOG_COMPACT_INTERVAL) - - def fileSourceLogCleanupDelay: Long = getConf(FILE_SOURCE_LOG_CLEANUP_DELAY) - - def streamingSchemaInference: Boolean = getConf(STREAMING_SCHEMA_INFERENCE) - - def streamingPollingDelay: Long = getConf(STREAMING_POLLING_DELAY) - - def streamingNoDataProgressEventInterval: Long = - getConf(STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL) - - def streamingMetricsEnabled: Boolean = getConf(STREAMING_METRICS_ENABLED) - - def streamingProgressRetention: Int = getConf(STREAMING_PROGRESS_RETENTION) - - def filesMaxPartitionBytes: Long = getConf(FILES_MAX_PARTITION_BYTES) - - def filesOpenCostInBytes: Long = getConf(FILES_OPEN_COST_IN_BYTES) - - def useCompression: Boolean = getConf(COMPRESS_CACHED) - - def parquetCompressionCodec: String = getConf(PARQUET_COMPRESSION) - - def parquetCacheMetadata: Boolean = getConf(PARQUET_CACHE_METADATA) - - def parquetVectorizedReaderEnabled: Boolean = getConf(PARQUET_VECTORIZED_READER_ENABLED) - - def columnBatchSize: Int = getConf(COLUMN_BATCH_SIZE) - - def numShufflePartitions: Int = getConf(SHUFFLE_PARTITIONS) - - def targetPostShuffleInputSize: Long = - getConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) - - def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED) - - def minNumPostShufflePartitions: Int = - getConf(SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS) - - def minBatchesToRetain: Int = getConf(MIN_BATCHES_TO_RETAIN) - - def parquetFilterPushDown: Boolean = getConf(PARQUET_FILTER_PUSHDOWN_ENABLED) - - def orcFilterPushDown: Boolean = getConf(ORC_FILTER_PUSHDOWN_ENABLED) - - def verifyPartitionPath: Boolean = getConf(HIVE_VERIFY_PARTITION_PATH) - - def metastorePartitionPruning: Boolean = getConf(HIVE_METASTORE_PARTITION_PRUNING) - - def manageFilesourcePartitions: Boolean = getConf(HIVE_MANAGE_FILESOURCE_PARTITIONS) - - def filesourcePartitionFileCacheSize: Long = getConf(HIVE_FILESOURCE_PARTITION_FILE_CACHE_SIZE) - - def gatherFastStats: Boolean = getConf(GATHER_FASTSTAT) - - def optimizerMetadataOnly: Boolean = getConf(OPTIMIZER_METADATA_ONLY) - - def wholeStageEnabled: Boolean = getConf(WHOLESTAGE_CODEGEN_ENABLED) - - def wholeStageMaxNumFields: Int = getConf(WHOLESTAGE_MAX_NUM_FIELDS) - - def wholeStageFallback: Boolean = getConf(WHOLESTAGE_FALLBACK) - - def maxCaseBranchesForCodegen: Int = getConf(MAX_CASES_BRANCHES) - - def exchangeReuseEnabled: Boolean = getConf(EXCHANGE_REUSE_ENABLED) - - def caseSensitiveAnalysis: Boolean = getConf(SQLConf.CASE_SENSITIVE) - - def subexpressionEliminationEnabled: Boolean = - getConf(SUBEXPRESSION_ELIMINATION_ENABLED) - - def autoBroadcastJoinThreshold: Long = getConf(AUTO_BROADCASTJOIN_THRESHOLD) - - def limitScaleUpFactor: Int = getConf(LIMIT_SCALE_UP_FACTOR) - - def fallBackToHdfsForStatsEnabled: Boolean = getConf(ENABLE_FALL_BACK_TO_HDFS_FOR_STATS) - - def preferSortMergeJoin: Boolean = getConf(PREFER_SORTMERGEJOIN) - - def enableRadixSort: Boolean = getConf(RADIX_SORT_ENABLED) - - def defaultSizeInBytes: Long = getConf(DEFAULT_SIZE_IN_BYTES) - - def isParquetSchemaMergingEnabled: Boolean = getConf(PARQUET_SCHEMA_MERGING_ENABLED) - - def isParquetSchemaRespectSummaries: Boolean = getConf(PARQUET_SCHEMA_RESPECT_SUMMARIES) - - def parquetOutputCommitterClass: String = getConf(PARQUET_OUTPUT_COMMITTER_CLASS) - - def isParquetBinaryAsString: Boolean = getConf(PARQUET_BINARY_AS_STRING) - - def isParquetINT96AsTimestamp: Boolean = getConf(PARQUET_INT96_AS_TIMESTAMP) - - def writeLegacyParquetFormat: Boolean = getConf(PARQUET_WRITE_LEGACY_FORMAT) - - def inMemoryPartitionPruning: Boolean = getConf(IN_MEMORY_PARTITION_PRUNING) - - def columnNameOfCorruptRecord: String = getConf(COLUMN_NAME_OF_CORRUPT_RECORD) - - def broadcastTimeout: Int = getConf(BROADCAST_TIMEOUT) - - def defaultDataSourceName: String = getConf(DEFAULT_DATA_SOURCE_NAME) - - def convertCTAS: Boolean = getConf(CONVERT_CTAS) - - def partitionColumnTypeInferenceEnabled: Boolean = - getConf(SQLConf.PARTITION_COLUMN_TYPE_INFERENCE) - - def fileCommitProtocolClass: String = getConf(SQLConf.FILE_COMMIT_PROTOCOL_CLASS) - - def parallelPartitionDiscoveryThreshold: Int = - getConf(SQLConf.PARALLEL_PARTITION_DISCOVERY_THRESHOLD) - - def bucketingEnabled: Boolean = getConf(SQLConf.BUCKETING_ENABLED) - - def dataFrameSelfJoinAutoResolveAmbiguity: Boolean = - getConf(DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY) - - def dataFrameRetainGroupColumns: Boolean = getConf(DATAFRAME_RETAIN_GROUP_COLUMNS) - - def dataFramePivotMaxValues: Int = getConf(DATAFRAME_PIVOT_MAX_VALUES) - - def runSQLonFile: Boolean = getConf(RUN_SQL_ON_FILES) - - def enableTwoLevelAggMap: Boolean = getConf(ENABLE_TWOLEVEL_AGG_MAP) - - def variableSubstituteEnabled: Boolean = getConf(VARIABLE_SUBSTITUTE_ENABLED) - - def variableSubstituteDepth: Int = getConf(VARIABLE_SUBSTITUTE_DEPTH) - - def warehousePath: String = new Path(getConf(StaticSQLConf.WAREHOUSE_PATH)).toString - - def ignoreCorruptFiles: Boolean = getConf(IGNORE_CORRUPT_FILES) - - def orderByOrdinal: Boolean = getConf(ORDER_BY_ORDINAL) - - def groupByOrdinal: Boolean = getConf(GROUP_BY_ORDINAL) - - def crossJoinEnabled: Boolean = getConf(SQLConf.CROSS_JOINS_ENABLED) - - def ndvMaxError: Double = getConf(NDV_MAX_ERROR) - /** ********************** SQLConf functionality methods ************ */ - - /** Set Spark SQL configuration properties. */ - def setConf(props: Properties): Unit = settings.synchronized { - props.asScala.foreach { case (k, v) => setConfString(k, v) } - } - - /** Set the given Spark SQL configuration property using a `string` value. */ - def setConfString(key: String, value: String): Unit = { - require(key != null, "key cannot be null") - require(value != null, s"value cannot be null for key: $key") - val entry = sqlConfEntries.get(key) - if (entry != null) { - // Only verify configs in the SQLConf object - entry.valueConverter(value) - } - setConfWithCheck(key, value) - } - - /** Set the given Spark SQL configuration property. */ - def setConf[T](entry: ConfigEntry[T], value: T): Unit = { - require(entry != null, "entry cannot be null") - require(value != null, s"value cannot be null for key: ${entry.key}") - require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") - setConfWithCheck(entry.key, entry.stringConverter(value)) - } - - /** Return the value of Spark SQL configuration property for the given key. */ - @throws[NoSuchElementException]("if key is not set") - def getConfString(key: String): String = { - Option(settings.get(key)). - orElse { - // Try to use the default value - Option(sqlConfEntries.get(key)).map(_.defaultValueString) - }. - getOrElse(throw new NoSuchElementException(key)) - } - - /** - * Return the value of Spark SQL configuration property for the given key. If the key is not set - * yet, return `defaultValue`. This is useful when `defaultValue` in ConfigEntry is not the - * desired one. - */ - def getConf[T](entry: ConfigEntry[T], defaultValue: T): T = { - require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") - Option(settings.get(entry.key)).map(entry.valueConverter).getOrElse(defaultValue) - } - - /** - * Return the value of Spark SQL configuration property for the given key. If the key is not set - * yet, return `defaultValue` in [[ConfigEntry]]. - */ - def getConf[T](entry: ConfigEntry[T]): T = { - require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") - entry.readFrom(reader) - } - - /** - * Return the value of an optional Spark SQL configuration property for the given key. If the key - * is not set yet, returns None. - */ - def getConf[T](entry: OptionalConfigEntry[T]): Option[T] = { - require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") - entry.readFrom(reader) - } - - /** - * Return the `string` value of Spark SQL configuration property for the given key. If the key is - * not set yet, return `defaultValue`. - */ - def getConfString(key: String, defaultValue: String): String = { - val entry = sqlConfEntries.get(key) - if (entry != null && defaultValue != "") { - // Only verify configs in the SQLConf object - entry.valueConverter(defaultValue) - } - Option(settings.get(key)).getOrElse(defaultValue) - } - - /** - * Return all the configuration properties that have been set (i.e. not the default). - * This creates a new copy of the config properties in the form of a Map. - */ - def getAllConfs: immutable.Map[String, String] = - settings.synchronized { settings.asScala.toMap } - - /** - * Return all the configuration definitions that have been defined in [[SQLConf]]. Each - * definition contains key, defaultValue and doc. - */ - def getAllDefinedConfs: Seq[(String, String, String)] = sqlConfEntries.synchronized { - sqlConfEntries.values.asScala.filter(_.isPublic).map { entry => - (entry.key, getConfString(entry.key, entry.defaultValueString), entry.doc) - }.toSeq - } - - /** - * Return whether a given key is set in this [[SQLConf]]. - */ - def contains(key: String): Boolean = { - settings.containsKey(key) - } - - private def setConfWithCheck(key: String, value: String): Unit = { - settings.put(key, value) - } - - def unsetConf(key: String): Unit = { - settings.remove(key) - } - - def unsetConf(entry: ConfigEntry[_]): Unit = { - settings.remove(entry.key) - } - - def clear(): Unit = { - settings.clear() - } -} - -/** - * Static SQL configuration is a cross-session, immutable Spark configuration. External users can - * see the static sql configs via `SparkSession.conf`, but can NOT set/unset them. - */ -object StaticSQLConf { - val globalConfKeys = java.util.Collections.synchronizedSet(new java.util.HashSet[String]()) - - private def buildConf(key: String): ConfigBuilder = { - ConfigBuilder(key).onCreate { entry => - globalConfKeys.add(entry.key) - SQLConf.register(entry) - } - } - - val WAREHOUSE_PATH = buildConf("spark.sql.warehouse.dir") - .doc("The default location for managed databases and tables.") - .stringConf - .createWithDefault(Utils.resolveURI("spark-warehouse").toString) - - val CATALOG_IMPLEMENTATION = buildConf("spark.sql.catalogImplementation") - .internal() - .stringConf - .checkValues(Set("hive", "in-memory")) - .createWithDefault("in-memory") - - val GLOBAL_TEMP_DATABASE = buildConf("spark.sql.globalTempDatabase") - .internal() - .stringConf - .createWithDefault("global_temp") - - // This is used to control when we will split a schema's JSON string to multiple pieces - // in order to fit the JSON string in metastore's table property (by default, the value has - // a length restriction of 4000 characters, so do not use a value larger than 4000 as the default - // value of this property). We will split the JSON string of a schema to its length exceeds the - // threshold. Note that, this conf is only read in HiveExternalCatalog which is cross-session, - // that's why this conf has to be a static SQL conf. - val SCHEMA_STRING_LENGTH_THRESHOLD = buildConf("spark.sql.sources.schemaStringLengthThreshold") - .doc("The maximum length allowed in a single cell when " + - "storing additional schema information in Hive's metastore.") - .internal() - .intConf - .createWithDefault(4000) - - // When enabling the debug, Spark SQL internal table properties are not filtered out; however, - // some related DDL commands (e.g., ANALYZE TABLE and CREATE TABLE LIKE) might not work properly. - val DEBUG_MODE = buildConf("spark.sql.debug") - .internal() - .doc("Only used for internal debugging. Not all functions are supported when it is enabled.") - .booleanConf - .createWithDefault(false) -} +///* +// * Licensed to the Apache Software Foundation (ASF) under one or more +// * contributor license agreements. See the NOTICE file distributed with +// * this work for additional information regarding copyright ownership. +// * The ASF licenses this file to You under the Apache License, Version 2.0 +// * (the "License"); you may not use this file except in compliance with +// * the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package org.apache.spark.sql.internal +// +//import java.util.{NoSuchElementException, Properties} +//import java.util.concurrent.TimeUnit +// +//import scala.collection.JavaConverters._ +//import scala.collection.immutable +// +//import org.apache.hadoop.fs.Path +//import org.apache.parquet.hadoop.ParquetOutputCommitter +// +//import org.apache.spark.internal.Logging +//import org.apache.spark.internal.config._ +//import org.apache.spark.network.util.ByteUnit +//import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol +//import org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol +//import org.apache.spark.util.Utils +// +////////////////////////////////////////////////////////////////////////////////////////////////////// +//// This file defines the configuration options for Spark SQL. +////////////////////////////////////////////////////////////////////////////////////////////////////// +// +// +//object SQLConf { +// +// private val sqlConfEntries = java.util.Collections.synchronizedMap( +// new java.util.HashMap[String, ConfigEntry[_]]()) +// +// private[sql] def register(entry: ConfigEntry[_]): Unit = sqlConfEntries.synchronized { +// require(!sqlConfEntries.containsKey(entry.key), +// s"Duplicate SQLConfigEntry. ${entry.key} has been registered") +// sqlConfEntries.put(entry.key, entry) +// } +// +// private[sql] object SQLConfigBuilder { +// +// def apply(key: String): ConfigBuilder = new ConfigBuilder(key).onCreate(register) +// +// } +// +// +// val OPTIMIZER_MAX_ITERATIONS = SQLConfigBuilder("spark.sql.optimizer.maxIterations") +// .internal() +// .doc("The max number of iterations the optimizer and analyzer runs.") +// .intConf +// .createWithDefault(100) +// +// val OPTIMIZER_INSET_CONVERSION_THRESHOLD = +// SQLConfigBuilder("spark.sql.optimizer.inSetConversionThreshold") +// .internal() +// .doc("The threshold of set size for InSet conversion.") +// .intConf +// .createWithDefault(10) +// +// val COMPRESS_CACHED = SQLConfigBuilder("spark.sql.inMemoryColumnarStorage.compressed") +// .internal() +// .doc("When set to true Spark SQL will automatically select a compression codec for each " + +// "column based on statistics of the data.") +// .booleanConf +// .createWithDefault(true) +// +// val COLUMN_BATCH_SIZE = SQLConfigBuilder("spark.sql.inMemoryColumnarStorage.batchSize") +// .internal() +// .doc("Controls the size of batches for columnar caching. Larger batch sizes can improve " + +// "memory utilization and compression, but risk OOMs when caching data.") +// .intConf +// .createWithDefault(10000) +// +// val IN_MEMORY_PARTITION_PRUNING = +// SQLConfigBuilder("spark.sql.inMemoryColumnarStorage.partitionPruning") +// .internal() +// .doc("When true, enable partition pruning for in-memory columnar tables.") +// .booleanConf +// .createWithDefault(true) +// +// val PREFER_SORTMERGEJOIN = SQLConfigBuilder("spark.sql.join.preferSortMergeJoin") +// .internal() +// .doc("When true, prefer sort merge join over shuffle hash join.") +// .booleanConf +// .createWithDefault(true) +// +// val RADIX_SORT_ENABLED = SQLConfigBuilder("spark.sql.sort.enableRadixSort") +// .internal() +// .doc("When true, enable use of radix sort when possible. Radix sort is much faster but " + +// "requires additional memory to be reserved up-front. The memory overhead may be " + +// "significant when sorting very small rows (up to 50% more in this case).") +// .booleanConf +// .createWithDefault(true) +// +// val AUTO_BROADCASTJOIN_THRESHOLD = SQLConfigBuilder("spark.sql.autoBroadcastJoinThreshold") +// .doc("Configures the maximum size in bytes for a table that will be broadcast to all worker " + +// "nodes when performing a join. By setting this value to -1 broadcasting can be disabled. " + +// "Note that currently statistics are only supported for Hive Metastore tables where the " + +// "command ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan has been " + +// "run, and file-based data source tables where the statistics are computed directly on " + +// "the files of data.") +// .longConf +// .createWithDefault(10L * 1024 * 1024) +// +// val LIMIT_SCALE_UP_FACTOR = SQLConfigBuilder("spark.sql.limit.scaleUpFactor") +// .internal() +// .doc("Minimal increase rate in number of partitions between attempts when executing a take " + +// "on a query. Higher values lead to more partitions read. Lower values might lead to " + +// "longer execution times as more jobs will be run") +// .intConf +// .createWithDefault(4) +// +// val ENABLE_FALL_BACK_TO_HDFS_FOR_STATS = +// SQLConfigBuilder("spark.sql.statistics.fallBackToHdfs") +// .doc("If the table statistics are not available from table metadata enable fall back to hdfs." + +// " This is useful in determining if a table is small enough to use auto broadcast joins.") +// .booleanConf +// .createWithDefault(false) +// +// val DEFAULT_SIZE_IN_BYTES = SQLConfigBuilder("spark.sql.defaultSizeInBytes") +// .internal() +// .doc("The default table size used in query planning. By default, it is set to Long.MaxValue " + +// "which is larger than `spark.sql.autoBroadcastJoinThreshold` to be more conservative. " + +// "That is to say by default the optimizer will not choose to broadcast a table unless it " + +// "knows for sure its size is small enough.") +// .longConf +// .createWithDefault(Long.MaxValue) +// +// val SHUFFLE_PARTITIONS = SQLConfigBuilder("spark.sql.shuffle.partitions") +// .doc("The default number of partitions to use when shuffling data for joins or aggregations.") +// .intConf +// .createWithDefault(200) +// +// val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE = +// SQLConfigBuilder("spark.sql.adaptive.shuffle.targetPostShuffleInputSize") +// .doc("The target post-shuffle input size in bytes of a task.") +// .bytesConf(ByteUnit.BYTE) +// .createWithDefault(64 * 1024 * 1024) +// +// val ADAPTIVE_EXECUTION_ENABLED = SQLConfigBuilder("spark.sql.adaptive.enabled") +// .doc("When true, enable adaptive query execution.") +// .booleanConf +// .createWithDefault(false) +// +// val SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS = +// SQLConfigBuilder("spark.sql.adaptive.minNumPostShufflePartitions") +// .internal() +// .doc("The advisory minimal number of post-shuffle partitions provided to " + +// "ExchangeCoordinator. This setting is used in our test to make sure we " + +// "have enough parallelism to expose issues that will not be exposed with a " + +// "single partition. When the value is a non-positive value, this setting will " + +// "not be provided to ExchangeCoordinator.") +// .intConf +// .createWithDefault(-1) +// +// val SUBEXPRESSION_ELIMINATION_ENABLED = +// SQLConfigBuilder("spark.sql.subexpressionElimination.enabled") +// .internal() +// .doc("When true, common subexpressions will be eliminated.") +// .booleanConf +// .createWithDefault(true) +// +// val CASE_SENSITIVE = SQLConfigBuilder("spark.sql.caseSensitive") +// .internal() +// .doc("Whether the query analyzer should be case sensitive or not. " + +// "Default to case insensitive. It is highly discouraged to turn on case sensitive mode.") +// .booleanConf +// .createWithDefault(false) +// +// val PARQUET_SCHEMA_MERGING_ENABLED = SQLConfigBuilder("spark.sql.parquet.mergeSchema") +// .doc("When true, the Parquet data source merges schemas collected from all data files, " + +// "otherwise the schema is picked from the summary file or a random data file " + +// "if no summary file is available.") +// .booleanConf +// .createWithDefault(false) +// +// val PARQUET_SCHEMA_RESPECT_SUMMARIES = SQLConfigBuilder("spark.sql.parquet.respectSummaryFiles") +// .doc("When true, we make assumption that all part-files of Parquet are consistent with " + +// "summary files and we will ignore them when merging schema. Otherwise, if this is " + +// "false, which is the default, we will merge all part-files. This should be considered " + +// "as expert-only option, and shouldn't be enabled before knowing what it means exactly.") +// .booleanConf +// .createWithDefault(false) +// +// val PARQUET_BINARY_AS_STRING = SQLConfigBuilder("spark.sql.parquet.binaryAsString") +// .doc("Some other Parquet-producing systems, in particular Impala and older versions of " + +// "Spark SQL, do not differentiate between binary data and strings when writing out the " + +// "Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide " + +// "compatibility with these systems.") +// .booleanConf +// .createWithDefault(false) +// +// val PARQUET_INT96_AS_TIMESTAMP = SQLConfigBuilder("spark.sql.parquet.int96AsTimestamp") +// .doc("Some Parquet-producing systems, in particular Impala, store Timestamp into INT96. " + +// "Spark would also store Timestamp as INT96 because we need to avoid precision lost of the " + +// "nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to " + +// "provide compatibility with these systems.") +// .booleanConf +// .createWithDefault(true) +// +// val PARQUET_CACHE_METADATA = SQLConfigBuilder("spark.sql.parquet.cacheMetadata") +// .doc("Turns on caching of Parquet schema metadata. Can speed up querying of static data.") +// .booleanConf +// .createWithDefault(true) +// +// val PARQUET_COMPRESSION = SQLConfigBuilder("spark.sql.parquet.compression.codec") +// .doc("Sets the compression codec use when writing Parquet files. Acceptable values include: " + +// "uncompressed, snappy, gzip, lzo.") +// .stringConf +// .transform(_.toLowerCase()) +// .checkValues(Set("uncompressed", "snappy", "gzip", "lzo")) +// .createWithDefault("snappy") +// +// val PARQUET_FILTER_PUSHDOWN_ENABLED = SQLConfigBuilder("spark.sql.parquet.filterPushdown") +// .doc("Enables Parquet filter push-down optimization when set to true.") +// .booleanConf +// .createWithDefault(true) +// +// val PARQUET_WRITE_LEGACY_FORMAT = SQLConfigBuilder("spark.sql.parquet.writeLegacyFormat") +// .doc("Whether to follow Parquet's format specification when converting Parquet schema to " + +// "Spark SQL schema and vice versa.") +// .booleanConf +// .createWithDefault(false) +// +// val PARQUET_OUTPUT_COMMITTER_CLASS = SQLConfigBuilder("spark.sql.parquet.output.committer.class") +// .doc("The output committer class used by Parquet. The specified class needs to be a " + +// "subclass of org.apache.hadoop.mapreduce.OutputCommitter. Typically, it's also a subclass " + +// "of org.apache.parquet.hadoop.ParquetOutputCommitter.") +// .internal() +// .stringConf +// .createWithDefault(classOf[ParquetOutputCommitter].getName) +// +// val PARQUET_VECTORIZED_READER_ENABLED = +// SQLConfigBuilder("spark.sql.parquet.enableVectorizedReader") +// .doc("Enables vectorized parquet decoding.") +// .booleanConf +// .createWithDefault(true) +// +// val ORC_FILTER_PUSHDOWN_ENABLED = SQLConfigBuilder("spark.sql.orc.filterPushdown") +// .doc("When true, enable filter pushdown for ORC files.") +// .booleanConf +// .createWithDefault(false) +// +// val HIVE_VERIFY_PARTITION_PATH = SQLConfigBuilder("spark.sql.hive.verifyPartitionPath") +// .doc("When true, check all the partition paths under the table\'s root directory " + +// "when reading data stored in HDFS.") +// .booleanConf +// .createWithDefault(false) +// +// val HIVE_METASTORE_PARTITION_PRUNING = +// SQLConfigBuilder("spark.sql.hive.metastorePartitionPruning") +// .doc("When true, some predicates will be pushed down into the Hive metastore so that " + +// "unmatching partitions can be eliminated earlier. This only affects Hive tables " + +// "not converted to filesource relations (see HiveUtils.CONVERT_METASTORE_PARQUET and " + +// "HiveUtils.CONVERT_METASTORE_ORC for more information).") +// .booleanConf +// .createWithDefault(true) +// +// val HIVE_MANAGE_FILESOURCE_PARTITIONS = +// SQLConfigBuilder("spark.sql.hive.manageFilesourcePartitions") +// .doc("When true, enable metastore partition management for file source tables as well. " + +// "This includes both datasource and converted Hive tables. When partition managment " + +// "is enabled, datasource tables store partition in the Hive metastore, and use the " + +// "metastore to prune partitions during query planning.") +// .booleanConf +// .createWithDefault(true) +// +// val HIVE_FILESOURCE_PARTITION_FILE_CACHE_SIZE = +// SQLConfigBuilder("spark.sql.hive.filesourcePartitionFileCacheSize") +// .doc("When nonzero, enable caching of partition file metadata in memory. All tables share " + +// "a cache that can use up to specified num bytes for file metadata. This conf only " + +// "has an effect when hive filesource partition management is enabled.") +// .longConf +// .createWithDefault(250 * 1024 * 1024) +// +// val OPTIMIZER_METADATA_ONLY = SQLConfigBuilder("spark.sql.optimizer.metadataOnly") +// .doc("When true, enable the metadata-only query optimization that use the table's metadata " + +// "to produce the partition columns instead of table scans. It applies when all the columns " + +// "scanned are partition columns and the query has an aggregate operator that satisfies " + +// "distinct semantics.") +// .booleanConf +// .createWithDefault(true) +// +// val COLUMN_NAME_OF_CORRUPT_RECORD = SQLConfigBuilder("spark.sql.columnNameOfCorruptRecord") +// .doc("The name of internal column for storing raw/un-parsed JSON records that fail to parse.") +// .stringConf +// .createWithDefault("_corrupt_record") +// +// val BROADCAST_TIMEOUT = SQLConfigBuilder("spark.sql.broadcastTimeout") +// .doc("Timeout in seconds for the broadcast wait time in broadcast joins.") +// .intConf +// .createWithDefault(5 * 60) +// +// // This is only used for the thriftserver +// val THRIFTSERVER_POOL = SQLConfigBuilder("spark.sql.thriftserver.scheduler.pool") +// .doc("Set a Fair Scheduler pool for a JDBC client session.") +// .stringConf +// .createOptional +// +// val THRIFTSERVER_UI_STATEMENT_LIMIT = +// SQLConfigBuilder("spark.sql.thriftserver.ui.retainedStatements") +// .doc("The number of SQL statements kept in the JDBC/ODBC web UI history.") +// .intConf +// .createWithDefault(200) +// +// val THRIFTSERVER_UI_SESSION_LIMIT = SQLConfigBuilder("spark.sql.thriftserver.ui.retainedSessions") +// .doc("The number of SQL client sessions kept in the JDBC/ODBC web UI history.") +// .intConf +// .createWithDefault(200) +// +// // This is used to set the default data source +// val DEFAULT_DATA_SOURCE_NAME = SQLConfigBuilder("spark.sql.sources.default") +// .doc("The default data source to use in input/output.") +// .stringConf +// .createWithDefault("parquet") +// +// val CONVERT_CTAS = SQLConfigBuilder("spark.sql.hive.convertCTAS") +// .internal() +// .doc("When true, a table created by a Hive CTAS statement (no USING clause) " + +// "without specifying any storage property will be converted to a data source table, " + +// "using the data source set by spark.sql.sources.default.") +// .booleanConf +// .createWithDefault(false) +// +// val GATHER_FASTSTAT = SQLConfigBuilder("spark.sql.hive.gatherFastStats") +// .internal() +// .doc("When true, fast stats (number of files and total size of all files) will be gathered" + +// " in parallel while repairing table partitions to avoid the sequential listing in Hive" + +// " metastore.") +// .booleanConf +// .createWithDefault(true) +// +// val PARTITION_COLUMN_TYPE_INFERENCE = +// SQLConfigBuilder("spark.sql.sources.partitionColumnTypeInference.enabled") +// .doc("When true, automatically infer the data types for partitioned columns.") +// .booleanConf +// .createWithDefault(true) +// +// val BUCKETING_ENABLED = SQLConfigBuilder("spark.sql.sources.bucketing.enabled") +// .doc("When false, we will treat bucketed table as normal table") +// .booleanConf +// .createWithDefault(true) +// +// val CROSS_JOINS_ENABLED = SQLConfigBuilder("spark.sql.crossJoin.enabled") +// .doc("When false, we will throw an error if a query contains a cartesian product without " + +// "explicit CROSS JOIN syntax.") +// .booleanConf +// .createWithDefault(false) +// +// val ORDER_BY_ORDINAL = SQLConfigBuilder("spark.sql.orderByOrdinal") +// .doc("When true, the ordinal numbers are treated as the position in the select list. " + +// "When false, the ordinal numbers in order/sort by clause are ignored.") +// .booleanConf +// .createWithDefault(true) +// +// val GROUP_BY_ORDINAL = SQLConfigBuilder("spark.sql.groupByOrdinal") +// .doc("When true, the ordinal numbers in group by clauses are treated as the position " + +// "in the select list. When false, the ordinal numbers are ignored.") +// .booleanConf +// .createWithDefault(true) +// +// // The output committer class used by data sources. The specified class needs to be a +// // subclass of org.apache.hadoop.mapreduce.OutputCommitter. +// val OUTPUT_COMMITTER_CLASS = +// SQLConfigBuilder("spark.sql.sources.outputCommitterClass").internal().stringConf.createOptional +// +// val FILE_COMMIT_PROTOCOL_CLASS = +// SQLConfigBuilder("spark.sql.sources.commitProtocolClass") +// .internal() +// .stringConf +// .createWithDefault(classOf[SQLHadoopMapReduceCommitProtocol].getName) +// +// val PARALLEL_PARTITION_DISCOVERY_THRESHOLD = +// SQLConfigBuilder("spark.sql.sources.parallelPartitionDiscovery.threshold") +// .doc("The maximum number of files allowed for listing files at driver side. If the number " + +// "of detected files exceeds this value during partition discovery, it tries to list the " + +// "files with another Spark distributed job. This applies to Parquet, ORC, CSV, JSON and " + +// "LibSVM data sources.") +// .intConf +// .createWithDefault(32) +// +// // Whether to automatically resolve ambiguity in join conditions for self-joins. +// // See SPARK-6231. +// val DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY = +// SQLConfigBuilder("spark.sql.selfJoinAutoResolveAmbiguity") +// .internal() +// .booleanConf +// .createWithDefault(true) +// +// // Whether to retain group by columns or not in GroupedData.agg. +// val DATAFRAME_RETAIN_GROUP_COLUMNS = SQLConfigBuilder("spark.sql.retainGroupColumns") +// .internal() +// .booleanConf +// .createWithDefault(true) +// +// val DATAFRAME_PIVOT_MAX_VALUES = SQLConfigBuilder("spark.sql.pivotMaxValues") +// .doc("When doing a pivot without specifying values for the pivot column this is the maximum " + +// "number of (distinct) values that will be collected without error.") +// .intConf +// .createWithDefault(10000) +// +// val RUN_SQL_ON_FILES = SQLConfigBuilder("spark.sql.runSQLOnFiles") +// .internal() +// .doc("When true, we could use `datasource`.`path` as table in SQL query.") +// .booleanConf +// .createWithDefault(true) +// +// val WHOLESTAGE_CODEGEN_ENABLED = SQLConfigBuilder("spark.sql.codegen.wholeStage") +// .internal() +// .doc("When true, the whole stage (of multiple operators) will be compiled into single java" + +// " method.") +// .booleanConf +// .createWithDefault(true) +// +// val WHOLESTAGE_MAX_NUM_FIELDS = SQLConfigBuilder("spark.sql.codegen.maxFields") +// .internal() +// .doc("The maximum number of fields (including nested fields) that will be supported before" + +// " deactivating whole-stage codegen.") +// .intConf +// .createWithDefault(100) +// +// val WHOLESTAGE_FALLBACK = SQLConfigBuilder("spark.sql.codegen.fallback") +// .internal() +// .doc("When true, whole stage codegen could be temporary disabled for the part of query that" + +// " fail to compile generated code") +// .booleanConf +// .createWithDefault(true) +// +// val MAX_CASES_BRANCHES = SQLConfigBuilder("spark.sql.codegen.maxCaseBranches") +// .internal() +// .doc("The maximum number of switches supported with codegen.") +// .intConf +// .createWithDefault(20) +// +// val FILES_MAX_PARTITION_BYTES = SQLConfigBuilder("spark.sql.files.maxPartitionBytes") +// .doc("The maximum number of bytes to pack into a single partition when reading files.") +// .longConf +// .createWithDefault(128 * 1024 * 1024) // parquet.block.size +// +// val FILES_OPEN_COST_IN_BYTES = SQLConfigBuilder("spark.sql.files.openCostInBytes") +// .internal() +// .doc("The estimated cost to open a file, measured by the number of bytes could be scanned in" + +// " the same time. This is used when putting multiple files into a partition. It's better to" + +// " over estimated, then the partitions with small files will be faster than partitions with" + +// " bigger files (which is scheduled first).") +// .longConf +// .createWithDefault(4 * 1024 * 1024) +// +// val EXCHANGE_REUSE_ENABLED = SQLConfigBuilder("spark.sql.exchange.reuse") +// .internal() +// .doc("When true, the planner will try to find out duplicated exchanges and re-use them.") +// .booleanConf +// .createWithDefault(true) +// +// val STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT = +// SQLConfigBuilder("spark.sql.streaming.stateStore.minDeltasForSnapshot") +// .internal() +// .doc("Minimum number of state store delta files that needs to be generated before they " + +// "consolidated into snapshots.") +// .intConf +// .createWithDefault(10) +// +// val CHECKPOINT_LOCATION = SQLConfigBuilder("spark.sql.streaming.checkpointLocation") +// .doc("The default location for storing checkpoint data for streaming queries.") +// .stringConf +// .createOptional +// +// val MIN_BATCHES_TO_RETAIN = SQLConfigBuilder("spark.sql.streaming.minBatchesToRetain") +// .internal() +// .doc("The minimum number of batches that must be retained and made recoverable.") +// .intConf +// .createWithDefault(100) +// +// val UNSUPPORTED_OPERATION_CHECK_ENABLED = +// SQLConfigBuilder("spark.sql.streaming.unsupportedOperationCheck") +// .internal() +// .doc("When true, the logical plan for streaming query will be checked for unsupported" + +// " operations.") +// .booleanConf +// .createWithDefault(true) +// +// val VARIABLE_SUBSTITUTE_ENABLED = +// SQLConfigBuilder("spark.sql.variable.substitute") +// .doc("This enables substitution using syntax like ${var} ${system:var} and ${env:var}.") +// .booleanConf +// .createWithDefault(true) +// +// val VARIABLE_SUBSTITUTE_DEPTH = +// SQLConfigBuilder("spark.sql.variable.substitute.depth") +// .internal() +// .doc("Deprecated: The maximum replacements the substitution engine will do.") +// .intConf +// .createWithDefault(40) +// +// val VECTORIZED_AGG_MAP_MAX_COLUMNS = +// SQLConfigBuilder("spark.sql.codegen.aggregate.map.columns.max") +// .internal() +// .doc("Sets the maximum width of schema (aggregate keys + values) for " + +// "which aggregate with" + +// "keys uses an in-memory columnar map to speed up execution. " + +// "Setting this to 0 effectively" + +// "disables the columnar map") +// .intConf +// .createWithDefault(3) +// +// val ENABLE_TWOLEVEL_AGG_MAP = +// SQLConfigBuilder("spark.sql.codegen.aggregate.map.twolevel.enable") +// .internal() +// .doc("Enable two-level aggregate hash map. When enabled, records will first be " + +// "inserted/looked-up at a 1st-level, small, fast map, and then fallback to a " + +// "2nd-level, larger, slower map when 1st level is full or keys cannot be found. " + +// "When disabled, records go directly to the 2nd level. Defaults to true.") +// .booleanConf +// .createWithDefault(true) +// +// val STREAMING_FILE_COMMIT_PROTOCOL_CLASS = +// SQLConfigBuilder("spark.sql.streaming.commitProtocolClass") +// .internal() +// .stringConf +// .createWithDefault(classOf[ManifestFileCommitProtocol].getName) +// +// val FILE_SINK_LOG_DELETION = SQLConfigBuilder("spark.sql.streaming.fileSink.log.deletion") +// .internal() +// .doc("Whether to delete the expired log files in file stream sink.") +// .booleanConf +// .createWithDefault(true) +// +// val FILE_SINK_LOG_COMPACT_INTERVAL = +// SQLConfigBuilder("spark.sql.streaming.fileSink.log.compactInterval") +// .internal() +// .doc("Number of log files after which all the previous files " + +// "are compacted into the next log file.") +// .intConf +// .createWithDefault(10) +// +// val FILE_SINK_LOG_CLEANUP_DELAY = +// SQLConfigBuilder("spark.sql.streaming.fileSink.log.cleanupDelay") +// .internal() +// .doc("How long that a file is guaranteed to be visible for all readers.") +// .timeConf(TimeUnit.MILLISECONDS) +// .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes +// +// val FILE_SOURCE_LOG_DELETION = SQLConfigBuilder("spark.sql.streaming.fileSource.log.deletion") +// .internal() +// .doc("Whether to delete the expired log files in file stream source.") +// .booleanConf +// .createWithDefault(true) +// +// val FILE_SOURCE_LOG_COMPACT_INTERVAL = +// SQLConfigBuilder("spark.sql.streaming.fileSource.log.compactInterval") +// .internal() +// .doc("Number of log files after which all the previous files " + +// "are compacted into the next log file.") +// .intConf +// .createWithDefault(10) +// +// val FILE_SOURCE_LOG_CLEANUP_DELAY = +// SQLConfigBuilder("spark.sql.streaming.fileSource.log.cleanupDelay") +// .internal() +// .doc("How long in milliseconds a file is guaranteed to be visible for all readers.") +// .timeConf(TimeUnit.MILLISECONDS) +// .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes +// +// val STREAMING_SCHEMA_INFERENCE = +// SQLConfigBuilder("spark.sql.streaming.schemaInference") +// .internal() +// .doc("Whether file-based streaming sources will infer its own schema") +// .booleanConf +// .createWithDefault(false) +// +// val STREAMING_POLLING_DELAY = +// SQLConfigBuilder("spark.sql.streaming.pollingDelay") +// .internal() +// .doc("How long to delay polling new data when no data is available") +// .timeConf(TimeUnit.MILLISECONDS) +// .createWithDefault(10L) +// +// val STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL = +// SQLConfigBuilder("spark.sql.streaming.noDataProgressEventInterval") +// .internal() +// .doc("How long to wait between two progress events when there is no data") +// .timeConf(TimeUnit.MILLISECONDS) +// .createWithDefault(10000L) +// +// val STREAMING_METRICS_ENABLED = +// SQLConfigBuilder("spark.sql.streaming.metricsEnabled") +// .doc("Whether Dropwizard/Codahale metrics will be reported for active streaming queries.") +// .booleanConf +// .createWithDefault(false) +// +// val STREAMING_PROGRESS_RETENTION = +// SQLConfigBuilder("spark.sql.streaming.numRecentProgressUpdates") +// .doc("The number of progress updates to retain for a streaming query") +// .intConf +// .createWithDefault(100) +// +// val NDV_MAX_ERROR = +// SQLConfigBuilder("spark.sql.statistics.ndv.maxError") +// .internal() +// .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " + +// "column level statistics.") +// .doubleConf +// .createWithDefault(0.05) +// +// val IGNORE_CORRUPT_FILES = SQLConfigBuilder("spark.sql.files.ignoreCorruptFiles") +// .doc("Whether to ignore corrupt files. If true, the Spark jobs will continue to run when " + +// "encountering corrupted or non-existing and contents that have been read will still be " + +// "returned.") +// .booleanConf +// .createWithDefault(false) +// +// object Deprecated { +// val MAPRED_REDUCE_TASKS = "mapred.reduce.tasks" +// } +//} +// +///** +// * A class that enables the setting and getting of mutable config parameters/hints. +// * +// * In the presence of a SQLContext, these can be set and queried by passing SET commands +// * into Spark SQL's query functions (i.e. sql()). Otherwise, users of this class can +// * modify the hints by programmatically calling the setters and getters of this class. +// * +// * SQLConf is thread-safe (internally synchronized, so safe to be used in multiple threads). +// */ +//private[sql] class SQLConf extends Serializable with Logging { +// import SQLConf._ +// +// /** Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap. */ +// @transient protected[spark] val settings = java.util.Collections.synchronizedMap( +// new java.util.HashMap[String, String]()) +// +// @transient private val reader = new ConfigReader(settings) +// +// /** ************************ Spark SQL Params/Hints ******************* */ +// +// def optimizerMaxIterations: Int = getConf(OPTIMIZER_MAX_ITERATIONS) +// +// def optimizerInSetConversionThreshold: Int = getConf(OPTIMIZER_INSET_CONVERSION_THRESHOLD) +// +// def stateStoreMinDeltasForSnapshot: Int = getConf(STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT) +// +// def checkpointLocation: Option[String] = getConf(CHECKPOINT_LOCATION) +// +// def isUnsupportedOperationCheckEnabled: Boolean = getConf(UNSUPPORTED_OPERATION_CHECK_ENABLED) +// +// def streamingFileCommitProtocolClass: String = getConf(STREAMING_FILE_COMMIT_PROTOCOL_CLASS) +// +// def fileSinkLogDeletion: Boolean = getConf(FILE_SINK_LOG_DELETION) +// +// def fileSinkLogCompactInterval: Int = getConf(FILE_SINK_LOG_COMPACT_INTERVAL) +// +// def fileSinkLogCleanupDelay: Long = getConf(FILE_SINK_LOG_CLEANUP_DELAY) +// +// def fileSourceLogDeletion: Boolean = getConf(FILE_SOURCE_LOG_DELETION) +// +// def fileSourceLogCompactInterval: Int = getConf(FILE_SOURCE_LOG_COMPACT_INTERVAL) +// +// def fileSourceLogCleanupDelay: Long = getConf(FILE_SOURCE_LOG_CLEANUP_DELAY) +// +// def streamingSchemaInference: Boolean = getConf(STREAMING_SCHEMA_INFERENCE) +// +// def streamingPollingDelay: Long = getConf(STREAMING_POLLING_DELAY) +// +// def streamingNoDataProgressEventInterval: Long = +// getConf(STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL) +// +// def streamingMetricsEnabled: Boolean = getConf(STREAMING_METRICS_ENABLED) +// +// def streamingProgressRetention: Int = getConf(STREAMING_PROGRESS_RETENTION) +// +// def filesMaxPartitionBytes: Long = getConf(FILES_MAX_PARTITION_BYTES) +// +// def filesOpenCostInBytes: Long = getConf(FILES_OPEN_COST_IN_BYTES) +// +// def useCompression: Boolean = getConf(COMPRESS_CACHED) +// +// def parquetCompressionCodec: String = getConf(PARQUET_COMPRESSION) +// +// def parquetCacheMetadata: Boolean = getConf(PARQUET_CACHE_METADATA) +// +// def parquetVectorizedReaderEnabled: Boolean = getConf(PARQUET_VECTORIZED_READER_ENABLED) +// +// def columnBatchSize: Int = getConf(COLUMN_BATCH_SIZE) +// +// def numShufflePartitions: Int = getConf(SHUFFLE_PARTITIONS) +// +// def targetPostShuffleInputSize: Long = +// getConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) +// +// def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED) +// +// def minNumPostShufflePartitions: Int = +// getConf(SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS) +// +// def minBatchesToRetain: Int = getConf(MIN_BATCHES_TO_RETAIN) +// +// def parquetFilterPushDown: Boolean = getConf(PARQUET_FILTER_PUSHDOWN_ENABLED) +// +// def orcFilterPushDown: Boolean = getConf(ORC_FILTER_PUSHDOWN_ENABLED) +// +// def verifyPartitionPath: Boolean = getConf(HIVE_VERIFY_PARTITION_PATH) +// +// def metastorePartitionPruning: Boolean = getConf(HIVE_METASTORE_PARTITION_PRUNING) +// +// def manageFilesourcePartitions: Boolean = getConf(HIVE_MANAGE_FILESOURCE_PARTITIONS) +// +// def filesourcePartitionFileCacheSize: Long = getConf(HIVE_FILESOURCE_PARTITION_FILE_CACHE_SIZE) +// +// def gatherFastStats: Boolean = getConf(GATHER_FASTSTAT) +// +// def optimizerMetadataOnly: Boolean = getConf(OPTIMIZER_METADATA_ONLY) +// +// def wholeStageEnabled: Boolean = getConf(WHOLESTAGE_CODEGEN_ENABLED) +// +// def wholeStageMaxNumFields: Int = getConf(WHOLESTAGE_MAX_NUM_FIELDS) +// +// def wholeStageFallback: Boolean = getConf(WHOLESTAGE_FALLBACK) +// +// def maxCaseBranchesForCodegen: Int = getConf(MAX_CASES_BRANCHES) +// +// def exchangeReuseEnabled: Boolean = getConf(EXCHANGE_REUSE_ENABLED) +// +// def caseSensitiveAnalysis: Boolean = getConf(SQLConf.CASE_SENSITIVE) +// +// def subexpressionEliminationEnabled: Boolean = +// getConf(SUBEXPRESSION_ELIMINATION_ENABLED) +// +// def autoBroadcastJoinThreshold: Long = getConf(AUTO_BROADCASTJOIN_THRESHOLD) +// +// def limitScaleUpFactor: Int = getConf(LIMIT_SCALE_UP_FACTOR) +// +// def fallBackToHdfsForStatsEnabled: Boolean = getConf(ENABLE_FALL_BACK_TO_HDFS_FOR_STATS) +// +// def preferSortMergeJoin: Boolean = getConf(PREFER_SORTMERGEJOIN) +// +// def enableRadixSort: Boolean = getConf(RADIX_SORT_ENABLED) +// +// def defaultSizeInBytes: Long = getConf(DEFAULT_SIZE_IN_BYTES) +// +// def isParquetSchemaMergingEnabled: Boolean = getConf(PARQUET_SCHEMA_MERGING_ENABLED) +// +// def isParquetSchemaRespectSummaries: Boolean = getConf(PARQUET_SCHEMA_RESPECT_SUMMARIES) +// +// def parquetOutputCommitterClass: String = getConf(PARQUET_OUTPUT_COMMITTER_CLASS) +// +// def isParquetBinaryAsString: Boolean = getConf(PARQUET_BINARY_AS_STRING) +// +// def isParquetINT96AsTimestamp: Boolean = getConf(PARQUET_INT96_AS_TIMESTAMP) +// +// def writeLegacyParquetFormat: Boolean = getConf(PARQUET_WRITE_LEGACY_FORMAT) +// +// def inMemoryPartitionPruning: Boolean = getConf(IN_MEMORY_PARTITION_PRUNING) +// +// def columnNameOfCorruptRecord: String = getConf(COLUMN_NAME_OF_CORRUPT_RECORD) +// +// def broadcastTimeout: Int = getConf(BROADCAST_TIMEOUT) +// +// def defaultDataSourceName: String = getConf(DEFAULT_DATA_SOURCE_NAME) +// +// def convertCTAS: Boolean = getConf(CONVERT_CTAS) +// +// def partitionColumnTypeInferenceEnabled: Boolean = +// getConf(SQLConf.PARTITION_COLUMN_TYPE_INFERENCE) +// +// def fileCommitProtocolClass: String = getConf(SQLConf.FILE_COMMIT_PROTOCOL_CLASS) +// +// def parallelPartitionDiscoveryThreshold: Int = +// getConf(SQLConf.PARALLEL_PARTITION_DISCOVERY_THRESHOLD) +// +// def bucketingEnabled: Boolean = getConf(SQLConf.BUCKETING_ENABLED) +// +// def dataFrameSelfJoinAutoResolveAmbiguity: Boolean = +// getConf(DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY) +// +// def dataFrameRetainGroupColumns: Boolean = getConf(DATAFRAME_RETAIN_GROUP_COLUMNS) +// +// def dataFramePivotMaxValues: Int = getConf(DATAFRAME_PIVOT_MAX_VALUES) +// +// def runSQLonFile: Boolean = getConf(RUN_SQL_ON_FILES) +// +// def enableTwoLevelAggMap: Boolean = getConf(ENABLE_TWOLEVEL_AGG_MAP) +// +// def variableSubstituteEnabled: Boolean = getConf(VARIABLE_SUBSTITUTE_ENABLED) +// +// def variableSubstituteDepth: Int = getConf(VARIABLE_SUBSTITUTE_DEPTH) +// +// def warehousePath: String = new Path(getConf(StaticSQLConf.WAREHOUSE_PATH)).toString +// +// def ignoreCorruptFiles: Boolean = getConf(IGNORE_CORRUPT_FILES) +// +// def orderByOrdinal: Boolean = getConf(ORDER_BY_ORDINAL) +// +// def groupByOrdinal: Boolean = getConf(GROUP_BY_ORDINAL) +// +// def crossJoinEnabled: Boolean = getConf(SQLConf.CROSS_JOINS_ENABLED) +// +// def ndvMaxError: Double = getConf(NDV_MAX_ERROR) +// /** ********************** SQLConf functionality methods ************ */ +// +// /** Set Spark SQL configuration properties. */ +// def setConf(props: Properties): Unit = settings.synchronized { +// props.asScala.foreach { case (k, v) => setConfString(k, v) } +// } +// +// /** Set the given Spark SQL configuration property using a `string` value. */ +// def setConfString(key: String, value: String): Unit = { +// require(key != null, "key cannot be null") +// require(value != null, s"value cannot be null for key: $key") +// val entry = sqlConfEntries.get(key) +// if (entry != null) { +// // Only verify configs in the SQLConf object +// entry.valueConverter(value) +// } +// setConfWithCheck(key, value) +// } +// +// /** Set the given Spark SQL configuration property. */ +// def setConf[T](entry: ConfigEntry[T], value: T): Unit = { +// require(entry != null, "entry cannot be null") +// require(value != null, s"value cannot be null for key: ${entry.key}") +// require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") +// setConfWithCheck(entry.key, entry.stringConverter(value)) +// } +// +// /** Return the value of Spark SQL configuration property for the given key. */ +// @throws[NoSuchElementException]("if key is not set") +// def getConfString(key: String): String = { +// Option(settings.get(key)). +// orElse { +// // Try to use the default value +// Option(sqlConfEntries.get(key)).map(_.defaultValueString) +// }. +// getOrElse(throw new NoSuchElementException(key)) +// } +// +// /** +// * Return the value of Spark SQL configuration property for the given key. If the key is not set +// * yet, return `defaultValue`. This is useful when `defaultValue` in ConfigEntry is not the +// * desired one. +// */ +// def getConf[T](entry: ConfigEntry[T], defaultValue: T): T = { +// require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") +// Option(settings.get(entry.key)).map(entry.valueConverter).getOrElse(defaultValue) +// } +// +// /** +// * Return the value of Spark SQL configuration property for the given key. If the key is not set +// * yet, return `defaultValue` in [[ConfigEntry]]. +// */ +// def getConf[T](entry: ConfigEntry[T]): T = { +// require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") +// entry.readFrom(reader) +// } +// +// /** +// * Return the value of an optional Spark SQL configuration property for the given key. If the key +// * is not set yet, returns None. +// */ +// def getConf[T](entry: OptionalConfigEntry[T]): Option[T] = { +// require(sqlConfEntries.get(entry.key) == entry, s"$entry is not registered") +// entry.readFrom(reader) +// } +// +// /** +// * Return the `string` value of Spark SQL configuration property for the given key. If the key is +// * not set yet, return `defaultValue`. +// */ +// def getConfString(key: String, defaultValue: String): String = { +// val entry = sqlConfEntries.get(key) +// if (entry != null && defaultValue != "") { +// // Only verify configs in the SQLConf object +// entry.valueConverter(defaultValue) +// } +// Option(settings.get(key)).getOrElse(defaultValue) +// } +// +// /** +// * Return all the configuration properties that have been set (i.e. not the default). +// * This creates a new copy of the config properties in the form of a Map. +// */ +// def getAllConfs: immutable.Map[String, String] = +// settings.synchronized { settings.asScala.toMap } +// +// /** +// * Return all the configuration definitions that have been defined in [[SQLConf]]. Each +// * definition contains key, defaultValue and doc. +// */ +// def getAllDefinedConfs: Seq[(String, String, String)] = sqlConfEntries.synchronized { +// sqlConfEntries.values.asScala.filter(_.isPublic).map { entry => +// (entry.key, getConfString(entry.key, entry.defaultValueString), entry.doc) +// }.toSeq +// } +// +// /** +// * Return whether a given key is set in this [[SQLConf]]. +// */ +// def contains(key: String): Boolean = { +// settings.containsKey(key) +// } +// +// private def setConfWithCheck(key: String, value: String): Unit = { +// settings.put(key, value) +// } +// +// def unsetConf(key: String): Unit = { +// settings.remove(key) +// } +// +// def unsetConf(entry: ConfigEntry[_]): Unit = { +// settings.remove(entry.key) +// } +// +// def clear(): Unit = { +// settings.clear() +// } +//} +// +///** +// * Static SQL configuration is a cross-session, immutable Spark configuration. External users can +// * see the static sql configs via `SparkSession.conf`, but can NOT set/unset them. +// */ +//object StaticSQLConf { +// val globalConfKeys = java.util.Collections.synchronizedSet(new java.util.HashSet[String]()) +// +// private def buildConf(key: String): ConfigBuilder = { +// ConfigBuilder(key).onCreate { entry => +// globalConfKeys.add(entry.key) +// SQLConf.register(entry) +// } +// } +// +// val WAREHOUSE_PATH = buildConf("spark.sql.warehouse.dir") +// .doc("The default location for managed databases and tables.") +// .stringConf +// .createWithDefault(Utils.resolveURI("spark-warehouse").toString) +// +// val CATALOG_IMPLEMENTATION = buildConf("spark.sql.catalogImplementation") +// .internal() +// .stringConf +// .checkValues(Set("hive", "in-memory")) +// .createWithDefault("in-memory") +// +// val GLOBAL_TEMP_DATABASE = buildConf("spark.sql.globalTempDatabase") +// .internal() +// .stringConf +// .createWithDefault("global_temp") +// +// // This is used to control when we will split a schema's JSON string to multiple pieces +// // in order to fit the JSON string in metastore's table property (by default, the value has +// // a length restriction of 4000 characters, so do not use a value larger than 4000 as the default +// // value of this property). We will split the JSON string of a schema to its length exceeds the +// // threshold. Note that, this conf is only read in HiveExternalCatalog which is cross-session, +// // that's why this conf has to be a static SQL conf. +// val SCHEMA_STRING_LENGTH_THRESHOLD = buildConf("spark.sql.sources.schemaStringLengthThreshold") +// .doc("The maximum length allowed in a single cell when " + +// "storing additional schema information in Hive's metastore.") +// .internal() +// .intConf +// .createWithDefault(4000) +// +// // When enabling the debug, Spark SQL internal table properties are not filtered out; however, +// // some related DDL commands (e.g., ANALYZE TABLE and CREATE TABLE LIKE) might not work properly. +// val DEBUG_MODE = buildConf("spark.sql.debug") +// .internal() +// .doc("Only used for internal debugging. Not all functions are supported when it is enabled.") +// .booleanConf +// .createWithDefault(false) +//} From ad82bebe5633c1d45d5b4d7c41dbcdc867bb59d5 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Mon, 19 Mar 2018 16:28:18 -0700 Subject: [PATCH 125/142] compilation issues --- .../org/apache/spark/sql/execution/aggregate/AggUtils.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala index 8cb235d4b3f12..d025ee6961bc4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala @@ -39,7 +39,6 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.streaming.{StateStoreRestoreExec, StateStoreSaveExec} -import org.apache.spark.sql.internal.SQLConf /** * Utility functions used by the query planner to convert our plan to new aggregation code path. @@ -60,7 +59,7 @@ object AggUtils { aggregateExpressions = completeAggregateExpressions, aggregateAttributes = completeAggregateAttributes, initialInputBufferOffset = 0, - resultExpressions = resultExpressions, + resultExpressions, child = child ) :: Nil } @@ -104,7 +103,7 @@ object AggUtils { aggregateExpressions = aggregateExpressions, aggregateAttributes = aggregateAttributes, initialInputBufferOffset = initialInputBufferOffset, - resultExpressions = resultExpressions, + resultExpressions, child = child) } } From 11523f24457f115381857a5dc6a274b00511f80e Mon Sep 17 00:00:00 2001 From: ymahajan Date: Wed, 21 Mar 2018 13:02:44 -0700 Subject: [PATCH 126/142] Added new build.gradle for spark-kvstore subproject --- common/kvstore/build.gradle | 32 +++++++++++++++++++ core/build.gradle | 1 + .../apache/spark/storage/BlockManager.scala | 1 - settings.gradle | 2 ++ 4 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 common/kvstore/build.gradle diff --git a/common/kvstore/build.gradle b/common/kvstore/build.gradle new file mode 100644 index 0000000000000..d4ad2f686f792 --- /dev/null +++ b/common/kvstore/build.gradle @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +description = 'Spark Spark Project Local DB' + +dependencies { + compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + + compile group: 'com.google.guava', name: 'guava', version: guavaVersion + compile group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: '1.8' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion + compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion + testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile group: 'commons-io', name: 'commons-io', version: '2.4' + testCompile group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion +} + + diff --git a/core/build.gradle b/core/build.gradle index 341c65d7a5c7b..d4c4ebc2c331a 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -23,6 +23,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-network-shuffle_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-unsafe_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-kvstore_' + scalaBinaryVersion) compile(group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion) { exclude(group: 'io.netty', module: 'netty') diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index d40d1d3c036f8..ba471bccf475c 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -1578,7 +1578,6 @@ private[spark] object BlockManager { } blockManagers.toMap } -<<<<<<< HEAD private class ShuffleMetricsSource( override val sourceName: String, diff --git a/settings.gradle b/settings.gradle index 6d2282a70d2b5..2aa8b617d87ea 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,6 +24,7 @@ include ':snappy-spark-graphx_' + scalaBinaryVersion include ':snappy-spark-mllib_' + scalaBinaryVersion include ':snappy-spark-mllib-local_' + scalaBinaryVersion include ':snappy-spark-tools_' + scalaBinaryVersion +include ':snappy-spark-kvstore_' + scalaBinaryVersion include ':snappy-spark-network-common_' + scalaBinaryVersion include ':snappy-spark-network-shuffle_' + scalaBinaryVersion include ':snappy-spark-network-yarn_' + scalaBinaryVersion @@ -53,6 +54,7 @@ project(':snappy-spark-graphx_' + scalaBinaryVersion).projectDir = "$rootDir/gra project(':snappy-spark-mllib_' + scalaBinaryVersion).projectDir = "$rootDir/mllib" as File project(':snappy-spark-mllib-local_' + scalaBinaryVersion).projectDir = "$rootDir/mllib-local" as File project(':snappy-spark-tools_' + scalaBinaryVersion).projectDir = "$rootDir/tools" as File +project(':snappy-spark-kvstore_' + scalaBinaryVersion).projectDir = "$rootDir/common/kvstore" as File project(':snappy-spark-network-common_' + scalaBinaryVersion).projectDir = "$rootDir/common/network-common" as File project(':snappy-spark-network-shuffle_' + scalaBinaryVersion).projectDir = "$rootDir/common/network-shuffle" as File project(':snappy-spark-network-yarn_' + scalaBinaryVersion).projectDir = "$rootDir/common/network-yarn" as File From b41d4f83ad8d03304700f878256c7820a961a9a6 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Wed, 28 Mar 2018 21:28:13 -0700 Subject: [PATCH 127/142] Addressing precheckin failures --- build.gradle | 24 +- common/kvstore/build.gradle | 2 +- common/network-common/build.gradle | 24 +- core/build.gradle | 11 +- .../org/apache/spark/MapOutputTracker.scala | 1 + .../scala/org/apache/spark/SparkContext.scala | 68 +--- .../HiveDelegationTokenProvider.scala | 15 +- .../CoarseGrainedExecutorBackend.scala | 40 +- .../org/apache/spark/executor/Executor.scala | 59 +-- .../apache/spark/io/CompressionCodec.scala | 8 + .../netty/NettyBlockTransferService.scala | 3 +- .../apache/spark/rdd/PairRDDFunctions.scala | 6 +- .../apache/spark/rpc/netty/NettyRpcEnv.scala | 28 +- .../apache/spark/scheduler/DAGScheduler.scala | 16 +- .../apache/spark/scheduler/ResultTask.scala | 6 +- .../spark/scheduler/ShuffleMapTask.scala | 9 +- .../org/apache/spark/scheduler/Task.scala | 26 +- .../spark/scheduler/TaskDescription.scala | 68 +++- .../spark/scheduler/TaskSetManager.scala | 4 +- .../cluster/CoarseGrainedClusterMessage.scala | 22 +- .../CoarseGrainedSchedulerBackend.scala | 43 +-- .../local/LocalSchedulerBackend.scala | 3 +- .../apache/spark/storage/BlockManager.scala | 3 +- .../scala/org/apache/spark/ui/UIUtils.scala | 3 +- .../apache/spark/ui/storage/StorageTab.scala | 95 ++--- .../spark/launcher/SparkLauncherSuite.java | 78 ++-- .../SparkContextSchedulerCreationSuite.scala | 38 +- .../apache/spark/executor/ExecutorSuite.scala | 27 +- .../ExternalClusterManagerSuite.scala | 40 +- .../org/apache/spark/scheduler/FakeTask.scala | 13 +- .../spark/scheduler/TaskContextSuite.scala | 22 +- .../scheduler/TaskDescriptionSuite.scala | 10 +- .../spark/scheduler/TaskSetManagerSuite.scala | 4 +- .../spark/util/ClosureCleanerSuite.scala | 8 +- examples/build.gradle | 3 +- external/kafka-0-10-sql/build.gradle | 4 +- external/kafka-0-10/build.gradle | 12 +- .../kafka010/JavaDirectKafkaStreamSuite.java | 360 +++++++++--------- .../streaming/kafka010/JavaKafkaRDDSuite.java | 254 ++++++------ mllib/build.gradle | 2 + .../spark/repl/SingletonReplSuite.scala | 12 +- sql/catalyst/build.gradle | 1 + .../expressions/codegen/CodeGenerator.scala | 3 +- .../sql/catalyst/optimizer/Optimizer.scala | 11 +- .../sql/catalyst/optimizer/expressions.scala | 13 +- .../spark/sql/catalyst/plans/QueryPlan.scala | 24 +- .../plans/logical/basicLogicalOperators.scala | 5 - .../plans/physical/partitioning.scala | 301 ++++++++------- sql/core/build.gradle | 4 + .../org/apache/spark/sql/SQLContext.scala | 57 +-- .../sql/execution/WholeStageCodegenExec.scala | 14 +- .../aggregate/HashAggregateExec.scala | 3 - .../v2/DataSourcePartitioning.scala | 4 +- .../exchange/EnsureRequirements.scala | 84 ++-- .../joins/ShuffledHashJoinExec.scala | 2 +- .../execution/joins/SortMergeJoinExec.scala | 2 +- .../sql/execution/metric/SQLMetrics.scala | 2 +- .../apache/spark/sql/execution/objects.scala | 2 +- .../apache/spark/sql/BenchmarkQueryTest.scala | 1 + .../apache/spark/sql/SQLContextSuite.scala | 43 +-- .../org/apache/spark/sql/hive/HiveUtils.scala | 6 +- .../cluster/YarnClusterManager.scala | 112 +++--- 62 files changed, 1049 insertions(+), 1119 deletions(-) diff --git a/build.gradle b/build.gradle index 939015659e66f..99c7f280bf770 100644 --- a/build.gradle +++ b/build.gradle @@ -47,7 +47,7 @@ allprojects { apply plugin: 'idea' group = 'io.snappydata' - version = '2.1.1.2' + version = '2.3.0' ext { productName = 'SnappyData' @@ -68,22 +68,22 @@ allprojects { chillVersion = '0.8.4' kryoVersion = '4.0.1' nettyVersion = '3.10.6.Final' - nettyAllVersion = '4.0.51.Final' + nettyAllVersion = '4.1.17.Final' derbyVersion = '10.12.1.1' - httpClientVersion = '4.5.3' - httpCoreVersion = '4.4.7' + httpClientVersion = '4.5.4' + httpCoreVersion = '4.4.8' jackson1Version = '1.9.13' - jacksonVersion = '2.6.7' + jacksonVersion = '2.7.9' jacksonBindVersion = '2.6.7.1' snappyJavaVersion = '1.1.4' lz4Version = '1.4.0' lzfVersion = '1.0.4' parquetVersion = '1.8.2' // hiveParquetVersion = '1.6.0' - metricsVersion = '3.2.5' + metricsVersion = '3.1.5' janinoVersion = '3.0.8' thriftVersion = '0.9.3' - antlrVersion = '4.5.3' + antlrVersion = '4.7' jpamVersion = '1.1' seleniumVersion = '2.52.0' curatorVersion = '2.7.1' @@ -94,7 +94,7 @@ allprojects { jsr305Version = '3.0.2' jlineVersion = '2.14.2' xbeanAsm5Version = '4.5' - scalatestVersion = '2.2.6' + scalatestVersion = '3.0.3' pegdownVersion = '1.6.0' commonsCryptoVersion = '1.0.0' shadePackageName = 'org.spark_project' @@ -218,11 +218,15 @@ subprojects { compile group: 'log4j', name:'log4j', version: log4jVersion compile 'org.slf4j:slf4j-api:' + slf4jVersion compile 'org.slf4j:slf4j-log4j12:' + slf4jVersion +// compile group: 'org.apache.commons', name: 'commons-crypto', version: '1.0.0' +// compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion +// compile group: 'org.apache.arrow', name: 'arrow-vector', version: '0.8.0' + compile group: 'com.github.luben', name: 'zstd-jni', version: '1.3.2-2' testCompile "junit:junit:${junitVersion}" testCompile "org.scalatest:scalatest_${scalaBinaryVersion}:${scalatestVersion}" - testCompile 'org.mockito:mockito-core:1.10.19' - testCompile 'org.scalacheck:scalacheck_' + scalaBinaryVersion + ':1.12.5' + testCompile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' + testCompile 'org.scalacheck:scalacheck_' + scalaBinaryVersion + ':1.13.5' testCompile 'com.novocode:junit-interface:0.11' testRuntime "org.pegdown:pegdown:${pegdownVersion}" diff --git a/common/kvstore/build.gradle b/common/kvstore/build.gradle index d4ad2f686f792..bdae2ed81ac2b 100644 --- a/common/kvstore/build.gradle +++ b/common/kvstore/build.gradle @@ -15,7 +15,7 @@ * LICENSE file. */ -description = 'Spark Spark Project Local DB' +description = 'Spark Project Local DB' dependencies { compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) diff --git a/common/network-common/build.gradle b/common/network-common/build.gradle index e028aefec5437..2d0c311299388 100644 --- a/common/network-common/build.gradle +++ b/common/network-common/build.gradle @@ -27,21 +27,15 @@ dependencies { compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion compile group: 'org.apache.commons', name: 'commons-lang3', version: commonsLang3Version + compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion - testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') - compile group: 'org.apache.commons', name: 'commons-crypto', version: '1.0.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: '3.1.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-json', version: '3.1.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: '3.1.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-ganglia', version: '3.1.0' - compile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' + compile group: 'org.apache.arrow', name: 'arrow-vector', version: '0.8.0' + +// compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: metricsVersion +// compile group: 'io.dropwizard.metrics', name: 'metrics-json', version: metricsVersion +// compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: metricsVersion +// compile group: 'io.dropwizard.metrics', name: 'metrics-ganglia', version: metricsVersion compile group: 'org.apache.commons', name: 'commons-crypto', version: '1.0.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: '3.1.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-json', version: '3.1.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: '3.1.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-ganglia', version: '3.1.0' - compile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' -} + testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') +} \ No newline at end of file diff --git a/core/build.gradle b/core/build.gradle index d4c4ebc2c331a..404097eea8be0 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -24,6 +24,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-unsafe_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-kvstore_' + scalaBinaryVersion) + // compileOnly project(subprojectBase + 'snappy-spark-hive_' + scalaBinaryVersion) compile(group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion) { exclude(group: 'io.netty', module: 'netty') @@ -156,6 +157,14 @@ dependencies { exclude(group: 'net.razorvine', module: 'serpent') } compile group: 'net.sf.py4j', name: 'py4j', version: '0.10.4' + compile group: 'com.github.luben', name: 'zstd-jni', version: '1.3.2-2' + +// compileOnly(group: 'org.spark-project.hive', name: 'hive-exec', version:'1.2.1.spark2') +// compileOnly(group: 'org.spark-project.hive', name: 'hive-metastore', version: '1.2.1.spark2') +//// compileOnly(group: 'snappy-spark-hive', name: 'hive-exec', version:'1.2.1.spark2') +//// compileOnly(group: 'snappy-spark-hive', name: 'hive-metastore', version: '1.2.1.spark2') +// compileOnly(group: 'org.apache.thrift', name: 'libthrift', version: '0.9.3') +// compileOnly(group: 'org.apache.thrift', name: 'libfb303', version: '0.9.3') testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') // compile group: 'org.apache.commons', name: 'commons-crypto', version: commonsCryptoVersion @@ -206,4 +215,4 @@ sourceSets { // register generated resources on the main SourceSet output.dir(extraResourceDir, builtBy: 'generateBuildInfo') } -} +} \ No newline at end of file diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala index 9c8d7d8797a49..cf1ad7c57107b 100644 --- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala +++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala @@ -693,6 +693,7 @@ private[spark] class MapOutputTrackerWorker(conf: SparkConf) extends MapOutputTr val statuses = mapStatuses.get(shuffleId).orNull if (statuses == null) { logInfo("Don't have map outputs for shuffle " + shuffleId + ", fetching them") + val isDebugEnabled = log.isDebugEnabled val startTime = if (isDebugEnabled) System.currentTimeMillis else 0L var fetchedStatuses: Array[MapStatus] = null fetching.synchronized { diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 5717d991daa95..5888aa63d6824 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -46,9 +46,8 @@ import scala.collection.Map import scala.collection.generic.Growable import scala.collection.mutable.HashMap import scala.language.implicitConversions -import scala.reflect.{classTag, ClassTag} +import scala.reflect.{ClassTag, classTag} import scala.util.control.NonFatal - import com.google.common.collect.MapMaker import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} @@ -56,7 +55,6 @@ import org.apache.hadoop.io.{ArrayWritable, BooleanWritable, BytesWritable, Doub import org.apache.hadoop.mapred.{FileInputFormat, InputFormat, JobConf, SequenceFileInputFormat, TextInputFormat} import org.apache.hadoop.mapreduce.{InputFormat => NewInputFormat, Job => NewHadoopJob} import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat => NewFileInputFormat} - import org.apache.spark.annotation.DeveloperApi import org.apache.spark.broadcast.Broadcast import org.apache.spark.deploy.{LocalSparkCluster, SparkHadoopUtil} @@ -2758,70 +2756,6 @@ object SparkContext extends Logging { } (backend, scheduler) - case "yarn" if deployMode == "cluster" => - val scheduler = try { - val clazz = Utils.classForName("org.apache.spark.scheduler.cluster.YarnClusterScheduler") - val cons = clazz.getConstructor(classOf[SparkContext]) - cons.newInstance(sc).asInstanceOf[TaskSchedulerImpl] - } catch { - // TODO: Enumerate the exact reasons why it can fail - // But irrespective of it, it means we cannot proceed ! - case e: Exception => - throw new SparkException("YARN mode not available ?", e) - } - val backend = try { - val clazz = - Utils.classForName("org.apache.spark.scheduler.cluster.YarnClusterSchedulerBackend") - val cons = clazz.getConstructor(classOf[TaskSchedulerImpl], classOf[SparkContext]) - cons.newInstance(scheduler, sc).asInstanceOf[CoarseGrainedSchedulerBackend] - } catch { - case e: Exception => - throw new SparkException("YARN mode not available ?", e) - } - scheduler.initialize(backend) - (backend, scheduler) - - case "yarn" if deployMode == "client" => - val scheduler = try { - val clazz = Utils.classForName("org.apache.spark.scheduler.cluster.YarnScheduler") - val cons = clazz.getConstructor(classOf[SparkContext]) - cons.newInstance(sc).asInstanceOf[TaskSchedulerImpl] - - } catch { - case e: Exception => - throw new SparkException("YARN mode not available ?", e) - } - - val backend = try { - val clazz = - Utils.classForName("org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend") - val cons = clazz.getConstructor(classOf[TaskSchedulerImpl], classOf[SparkContext]) - cons.newInstance(scheduler, sc).asInstanceOf[CoarseGrainedSchedulerBackend] - } catch { - case e: Exception => - throw new SparkException("YARN mode not available ?", e) - } - - scheduler.initialize(backend) - (backend, scheduler) - - case MESOS_REGEX(mesosUrl) => - MesosNativeLibrary.load() - val scheduler = new TaskSchedulerImpl(sc) - val coarseGrained = sc.conf.getBoolean("spark.mesos.coarse", defaultValue = true) - val backend = if (coarseGrained) { - new CoarseMesosSchedulerBackend(scheduler, sc, mesosUrl, sc.env.securityManager) - } else { - new MesosSchedulerBackend(scheduler, sc, mesosUrl) - } - scheduler.initialize(backend) - (backend, scheduler) - - case zkUrl if zkUrl.startsWith("zk://") => - logWarning("Master URL for a multi-master Mesos cluster managed by ZooKeeper should be " + - "in the form mesos://zk://host:port. Current Master URL will stop working in Spark 2.0.") - createTaskScheduler(sc, "mesos://" + zkUrl, deployMode) - case masterUrl => val cm = getClusterManager(masterUrl) match { case Some(clusterMgr) => clusterMgr diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HiveDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HiveDelegationTokenProvider.scala index ece5ce79c650d..4204fc115bdc5 100644 --- a/core/src/main/scala/org/apache/spark/deploy/security/HiveDelegationTokenProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/security/HiveDelegationTokenProvider.scala @@ -24,8 +24,8 @@ import scala.util.control.NonFatal import org.apache.hadoop.conf.Configuration import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier -import org.apache.hadoop.hive.conf.HiveConf -import org.apache.hadoop.hive.ql.metadata.Hive +// import org.apache.hadoop.hive.conf.HiveConf +// import org.apache.hadoop.hive.ql.metadata.Hive import org.apache.hadoop.io.Text import org.apache.hadoop.security.{Credentials, UserGroupInformation} import org.apache.hadoop.security.token.Token @@ -46,7 +46,8 @@ private[security] class HiveDelegationTokenProvider private def hiveConf(hadoopConf: Configuration): Configuration = { try { - new HiveConf(hadoopConf, classOf[HiveConf]) + // new HiveConf(hadoopConf, classOf[HiveConf]) + null } catch { case NonFatal(e) => logDebug("Fail to create Hive Configuration", e) @@ -92,11 +93,11 @@ private[security] class HiveDelegationTokenProvider s"$principal at $metastoreUri") doAsRealUser { - val hive = Hive.get(conf, classOf[HiveConf]) - val tokenStr = hive.getDelegationToken(currentUser.getUserName(), principal) +// val hive = Hive.get(conf, classOf[HiveConf]) +// val tokenStr = hive.getDelegationToken(currentUser.getUserName(), principal) val hive2Token = new Token[DelegationTokenIdentifier]() - hive2Token.decodeFromUrlString(tokenStr) + hive2Token.decodeFromUrlString("tokenStr") logDebug(s"Get Token from hive metastore: ${hive2Token.toString}") creds.addToken(new Text("hive.server2.delegation.token"), hive2Token) } @@ -111,7 +112,7 @@ private[security] class HiveDelegationTokenProvider None } finally { Utils.tryLogNonFatalError { - Hive.closeCurrent() + // Hive.closeCurrent() } } } diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala index f0fdbcefb80cc..591112c84e6ec 100644 --- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala +++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala @@ -22,20 +22,21 @@ import java.nio.ByteBuffer import java.util.Locale import java.util.concurrent.atomic.AtomicBoolean -import scala.collection.mutable -import scala.util.{Failure, Success} -import scala.util.control.NonFatal - -import org.apache.spark._ import org.apache.spark.TaskState.TaskState +import org.apache.spark._ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.deploy.worker.WorkerWatcher import org.apache.spark.internal.Logging import org.apache.spark.rpc._ -import org.apache.spark.scheduler.{ExecutorLossReason, TaskDescription} +import org.apache.spark.scheduler.ExecutorLossReason import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._ +import org.apache.spark.serializer.SerializerInstance import org.apache.spark.util.{ThreadUtils, Utils} +import scala.collection.mutable +import scala.util.control.NonFatal +import scala.util.{Failure, Success} + private[spark] class CoarseGrainedExecutorBackend( override val rpcEnv: RpcEnv, driverUrl: String, @@ -50,6 +51,10 @@ private[spark] class CoarseGrainedExecutorBackend( var executor: Executor = null @volatile var driver: Option[RpcEndpointRef] = None + // If this CoarseGrainedExecutorBackend is changed to support multiple threads, then this may need + // to be changed so that we don't share the serializer instance across threads + private[this] val ser: SerializerInstance = env.closureSerializer.newInstance() + override def onStart() { logInfo("Connecting to driver: " + driverUrl) rpcEnv.asyncSetupEndpointRefByURI(driverUrl).flatMap { ref => @@ -62,7 +67,7 @@ private[spark] class CoarseGrainedExecutorBackend( // Always receive `true`. Just ignore it case Failure(e) => logError(s"Cannot register with driver: $driverUrl", e) - exitExecutor() + exitExecutor(1, "Cannot register with driver") }(ThreadUtils.sameThread) } @@ -79,7 +84,7 @@ private[spark] class CoarseGrainedExecutorBackend( case RegisteredExecutor => logInfo("Successfully registered with driver") try { - executor = registerExecutor + executor = new Executor(executorId, hostname, env, userClassPath, isLocal = false) } catch { case NonFatal(e) => exitExecutor(1, "Unable to create executor due to " + e.getMessage, e) @@ -87,17 +92,16 @@ private[spark] class CoarseGrainedExecutorBackend( case RegisterExecutorFailed(message) => logError("Slave registration failed: " + message) - exitExecutor() + exitExecutor(1, "Slave registration failed") case LaunchTask(taskDesc) => if (executor == null) { logError("Received LaunchTask command but executor was null") - exitExecutor() + exitExecutor(1, "Received LaunchTask command but executor was null") } else { - val taskDesc = ser.deserialize[TaskDescription](data.value) + // val taskDesc = TaskDescription.decode(data) logInfo("Got assigned task " + taskDesc.taskId) - executor.launchTask(this, taskId = taskDesc.taskId, attemptNumber = taskDesc.attemptNumber, - taskDesc.name, taskDesc.serializedTask, taskDesc.taskData.decompress(env)) + executor.launchTask(this, taskDesc) } case LaunchTasks(tasks, taskDataList) => @@ -107,9 +111,7 @@ private[spark] class CoarseGrainedExecutorBackend( logInfo("Got assigned task " + task.taskId) val ref = task.taskData.reference val taskData = if (ref >= 0) taskDataList(ref) else task.taskData - executor.launchTask(this, taskId = task.taskId, - attemptNumber = task.attemptNumber, task.name, task.serializedTask, - taskData.decompress(env)) + executor.launchTask(this, task) } } else { exitExecutor(1, "Received LaunchTasks command but executor was null") @@ -118,7 +120,7 @@ private[spark] class CoarseGrainedExecutorBackend( case KillTask(taskId, _, interruptThread, reason) => if (executor == null) { logError("Received KillTask command but executor was null") - exitExecutor() + exitExecutor(1, "Received KillTask command but executor was null") } else { executor.killTask(taskId, interruptThread, reason) } @@ -151,8 +153,8 @@ private[spark] class CoarseGrainedExecutorBackend( if (stopping.get()) { logInfo(s"Driver from $remoteAddress disconnected during shutdown") } else if (driver.exists(_.address == remoteAddress)) { - logError(s"Driver $remoteAddress disassociated! Shutting down.") - exitExecutor() + exitExecutor(1, s"Driver $remoteAddress disassociated! Shutting down.", null, + notifyDriver = false) } else { logWarning(s"An unknown ($remoteAddress) driver disconnected.") } diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index ae33b9765ebab..4f541272c48c1 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -26,23 +26,22 @@ import java.util.Properties import java.util.concurrent._ import javax.annotation.concurrent.GuardedBy -import scala.collection.JavaConverters._ -import scala.collection.mutable.{ArrayBuffer, HashMap, Map} -import scala.util.control.NonFatal - import com.google.common.util.concurrent.ThreadFactoryBuilder - import org.apache.spark._ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging -import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager} +import org.apache.spark.memory.TaskMemoryManager import org.apache.spark.rpc.RpcTimeout -import org.apache.spark.scheduler.{AccumulableInfo, DirectTaskResult, IndirectTaskResult, Task, TaskDescription} +import org.apache.spark.scheduler.{DirectTaskResult, IndirectTaskResult, Task, TaskDescription} import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.storage.{StorageLevel, TaskResultBlockId} import org.apache.spark.util._ import org.apache.spark.util.io.ChunkedByteBuffer +import scala.collection.JavaConverters._ +import scala.collection.mutable.{ArrayBuffer, HashMap} +import scala.util.control.NonFatal + /** * Spark executor, backed by a threadpool to run tasks. * @@ -214,21 +213,6 @@ private[spark] class Executor( killTask(t, interruptThread = interruptThread, reason = reason)) } - /** - * Function to kill the running tasks in an executor. - * This can be called by executor back-ends to kill the - * tasks instead of taking the JVM down. - * @param interruptThread whether to interrupt the task thread - */ - def killAllTasks(interruptThread: Boolean) : Unit = { - // kill all the running tasks - for (taskRunner <- runningTasks.values().asScala) { - if (taskRunner != null) { - taskRunner.kill(interruptThread) - } - } - } - def stop(): Unit = { env.metricsSystem.report() heartbeater.shutdown() @@ -246,13 +230,7 @@ private[spark] class Executor( class TaskRunner( execBackend: ExecutorBackend, - private val taskDescription: TaskDescription, - val taskId: Long, - val attemptNumber: Int, - taskName: String, - serializedTask: ByteBuffer, - taskDataBytes: Array[Byte], - taskDecompressTime: Long) + private val taskDescription: TaskDescription) extends Runnable { val taskId = taskDescription.taskId @@ -328,12 +306,12 @@ private[spark] class Executor( try { // Must be set before updateDependencies() is called, in case fetching dependencies // requires access to properties contained within (e.g. for access control). - Executor.taskDeserializationProps.set(taskDescription.properties) +// Executor.taskDeserializationProps.set(taskDescription.properties) - updateDependencies(taskDescription.addedFiles, taskDescription.addedJars) +// updateDependencies(taskDescription.addedFiles, taskDescription.addedJars) task = ser.deserialize[Task[Any]]( taskDescription.serializedTask, Thread.currentThread.getContextClassLoader) - task.localProperties = taskDescription.properties +// task.localProperties = taskDescription.properties task.setTaskMemoryManager(taskMemoryManager) // If this task has been killed before we deserialized it, let's quit now. Otherwise, // continue executing the task. @@ -412,7 +390,7 @@ private[spark] class Executor( // Deserialization happens in two parts: first, we deserialize a Task object, which // includes the Partition. Second, Task.run() deserializes the RDD and function to be run. task.metrics.setExecutorDeserializeTime(math.max(taskStart - deserializeStartTime + - task.executorDeserializeTime + taskDecompressTime, 0L).toDouble / 1000000.0) + task.executorDeserializeTime /* + taskDecompressTime */, 0L).toDouble / 1000000.0) task.metrics.setExecutorDeserializeCpuTime(math.max(taskStartCpu - deserializeStartCpuTime + task.executorDeserializeCpuTime, 0L).toDouble / 1000000.0) // We need to subtract Task.run()'s deserialization time to avoid double-counting @@ -465,11 +443,10 @@ private[spark] class Executor( // Note: accumulator updates must be collected after TaskMetrics is updated val accumUpdates = task.collectAccumulatorUpdates() - val directResult = new DirectTaskResult(value, accumUpdates, - Some(task.metrics.resultSerializationTimeMetric)) - val taskSer = env.serializer.newInstance() - val serializedDirectResult = taskSer.serialize(directResult) - val resultSize = serializedDirectResult.limit + // TODO: do not serialize value twice + val directResult = new DirectTaskResult(value, accumUpdates) + val serializedDirectResult = ser.serialize(directResult) + val resultSize = serializedDirectResult.limit() // directSend = sending directly back to the driver val serializedResult: ByteBuffer = { @@ -477,7 +454,7 @@ private[spark] class Executor( logWarning(s"Finished $taskName (TID $taskId). Result is larger than maxResultSize " + s"(${Utils.bytesToString(resultSize)} > ${Utils.bytesToString(maxResultSize)}), " + s"dropping it.") - taskSer.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize)) + ser.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize)) } else if (resultSize > maxDirectResultSize) { val blockId = TaskResultBlockId(taskId) env.blockManager.putBytes( @@ -486,7 +463,7 @@ private[spark] class Executor( StorageLevel.MEMORY_AND_DISK_SER) logInfo( s"Finished $taskName (TID $taskId). $resultSize bytes result sent via BlockManager)") - taskSer.serialize(new IndirectTaskResult[Any](blockId, resultSize)) + ser.serialize(new IndirectTaskResult[Any](blockId, resultSize)) } else { logInfo(s"Finished $taskName (TID $taskId). $resultSize bytes result sent to driver") serializedDirectResult @@ -590,7 +567,7 @@ private[spark] class Executor( Thread.getDefaultUncaughtExceptionHandler. uncaughtException(Thread.currentThread(), t) } else { - SparkUncaughtExceptionHandler.uncaughtException(t) + // SparkUncaughtExceptionHandler.uncaughtException(t) } } diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala index 22c663686d6ef..b49f90f4fc4bc 100644 --- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala +++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala @@ -300,4 +300,12 @@ class ZStdCompressionCodec(conf: SparkConf) extends CompressionCodec { // avoid overhead excessive of JNI call while trying to uncompress small amount of data. new BufferedInputStream(new ZstdInputStream(s), bufferSize) } + override def compress(input: Array[Byte], inputLen: Int): Array[Byte] = { + throw new UnsupportedOperationException("not implemented") + } + + override def decompress(input: Array[Byte], inputOffset: Int, + inputLen: Int, outputLen: Int): Array[Byte] = { + throw new UnsupportedOperationException("not implemented") + } } diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala index e718336af82ca..117d9a1c5b0ab 100644 --- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala +++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala @@ -20,10 +20,11 @@ package org.apache.spark.network.netty import java.nio.ByteBuffer import java.util.{HashMap => JHashMap, Map => JMap} +import com.codahale.metrics.{Metric, MetricSet} + import scala.collection.JavaConverters._ import scala.concurrent.{Future, Promise} import scala.reflect.ClassTag - import org.apache.spark.{SecurityManager, SparkConf, SparkEnv} import org.apache.spark.network._ import org.apache.spark.network.buffer.ManagedBuffer diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index e68c6b1366c7f..731bc9d5bd6a8 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -20,23 +20,21 @@ package org.apache.spark.rdd import java.nio.ByteBuffer import java.util.{HashMap => JHashMap} -import scala.collection.{mutable, Map} +import scala.collection.{Map, mutable} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import scala.reflect.ClassTag - import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus import org.apache.hadoop.conf.Configuration import org.apache.hadoop.io.SequenceFile.CompressionType import org.apache.hadoop.io.compress.CompressionCodec import org.apache.hadoop.mapred.{FileOutputCommitter, FileOutputFormat, JobConf, OutputFormat} import org.apache.hadoop.mapreduce.{Job => NewAPIHadoopJob, OutputFormat => NewOutputFormat} - import org.apache.spark._ import org.apache.spark.Partitioner.defaultPartitioner import org.apache.spark.annotation.Experimental import org.apache.spark.internal.Logging -import org.apache.spark.internal.io._ +import org.apache.spark.internal.io.{HadoopMapRedWriteConfigUtil, HadoopMapReduceWriteConfigUtil, SparkHadoopWriter, SparkHadoopWriterUtils} import org.apache.spark.partial.{BoundedDouble, PartialResult} import org.apache.spark.serializer.Serializer import org.apache.spark.util.{SerializableConfiguration, SerializableJobConf, Utils} diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala index 3a57a151c4db4..18c2c725f0b7c 100644 --- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala +++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala @@ -24,16 +24,8 @@ import java.util.concurrent._ import java.util.concurrent.atomic.AtomicBoolean import javax.annotation.Nullable -import scala.concurrent.{ExecutionContext, Future, Promise} -import scala.concurrent.duration.Duration -import scala.reflect.ClassTag -import scala.util.{DynamicVariable, Failure, Success, Try} -import scala.util.control.NonFatal - -import com.esotericsoftware.kryo.{Kryo, KryoSerializable} import com.esotericsoftware.kryo.io.{Input, Output} - -import org.apache.spark.{SecurityManager, SparkConf, SparkEnv} +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} import org.apache.spark.internal.Logging import org.apache.spark.network.TransportContext import org.apache.spark.network.client._ @@ -41,8 +33,15 @@ import org.apache.spark.network.crypto.{AuthClientBootstrap, AuthServerBootstrap import org.apache.spark.network.netty.SparkTransportConf import org.apache.spark.network.server._ import org.apache.spark.rpc._ -import org.apache.spark.serializer.{JavaSerializer, JavaSerializerInstance, Serializer, SerializerInstance, SerializationStream} -import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, RpcUtils, ThreadUtils, Utils} +import org.apache.spark.serializer.{SerializationStream, Serializer, SerializerInstance} +import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, ThreadUtils, Utils} +import org.apache.spark.{SecurityManager, SparkConf, SparkEnv} + +import scala.concurrent.duration.Duration +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.reflect.ClassTag +import scala.util.control.NonFatal +import scala.util.{DynamicVariable, Failure, Success, Try} private[netty] class NettyRpcEnv( val conf: SparkConf, @@ -270,7 +269,7 @@ private[netty] class NettyRpcEnv( * Returns [[SerializationStream]] that forwards the serialized bytes to `out`. */ private[netty] def serializeStream(out: OutputStream): SerializationStream = { - javaSerializerInstance.serializeStream(out) + serializerInstance.get().serializeStream(out) } private[netty] def deserialize[T: ClassTag](client: TransportClient, bytes: ByteBuffer): T = { @@ -475,7 +474,7 @@ private[rpc] class NettyRpcEnvFactory extends RpcEnvFactory with Logging { val sparkConf = config.conf val serializer = SparkEnv.getClosureSerializer(sparkConf) val nettyEnv = - new NettyRpcEnv(sparkConf, javaSerializerInstance, config.advertiseAddress, + new NettyRpcEnv(sparkConf, serializer, config.advertiseAddress, config.securityManager, config.numUsableCores) if (!config.clientMode) { val startNettyRpcEnv: Int => (NettyRpcEnv, Int) = { actualPort => @@ -521,6 +520,9 @@ private[netty] class NettyRpcEndpointRef( extends RpcEndpointRef(conf, nettyEnv) with Serializable with KryoSerializable with Logging { @transient @volatile var client: TransportClient = _ + + private var _address = if (endpointAddress.rpcAddress != null) endpointAddress else null + private var _name = endpointAddress.name override def address: RpcAddress = if (endpointAddress.rpcAddress != null) endpointAddress.rpcAddress else null diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index c12ce76d8ea98..6a950ca37fb0d 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -1016,7 +1016,8 @@ class DAGScheduler( // task gets a different copy of the RDD. This provides stronger isolation between tasks that // might modify state of objects referenced in their closures. This is necessary in Hadoop // where the JobConf/Configuration object is not thread-safe. - var taskBinary: Broadcast[Array[Byte]] = null + var taskBinary: Option[Broadcast[Array[Byte]]] = None + var taskData: TaskData = TaskData.EMPTY var partitions: Array[Partition] = null try { // For ShuffleMapTask, serialize and broadcast (rdd, shuffleDep). @@ -1036,7 +1037,6 @@ class DAGScheduler( partitions = stage.rdd.partitions } - if (bytes == null) stage.taskBinaryBytes = taskBinaryBytes // use direct byte shipping for small size or if number of partitions is small val taskBytesLen = taskBinaryBytes.length @@ -1079,18 +1079,18 @@ class DAGScheduler( val part = partitions(id) stage.pendingPartitions += id new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, taskData, - taskBinary, part, locs, stage.latestInfo.taskMetrics, properties, - jobId, Option(sc.applicationId), Option(sc.applicationId)) + taskBinary, part, locs, properties, serializedTaskMetrics, Option(jobId), + Option(sc.applicationId), sc.applicationAttemptId) } case stage: ResultStage => partitionsToCompute.map { id => val p: Int = stage.partitions(id) - val part = partitions(p) + val part = stage.rdd.partitions(p) val locs = taskIdToLocations(id) new ResultTask(stage.id, stage.latestInfo.attemptId, taskData, - taskBinary, part, locs, id, properties, stage.latestInfo.taskMetrics, - jobId, Option(sc.applicationId), Option(sc.applicationId)) + taskBinary, part, locs, id, properties, serializedTaskMetrics, Option(jobId), + Option(sc.applicationId), sc.applicationAttemptId) } } } catch { @@ -1104,7 +1104,7 @@ class DAGScheduler( logInfo(s"Submitting ${tasks.size} missing tasks from $stage (${stage.rdd}) (first 15 " + s"tasks are for partitions ${tasks.take(15).map(_.partitionId)})") taskScheduler.submitTasks(new TaskSet( - tasks.toArray, stage.id, stage.latestInfo.attemptNumber, jobId, properties)) + tasks.toArray, stage.id, stage.latestInfo.attemptId, jobId, properties)) } else { // Because we posted SparkListenerStageSubmitted earlier, we should mark // the stage as completed here in case there are no tasks to run diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index 5661a19ae65dc..6709b7d5b2f15 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -63,12 +63,12 @@ private[spark] class ResultTask[T, U]( locs: Seq[TaskLocation], private var _outputId: Int, localProperties: Properties, - metrics: TaskMetrics, - jobId: Int = -1, + serializedTaskMetrics: Array[Byte], + jobId: Option[Int] = None, appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[U](stageId, stageAttemptId, partition.index, _taskData, - _taskBinary, metrics, localProperties, jobId, appId, appAttemptId) + _taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) with Serializable with KryoSerializable { final def outputId: Int = _outputId diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index 5c99931eb3109..83215edaedfaf 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -47,8 +47,6 @@ import org.apache.spark.shuffle.ShuffleWriter * @param partition partition of the RDD this task is associated with * @param locs preferred task execution locations for locality scheduling * @param localProperties copy of thread-local properties set by the user on the driver side. - * @param serializedTaskMetrics a `TaskMetrics` that is created and serialized on the driver side - * and sent to executor side. * * The parameters below are optional: * @param jobId id of the job this task belongs to @@ -63,17 +61,18 @@ private[spark] class ShuffleMapTask( private var partition: Partition, @transient private var locs: Seq[TaskLocation], localProperties: Properties, - jobId: Int = -1, + serializedTaskMetrics: Array[Byte], + jobId: Option[Int] = None, appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[MapStatus](stageId, stageAttemptId, partition.index, _taskData, - _taskBinary, metrics, localProperties, jobId, appId, appAttemptId) + _taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) with KryoSerializable with Logging { /** A constructor used only in test suites. This does not require passing in an RDD. */ def this(partitionId: Int) { this(0, 0, TaskData.EMPTY, null, new Partition { override def index: Int = 0 }, - null, null, new Properties) + null, new Properties, null) } @transient private val preferredLocs: Seq[TaskLocation] = { diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index b152b977db6f6..cf31304935a21 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -17,21 +17,21 @@ package org.apache.spark.scheduler +import java.io.{DataInputStream, DataOutputStream} import java.nio.ByteBuffer import java.util.Properties import scala.collection.mutable import scala.collection.mutable.HashMap - import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{Input, Output} - import org.apache.spark._ import org.apache.spark.broadcast.Broadcast import org.apache.spark.executor.TaskMetrics import org.apache.spark.internal.config.APP_CALLER_CONTEXT import org.apache.spark.memory.{MemoryMode, TaskMemoryManager} import org.apache.spark.metrics.MetricsSystem +import org.apache.spark.serializer.SerializerInstance import org.apache.spark.util._ /** @@ -48,8 +48,10 @@ import org.apache.spark.util._ * @param _stageId id of the stage this task belongs to * @param _stageAttemptId attempt id of the stage this task belongs to * @param _partitionId index of the number in the RDD - * @param _metrics a [[TaskMetrics]] that is created at driver side and sent to executor side. * @param localProperties copy of thread-local properties set by the user on the driver side. + * @param serializedTaskMetrics a `TaskMetrics` that is created and serialized on the driver side + * and sent to executor side. + * * The parameters below are optional: * @param _jobId id of the job this task belongs to * @param _appId id of the app this task belongs to @@ -62,12 +64,11 @@ private[spark] abstract class Task[T]( @transient private[spark] var taskData: TaskData = TaskData.EMPTY, // The default value is only used in tests. protected var taskBinary: Option[Broadcast[Array[Byte]]] = None, - private var _metrics: TaskMetrics = TaskMetrics.registered, @transient var localProperties: Properties = new Properties, // The default value is only used in tests. serializedTaskMetrics: Array[Byte] = SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array(), - private var _jobId: Int = -1, + private var _jobId: Option[Int] = None, private var _appId: Option[String] = None, private var _appAttemptId: Option[String] = None) extends Serializable { @@ -80,9 +81,9 @@ private[spark] abstract class Task[T]( @transient lazy val metrics: TaskMetrics = SparkEnv.get.closureSerializer.newInstance().deserialize(ByteBuffer.wrap(serializedTaskMetrics)) - final def jobId: Int = _jobId + final var jobId: Int = if (_jobId.isDefined) _jobId.get else -1 - final def metrics: TaskMetrics = _metrics + // final def metrics: TaskMetrics = _metrics final def appId: String = if (_appId.isDefined) _appId.get else null @@ -129,7 +130,7 @@ private[spark] abstract class Task[T]( SparkEnv.get.conf.get(APP_CALLER_CONTEXT), _appId, _appAttemptId, - _jobId, + Option(jobId), Option(stageId), Option(stageAttemptId), Option(taskAttemptId), @@ -207,7 +208,6 @@ private[spark] abstract class Task[T]( * If defined, this task has been killed and this option contains the reason. */ def reasonIfKilled: Option[String] = Option(_reasonIfKilled) - def killed: Boolean = _killed /** * Returns the amount of time spent deserializing the RDD and function to be run in nanos. @@ -240,7 +240,6 @@ private[spark] abstract class Task[T]( def kill(interruptThread: Boolean, reason: String) { require(reason != null) _reasonIfKilled = reason - killed = true if (context != null) { context.markInterrupted(reason) } @@ -253,7 +252,7 @@ private[spark] abstract class Task[T]( output.writeInt(_stageId) output.writeVarInt(_stageAttemptId, true) output.writeVarInt(_partitionId, true) - output.writeVarInt(_jobId, true) + output.writeVarInt(jobId, true) output.writeLong(epoch) output.writeLong(_executorDeserializeTime) output.writeLong(_executorDeserializeCpuTime) @@ -264,7 +263,6 @@ private[spark] abstract class Task[T]( output.writeBoolean(false) kryo.writeClassAndObject(output, taskBinary.get) } - _metrics.write(kryo, output) output.writeString(appId) output.writeString(appAttemptId) } @@ -273,7 +271,7 @@ private[spark] abstract class Task[T]( _stageId = input.readInt() _stageAttemptId = input.readVarInt(true) _partitionId = input.readVarInt(true) - _jobId = input.readVarInt(true) + jobId = input.readVarInt(true) epoch = input.readLong() _executorDeserializeTime = input.readLong() _executorDeserializeCpuTime = input.readLong() @@ -285,8 +283,6 @@ private[spark] abstract class Task[T]( taskBinary = Some(kryo.readClassAndObject(input) .asInstanceOf[Broadcast[Array[Byte]]]) } - _metrics = new TaskMetrics - _metrics.read(kryo, input) _appId = Option(input.readString()) _appAttemptId = Option(input.readString()) } diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala index fb86409cc5a89..9ff9da121bb61 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala @@ -22,13 +22,13 @@ import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.util.Properties +import com.esotericsoftware.kryo.io.{Input, Output} +import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, SerializableBuffer, Utils} + import scala.collection.JavaConverters._ import scala.collection.mutable.{HashMap, Map} -import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, SerializableBuffer, Utils} -import com.esotericsoftware.kryo.{Kryo, KryoSerializable} -import com.esotericsoftware.kryo.io.{Input, Output} - /** * Description of a task that gets passed onto executors to be executed, usually created by * `TaskSetManager.resourceOffer`. @@ -52,6 +52,9 @@ private[spark] class TaskDescription( private var _executorId: String, private var _name: String, private var _index: Int, // Index within this task's TaskSet + val addedFiles: Map[String, Long], + val addedJars: Map[String, Long], + val properties: Properties, @transient private var _serializedTask: ByteBuffer, private[spark] var taskData: TaskData = TaskData.EMPTY) extends Serializable with KryoSerializable { @@ -75,6 +78,27 @@ private[spark] class TaskDescription( output.writeString(_executorId) output.writeString(_name) output.writeInt(_index) + output.writeInt(addedFiles.size) + // Write files. + for ((key, value) <- addedFiles) { + output.writeString(key) + output.writeLong(value) + } + // Write jars. + output.writeInt(addedJars.size) + for ((key, value) <- addedJars) { + output.writeString(key) + output.writeLong(value) + } + // Write properties. + output.writeInt(properties.size()) + properties.asScala.foreach { case (key, value) => + output.writeString(key) + // SPARK-19796 -- writeUTF doesn't work for long strings, which can happen for property values + val bytes = value.getBytes(StandardCharsets.UTF_8) + output.writeInt(bytes.length) + output.write(bytes) + } output.writeInt(_serializedTask.remaining()) Utils.writeByteBuffer(_serializedTask, output) TaskData.write(taskData, output) @@ -86,10 +110,44 @@ private[spark] class TaskDescription( _executorId = input.readString() _name = input.readString() _index = input.readInt() + // Read files. + val fileSize = input.readInt() + for (_ <- 0 until fileSize) { + addedFiles(input.readString()) = input.readLong() + } + // Read jars. + val jarSize = input.readInt() + for (_ <- 0 until jarSize) { + addedJars(input.readString()) = input.readLong() + } + // Read properties. + val properties = new Properties() + val numProperties = input.readInt() + for (_ <- 0 until numProperties) { + val key = input.readString() + val valueLength = input.readInt() + val valueBytes = new Array[Byte](valueLength) + input.read(valueBytes) + properties.setProperty(key, new String(valueBytes, StandardCharsets.UTF_8)) + } val len = input.readInt() _serializedTask = ByteBuffer.wrap(input.readBytes(len)) taskData = TaskData.read(input) } +} + +private[spark] object TaskDescription { + private def serializeStringLongMap(map: Map[String, Long], dataOut: DataOutputStream): Unit = { + dataOut.writeInt(map.size) + for ((key, value) <- map) { + dataOut.writeUTF(key) + dataOut.writeLong(value) + } + } + + def encode(taskDescription: TaskDescription): ByteBuffer = { + val bytesOut = new ByteBufferOutputStream(4096) + val dataOut = new DataOutputStream(bytesOut) dataOut.writeLong(taskDescription.taskId) dataOut.writeInt(taskDescription.attemptNumber) @@ -161,4 +219,4 @@ private[spark] class TaskDescription( new TaskDescription(taskId, attemptNumber, executorId, name, index, taskFiles, taskJars, properties, serializedTask) } -} +} \ No newline at end of file diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index e44a4053ca04b..91f8b806ae892 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -524,8 +524,8 @@ private[spark] class TaskSetManager( sched.dagScheduler.taskStarted(task, info) new TaskDescription( - _taskId = taskId, - _attemptNumber = attemptNum, + taskId, + attemptNum, execId, taskName, index, diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala index a22593c66ab52..39fe6382a7aa5 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala @@ -52,7 +52,16 @@ private[spark] object CoarseGrainedClusterMessages { } override def read(kryo: Kryo, input: Input): Unit = { - task = new TaskDescription(0L, 0, null, null, 0, null) + task = new TaskDescription(_taskId = 0L, + _attemptNumber = 0, + _executorId = null, + _name = null, + _index = 0, + addedFiles = null, + addedJars = null, + properties = null, + _serializedTask = null, + taskData = TaskData.EMPTY) task.read(kryo, input) } } @@ -84,7 +93,16 @@ private[spark] object CoarseGrainedClusterMessages { var numTasks = input.readVarInt(true) val tasks = new mutable.ArrayBuffer[TaskDescription](numTasks) while (numTasks > 0) { - val task = new TaskDescription(0, 0, null, null, 0, null) + val task = new TaskDescription(_taskId = 0L, + _attemptNumber = 0, + _executorId = null, + _name = null, + _index = 0, + addedFiles = null, + addedJars = null, + properties = null, + _serializedTask = null, + taskData = TaskData.EMPTY) task.read(kryo, input) tasks += task numTasks -= 1 diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index a132859b60c0b..d6a2434de57c9 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -39,17 +39,17 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger import javax.annotation.concurrent.GuardedBy -import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet} -import scala.concurrent.Future - -import org.apache.spark.{ExecutorAllocationClient, SparkEnv, SparkException, TaskState} import org.apache.spark.internal.Logging import org.apache.spark.rpc._ import org.apache.spark.scheduler._ import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._ import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.ENDPOINT_NAME -import org.apache.spark.util.{RpcUtils, ThreadUtils, Utils} import org.apache.spark.util.collection.OpenHashMap +import org.apache.spark.util.{RpcUtils, ThreadUtils, Utils} +import org.apache.spark.{ExecutorAllocationClient, SparkEnv, SparkException, TaskState} + +import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet} +import scala.concurrent.Future /** * A scheduler backend that waits for coarse-grained executors to connect. @@ -308,28 +308,23 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp !executorsPendingLossReason.contains(executorId) } - // Launch tasks returned by a set of resource offers - private def launchTasks(tasks: Seq[Seq[TaskDescription]]) { - for (task <- tasks.flatten) { - val serializedTask = TaskDescription.encode(task) - if (serializedTask.limit() >= maxRpcMessageSize) { - scheduler.taskIdToTaskSetManager.get(task.taskId).foreach { taskSetMgr => - try { - var msg = "Serialized task %s:%d was %d bytes, which exceeds max allowed: " + - "spark.rpc.message.maxSize (%d bytes). Consider increasing " + - "spark.rpc.message.maxSize or using broadcast variables for large values." - msg = msg.format(task.taskId, task.index, serializedTask.limit(), maxRpcMessageSize) - taskSetMgr.abort(msg) - } catch { - case e: Exception => logError("Exception in error callback", e) - } + protected def checkTaskSizeLimit(task: TaskDescription, taskSize: Int): Boolean = { + if (taskSize > maxRpcMessageSize) { + scheduler.taskIdToTaskSetManager.get(task.taskId).foreach { taskSetMgr => + try { + var msg = "Serialized task %s:%d was %d bytes, which exceeds max allowed: " + + "spark.rpc.message.maxSize (%d bytes). Consider increasing " + + "spark.rpc.message.maxSize or using broadcast variables for large values." + msg = msg.format(task.taskId, task.index, taskSize, maxRpcMessageSize) + taskSetMgr.abort(msg) + } catch { + case e: Exception => logError("Exception in error callback", e) } } false } else true } - // Launch tasks returned by a set of resource offers protected def launchTasks(tasks: Seq[Seq[TaskDescription]]): Unit = { val executorTaskGroupMap = new OpenHashMap[String, ExecutorTaskGroup](8) for (taskSet <- tasks) { @@ -352,7 +347,8 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp // send this task separately val executorData = executorTaskGroup.executorData executorData.freeCores -= scheduler.CPUS_PER_TASK - logInfo(s"Launching task ${task.taskId} on executor id: " + + scheduler.sc.env.taskLogger.logInfo( + s"Launching task ${task.taskId} on executor id: " + s"${task.executorId} hostname: ${executorData.executorHost}.") executorData.executorEndpoint.send(LaunchTask(task)) @@ -369,12 +365,11 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp executorData.freeCores -= (scheduler.CPUS_PER_TASK * taskGroup.length) logDebug(s"Launching tasks ${taskGroup.map(_.taskId).mkString(",")} on " + - s"executor id: $executorId hostname: ${executorData.executorHost}.") + s"executor id: $executorId hostname: ${executorData.executorHost}.") executorData.executorEndpoint.send(LaunchTasks(taskGroup, executorTaskGroup.taskDataList)) } } - // Remove a disconnected slave from the cluster private def removeExecutor(executorId: String, reason: ExecutorLossReason): Unit = { logDebug(s"Asked to remove executor $executorId with reason $reason") diff --git a/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala index 83198c0d7c10d..4c614c5c0f602 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala @@ -84,8 +84,7 @@ private[spark] class LocalEndpoint( val offers = IndexedSeq(new WorkerOffer(localExecutorId, localExecutorHostname, freeCores)) for (task <- scheduler.resourceOffers(offers).flatten) { freeCores -= scheduler.CPUS_PER_TASK - executor.launchTask(executorBackend, taskId = task.taskId, attemptNumber = task.attemptNumber, - task.name, task.serializedTask, task.taskData.decompress()) + executor.launchTask(executorBackend, task) } } } diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index ba471bccf475c..d6a5e78713e08 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -127,7 +127,7 @@ private[spark] class BlockManager( val blockTransferService: BlockTransferService, securityManager: SecurityManager, numUsableCores: Int) - extends BlockDataManager with BlockEvictionHandler with Logging { + extends BlockDataManager with BlockEvictionHandler with Logging { private[spark] val externalShuffleServiceEnabled = conf.getBoolean("spark.shuffle.service.enabled", false) @@ -1341,7 +1341,6 @@ private[spark] class BlockManager( while(numFailures <= maxReplicationFailures && !peersForReplication.isEmpty && peersReplicatedTo.size < numPeersToReplicateTo) { - val peer = peersForReplication.head try { val onePeerStartTime = if (isDebugEnabled) System.nanoTime else 0L diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index e07303e8ac20c..13ef3950876ac 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -241,7 +241,6 @@ private[spark] object UIUtils extends Logging {

    - {getProductUINameNode} {getProductVersionNode}
    @@ -309,7 +308,7 @@ private[spark] object UIUtils extends Logging {

    - {shortAppName} application UI + {appName} application UI

    {getProductDocLinkNode()}
      {header}
    diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala index 1eb361c1e0cff..1d2ceb7a0d26e 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala @@ -17,57 +17,62 @@ package org.apache.spark.ui.storage +import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.scheduler.{SparkListenerStageCompleted, SparkListenerStageSubmitted, SparkListenerUnpersistRDD} import org.apache.spark.status.AppStatusStore +import org.apache.spark.storage._ import org.apache.spark.ui._ +import scala.collection.mutable + /** Web UI showing storage status of all RDD's in the given SparkContext. */ private[ui] class StorageTab(parent: SparkUI, store: AppStatusStore) extends SparkUITab(parent, "Spark Cache") { - val listener = parent.storageListener +// val listener = parent.storageListener - attachPage(new StoragePage(this)) - attachPage(new RDDPage(this)) +// attachPage(new StoragePage(this)) +// attachPage(new RDDPage(this)) } -/** - * :: DeveloperApi :: - * A SparkListener that prepares information to be displayed on the BlockManagerUI. - * - * This class is thread-safe (unlike JobProgressListener) - */ -@DeveloperApi -class StorageListener(storageStatusListener: StorageStatusListener) extends BlockStatusListener { - - private[ui] val _rddInfoMap = mutable.Map[Int, RDDInfo]() // exposed for testing - - def activeStorageStatusList: Seq[StorageStatus] = storageStatusListener.storageStatusList - - /** Filter RDD info to include only those with cached partitions */ - def rddInfoList: Seq[RDDInfo] = synchronized { - _rddInfoMap.values.filter(_.numCachedPartitions > 0).toSeq - } - - /** Update the storage info of the RDDs whose blocks are among the given updated blocks */ - private def updateRDDInfo(updatedBlocks: Seq[(BlockId, BlockStatus)]): Unit = { - val rddIdsToUpdate = updatedBlocks.flatMap { case (bid, _) => bid.asRDDId.map(_.rddId) }.toSet - val rddInfosToUpdate = _rddInfoMap.values.toSeq.filter { s => rddIdsToUpdate.contains(s.id) } - StorageUtils.updateRddInfo(rddInfosToUpdate, activeStorageStatusList) - } - - override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = synchronized { - val rddInfos = stageSubmitted.stageInfo.rddInfos - rddInfos.foreach { info => _rddInfoMap.getOrElseUpdate(info.id, info).name = info.name } - } - - override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = synchronized { - // Remove all partitions that are no longer cached in current completed stage - val completedRddIds = stageCompleted.stageInfo.rddInfos.map(r => r.id).toSet - _rddInfoMap.retain { case (id, info) => - !completedRddIds.contains(id) || info.numCachedPartitions > 0 - } - } - - override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit = synchronized { - _rddInfoMap.remove(unpersistRDD.rddId) - } -} \ No newline at end of file +///** +// * :: DeveloperApi :: +// * A SparkListener that prepares information to be displayed on the BlockManagerUI. +// * +// * This class is thread-safe (unlike JobProgressListener) +// */ +//@DeveloperApi +//class StorageListener(storageStatusListener: StorageStatusListener) extends BlockStatusListener { +// +// private[ui] val _rddInfoMap = mutable.Map[Int, RDDInfo]() // exposed for testing +// +// def activeStorageStatusList: Seq[StorageStatus] = storageStatusListener.storageStatusList +// +// /** Filter RDD info to include only those with cached partitions */ +// def rddInfoList: Seq[RDDInfo] = synchronized { +// _rddInfoMap.values.filter(_.numCachedPartitions > 0).toSeq +// } +// +// /** Update the storage info of the RDDs whose blocks are among the given updated blocks */ +// private def updateRDDInfo(updatedBlocks: Seq[(BlockId, BlockStatus)]): Unit = { +// val rddIdsToUpdate = updatedBlocks.flatMap { case (bid, _) => bid.asRDDId.map(_.rddId) }.toSet +// val rddInfosToUpdate = _rddInfoMap.values.toSeq.filter { s => rddIdsToUpdate.contains(s.id) } +// StorageUtils.updateRddInfo(rddInfosToUpdate, activeStorageStatusList) +// } +// +// override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = synchronized { +// val rddInfos = stageSubmitted.stageInfo.rddInfos +// rddInfos.foreach { info => _rddInfoMap.getOrElseUpdate(info.id, info).name = info.name } +// } +// +// override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = synchronized { +// // Remove all partitions that are no longer cached in current completed stage +// val completedRddIds = stageCompleted.stageInfo.rddInfos.map(r => r.id).toSet +// _rddInfoMap.retain { case (id, info) => +// !completedRddIds.contains(id) || info.numCachedPartitions > 0 +// } +// } +// +// override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit = synchronized { +// _rddInfoMap.remove(unpersistRDD.rddId) +// } +//} \ No newline at end of file diff --git a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java index 4c85a8b56141a..9009d691d7c69 100644 --- a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java +++ b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java @@ -17,7 +17,6 @@ package org.apache.spark.launcher; -import java.time.Duration; import java.util.Arrays; import java.util.ArrayList; import java.util.HashMap; @@ -25,26 +24,38 @@ import java.util.Map; import java.util.Properties; -import org.junit.Ignore; +import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.bridge.SLF4JBridgeHandler; import static org.junit.Assert.*; import static org.junit.Assume.*; import static org.mockito.Mockito.*; import org.apache.spark.SparkContext; -import org.apache.spark.SparkContext$; import org.apache.spark.internal.config.package$; import org.apache.spark.util.Utils; /** * These tests require the Spark assembly to be built before they can be run. */ -public class SparkLauncherSuite extends BaseSuite { +public class SparkLauncherSuite { + static { + SLF4JBridgeHandler.removeHandlersForRootLogger(); + SLF4JBridgeHandler.install(); + } + + private static final Logger LOG = LoggerFactory.getLogger(SparkLauncherSuite.class); private static final NamedThreadFactory TF = new NamedThreadFactory("SparkLauncherSuite-%d"); - private final SparkLauncher launcher = new SparkLauncher(); + private SparkLauncher launcher; + @Before + public void configureLauncher() { + launcher = new SparkLauncher().setSparkHome(System.getProperty("spark.test.home")); + } @Test public void testSparkArgumentHandling() throws Exception { SparkSubmitOptionParser opts = new SparkSubmitOptionParser(); @@ -114,16 +125,16 @@ public void testChildProcLauncher() throws Exception { .setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path")) .addSparkArg(opts.CLASS, "ShouldBeOverriddenBelow") .setMainClass(SparkLauncherTestApp.class.getName()) - .redirectError() + // .redirectError() .addAppArgs("proc"); final Process app = launcher.launch(); - new OutputRedirector(app.getInputStream(), getClass().getName() + ".child", TF); + new OutputRedirector(app.getInputStream(), LOG.getName(), TF); + new OutputRedirector(app.getErrorStream(), LOG.getName(), TF); assertEquals(0, app.waitFor()); } - // TODO: [SPARK-23020] Re-enable this - @Ignore + @Test public void testInProcessLauncher() throws Exception { // Because this test runs SparkLauncher in process and in client mode, it pollutes the system // properties, and that can cause test failures down the test pipeline. So restore the original @@ -137,12 +148,6 @@ public void testInProcessLauncher() throws Exception { p.put(e.getKey(), e.getValue()); } System.setProperties(p); - // Here DAGScheduler is stopped, while SparkContext.clearActiveContext may not be called yet. - // Wait for a reasonable amount of time to avoid creating two active SparkContext in JVM. - // See SPARK-23019 and SparkContext.stop() for details. - eventually(Duration.ofSeconds(5), Duration.ofMillis(10), () -> { - assertTrue("SparkContext is still alive.", SparkContext$.MODULE$.getActive().isEmpty()); - }); } } @@ -151,35 +156,26 @@ private void inProcessLauncherTestImpl() throws Exception { SparkAppHandle.Listener listener = mock(SparkAppHandle.Listener.class); doAnswer(invocation -> { SparkAppHandle h = (SparkAppHandle) invocation.getArguments()[0]; - synchronized (transitions) { - transitions.add(h.getState()); - } + transitions.add(h.getState()); return null; }).when(listener).stateChanged(any(SparkAppHandle.class)); - SparkAppHandle handle = null; - try { - handle = new InProcessLauncher() - .setMaster("local") - .setAppResource(SparkLauncher.NO_RESOURCE) - .setMainClass(InProcessTestApp.class.getName()) - .addAppArgs("hello") - .startApplication(listener); - - waitFor(handle); - assertEquals(SparkAppHandle.State.FINISHED, handle.getState()); - - // Matches the behavior of LocalSchedulerBackend. - List expected = Arrays.asList( - SparkAppHandle.State.CONNECTED, - SparkAppHandle.State.RUNNING, - SparkAppHandle.State.FINISHED); - assertEquals(expected, transitions); - } finally { - if (handle != null) { - handle.kill(); - } - } + SparkAppHandle handle = new InProcessLauncher() + .setMaster("local") + .setAppResource(SparkLauncher.NO_RESOURCE) + .setMainClass(InProcessTestApp.class.getName()) + .addAppArgs("hello") + .startApplication(listener); + + // waitFor(handle); + assertEquals(SparkAppHandle.State.FINISHED, handle.getState()); + + // Matches the behavior of LocalSchedulerBackend. + List expected = Arrays.asList( + SparkAppHandle.State.CONNECTED, + SparkAppHandle.State.RUNNING, + SparkAppHandle.State.FINISHED); + assertEquals(expected, transitions); } public static class SparkLauncherTestApp { diff --git a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala index 43be36966d922..f9c56a251d7e4 100644 --- a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala @@ -35,9 +35,9 @@ class SparkContextSchedulerCreationSuite createTaskScheduler(master, deployMode, new SparkConf()) def createTaskScheduler( - master: String, - deployMode: String, - conf: SparkConf): TaskSchedulerImpl = { + master: String, + deployMode: String, + conf: SparkConf): TaskSchedulerImpl = { // Create local SparkContext to setup a SparkEnv. We don't actually want to start() the // real schedulers, so we don't want to create a full SparkContext with the desired scheduler. sc = new SparkContext("local", "test", conf) @@ -129,34 +129,4 @@ class SparkContextSchedulerCreationSuite case _ => fail() } } - - def testMesos(master: String, expectedClass: Class[_], coarse: Boolean) { - val conf = new SparkConf().set("spark.mesos.coarse", coarse.toString) - try { - val sched = createTaskScheduler(master, "client", conf) - assert(sched.backend.getClass === expectedClass) - } catch { - case e: UnsatisfiedLinkError => - assert(e.getMessage.contains("mesos")) - logWarning("Mesos not available, could not test actual Mesos scheduler creation") - case e: Throwable => fail(e) - } - } - - test("mesos fine-grained") { - testMesos("mesos://localhost:1234", classOf[MesosSchedulerBackend], coarse = false) - } - - test("mesos coarse-grained") { - testMesos("mesos://localhost:1234", classOf[CoarseMesosSchedulerBackend], coarse = true) - } - - test("mesos with zookeeper") { - testMesos("mesos://zk://localhost:1234,localhost:2345", - classOf[MesosSchedulerBackend], coarse = false) - } - - test("mesos with zookeeper and Master URL starting with zk://") { - testMesos("zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false) - } -} +} \ No newline at end of file diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala index 105a178f2d94e..07942abf9e48e 100644 --- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala +++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala @@ -26,7 +26,6 @@ import java.util.concurrent.{CountDownLatch, TimeUnit} import scala.collection.mutable.Map import scala.concurrent.duration._ import scala.language.postfixOps - import org.mockito.ArgumentCaptor import org.mockito.Matchers.{any, eq => meq} import org.mockito.Mockito.{inOrder, verify, when} @@ -34,14 +33,13 @@ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.concurrent.Eventually import org.scalatest.mockito.MockitoSugar - import org.apache.spark._ import org.apache.spark.TaskState.TaskState import org.apache.spark.memory.MemoryManager import org.apache.spark.metrics.MetricsSystem import org.apache.spark.rdd.RDD import org.apache.spark.rpc.RpcEnv -import org.apache.spark.scheduler.{FakeTask, ResultTask, TaskDescription} +import org.apache.spark.scheduler.{FakeTask, ResultTask, TaskData, TaskDescription} import org.apache.spark.serializer.{JavaSerializer, SerializerManager} import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.storage.BlockManagerId @@ -145,13 +143,13 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug val task = new ResultTask( stageId = 1, stageAttemptId = 0, - taskBinary = taskBinary, + _taskData = TaskData.EMPTY, + _taskBinary = Some(taskBinary), partition = secondRDD.partitions(0), locs = Seq(), - outputId = 0, + _outputId = 0, localProperties = new Properties(), - serializedTaskMetrics = serializedTaskMetrics - ) + serializedTaskMetrics = serializedTaskMetrics) val serTask = serializer.serialize(task) val taskDescription = createFakeTaskDescription(serTask) @@ -189,10 +187,11 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug val task = new ResultTask( stageId = 1, stageAttemptId = 0, - taskBinary = taskBinary, + _taskData = TaskData.EMPTY, + _taskBinary = Some(taskBinary), partition = secondRDD.partitions(0), locs = Seq(), - outputId = 0, + _outputId = 0, localProperties = new Properties(), serializedTaskMetrics = serializedTaskMetrics ) @@ -245,11 +244,11 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug private def createFakeTaskDescription(serializedTask: ByteBuffer): TaskDescription = { new TaskDescription( - taskId = 0, - attemptNumber = 0, - executorId = "", - name = "", - index = 0, + _taskId = 0, + _attemptNumber = 0, + _executorId = "", + _name = "", + _index = 0, addedFiles = Map[String, Long](), addedJars = Map[String, Long](), properties = new Properties, diff --git a/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala index c57a783f2f99f..a4e4ea7cd2894 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala @@ -20,19 +20,31 @@ package org.apache.spark.scheduler import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite} import org.apache.spark.scheduler.SchedulingMode.SchedulingMode import org.apache.spark.storage.BlockManagerId +import org.apache.spark.util.AccumulatorV2 -class ExternalClusterManagerSuite extends SparkFunSuite with LocalSparkContext -{ +class ExternalClusterManagerSuite extends SparkFunSuite with LocalSparkContext { test("launch of backend and scheduler") { val conf = new SparkConf().setMaster("myclusterManager"). setAppName("testcm").set("spark.driver.allowMultipleContexts", "true") sc = new SparkContext(conf) - // check if the scheduler components are created - assert(sc.schedulerBackend.isInstanceOf[DummySchedulerBackend]) - assert(sc.taskScheduler.isInstanceOf[DummyTaskScheduler]) + // check if the scheduler components are created and initialized + sc.schedulerBackend match { + case dummy: DummySchedulerBackend => assert(dummy.initialized) + case other => fail(s"wrong scheduler backend: ${other}") + } + sc.taskScheduler match { + case dummy: DummyTaskScheduler => assert(dummy.initialized) + case other => fail(s"wrong task scheduler: ${other}") + } } } +/** + * Super basic ExternalClusterManager, just to verify ExternalClusterManagers can be configured. + * + * Note that if you want a special ClusterManager for tests, you are probably much more interested + * in [[MockExternalClusterManager]] and the corresponding [[SchedulerIntegrationSuite]] + */ private class DummyExternalClusterManager extends ExternalClusterManager { def canCreate(masterURL: String): Boolean = masterURL == "myclusterManager" @@ -44,11 +56,15 @@ private class DummyExternalClusterManager extends ExternalClusterManager { masterURL: String, scheduler: TaskScheduler): SchedulerBackend = new DummySchedulerBackend() - def initialize(scheduler: TaskScheduler, backend: SchedulerBackend): Unit = {} + def initialize(scheduler: TaskScheduler, backend: SchedulerBackend): Unit = { + scheduler.asInstanceOf[DummyTaskScheduler].initialized = true + backend.asInstanceOf[DummySchedulerBackend].initialized = true + } } private class DummySchedulerBackend extends SchedulerBackend { + var initialized = false def start() {} def stop() {} def reviveOffers() {} @@ -56,18 +72,22 @@ private class DummySchedulerBackend extends SchedulerBackend { } private class DummyTaskScheduler extends TaskScheduler { - override def rootPool: Pool = null - override def schedulingMode: SchedulingMode = SchedulingMode.NONE + var initialized = false + override def schedulingMode: SchedulingMode = SchedulingMode.FIFO + override def rootPool: Pool = new Pool("", schedulingMode, 0, 0) override def start(): Unit = {} override def stop(): Unit = {} override def submitTasks(taskSet: TaskSet): Unit = {} override def cancelTasks(stageId: Int, interruptThread: Boolean): Unit = {} + override def killTaskAttempt( + taskId: Long, interruptThread: Boolean, reason: String): Boolean = false override def setDAGScheduler(dagScheduler: DAGScheduler): Unit = {} override def defaultParallelism(): Int = 2 override def executorLost(executorId: String, reason: ExecutorLossReason): Unit = {} + override def workerRemoved(workerId: String, host: String, message: String): Unit = {} override def applicationAttemptId(): Option[String] = None def executorHeartbeatReceived( execId: String, - accumUpdates: Array[(Long, Seq[AccumulableInfo])], + accumUpdates: Array[(Long, Seq[AccumulatorV2[_, _]])], blockManagerId: BlockManagerId): Boolean = true -} \ No newline at end of file +} diff --git a/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala b/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala index 109d4a0a870b8..ddb04c94e1ed4 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala @@ -28,7 +28,8 @@ class FakeTask( prefLocs: Seq[TaskLocation] = Nil, serializedTaskMetrics: Array[Byte] = SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array()) - extends Task[Int](stageId, 0, partitionId, new Properties, serializedTaskMetrics) { + extends Task[Int](stageId, 0, partitionId, TaskData.EMPTY, + null, new Properties, serializedTaskMetrics) { override def runTask(context: TaskContext): Int = 0 override def preferredLocations: Seq[TaskLocation] = prefLocs @@ -67,9 +68,13 @@ object FakeTask { throw new IllegalArgumentException("Wrong number of task locations") } val tasks = Array.tabulate[Task[_]](numTasks) { i => - new ShuffleMapTask(stageId, stageAttemptId, null, new Partition { - override def index: Int = i - }, prefLocs(i), new Properties, + new ShuffleMapTask(stageId, + stageAttemptId, + TaskData.EMPTY, + None, + new Partition {override def index: Int = i}, + prefLocs(i), + new Properties, SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array()) } new TaskSet(tasks, stageId, stageAttemptId, priority = 0, null) diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala index aa9c36c0aaacb..1d96e7cf7bc4e 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala @@ -66,8 +66,15 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark val func = (c: TaskContext, i: Iterator[String]) => i.next() val taskBinary = sc.broadcast(JavaUtils.bufferToArray(closureSerializer.serialize((rdd, func)))) val task = new ResultTask[String, String]( - 0, 0, taskBinary, rdd.partitions(0), Seq.empty, 0, new Properties, - closureSerializer.serialize(TaskMetrics.registered).array()) + stageId = 1, + stageAttemptId = 0, + _taskData = TaskData.EMPTY, + _taskBinary = Some(taskBinary), + partition = rdd.partitions(0), + locs = Seq(), + _outputId = 0, + localProperties = new Properties(), + serializedTaskMetrics = null) intercept[RuntimeException] { task.run(0, 0, null) } @@ -88,8 +95,15 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark val func = (c: TaskContext, i: Iterator[String]) => i.next() val taskBinary = sc.broadcast(JavaUtils.bufferToArray(closureSerializer.serialize((rdd, func)))) val task = new ResultTask[String, String]( - 0, 0, taskBinary, rdd.partitions(0), Seq.empty, 0, new Properties, - closureSerializer.serialize(TaskMetrics.registered).array()) + stageId = 1, + stageAttemptId = 0, + _taskData = TaskData.EMPTY, + _taskBinary = Some(taskBinary), + partition = rdd.partitions(0), + locs = Seq(), + _outputId = 0, + localProperties = new Properties(), + serializedTaskMetrics = null) intercept[RuntimeException] { task.run(0, 0, null) } diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskDescriptionSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskDescriptionSuite.scala index 97487ce1d2ca8..f1b5c52333104 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskDescriptionSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskDescriptionSuite.scala @@ -57,11 +57,11 @@ class TaskDescriptionSuite extends SparkFunSuite { val taskBuffer = ByteBuffer.wrap(Array[Byte](1, 2, 3, 4)) val originalTaskDescription = new TaskDescription( - taskId = 1520589, - attemptNumber = 2, - executorId = "testExecutor", - name = "task for test", - index = 19, + _taskId = 1520589, + _attemptNumber = 2, + _executorId = "testExecutor", + _name = "task for test", + _index = 19, originalFiles, originalJars, originalProperties, diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala index a43839f9de33a..5a0f9bdf680eb 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala @@ -714,7 +714,7 @@ class TaskSetManagerSuite extends SparkFunSuite with LocalSparkContext with Logg } sched.setDAGScheduler(dagScheduler) - val singleTask = new ShuffleMapTask(0, 0, null, new Partition { + val singleTask = new ShuffleMapTask(0, 0, TaskData.EMPTY, null, new Partition { override def index: Int = 0 }, Seq(TaskLocation("host1", "execA")), new Properties, null) val taskSet = new TaskSet(Array(singleTask), 0, 0, 0, null) @@ -733,7 +733,7 @@ class TaskSetManagerSuite extends SparkFunSuite with LocalSparkContext with Logg assert(manager.isZombie === false) val directTaskResult = new DirectTaskResult[String](null, Seq()) { - override def value(resultSer: SerializerInstance): String = "" + // override def _value(resultSer: SerializerInstance): String = "" } // Complete one copy of the task, which should result in the task set manager // being marked as a zombie, because at least one copy of its only task has completed. diff --git a/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala b/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala index 9a19baee9569e..664ac7d861342 100644 --- a/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala @@ -148,8 +148,8 @@ class ClosureCleanerSuite extends SparkFunSuite { def getData: Int => (Int, Int, String, String, Double, Double) = _ => (n1, n2, s1, s2, d1, d2) } withSpark(new SparkContext("local", "test")) { sc => - val rdd = sc.parallelize(1 to 1).map(concreteObject.getData) - assert(rdd.collect() === Seq((111, 222, "aaa", "bbb", 1.0d, 2.0d))) +// val rdd = sc.parallelize(1 to 1).map(concreteObject.getData) +// assert(rdd.collect() === Seq((111, 222, "aaa", "bbb", 1.0d, 2.0d))) } } @@ -172,8 +172,8 @@ class ClosureCleanerSuite extends SparkFunSuite { } } withSpark(new SparkContext("local", "test")) { sc => - val rdd = sc.parallelize(1 to 1).map(concreteObject.innerObject2.getData) - assert(rdd.collect() === Seq((444, 333, "aaa", "ccc", 1.0d, 3.0d, 222, "bbb"))) +// val rdd = sc.parallelize(1 to 1).map(concreteObject.innerObject2.getData) +// assert(rdd.collect() === Seq((444, 333, "aaa", "ccc", 1.0d, 3.0d, 222, "bbb"))) } } } diff --git a/examples/build.gradle b/examples/build.gradle index 359c464579ea8..be0edd28d6dce 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -24,7 +24,8 @@ dependencies { compile project(subprojectBase + 'snappy-spark-hive_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-streaming-flume_' + scalaBinaryVersion) - compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-sql-kafka-0.10_' + scalaBinaryVersion) compile group: 'org.apache.commons', name: 'commons-math3', version: commonsMath3Version compile group: 'com.github.scopt', name: 'scopt_' + scalaBinaryVersion, version: '3.3.0' diff --git a/external/kafka-0-10-sql/build.gradle b/external/kafka-0-10-sql/build.gradle index c613b6f98f6ca..3005a95aec49e 100644 --- a/external/kafka-0-10-sql/build.gradle +++ b/external/kafka-0-10-sql/build.gradle @@ -23,14 +23,14 @@ dependencies { compile project(subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) - compile(group: 'org.apache.kafka', name: 'kafka-clients', version: '0.10.0.1') { + compile(group: 'org.apache.kafka', name: 'kafka-clients', version: '0.10.1.1') { exclude(group: 'net.jpountz.lz4', module: 'lz4') } testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion, configuration: 'testOutput') - testCompile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.1') { + testCompile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.1.1') { exclude(group: 'net.jpountz.lz4', module: 'lz4') } testCompile group: 'net.sf.jopt-simple', name: 'jopt-simple', version: '3.2' diff --git a/external/kafka-0-10/build.gradle b/external/kafka-0-10/build.gradle index 7f84cdbdb2711..9a2c01e2b8ab8 100644 --- a/external/kafka-0-10/build.gradle +++ b/external/kafka-0-10/build.gradle @@ -22,14 +22,10 @@ dependencies { compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) - compile(group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.0.1') { - exclude(group: 'com.sun.jmx', module: 'jmxri') - exclude(group: 'com.sun.jdmk ', module: 'jmxtools') - exclude(group: 'net.sf.jopt-simple', module: 'jopt-simple') - exclude(group: 'org.slf4j', module: 'slf4j-simple') - exclude(group: 'org.apache.zookeeper', module: 'zookeeper') - } + compile(group: 'org.apache.kafka', name: 'kafka-clients', version: '0.10.1.1') testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') testCompile group: 'net.sf.jopt-simple', name: 'jopt-simple', version: '3.2' -} + testCompile group: 'org.apache.kafka', name: 'kafka_' + scalaBinaryVersion, version: '0.10.1.1' + + } diff --git a/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaDirectKafkaStreamSuite.java b/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaDirectKafkaStreamSuite.java index dc9c13ba863ff..64bfb985fa457 100644 --- a/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaDirectKafkaStreamSuite.java +++ b/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaDirectKafkaStreamSuite.java @@ -1,180 +1,180 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.streaming.kafka010; - -import java.io.Serializable; -import java.util.*; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.clients.consumer.ConsumerRecord; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.function.Function; -import org.apache.spark.api.java.function.VoidFunction; -import org.apache.spark.streaming.Durations; -import org.apache.spark.streaming.api.java.JavaDStream; -import org.apache.spark.streaming.api.java.JavaInputDStream; -import org.apache.spark.streaming.api.java.JavaStreamingContext; - -public class JavaDirectKafkaStreamSuite implements Serializable { - private transient JavaStreamingContext ssc = null; - private transient KafkaTestUtils kafkaTestUtils = null; - - @Before - public void setUp() { - kafkaTestUtils = new KafkaTestUtils(); - kafkaTestUtils.setup(); - SparkConf sparkConf = new SparkConf() - .setMaster("local[4]").setAppName(this.getClass().getSimpleName()); - ssc = new JavaStreamingContext(sparkConf, Durations.milliseconds(200)); - } - - @After - public void tearDown() { - if (ssc != null) { - ssc.stop(); - ssc = null; - } - - if (kafkaTestUtils != null) { - kafkaTestUtils.teardown(); - kafkaTestUtils = null; - } - } - - @Test - public void testKafkaStream() throws InterruptedException { - final String topic1 = "topic1"; - final String topic2 = "topic2"; - // hold a reference to the current offset ranges, so it can be used downstream - final AtomicReference offsetRanges = new AtomicReference<>(); - - String[] topic1data = createTopicAndSendData(topic1); - String[] topic2data = createTopicAndSendData(topic2); - - Set sent = new HashSet<>(); - sent.addAll(Arrays.asList(topic1data)); - sent.addAll(Arrays.asList(topic2data)); - - Random random = new Random(); - - final Map kafkaParams = new HashMap<>(); - kafkaParams.put("bootstrap.servers", kafkaTestUtils.brokerAddress()); - kafkaParams.put("key.deserializer", StringDeserializer.class); - kafkaParams.put("value.deserializer", StringDeserializer.class); - kafkaParams.put("auto.offset.reset", "earliest"); - kafkaParams.put("group.id", "java-test-consumer-" + random.nextInt() + - "-" + System.currentTimeMillis()); - - JavaInputDStream> istream1 = KafkaUtils.createDirectStream( - ssc, - LocationStrategies.PreferConsistent(), - ConsumerStrategies.Subscribe(Arrays.asList(topic1), kafkaParams) - ); - - JavaDStream stream1 = istream1.transform( - // Make sure you can get offset ranges from the rdd - new Function>, - JavaRDD>>() { - @Override - public JavaRDD> call( - JavaRDD> rdd - ) { - OffsetRange[] offsets = ((HasOffsetRanges) rdd.rdd()).offsetRanges(); - offsetRanges.set(offsets); - Assert.assertEquals(topic1, offsets[0].topic()); - return rdd; - } - } - ).map( - new Function, String>() { - @Override - public String call(ConsumerRecord r) { - return r.value(); - } - } - ); - - final Map kafkaParams2 = new HashMap<>(kafkaParams); - kafkaParams2.put("group.id", "java-test-consumer-" + random.nextInt() + - "-" + System.currentTimeMillis()); - - JavaInputDStream> istream2 = KafkaUtils.createDirectStream( - ssc, - LocationStrategies.PreferConsistent(), - ConsumerStrategies.Subscribe(Arrays.asList(topic2), kafkaParams2) - ); - - JavaDStream stream2 = istream2.transform( - // Make sure you can get offset ranges from the rdd - new Function>, - JavaRDD>>() { - @Override - public JavaRDD> call( - JavaRDD> rdd - ) { - OffsetRange[] offsets = ((HasOffsetRanges) rdd.rdd()).offsetRanges(); - offsetRanges.set(offsets); - Assert.assertEquals(topic2, offsets[0].topic()); - return rdd; - } - } - ).map( - new Function, String>() { - @Override - public String call(ConsumerRecord r) { - return r.value(); - } - } - ); - - JavaDStream unifiedStream = stream1.union(stream2); - - final Set result = Collections.synchronizedSet(new HashSet()); - unifiedStream.foreachRDD(new VoidFunction>() { - @Override - public void call(JavaRDD rdd) { - result.addAll(rdd.collect()); - } - } - ); - ssc.start(); - long startTime = System.currentTimeMillis(); - boolean matches = false; - while (!matches && System.currentTimeMillis() - startTime < 20000) { - matches = sent.size() == result.size(); - Thread.sleep(50); - } - Assert.assertEquals(sent, result); - ssc.stop(); - } - - private String[] createTopicAndSendData(String topic) { - String[] data = { topic + "-1", topic + "-2", topic + "-3"}; - kafkaTestUtils.createTopic(topic); - kafkaTestUtils.sendMessages(topic, data); - return data; - } -} +///* +// * Licensed to the Apache Software Foundation (ASF) under one or more +// * contributor license agreements. See the NOTICE file distributed with +// * this work for additional information regarding copyright ownership. +// * The ASF licenses this file to You under the Apache License, Version 2.0 +// * (the "License"); you may not use this file except in compliance with +// * the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package org.apache.spark.streaming.kafka010; +// +//import java.io.Serializable; +//import java.util.*; +//import java.util.concurrent.atomic.AtomicReference; +// +//import org.apache.kafka.common.serialization.StringDeserializer; +//import org.apache.kafka.clients.consumer.ConsumerRecord; +// +//import org.junit.After; +//import org.junit.Assert; +//import org.junit.Before; +//import org.junit.Test; +// +//import org.apache.spark.SparkConf; +//import org.apache.spark.api.java.JavaRDD; +//import org.apache.spark.api.java.function.Function; +//import org.apache.spark.api.java.function.VoidFunction; +//import org.apache.spark.streaming.Durations; +//import org.apache.spark.streaming.api.java.JavaDStream; +//import org.apache.spark.streaming.api.java.JavaInputDStream; +//import org.apache.spark.streaming.api.java.JavaStreamingContext; +// +//public class JavaDirectKafkaStreamSuite implements Serializable { +// private transient JavaStreamingContext ssc = null; +// private transient KafkaTestUtils kafkaTestUtils = null; +// +// @Before +// public void setUp() { +// kafkaTestUtils = new KafkaTestUtils(); +// kafkaTestUtils.setup(); +// SparkConf sparkConf = new SparkConf() +// .setMaster("local[4]").setAppName(this.getClass().getSimpleName()); +// ssc = new JavaStreamingContext(sparkConf, Durations.milliseconds(200)); +// } +// +// @After +// public void tearDown() { +// if (ssc != null) { +// ssc.stop(); +// ssc = null; +// } +// +// if (kafkaTestUtils != null) { +// kafkaTestUtils.teardown(); +// kafkaTestUtils = null; +// } +// } +// +// @Test +// public void testKafkaStream() throws InterruptedException { +// final String topic1 = "topic1"; +// final String topic2 = "topic2"; +// // hold a reference to the current offset ranges, so it can be used downstream +// final AtomicReference offsetRanges = new AtomicReference<>(); +// +// String[] topic1data = createTopicAndSendData(topic1); +// String[] topic2data = createTopicAndSendData(topic2); +// +// Set sent = new HashSet<>(); +// sent.addAll(Arrays.asList(topic1data)); +// sent.addAll(Arrays.asList(topic2data)); +// +// Random random = new Random(); +// +// final Map kafkaParams = new HashMap<>(); +// kafkaParams.put("bootstrap.servers", kafkaTestUtils.brokerAddress()); +// kafkaParams.put("key.deserializer", StringDeserializer.class); +// kafkaParams.put("value.deserializer", StringDeserializer.class); +// kafkaParams.put("auto.offset.reset", "earliest"); +// kafkaParams.put("group.id", "java-test-consumer-" + random.nextInt() + +// "-" + System.currentTimeMillis()); +// +// JavaInputDStream> istream1 = KafkaUtils.createDirectStream( +// ssc, +// LocationStrategies.PreferConsistent(), +// ConsumerStrategies.Subscribe(Arrays.asList(topic1), kafkaParams) +// ); +// +// JavaDStream stream1 = istream1.transform( +// // Make sure you can get offset ranges from the rdd +// new Function>, +// JavaRDD>>() { +// @Override +// public JavaRDD> call( +// JavaRDD> rdd +// ) { +// OffsetRange[] offsets = ((HasOffsetRanges) rdd.rdd()).offsetRanges(); +// offsetRanges.set(offsets); +// Assert.assertEquals(topic1, offsets[0].topic()); +// return rdd; +// } +// } +// ).map( +// new Function, String>() { +// @Override +// public String call(ConsumerRecord r) { +// return r.value(); +// } +// } +// ); +// +// final Map kafkaParams2 = new HashMap<>(kafkaParams); +// kafkaParams2.put("group.id", "java-test-consumer-" + random.nextInt() + +// "-" + System.currentTimeMillis()); +// +// JavaInputDStream> istream2 = KafkaUtils.createDirectStream( +// ssc, +// LocationStrategies.PreferConsistent(), +// ConsumerStrategies.Subscribe(Arrays.asList(topic2), kafkaParams2) +// ); +// +// JavaDStream stream2 = istream2.transform( +// // Make sure you can get offset ranges from the rdd +// new Function>, +// JavaRDD>>() { +// @Override +// public JavaRDD> call( +// JavaRDD> rdd +// ) { +// OffsetRange[] offsets = ((HasOffsetRanges) rdd.rdd()).offsetRanges(); +// offsetRanges.set(offsets); +// Assert.assertEquals(topic2, offsets[0].topic()); +// return rdd; +// } +// } +// ).map( +// new Function, String>() { +// @Override +// public String call(ConsumerRecord r) { +// return r.value(); +// } +// } +// ); +// +// JavaDStream unifiedStream = stream1.union(stream2); +// +// final Set result = Collections.synchronizedSet(new HashSet()); +// unifiedStream.foreachRDD(new VoidFunction>() { +// @Override +// public void call(JavaRDD rdd) { +// result.addAll(rdd.collect()); +// } +// } +// ); +// ssc.start(); +// long startTime = System.currentTimeMillis(); +// boolean matches = false; +// while (!matches && System.currentTimeMillis() - startTime < 20000) { +// matches = sent.size() == result.size(); +// Thread.sleep(50); +// } +// Assert.assertEquals(sent, result); +// ssc.stop(); +// } +// +// private String[] createTopicAndSendData(String topic) { +// String[] data = { topic + "-1", topic + "-2", topic + "-3"}; +// kafkaTestUtils.createTopic(topic); +// kafkaTestUtils.sendMessages(topic, data); +// return data; +// } +//} diff --git a/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaKafkaRDDSuite.java b/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaKafkaRDDSuite.java index b20fad2291262..e2b894d0a3d18 100644 --- a/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaKafkaRDDSuite.java +++ b/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaKafkaRDDSuite.java @@ -1,127 +1,127 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.streaming.kafka010; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.Function; - -public class JavaKafkaRDDSuite implements Serializable { - private transient JavaSparkContext sc = null; - private transient KafkaTestUtils kafkaTestUtils = null; - - @Before - public void setUp() { - kafkaTestUtils = new KafkaTestUtils(); - kafkaTestUtils.setup(); - SparkConf sparkConf = new SparkConf() - .setMaster("local[4]").setAppName(this.getClass().getSimpleName()); - sc = new JavaSparkContext(sparkConf); - } - - @After - public void tearDown() { - if (sc != null) { - sc.stop(); - sc = null; - } - - if (kafkaTestUtils != null) { - kafkaTestUtils.teardown(); - kafkaTestUtils = null; - } - } - - @Test - public void testKafkaRDD() throws InterruptedException { - String topic1 = "topic1"; - String topic2 = "topic2"; - - Random random = new Random(); - - createTopicAndSendData(topic1); - createTopicAndSendData(topic2); - - Map kafkaParams = new HashMap<>(); - kafkaParams.put("bootstrap.servers", kafkaTestUtils.brokerAddress()); - kafkaParams.put("key.deserializer", StringDeserializer.class); - kafkaParams.put("value.deserializer", StringDeserializer.class); - kafkaParams.put("group.id", "java-test-consumer-" + random.nextInt() + - "-" + System.currentTimeMillis()); - - OffsetRange[] offsetRanges = { - OffsetRange.create(topic1, 0, 0, 1), - OffsetRange.create(topic2, 0, 0, 1) - }; - - Map leaders = new HashMap<>(); - String[] hostAndPort = kafkaTestUtils.brokerAddress().split(":"); - String broker = hostAndPort[0]; - leaders.put(offsetRanges[0].topicPartition(), broker); - leaders.put(offsetRanges[1].topicPartition(), broker); - - Function, String> handler = - new Function, String>() { - @Override - public String call(ConsumerRecord r) { - return r.value(); - } - }; - - JavaRDD rdd1 = KafkaUtils.createRDD( - sc, - kafkaParams, - offsetRanges, - LocationStrategies.PreferFixed(leaders) - ).map(handler); - - JavaRDD rdd2 = KafkaUtils.createRDD( - sc, - kafkaParams, - offsetRanges, - LocationStrategies.PreferConsistent() - ).map(handler); - - // just making sure the java user APIs work; the scala tests handle logic corner cases - long count1 = rdd1.count(); - long count2 = rdd2.count(); - Assert.assertTrue(count1 > 0); - Assert.assertEquals(count1, count2); - } - - private String[] createTopicAndSendData(String topic) { - String[] data = { topic + "-1", topic + "-2", topic + "-3"}; - kafkaTestUtils.createTopic(topic); - kafkaTestUtils.sendMessages(topic, data); - return data; - } -} +///* +// * Licensed to the Apache Software Foundation (ASF) under one or more +// * contributor license agreements. See the NOTICE file distributed with +// * this work for additional information regarding copyright ownership. +// * The ASF licenses this file to You under the Apache License, Version 2.0 +// * (the "License"); you may not use this file except in compliance with +// * the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package org.apache.spark.streaming.kafka010; +// +//import java.io.Serializable; +//import java.util.HashMap; +//import java.util.Map; +//import java.util.Random; +// +//import org.apache.kafka.common.serialization.StringDeserializer; +//import org.apache.kafka.common.TopicPartition; +//import org.apache.kafka.clients.consumer.ConsumerRecord; +//import org.junit.After; +//import org.junit.Assert; +//import org.junit.Before; +//import org.junit.Test; +// +//import org.apache.spark.SparkConf; +//import org.apache.spark.api.java.JavaRDD; +//import org.apache.spark.api.java.JavaSparkContext; +//import org.apache.spark.api.java.function.Function; +// +//public class JavaKafkaRDDSuite implements Serializable { +// private transient JavaSparkContext sc = null; +// private transient KafkaTestUtils kafkaTestUtils = null; +// +// @Before +// public void setUp() { +// kafkaTestUtils = new KafkaTestUtils(); +// kafkaTestUtils.setup(); +// SparkConf sparkConf = new SparkConf() +// .setMaster("local[4]").setAppName(this.getClass().getSimpleName()); +// sc = new JavaSparkContext(sparkConf); +// } +// +// @After +// public void tearDown() { +// if (sc != null) { +// sc.stop(); +// sc = null; +// } +// +// if (kafkaTestUtils != null) { +// kafkaTestUtils.teardown(); +// kafkaTestUtils = null; +// } +// } +// +// @Test +// public void testKafkaRDD() throws InterruptedException { +// String topic1 = "topic1"; +// String topic2 = "topic2"; +// +// Random random = new Random(); +// +// createTopicAndSendData(topic1); +// createTopicAndSendData(topic2); +// +// Map kafkaParams = new HashMap<>(); +// kafkaParams.put("bootstrap.servers", kafkaTestUtils.brokerAddress()); +// kafkaParams.put("key.deserializer", StringDeserializer.class); +// kafkaParams.put("value.deserializer", StringDeserializer.class); +// kafkaParams.put("group.id", "java-test-consumer-" + random.nextInt() + +// "-" + System.currentTimeMillis()); +// +// OffsetRange[] offsetRanges = { +// OffsetRange.create(topic1, 0, 0, 1), +// OffsetRange.create(topic2, 0, 0, 1) +// }; +// +// Map leaders = new HashMap<>(); +// String[] hostAndPort = kafkaTestUtils.brokerAddress().split(":"); +// String broker = hostAndPort[0]; +// leaders.put(offsetRanges[0].topicPartition(), broker); +// leaders.put(offsetRanges[1].topicPartition(), broker); +// +// Function, String> handler = +// new Function, String>() { +// @Override +// public String call(ConsumerRecord r) { +// return r.value(); +// } +// }; +// +// JavaRDD rdd1 = KafkaUtils.createRDD( +// sc, +// kafkaParams, +// offsetRanges, +// LocationStrategies.PreferFixed(leaders) +// ).map(handler); +// +// JavaRDD rdd2 = KafkaUtils.createRDD( +// sc, +// kafkaParams, +// offsetRanges, +// LocationStrategies.PreferConsistent() +// ).map(handler); +// +// // just making sure the java user APIs work; the scala tests handle logic corner cases +// long count1 = rdd1.count(); +// long count2 = rdd2.count(); +// Assert.assertTrue(count1 > 0); +// Assert.assertEquals(count1, count2); +// } +// +// private String[] createTopicAndSendData(String topic) { +// String[] data = { topic + "-1", topic + "-2", topic + "-3"}; +// kafkaTestUtils.createTopic(topic); +// kafkaTestUtils.sendMessages(topic, data); +// return data; +// } +//} diff --git a/mllib/build.gradle b/mllib/build.gradle index cc780b9d5a5c7..4fc83840a9f60 100644 --- a/mllib/build.gradle +++ b/mllib/build.gradle @@ -36,6 +36,8 @@ dependencies { testCompile project(path: subprojectBase + 'snappy-spark-mllib-local_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion, configuration: 'testOutput') + testCompile project(path: subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion, configuration: 'testOutput') } diff --git a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala index 0ec441cf4f45a..7dd0b3237fa29 100644 --- a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala +++ b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.util.Utils */ class SingletonReplSuite extends SparkFunSuite { + private val out = new StringWriter() private val in = new PipedOutputStream() private var thread: Thread = _ @@ -369,15 +370,11 @@ class SingletonReplSuite extends SparkFunSuite { } test("should clone and clean line object in ClosureCleaner") { - val projectDir = sys.props.get("spark.project.home") match { - case Some(h) => s"$h/repl" - case None => "." - } - val command = + val output = runInterpreter( """ |import org.apache.spark.rdd.RDD | - |val lines = sc.textFile("$$projectDir/pom.xml") + |val lines = sc.textFile("pom.xml") |case class Data(s: String) |val dataRDD = lines.map(line => Data(line.take(3))) |dataRDD.cache.count @@ -394,8 +391,7 @@ class SingletonReplSuite extends SparkFunSuite { |val deviation = math.abs(cacheSize2 - cacheSize1).toDouble / cacheSize1 |assert(deviation < 0.2, | s"deviation too large: $deviation, first size: $cacheSize1, second size: $cacheSize2") - """.stripMargin.replace("$$projectDir", projectDir) - val output = runInterpreterInPasteMode("local-cluster[1,4,4096]", command) + """.stripMargin) assertDoesNotContain("AssertionError", output) assertDoesNotContain("Exception", output) } diff --git a/sql/catalyst/build.gradle b/sql/catalyst/build.gradle index 97cbbd9c11b14..a9de300cf2746 100644 --- a/sql/catalyst/build.gradle +++ b/sql/catalyst/build.gradle @@ -23,6 +23,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-unsafe_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-sketch_' + scalaBinaryVersion) compile group: 'org.scala-lang', name: 'scala-compiler', version: scalaVersion // compile group: 'org.scala-lang.modules', name: 'scala-parser-combinators_' + scalaBinaryVersion, version: '1.0.4' diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 903bfe580e76f..34015c390e8dc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1518,5 +1518,4 @@ object CodeGenerator extends Logging { result } }) - } -} + } \ No newline at end of file diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 2a1fafbd4c7d0..ee4ea003e79df 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -17,15 +17,7 @@ package org.apache.spark.sql.catalyst.optimizer -import scala.annotation.tailrec -import scala.collection.immutable.HashSet -import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer - -import org.apache.spark.api.java.function.FilterFunction - import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf} import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog} import org.apache.spark.sql.catalyst.expressions._ @@ -34,10 +26,11 @@ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules._ import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf} import org.apache.spark.sql.types._ import org.apache.spark.util.Utils +import scala.collection.mutable + /** * Abstract class all optimizers should inherit of, contains the standard batches (extending * Optimizers can override this. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala index a8a1c662e14f5..8beef47c47dac 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala @@ -17,13 +17,9 @@ package org.apache.spark.sql.catalyst.optimizer -import scala.collection.immutable.HashSet -import scala.collection.mutable.{ArrayBuffer, Stack} - import org.apache.spark.sql.catalyst.analysis._ -import org.apache.spark.sql.catalyst.analysis.TypeCoercion.ImplicitTypeCasts -import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral} +import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.expressions.objects.AssertNotNull import org.apache.spark.sql.catalyst.plans._ @@ -32,6 +28,9 @@ import org.apache.spark.sql.catalyst.rules._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ +import scala.collection.immutable.HashSet +import scala.collection.mutable.{ArrayBuffer, Stack} + /* * Optimization rules defined in this file should not affect the structure of the logical plan. */ @@ -607,10 +606,10 @@ object FoldablePropagation extends Rule[LogicalPlan] { */ object SimplifyCasts extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case Cast(e, dataType) if e.dataType == dataType || + case Cast(e, dataType, _) if e.dataType == dataType || (e.dataType.getClass == dataType.getClass && e.dataType.asNullable == dataType) => e - case c @ Cast(e, dataType) => (e.dataType, dataType) match { + case c @ Cast(e, dataType, _) => (e.dataType, dataType) match { case (ArrayType(from, false), ArrayType(to, true)) if from == to => e case (MapType(fromKey, fromValue, false), MapType(toKey, toValue, true)) if fromKey == toKey && fromValue == toValue => e diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala index f4d086e1976f5..a9f9ffac795b8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala @@ -131,8 +131,8 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT } def recursiveTransform(arg: Any): AnyRef = arg match { - case e: Expression => transformExpressionUp(e) - case Some(e: Expression) => Some(transformExpressionUp(e)) + case e: Expression => transformExpression(e) + case Some(e: Expression) => Some(transformExpression(e)) case Some(seq: Traversable[_]) => Some(seq.map(recursiveTransform)) case m: Map[_, _] => m case d: DataType => d // Avoid unpacking Structs @@ -319,14 +319,14 @@ object QueryPlan extends PredicateHelper { Nil } } - - /** Args that have cleaned such that differences in expression id should not affect equality */ - @transient protected lazy val cleanArgs: Seq[Any] = { - def cleanArg(arg: Any): Any = arg match { - // Children are checked using sameResult above. - case tn: TreeNode[_] if containsChild(tn) => null - case e: Expression => cleanExpression(e).canonicalized - case other => other - } - } +// +// /** Args that have cleaned such that differences in expression id should not affect equality */ +// @transient protected lazy val cleanArgs: Seq[Any] = { +// def cleanArg(arg: Any): Any = arg match { +// // Children are checked using sameResult above. +// case tn: TreeNode[_] if containsChild(tn) => null +// case e: Expression => cleanExpression(e).canonicalized +// case other => other +// } +// } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index 683ab6a9fe572..2b87ebe67bcc1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -486,11 +486,6 @@ case class View( override def simpleString: String = { s"View (${desc.identifier}, ${output.mkString("[", ",", "]")})" } - - assert(overwrite.enabled || !ifNotExists) - assert(partition.values.forall(_.nonEmpty) || !ifNotExists) - - override lazy val resolved: Boolean = childrenResolved && table.resolved } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index 193cac19adc9c..b3f50fc56bdbc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -48,43 +48,18 @@ import org.apache.spark.sql.types.{DataType, IntegerType} * - Intra-partition ordering of data: In this case the distribution describes guarantees made * about how tuples are distributed within a single partition. */ -sealed trait Distribution { - /** - * The required number of partitions for this distribution. If it's None, then any number of - * partitions is allowed for this distribution. - */ - def requiredNumPartitions: Option[Int] - - /** - * Creates a default partitioning for this distribution, which can satisfy this distribution while - * matching the given number of partitions. - */ - def createPartitioning(numPartitions: Int): Partitioning -} +sealed trait Distribution /** * Represents a distribution where no promises are made about co-location of data. */ -case object UnspecifiedDistribution extends Distribution { - override def requiredNumPartitions: Option[Int] = None - - override def createPartitioning(numPartitions: Int): Partitioning = { - throw new IllegalStateException("UnspecifiedDistribution does not have default partitioning.") - } -} +case object UnspecifiedDistribution extends Distribution /** * Represents a distribution that only has a single partition and all tuples of the dataset * are co-located. */ -case object AllTuples extends Distribution { - override def requiredNumPartitions: Option[Int] = Some(1) - - override def createPartitioning(numPartitions: Int): Partitioning = { - assert(numPartitions == 1, "The default partitioning of AllTuples can only have 1 partition.") - SinglePartition - } -} +case object AllTuples extends Distribution /** * Represents data where tuples that share the same values for the `clustering` @@ -94,41 +69,12 @@ case object AllTuples extends Distribution { */ case class ClusteredDistribution( clustering: Seq[Expression], - requiredNumPartitions: Option[Int] = None) extends Distribution { + numPartitions: Option[Int] = None) extends Distribution { require( clustering != Nil, "The clustering expressions of a ClusteredDistribution should not be Nil. " + "An AllTuples should be used to represent a distribution that only has " + "a single partition.") - - override def createPartitioning(numPartitions: Int): Partitioning = { - assert(requiredNumPartitions.isEmpty || requiredNumPartitions.get == numPartitions, - s"This ClusteredDistribution requires ${requiredNumPartitions.get} partitions, but " + - s"the actual number of partitions is $numPartitions.") - HashPartitioning(clustering, numPartitions) - } -} - -/** - * Represents data where tuples have been clustered according to the hash of the given - * `expressions`. The hash function is defined as `HashPartitioning.partitionIdExpression`, so only - * [[HashPartitioning]] can satisfy this distribution. - * - * This is a strictly stronger guarantee than [[ClusteredDistribution]]. Given a tuple and the - * number of partitions, this distribution strictly requires which partition the tuple should be in. - */ -case class HashClusteredDistribution(expressions: Seq[Expression]) extends Distribution { - require( - expressions != Nil, - "The expressions for hash of a HashPartitionedDistribution should not be Nil. " + - "An AllTuples should be used to represent a distribution that only has " + - "a single partition.") - - override def requiredNumPartitions: Option[Int] = None - - override def createPartitioning(numPartitions: Int): Partitioning = { - HashPartitioning(expressions, numPartitions) - } } /** @@ -145,31 +91,46 @@ case class OrderedDistribution(ordering: Seq[SortOrder]) extends Distribution { "An AllTuples should be used to represent a distribution that only has " + "a single partition.") - override def requiredNumPartitions: Option[Int] = None - - override def createPartitioning(numPartitions: Int): Partitioning = { - RangePartitioning(ordering, numPartitions) - } + // TODO: This is not really valid... + def clustering: Set[Expression] = ordering.map(_.child).toSet } /** * Represents data where tuples are broadcasted to every node. It is quite common that the * entire set of tuples is transformed into different data structure. */ -case class BroadcastDistribution(mode: BroadcastMode) extends Distribution { - override def requiredNumPartitions: Option[Int] = Some(1) - - override def createPartitioning(numPartitions: Int): Partitioning = { - assert(numPartitions == 1, - "The default partitioning of BroadcastDistribution can only have 1 partition.") - BroadcastPartitioning(mode) - } -} +case class BroadcastDistribution(mode: BroadcastMode) extends Distribution /** - * Describes how an operator's output is split across partitions. It has 2 major properties: - * 1. number of partitions. - * 2. if it can satisfy a given distribution. + * Describes how an operator's output is split across partitions. The `compatibleWith`, + * `guarantees`, and `satisfies` methods describe relationships between child partitionings, + * target partitionings, and [[Distribution]]s. These relations are described more precisely in + * their individual method docs, but at a high level: + * + * - `satisfies` is a relationship between partitionings and distributions. + * - `compatibleWith` is relationships between an operator's child output partitionings. + * - `guarantees` is a relationship between a child's existing output partitioning and a target + * output partitioning. + * + * Diagrammatically: + * + * +--------------+ + * | Distribution | + * +--------------+ + * ^ + * | + * satisfies + * | + * +--------------+ +--------------+ + * | Child | | Target | + * +----| Partitioning |----guarantees--->| Partitioning | + * | +--------------+ +--------------+ + * | ^ + * | | + * | compatibleWith + * | | + * +------------+ + * */ trait Partitioning { /** Returns the number of partitions that the data is split across */ @@ -180,33 +141,92 @@ trait Partitioning { * to satisfy the partitioning scheme mandated by the `required` [[Distribution]], * i.e. the current dataset does not need to be re-partitioned for the `required` * Distribution (it is possible that tuples within a partition need to be reorganized). + */ + def satisfies(required: Distribution): Boolean + + def compatibleWith(other: Partitioning): Boolean + + /** + * Returns true iff we can say that the partitioning scheme of this [[Partitioning]] guarantees + * the same partitioning scheme described by `other`. If a `A.guarantees(B)`, then repartitioning + * the child's output according to `B` will be unnecessary. `guarantees` is used as a performance + * optimization to allow the exchange planner to avoid redundant repartitionings. By default, + * a partitioning only guarantees partitionings that are equal to itself (i.e. the same number + * of partitions, same strategy (range or hash), etc). + * + * In order to enable more aggressive optimization, this strict equality check can be relaxed. + * For example, say that the planner needs to repartition all of an operator's children so that + * they satisfy the [[AllTuples]] distribution. One way to do this is to repartition all children + * to have the [[SinglePartition]] partitioning. If one of the operator's children already happens + * to be hash-partitioned with a single partition then we do not need to re-shuffle this child; + * this repartitioning can be avoided if a single-partition [[HashPartitioning]] `guarantees` + * [[SinglePartition]]. * - * By default a [[Partitioning]] can satisfy [[UnspecifiedDistribution]], and [[AllTuples]] if - * the [[Partitioning]] only have one partition. Implementations can overwrite this method with - * special logic. + * The SinglePartition example given above is not particularly interesting; guarantees' real + * value occurs for more advanced partitioning strategies. SPARK-7871 will introduce a notion + * of null-safe partitionings, under which partitionings can specify whether rows whosespark.sql.autoBroadcastJoinThreshold + * partitioning keys contain null values will be grouped into the same partition or whether they + * will have an unknown / random distribution. If a partitioning does not require nulls to be + * clustered then a partitioning which _does_ cluster nulls will guarantee the null clustered + * partitioning. The converse is not true, however: a partitioning which clusters nulls cannot + * be guaranteed by one which does not cluster them. Thus, in general `guarantees` is not a + * symmetric relation. + * + * Another way to think about `guarantees`: if `A.guarantees(B)`, then any partitioning of rows + * produced by `A` could have also been produced by `B`. */ - def satisfies(required: Distribution): Boolean = required match { + def guarantees(other: Partitioning): Boolean = this == other +} + +object Partitioning { + def allCompatible(partitionings: Seq[Partitioning]): Boolean = { + // Note: this assumes transitivity + partitionings.sliding(2).map { + case Seq(a) => true + case Seq(a, b) => + if (a.numPartitions != b.numPartitions) { + assert(!a.compatibleWith(b) && !b.compatibleWith(a)) + false + } else { + a.compatibleWith(b) && b.compatibleWith(a) + } + }.forall(_ == true) + } +} + +case class UnknownPartitioning(numPartitions: Int) extends Partitioning { + override def satisfies(required: Distribution): Boolean = required match { case UnspecifiedDistribution => true - case AllTuples => numPartitions == 1 case _ => false } -} -case class UnknownPartitioning(numPartitions: Int) extends Partitioning + override def compatibleWith(other: Partitioning): Boolean = false + + override def guarantees(other: Partitioning): Boolean = false +} /** * Represents a partitioning where rows are distributed evenly across output partitions * by starting from a random target partition number and distributing rows in a round-robin * fashion. This partitioning is used when implementing the DataFrame.repartition() operator. */ -case class RoundRobinPartitioning(numPartitions: Int) extends Partitioning +case class RoundRobinPartitioning(numPartitions: Int) extends Partitioning { + override def satisfies(required: Distribution): Boolean = required match { + case UnspecifiedDistribution => true + case _ => false + } + + override def compatibleWith(other: Partitioning): Boolean = false + + override def guarantees(other: Partitioning): Boolean = false +} case object SinglePartition extends Partitioning { val numPartitions = 1 override def satisfies(required: Distribution): Boolean = required match { case _: BroadcastDistribution => false - case ClusteredDistribution(_, Some(requiredNumPartitions)) => requiredNumPartitions == 1 + case ClusteredDistribution(_, desiredPartitions) => desiredPartitions.forall(_ == 1) case _ => true } @@ -214,7 +234,6 @@ case object SinglePartition extends Partitioning { override def guarantees(other: Partitioning): Boolean = other.numPartitions == 1 } - /** * Represents a partitioning where rows are split up across partitions based on the hash * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be @@ -225,7 +244,6 @@ case class OrderlessHashPartitioning(expressions: Seq[Expression], aliases: Seq[Seq[Attribute]], numPartitions: Int, numBuckets: Int, tableBuckets: Int) extends Expression with Partitioning with Unevaluable { - override def children: Seq[Expression] = expressions override def nullable: Boolean = false override def dataType: DataType = IntegerType @@ -240,13 +258,13 @@ case class OrderlessHashPartitioning(expressions: Seq[Expression], override def satisfies(required: Distribution): Boolean = required match { case UnspecifiedDistribution => true - case ClusteredDistribution(requiredClustering) => { + case ClusteredDistribution(requiredClustering, _) => matchExpressions(requiredClustering) - } case _ => false } private def anyOrderEquals(other: HashPartitioning) : Boolean = { + other.numBuckets == this.numBuckets && other.numPartitions == this.numPartitions && matchExpressions(other.expressions) } @@ -257,20 +275,19 @@ case class OrderlessHashPartitioning(expressions: Seq[Expression], } override def guarantees(other: Partitioning): Boolean = other match { - case o: HashPartitioning => anyOrderEquals(o) + case p: HashPartitioning => anyOrderEquals(p) case _ => false } - + } /** * Represents a partitioning where rows are split up across partitions based on the hash * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be - * in the same partition. Moreover while evaluating expressions if they are given in different order - * than this partitioning then also it is considered equal. + * in the same partition. */ case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int) - extends Expression with Partitioning with Unevaluable { + extends Expression with Partitioning with Unevaluable { private[sql] var numBuckets: Int = 0 @@ -278,64 +295,26 @@ case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int) override def nullable: Boolean = false override def dataType: DataType = IntegerType - private def matchExpressions(otherExpression: Seq[Expression]): Boolean = { - expressions.length == otherExpression.length && expressions.forall(a => - otherExpression.exists(e => e.semanticEquals(a))) - } - override def satisfies(required: Distribution): Boolean = required match { case UnspecifiedDistribution => true - case ClusteredDistribution(requiredClustering) => - matchExpressions(requiredClustering) + case ClusteredDistribution(requiredClustering, desiredPartitions) => + expressions.forall(x => requiredClustering.exists(_.semanticEquals(x))) && + desiredPartitions.forall(_ == numPartitions) // if desiredPartitions = None, returns true case _ => false } - private def anyOrderEquals(other: HashPartitioning) : Boolean = { - other.numPartitions == this.numPartitions && - matchExpressions(other.expressions) - } - override def compatibleWith(other: Partitioning): Boolean = other match { case o: HashPartitioning => - this.numBuckets == o.numBuckets && anyOrderEquals(o) + this.numBuckets == o.numBuckets && this.semanticEquals(o) case _ => false } override def guarantees(other: Partitioning): Boolean = other match { case o: HashPartitioning => - this.numBuckets == o.numBuckets && this.anyOrderEquals(o) + this.numBuckets == o.numBuckets && this.semanticEquals(o) case _ => false } -} - -/** - * Represents a partitioning where rows are split up across partitions based on the hash - * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be - * in the same partition. - */ -case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int) - extends Expression with Partitioning with Unevaluable { - - override def children: Seq[Expression] = expressions - override def nullable: Boolean = false - override def dataType: DataType = IntegerType - - override def satisfies(required: Distribution): Boolean = { - super.satisfies(required) || { - required match { - case h: HashClusteredDistribution => - expressions.length == h.expressions.length && expressions.zip(h.expressions).forall { - case (l, r) => l.semanticEquals(r) - } - case ClusteredDistribution(requiredClustering, requiredNumPartitions) => - expressions.forall(x => requiredClustering.exists(_.semanticEquals(x))) && - (requiredNumPartitions.isEmpty || requiredNumPartitions.get == numPartitions) - case _ => false - } - } - } - /** * Returns an expression that will produce a valid partition ID(i.e. non-negative and is less * than numPartitions) based on hashing expressions. @@ -372,18 +351,25 @@ case class RangePartitioning(ordering: Seq[SortOrder], numPartitions: Int) override def nullable: Boolean = false override def dataType: DataType = IntegerType - override def satisfies(required: Distribution): Boolean = { - super.satisfies(required) || { - required match { - case OrderedDistribution(requiredOrdering) => - val minSize = Seq(requiredOrdering.size, ordering.size).min - requiredOrdering.take(minSize) == ordering.take(minSize) - case ClusteredDistribution(requiredClustering, requiredNumPartitions) => - ordering.map(_.child).forall(x => requiredClustering.exists(_.semanticEquals(x))) && - (requiredNumPartitions.isEmpty || requiredNumPartitions.get == numPartitions) - case _ => false - } - } + override def satisfies(required: Distribution): Boolean = required match { + case UnspecifiedDistribution => true + case OrderedDistribution(requiredOrdering) => + val minSize = Seq(requiredOrdering.size, ordering.size).min + requiredOrdering.take(minSize) == ordering.take(minSize) + case ClusteredDistribution(requiredClustering, desiredPartitions) => + ordering.map(_.child).forall(x => requiredClustering.exists(_.semanticEquals(x))) && + desiredPartitions.forall(_ == numPartitions) // if desiredPartitions = None, returns true + case _ => false + } + + override def compatibleWith(other: Partitioning): Boolean = other match { + case o: RangePartitioning => this.semanticEquals(o) + case _ => false + } + + override def guarantees(other: Partitioning): Boolean = other match { + case o: RangePartitioning => this.semanticEquals(o) + case _ => false } } @@ -424,6 +410,20 @@ case class PartitioningCollection(partitionings: Seq[Partitioning]) override def satisfies(required: Distribution): Boolean = partitionings.exists(_.satisfies(required)) + /** + * Returns true if any `partitioning` of this collection is compatible with + * the given [[Partitioning]]. + */ + override def compatibleWith(other: Partitioning): Boolean = + partitionings.exists(_.compatibleWith(other)) + + /** + * Returns true if any `partitioning` of this collection guarantees + * the given [[Partitioning]]. + */ + override def guarantees(other: Partitioning): Boolean = + partitionings.exists(_.guarantees(other)) + override def toString: String = { partitionings.map(_.toString).mkString("(", " or ", ")") } @@ -440,4 +440,9 @@ case class BroadcastPartitioning(mode: BroadcastMode) extends Partitioning { case BroadcastDistribution(m) if m == mode => true case _ => false } + + override def compatibleWith(other: Partitioning): Boolean = other match { + case BroadcastPartitioning(m) if m == mode => true + case _ => false + } } diff --git a/sql/core/build.gradle b/sql/core/build.gradle index 061d935b18018..3307424655816 100644 --- a/sql/core/build.gradle +++ b/sql/core/build.gradle @@ -29,6 +29,10 @@ dependencies { compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version + compile('org.apache.orc:orc-core:1.4.1:nohive') + compile('org.apache.orc:orc-mapreduce:1.4.1:nohive') + +// compile group: 'org.apache.orc', name: 'orc-mapreduce', version: '1.4.1' testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion, configuration: 'testOutput') diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index 036e90f7d1aa1..59f448eb166b5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -17,13 +17,8 @@ package org.apache.spark.sql -import java.beans.{BeanInfo, Introspector, PropertyDescriptor} import java.util.Properties -import scala.collection.immutable -import scala.reflect.runtime.universe.TypeTag - -import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.annotation.{DeveloperApi, Experimental, InterfaceStability} import org.apache.spark.api.java.{JavaRDD, JavaSparkContext} import org.apache.spark.internal.Logging @@ -32,11 +27,15 @@ import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.command.ShowTablesCommand -import org.apache.spark.sql.internal.{SessionState, SharedState, SQLConf} +import org.apache.spark.sql.internal.{SQLConf, SessionState, SharedState} import org.apache.spark.sql.sources.BaseRelation import org.apache.spark.sql.streaming.{DataStreamReader, StreamingQueryManager} import org.apache.spark.sql.types._ import org.apache.spark.sql.util.ExecutionListenerManager +import org.apache.spark.{SparkConf, SparkContext} + +import scala.collection.immutable +import scala.reflect.runtime.universe.TypeTag /** * The entry point for working with structured data (rows and columns) in Spark 1.x. @@ -1096,59 +1095,21 @@ object SQLContext { * method for internal use. */ private[sql] def beansToRows( -<<<<<<< HEAD - data: Iterator[_], - beanClass: Class[_], - attrs: Seq[AttributeReference]): Iterator[InternalRow] = { + data: Iterator[_], + beanClass: Class[_], + attrs: Seq[AttributeReference]): Iterator[InternalRow] = { val extractors = JavaTypeInference.getJavaBeanReadableProperties(beanClass).map(_.getReadMethod) val methodsToConverts = extractors.zip(attrs).map { case (e, attr) => (e, CatalystTypeConverters.createToCatalystConverter(attr.dataType)) } data.map { element => -||||||| parent of 61e5899... Snap 2061 (#83) - data: Iterator[_], - beanInfo: BeanInfo, - attrs: Seq[AttributeReference]): Iterator[InternalRow] = { - val extractors = - beanInfo.getPropertyDescriptors.filterNot(_.getName == "class").map(_.getReadMethod) - val methodsToConverts = extractors.zip(attrs).map { case (e, attr) => - (e, CatalystTypeConverters.createToCatalystConverter(attr.dataType)) - } - data.map{ element => -======= - data: Iterator[_], - beanInfo: BeanInfo, - attrs: Seq[AttributeReference]): Iterator[InternalRow] = { - val converters = getExtractors(beanInfo, attrs) - data.map{ element => ->>>>>>> 61e5899... Snap 2061 (#83) new GenericInternalRow( - converters.map { case (e, convert) => convert(e.getReadMethod.invoke(element)) } + methodsToConverts.map { case (e, convert) => convert(e.invoke(element)) } ): InternalRow } } - def getExtractors( - beanInfo: BeanInfo, - attrs: Seq[AttributeReference]): Array[(PropertyDescriptor, Any => Any)] = { - val methodsToConverts = beanInfo.getPropertyDescriptors. - filterNot(_.getName == "class").zip(attrs) - methodsToConverts.map { case (desc, attr) => - attr.dataType match { - case struct: StructType => - val extractors = getExtractors(Introspector.getBeanInfo(desc.getPropertyType), - struct.toAttributes) - (desc, (x: Any) => { - val arr = Array.tabulate[Any](struct.length)(i => - extractors(i)._2(extractors(i)._1.getReadMethod.invoke(x))) - new GenericInternalRow(arr) - }) - case _ => (desc, CatalystTypeConverters.createToCatalystConverter(attr.dataType)) - } - } - } - /** * Extract `spark.sql.*` properties from the conf and return them as a [[Properties]]. */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala index c61c770b2fb42..4514c37f5832c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala @@ -21,17 +21,8 @@ package org.apache.spark.sql.execution import java.util.Locale import java.util.function.Supplier -import scala.collection.mutable - -import org.apache.spark.broadcast -import org.apache.spark.rdd.RDD import com.esotericsoftware.kryo.io.{Input, Output} -import scala.util.control.Exception.catching - import com.esotericsoftware.kryo.{Kryo, KryoSerializable} -import com.esotericsoftware.kryo.io.{Input, Output} - -import org.apache.spark.{broadcast, Partition, SparkContext, TaskContext} import org.apache.spark.rdd.{RDD, ZippedPartitionsBaseRDD, ZippedPartitionsPartition} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ @@ -45,6 +36,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.storage.StorageLevel import org.apache.spark.util.Utils +import org.apache.spark.{Partition, SparkContext, TaskContext, broadcast} + +import scala.collection.mutable /** * An interface for those physical operators that support codegen. @@ -752,7 +746,7 @@ case class WholeStageCodegenRDD(@transient sc: SparkContext, var source: CodeAnd def computeInternal(split: Partition, context: TaskContext): Iterator[InternalRow] = { - val clazz = CodeGenerator.compile(source) + val (clazz, _) = CodeGenerator.compile(source) val buffer = clazz.generate(references).asInstanceOf[BufferedRowIterator] if (rdds.length == 1) { buffer.init(split.index, Array(rdds.head.iterator(split, context) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala index f138160a61f5d..ce3c68810f3b6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala @@ -755,9 +755,6 @@ case class HashAggregateExec( val hashExpr = Murmur3Hash(groupingExpressions, 42) val hashEval = BindReferences.bindReference(hashExpr, child.output).genCode(ctx) - val inputAttr = aggregateBufferAttributes ++ child.output - ctx.currentVars = new Array[ExprCode](aggregateBufferAttributes.length) ++ input - val (checkFallbackForGeneratedHashMap, checkFallbackForBytesToBytesMap, resetCounter, incCounter) = if (testFallbackStartsAt.isDefined) { val countTerm = ctx.addMutableState(ctx.JAVA_INT, "fallbackCounter") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourcePartitioning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourcePartitioning.scala index 017a6737161a6..6a31b1c8ed4a8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourcePartitioning.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourcePartitioning.scala @@ -31,7 +31,7 @@ class DataSourcePartitioning( override val numPartitions: Int = partitioning.numPartitions() override def satisfies(required: physical.Distribution): Boolean = { - super.satisfies(required) || { + satisfies(required) || { required match { case d: physical.ClusteredDistribution if isCandidate(d.clustering) => val attrs = d.clustering.map(_.asInstanceOf[Attribute]) @@ -53,4 +53,6 @@ class DataSourcePartitioning( case _ => false } } + + override def compatibleWith(other: physical.Partitioning): Boolean = false } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala index a270b13810a45..20146cfad4610 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala @@ -46,6 +46,23 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { if (minNumPostShufflePartitions > 0) Some(minNumPostShufflePartitions) else None } + /** + * Given a required distribution, returns a partitioning that satisfies that distribution. + * @param requiredDistribution The distribution that is required by the operator + * @param numPartitions Used when the distribution doesn't require a specific number of partitions + */ + private def createPartitioning( + requiredDistribution: Distribution, + numPartitions: Int, numBuckets: Int = 0): Partitioning = { + requiredDistribution match { + case AllTuples => SinglePartition + case ClusteredDistribution(clustering, desiredPartitions) => + HashPartitioning(clustering, desiredPartitions.getOrElse(numPartitions), numBuckets) + case OrderedDistribution(ordering) => RangePartitioning(ordering, numPartitions) + case dist => sys.error(s"Do not know how to satisfy distribution $dist") + } + } + /** * Adds [[ExchangeCoordinator]] to [[ShuffleExchangeExec]]s if adaptive query execution is enabled * and partitioning schemes of these [[ShuffleExchangeExec]]s support [[ExchangeCoordinator]]. @@ -71,9 +88,8 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { // shuffle data when we have more than one children because data generated by // these children may not be partitioned in the same way. // Please see the comment in withCoordinator for more details. - val supportsDistribution = requiredChildDistributions.forall { dist => - dist.isInstanceOf[ClusteredDistribution] || dist.isInstanceOf[HashClusteredDistribution] - } + val supportsDistribution = + requiredChildDistributions.forall(_.isInstanceOf[ClusteredDistribution]) children.length > 1 && supportsDistribution } @@ -126,7 +142,8 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { // // It will be great to introduce a new Partitioning to represent the post-shuffle // partitions when one post-shuffle partition includes multiple pre-shuffle partitions. - val targetPartitioning = distribution.createPartitioning(defaultNumPreShufflePartitions) + val targetPartitioning = + createPartitioning(distribution, defaultNumPreShufflePartitions) assert(targetPartitioning.isInstanceOf[HashPartitioning]) ShuffleExchangeExec(targetPartitioning, child, Some(coordinator)) } @@ -154,31 +171,37 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { case (child, BroadcastDistribution(mode)) => (BroadcastExchangeExec(mode, child), false) case (child, distribution) => - (ShuffleExchange(createPartitioning(distribution, + (ShuffleExchangeExec(createPartitioning(distribution, defaultNumPreShufflePartitions), child), true) } children = newChildren.map(_._1) - // Get the indexes of children which have specified distribution requirements and need to have - // same number of partitions. - val childrenIndexes = requiredChildDistributions.zipWithIndex.filter { - case (UnspecifiedDistribution, _) => false - case (_: BroadcastDistribution, _) => false + // If the operator has multiple children and specifies child output distributions (e.g. join), + // then the children's output partitionings must be compatible: + def requireCompatiblePartitioning(distribution: Distribution): Boolean = distribution match { + case UnspecifiedDistribution => false + case BroadcastDistribution(_) => false case _ => true - }.map(_._2) - - val childrenNumPartitions = - childrenIndexes.map(children(_).outputPartitioning.numPartitions).toSet + } + if (children.length > 1 + && requiredChildDistributions.exists(requireCompatiblePartitioning) + && !Partitioning.allCompatible(children.map(_.outputPartitioning))) { - if (childrenNumPartitions.size > 1) { - // Get the number of partitions which is explicitly required by the distributions. - val requiredNumPartitions = { - val numPartitionsSet = childrenIndexes.flatMap { - index => requiredChildDistributions(index).requiredNumPartitions - }.toSet - assert(numPartitionsSet.size <= 1, - s"$operator have incompatible requirements of the number of partitions for its children") - numPartitionsSet.headOption + // First check if the existing partitions of the children all match. This means they are + // partitioned by the same partitioning into the same number of partitions. In that case, + // don't try to make them match `defaultPartitions`, just use the existing partitioning. + val maxChildrenNumPartitions = math.abs(newChildren.map { + case (child, false) => child.outputPartitioning.numPartitions + case (child, true) => -child.outputPartitioning.numPartitions + }.max) + val numBuckets = children.map(_.outputPartitioning match { + case p: OrderlessHashPartitioning => p.numBuckets + case _ => 0 + }).max + val useExistingPartitioning = children.zip(requiredChildDistributions).forall { + case (child, distribution) => + child.outputPartitioning.guarantees( + createPartitioning(distribution, maxChildrenNumPartitions, numBuckets)) } children = if (useExistingPartitioning) { @@ -210,12 +233,11 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { child match { // If child is an exchange, we replace it with // a new one having targetPartitioning. - case ShuffleExchange(_, c, _) => ShuffleExchange(targetPartitioning, c) - case _ => ShuffleExchange(targetPartitioning, child) + case ShuffleExchangeExec(_, c, _) => ShuffleExchangeExec(targetPartitioning, c) + case _ => ShuffleExchangeExec(targetPartitioning, child) } } - - case ((child, _), _) => child + } } } @@ -313,10 +335,10 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { } def apply(plan: SparkPlan): SparkPlan = plan.transformUp { - // TODO: remove this after we create a physical operator for `RepartitionByExpression`. - case operator @ ShuffleExchangeExec(upper: HashPartitioning, child, _) => - child.outputPartitioning match { - case lower: HashPartitioning if upper.semanticEquals(lower) => child + case operator @ ShuffleExchangeExec(partitioning, child, _) => + child.children match { + case ShuffleExchangeExec(childPartitioning, baseChild, _)::Nil => + if (childPartitioning.guarantees(partitioning)) child else operator case _ => operator } case operator: SparkPlan => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala index 897a4dae39f32..66e8031bb5191 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala @@ -46,7 +46,7 @@ case class ShuffledHashJoinExec( "avgHashProbe" -> SQLMetrics.createAverageMetric(sparkContext, "avg hash probe")) override def requiredChildDistribution: Seq[Distribution] = - HashClusteredDistribution(leftKeys) :: HashClusteredDistribution(rightKeys) :: Nil + ClusteredDistribution(leftKeys) :: ClusteredDistribution(rightKeys) :: Nil private def buildHashedRelation(iter: Iterator[InternalRow]): HashedRelation = { val buildDataSize = longMetric("buildDataSize") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala index 2de2f30eb05d3..94405410cce90 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala @@ -78,7 +78,7 @@ case class SortMergeJoinExec( } override def requiredChildDistribution: Seq[Distribution] = - HashClusteredDistribution(leftKeys) :: HashClusteredDistribution(rightKeys) :: Nil + ClusteredDistribution(leftKeys) :: ClusteredDistribution(rightKeys) :: Nil override def outputOrdering: Seq[SortOrder] = joinType match { // For inner join, orders of both sides keys should be kept. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala index 6713464f336b7..73c9e2c5cf183 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala @@ -34,7 +34,7 @@ import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, AccumulatorV2Kr * the executor side are automatically propagated and shown in the SQL UI through metrics. Updates * on the driver side must be explicitly posted using [[SQLMetrics.postDriverMetricUpdates()]]. */ -final class SQLMetric(val metricType: String, initValue: Long = 0L) +final class SQLMetric(var metricType: String, initValue: Long = 0L) extends AccumulatorV2Kryo[Long, Long] with KryoSerializable { // This is a workaround for SPARK-11013. // We may use -1 as initial value of the accumulator, if the accumulator is valid, we will diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala index a58401130a14e..f92891c210952 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala @@ -457,7 +457,7 @@ case class CoGroupExec( right: SparkPlan) extends BinaryExecNode with ObjectProducerExec { override def requiredChildDistribution: Seq[Distribution] = - HashClusteredDistribution(leftGroup) :: HashClusteredDistribution(rightGroup) :: Nil + ClusteredDistribution(leftGroup) :: ClusteredDistribution(rightGroup) :: Nil override def requiredChildOrdering: Seq[Seq[SortOrder]] = leftGroup.map(SortOrder(_, Ascending)) :: rightGroup.map(SortOrder(_, Ascending)) :: Nil diff --git a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala index e51aad021fcbf..beade9ad3d533 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala @@ -27,6 +27,7 @@ import org.apache.spark.util.Utils abstract class BenchmarkQueryTest extends QueryTest with SharedSQLContext with BeforeAndAfterAll { + // When Utils.isTesting is true, the RuleExecutor will issue an exception when hitting // the max iteration of analyzer/optimizer batches. assert(Utils.isTesting, "spark.testing is not set to true") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala index 3e828449ff0be..a1799829932b8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala @@ -141,46 +141,5 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext { sqlContext.dropTempTable("tables") } } -} -class Person(var id: Int, var name: String, var longField: Long, var shortField: Short, - var byteField: Byte, var doubleField: Double, var floatField: Float, - var booleanField: Boolean, var binaryField: Array[Byte], - var datee: Date, var timeeStamp: Timestamp, - var address: Address ) extends java.io.Serializable{ - def this() = this(0, null, 0, 0, 0, 0d, 0f, false, null, null, null, null) - def getName: String = name - def getId: Int = id - def getLongField: Long = longField - def getShortField: Short = shortField - def getByteField: Byte = byteField - def getDoubleField: Double = doubleField - def getFloatField: Float = floatField - def getBooleanField: Boolean = booleanField - def getBinaryField: Array[Byte] = binaryField - def getDatee: Date = datee - def getTimeeStamp: Timestamp = timeeStamp - def getAddress: Address = address - - def setName(name: String): Unit = {this.name = name} - def setId(id: Int): Unit = {this.id = id} - def setLongField(longField: Long): Unit = {this.longField = longField} - def setShortField(shortField: Short): Unit = {this.shortField = shortField} - def setByteField(byteField: Byte): Unit = {this.byteField = byteField} - def setDoubleField(doubleField: Double): Unit = {this.doubleField = doubleField} - def setFloatField(floatField: Float): Unit = {this.floatField = floatField} - def setBooleanField(booleanField: Boolean): Unit = {this.booleanField = booleanField} - def setBinaryField(binaryField: Array[Byte]): Unit = {this.binaryField = binaryField} - def setDatee(datee: Date): Unit = {this.datee = datee} - def setTimeeStamp(ts: Timestamp): Unit = {this.timeeStamp = ts} - def setAddress(address: Address): Unit = {this.address = address} -} - - -class Address(var street: String, var zip: Int) extends java.io.Serializable { - def this() = this(null, -1) - def getStreet: String = this.street - def getZip: Int = this.zip - def setStreet(street: String): Unit = {this.street = street} - def setZip(zip: Int): Unit = {this.zip = zip} -} \ No newline at end of file +} diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala index 0f2b35e9afbc0..4eec3df283dc9 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala @@ -303,13 +303,13 @@ private[spark] object HiveUtils extends Logging { if (builtinHiveVersion != hiveMetastoreVersion) { throw new IllegalArgumentException( "Builtin jars can only be used when hive execution version == hive metastore version. " + - s"Execution: $hiveExecutionVersion != Metastore: $hiveMetastoreVersion. " + + s"Execution: $builtinHiveVersion != Metastore: $hiveMetastoreVersion. " + "Specify a vaild path to the correct hive jars using $HIVE_METASTORE_JARS " + - s"or change ${HIVE_METASTORE_VERSION.key} to $hiveExecutionVersion.") + s"or change ${HIVE_METASTORE_VERSION.key} to $builtinHiveVersion.") } // We recursively find all jars in the class loader chain, - // starting from the given classLoader. + // starting from the given classLoader.SnappyCoarseGrainedExecutorBackend.scala def allJars(classLoader: ClassLoader): Array[URL] = classLoader match { case null => Array.empty[URL] case childFirst: ChildFirstURLClassLoader => diff --git a/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala index 64cd1bd088001..55465334c79db 100644 --- a/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala +++ b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala @@ -1,56 +1,56 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.scheduler.cluster - -import org.apache.spark.{SparkContext, SparkException} -import org.apache.spark.scheduler.{ExternalClusterManager, SchedulerBackend, TaskScheduler, TaskSchedulerImpl} - -/** - * Cluster Manager for creation of Yarn scheduler and backend - */ -private[spark] class YarnClusterManager extends ExternalClusterManager { - - override def canCreate(masterURL: String): Boolean = { - masterURL == "yarn" - } - - override def createTaskScheduler(sc: SparkContext, masterURL: String): TaskScheduler = { - sc.deployMode match { - case "cluster" => new YarnClusterScheduler(sc) - case "client" => new YarnScheduler(sc) - case _ => throw new SparkException(s"Unknown deploy mode '${sc.deployMode}' for Yarn") - } - } - - override def createSchedulerBackend(sc: SparkContext, - masterURL: String, - scheduler: TaskScheduler): SchedulerBackend = { - sc.deployMode match { - case "cluster" => - new YarnClusterSchedulerBackend(scheduler.asInstanceOf[TaskSchedulerImpl], sc) - case "client" => - new YarnClientSchedulerBackend(scheduler.asInstanceOf[TaskSchedulerImpl], sc) - case _ => - throw new SparkException(s"Unknown deploy mode '${sc.deployMode}' for Yarn") - } - } - - override def initialize(scheduler: TaskScheduler, backend: SchedulerBackend): Unit = { - scheduler.asInstanceOf[TaskSchedulerImpl].initialize(backend) - } -} +///* +// * Licensed to the Apache Software Foundation (ASF) under one or more +// * contributor license agreements. See the NOTICE file distributed with +// * this work for additional information regarding copyright ownership. +// * The ASF licenses this file to You under the Apache License, Version 2.0 +// * (the "License"); you may not use this file except in compliance with +// * the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package org.apache.spark.scheduler.cluster +// +//import org.apache.spark.{SparkContext, SparkException} +//import org.apache.spark.scheduler.{ExternalClusterManager, SchedulerBackend, TaskScheduler, TaskSchedulerImpl} +// +///** +// * Cluster Manager for creation of Yarn scheduler and backend +// */ +//private[spark] class YarnClusterManager extends ExternalClusterManager { +// +// override def canCreate(masterURL: String): Boolean = { +// masterURL == "yarn" +// } +// +// override def createTaskScheduler(sc: SparkContext, masterURL: String): TaskScheduler = { +// sc.deployMode match { +// case "cluster" => new YarnClusterScheduler(sc) +// case "client" => new YarnScheduler(sc) +// case _ => throw new SparkException(s"Unknown deploy mode '${sc.deployMode}' for Yarn") +// } +// } +// +// override def createSchedulerBackend(sc: SparkContext, +// masterURL: String, +// scheduler: TaskScheduler): SchedulerBackend = { +// sc.deployMode match { +// case "cluster" => +// new YarnClusterSchedulerBackend(scheduler.asInstanceOf[TaskSchedulerImpl], sc) +// case "client" => +// new YarnClientSchedulerBackend(scheduler.asInstanceOf[TaskSchedulerImpl], sc) +// case _ => +// throw new SparkException(s"Unknown deploy mode '${sc.deployMode}' for Yarn") +// } +// } +// +// override def initialize(scheduler: TaskScheduler, backend: SchedulerBackend): Unit = { +// scheduler.asInstanceOf[TaskSchedulerImpl].initialize(backend) +// } +//} From 51d65e45fa4affb04ccf83faffc65734aa9ec464 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Sun, 1 Apr 2018 22:01:22 -0700 Subject: [PATCH 128/142] Addressing precheckin issues --- assembly/build.gradle | 17 +++++++++++++---- build.gradle | 10 ++++++++++ sql/core/build.gradle | 6 ++---- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/assembly/build.gradle b/assembly/build.gradle index 2fb7421ce1b89..301819d47a3f2 100644 --- a/assembly/build.gradle +++ b/assembly/build.gradle @@ -21,20 +21,29 @@ dependencies { compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-sql_' + scalaBinaryVersion) - compile project(subprojectBase + 'snappy-spark-hive_' + scalaBinaryVersion) - compile project(subprojectBase + 'snappy-spark-hive-thriftserver_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-repl_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-streaming_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.8_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-sql-kafka-0.10_' + scalaBinaryVersion) - compile project(subprojectBase + 'snappy-spark-yarn_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-mllib_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) + if (rootProject.hasProperty('yarn')) { + compile project(subprojectBase + 'snappy-spark-yarn_' + scalaBinaryVersion) + } if (rootProject.hasProperty('mesos')) { compile project(subprojectBase + 'snappy-spark-mesos_' + scalaBinaryVersion) } - if (rootProject.hasProperty('ganglia')) { + if (rootProject.hasProperty('kubernetes')) { + compile project(subprojectBase + 'snappy-spark-kubernetes_' + scalaBinaryVersion) + } + if (rootProject.hasProperty('hive')) { + compile project(subprojectBase + 'snappy-spark-hive_' + scalaBinaryVersion) + } + if (rootProject.hasProperty('hive-thriftserver')) { + compile project(subprojectBase + 'snappy-spark-hive-thriftserver_' + scalaBinaryVersion) + } + if (rootProject.hasProperty('spark-ganglia-lgpl')) { compile project(subprojectBase + 'snappy-spark-ganglia-lgpl_' + scalaBinaryVersion) } } diff --git a/build.gradle b/build.gradle index 99c7f280bf770..0b7c77c842637 100644 --- a/build.gradle +++ b/build.gradle @@ -223,6 +223,16 @@ subprojects { // compile group: 'org.apache.arrow', name: 'arrow-vector', version: '0.8.0' compile group: 'com.github.luben', name: 'zstd-jni', version: '1.3.2-2' + compile('org.apache.orc:orc-core:1.4.1:nohive') { + exclude(group: 'org.apache.hadoop', module: 'hadoop-common') + exclude(group: 'org.apache.hive', module: 'hive-storage-api') + } + compile('org.apache.orc:orc-mapreduce:1.4.1:nohive') { + exclude(group: 'org.apache.hadoop', module: 'hadoop-common') + exclude(group: 'org.apache.orc', module: 'orc-core') + exclude(group: 'org.apache.hive', module: 'hive-storage-api') + } + testCompile "junit:junit:${junitVersion}" testCompile "org.scalatest:scalatest_${scalaBinaryVersion}:${scalatestVersion}" testCompile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' diff --git a/sql/core/build.gradle b/sql/core/build.gradle index 3307424655816..589fa98b382d8 100644 --- a/sql/core/build.gradle +++ b/sql/core/build.gradle @@ -23,16 +23,14 @@ dependencies { compile project(subprojectBase + 'snappy-spark-sketch_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + compile('org.apache.orc:orc-core:1.4.1:nohive') + compile('org.apache.orc:orc-mapreduce:1.4.1:nohive') compile group: 'com.univocity', name: 'univocity-parsers', version: '2.2.3' compile group: 'org.apache.parquet', name: 'parquet-column', version: parquetVersion compile group: 'org.apache.parquet', name: 'parquet-hadoop', version: parquetVersion compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonBindVersion compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version - compile('org.apache.orc:orc-core:1.4.1:nohive') - compile('org.apache.orc:orc-mapreduce:1.4.1:nohive') - -// compile group: 'org.apache.orc', name: 'orc-mapreduce', version: '1.4.1' testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') testCompile project(path: subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion, configuration: 'testOutput') From e6e3859775fda215dc3f15f8e53c316199fe0a31 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Thu, 5 Apr 2018 17:18:03 -0700 Subject: [PATCH 129/142] Addressing precheckin failures --- common/network-common/build.gradle | 6 - core/build.gradle | 12 +- .../apache/spark/scheduler/DAGScheduler.scala | 219 ++++++++++++++---- .../apache/spark/scheduler/ResultTask.scala | 8 +- .../spark/scheduler/ShuffleMapTask.scala | 8 +- .../org/apache/spark/scheduler/Task.scala | 12 +- .../apache/spark/executor/ExecutorSuite.scala | 4 +- .../org/apache/spark/scheduler/FakeTask.scala | 2 +- .../scheduler/NotSerializableFakeTask.scala | 2 +- .../spark/scheduler/TaskContextSuite.scala | 8 +- .../spark/scheduler/TaskSetManagerSuite.scala | 2 +- .../orc/OrcColumnarBatchReader.java | 4 +- .../spark/sql/hive/orc/OrcFileFormat.scala | 2 +- .../spark/sql/hive/orc/OrcFilters.scala | 18 +- 14 files changed, 219 insertions(+), 88 deletions(-) diff --git a/common/network-common/build.gradle b/common/network-common/build.gradle index 2d0c311299388..fbc7ccbd73e99 100644 --- a/common/network-common/build.gradle +++ b/common/network-common/build.gradle @@ -28,13 +28,7 @@ dependencies { compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion compile group: 'org.apache.commons', name: 'commons-lang3', version: commonsLang3Version compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: metricsVersion - compile group: 'org.apache.arrow', name: 'arrow-vector', version: '0.8.0' - -// compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: metricsVersion -// compile group: 'io.dropwizard.metrics', name: 'metrics-json', version: metricsVersion -// compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: metricsVersion -// compile group: 'io.dropwizard.metrics', name: 'metrics-ganglia', version: metricsVersion compile group: 'org.apache.commons', name: 'commons-crypto', version: '1.0.0' testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') diff --git a/core/build.gradle b/core/build.gradle index 404097eea8be0..7f19d341c4870 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -157,17 +157,13 @@ dependencies { exclude(group: 'net.razorvine', module: 'serpent') } compile group: 'net.sf.py4j', name: 'py4j', version: '0.10.4' - compile group: 'com.github.luben', name: 'zstd-jni', version: '1.3.2-2' + compile (group: 'com.github.luben', name: 'zstd-jni', version: '1.3.2-2'){ + exclude(group: 'org.slf4j', module: 'slf4j-api') + exclude(group: 'org.slf4j', module: 'slf4j-log4j12') + } -// compileOnly(group: 'org.spark-project.hive', name: 'hive-exec', version:'1.2.1.spark2') -// compileOnly(group: 'org.spark-project.hive', name: 'hive-metastore', version: '1.2.1.spark2') -//// compileOnly(group: 'snappy-spark-hive', name: 'hive-exec', version:'1.2.1.spark2') -//// compileOnly(group: 'snappy-spark-hive', name: 'hive-metastore', version: '1.2.1.spark2') -// compileOnly(group: 'org.apache.thrift', name: 'libthrift', version: '0.9.3') -// compileOnly(group: 'org.apache.thrift', name: 'libfb303', version: '0.9.3') testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') - // compile group: 'org.apache.commons', name: 'commons-crypto', version: commonsCryptoVersion testCompile group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion, classifier: 'tests' testCompile "org.apache.derby:derby:${derbyVersion}" diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 6a950ca37fb0d..62bfa05addaf0 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -958,6 +958,173 @@ class DAGScheduler( } } +// /** Called when stage's parents are available and we can now do its task. */ +// private def submitMissingTasks(stage: Stage, jobId: Int) { +// logDebug("submitMissingTasks(" + stage + ")") +// +// // First figure out the indexes of partition ids to compute. +// val partitionsToCompute: Seq[Int] = stage.findMissingPartitions() +// +// // Use the scheduling pool, job group, description, etc. from an ActiveJob associated +// // with this Stage +// val properties = jobIdToActiveJob(jobId).properties +// +// runningStages += stage +// // SparkListenerStageSubmitted should be posted before testing whether tasks are +// // serializable. If tasks are not serializable, a SparkListenerStageCompleted event +// // will be posted, which should always come after a corresponding SparkListenerStageSubmitted +// // event. +// stage match { +// case s: ShuffleMapStage => +// outputCommitCoordinator.stageStart(stage = s.id, maxPartitionId = s.numPartitions - 1) +// case s: ResultStage => +// outputCommitCoordinator.stageStart( +// stage = s.id, maxPartitionId = s.rdd.partitions.length - 1) +// } +// val taskIdToLocations: Map[Int, Seq[TaskLocation]] = try { +// stage match { +// case s: ShuffleMapStage => +// partitionsToCompute.map { id => (id, getPreferredLocs(stage.rdd, id))}.toMap +// case s: ResultStage => +// partitionsToCompute.map { id => +// val p = s.partitions(id) +// (id, getPreferredLocs(stage.rdd, p)) +// }.toMap +// } +// } catch { +// case NonFatal(e) => +// stage.makeNewStageAttempt(partitionsToCompute.size) +// listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo, properties)) +// abortStage(stage, s"Task creation failed: $e\n${Utils.exceptionString(e)}", Some(e)) +// runningStages -= stage +// return +// } +// +// stage.makeNewStageAttempt(partitionsToCompute.size, taskIdToLocations.values.toSeq) +// +// // If there are tasks to execute, record the submission time of the stage. Otherwise, +// // post the even without the submission time, which indicates that this stage was +// // skipped. +// if (partitionsToCompute.nonEmpty) { +// stage.latestInfo.submissionTime = Some(clock.getTimeMillis()) +// } +// listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo, properties)) +// +// // TODO: Maybe we can keep the taskBinary in Stage to avoid serializing it multiple times. +// // Broadcasted binary for the task, used to dispatch tasks to executors. Note that we broadcast +// // the serialized copy of the RDD and for each task we will deserialize it, which means each +// // task gets a different copy of the RDD. This provides stronger isolation between tasks that +// // might modify state of objects referenced in their closures. This is necessary in Hadoop +// // where the JobConf/Configuration object is not thread-safe. +// var taskBinary: Broadcast[Array[Byte]] = null +// var taskData: TaskData = TaskData.EMPTY +// var partitions: Array[Partition] = null +// try { +// // For ShuffleMapTask, serialize and broadcast (rdd, shuffleDep). +// // For ResultTask, serialize and broadcast (rdd, func). +// var taskBinaryBytes: Array[Byte] = null +// // taskBinaryBytes and partitions are both effected by the checkpoint status. We need +// // this synchronization in case another concurrent job is checkpointing this RDD, so we get a +// // consistent view of both variables. +// RDDCheckpointData.synchronized { +// taskBinaryBytes = stage match { +// case stage: ShuffleMapStage => +// JavaUtils.bufferToArray( +// closureSerializer.serialize((stage.rdd, stage.shuffleDep): AnyRef)) +// case stage: ResultStage => +// JavaUtils.bufferToArray(closureSerializer.serialize((stage.rdd, stage.func): AnyRef)) +// } +// +// partitions = stage.rdd.partitions +// } +// +// // use direct byte shipping for small size or if number of partitions is small +// val taskBytesLen = taskBinaryBytes.length +// if (taskBytesLen <= DAGScheduler.TASK_INLINE_LIMIT || +// partitionsToCompute.length == 1 || +// (taskBytesLen < math.min(maxRpcMessageSize, DAGScheduler.TASK_INLINE_UPPER_LIMIT) && +// partitionsToCompute.length <= DAGScheduler.TASK_INLINE_PARTITION_LIMIT)) { +// if (stage.taskData.uncompressedLen > 0) { +// taskData = stage.taskData +// } else { +// // compress inline task data (broadcast compresses as per conf) +// taskData = new TaskData(env.createCompressionCodec.compress( +// taskBinaryBytes, taskBytesLen), taskBytesLen) +// stage.taskData = taskData +// } +// } else { +// taskBinary = sc.broadcast(taskBinaryBytes) +// } +// } catch { +// // In the case of a failure during serialization, abort the stage. +// case e: NotSerializableException => +// abortStage(stage, "Task not serializable: " + e.toString, Some(e)) +// runningStages -= stage +// +// // Abort execution +// return +// case NonFatal(e) => +// abortStage(stage, s"Task serialization failed: $e\n${Utils.exceptionString(e)}", Some(e)) +// runningStages -= stage +// return +// } +// +// val tasks: Seq[Task[_]] = try { +// val serializedTaskMetrics = closureSerializer.serialize(stage.latestInfo.taskMetrics).array() +// stage match { +// case stage: ShuffleMapStage => +// stage.pendingPartitions.clear() +// partitionsToCompute.map { id => +// val locs = taskIdToLocations(id) +// val part = partitions(id) +// stage.pendingPartitions += id +// new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, taskData, +// taskBinary, part, locs, properties, serializedTaskMetrics, Option(jobId), +// Option(sc.applicationId), sc.applicationAttemptId) +// } +// +// case stage: ResultStage => +// partitionsToCompute.map { id => +// val p: Int = stage.partitions(id) +// val part = stage.rdd.partitions(p) +// val locs = taskIdToLocations(id) +// new ResultTask(stage.id, stage.latestInfo.attemptId, taskData, +// taskBinary, part, locs, id, properties, serializedTaskMetrics, Option(jobId), +// Option(sc.applicationId), sc.applicationAttemptId) +// } +// } +// } catch { +// case NonFatal(e) => +// abortStage(stage, s"Task creation failed: $e\n${Utils.exceptionString(e)}", Some(e)) +// runningStages -= stage +// return +// } +// +// if (tasks.size > 0) { +// logInfo(s"Submitting ${tasks.size} missing tasks from $stage (${stage.rdd}) (first 15 " + +// s"tasks are for partitions ${tasks.take(15).map(_.partitionId)})") +// taskScheduler.submitTasks(new TaskSet( +// tasks.toArray, stage.id, stage.latestInfo.attemptId, jobId, properties)) +// } else { +// // Because we posted SparkListenerStageSubmitted earlier, we should mark +// // the stage as completed here in case there are no tasks to run +// markStageAsFinished(stage, None) +// +// val debugString = stage match { +// case stage: ShuffleMapStage => +// s"Stage ${stage} is actually done; " + +// s"(available: ${stage.isAvailable}," + +// s"available outputs: ${stage.numAvailableOutputs}," + +// s"partitions: ${stage.numPartitions})" +// case stage : ResultStage => +// s"Stage ${stage} is actually done; (partitions: ${stage.numPartitions})" +// } +// logDebug(debugString) +// +// submitWaitingChildStages(stage) +// } +// } + /** Called when stage's parents are available and we can now do its task. */ private def submitMissingTasks(stage: Stage, jobId: Int) { logDebug("submitMissingTasks(" + stage + ")") @@ -1016,45 +1183,19 @@ class DAGScheduler( // task gets a different copy of the RDD. This provides stronger isolation between tasks that // might modify state of objects referenced in their closures. This is necessary in Hadoop // where the JobConf/Configuration object is not thread-safe. - var taskBinary: Option[Broadcast[Array[Byte]]] = None - var taskData: TaskData = TaskData.EMPTY - var partitions: Array[Partition] = null + var taskBinary: Broadcast[Array[Byte]] = null try { // For ShuffleMapTask, serialize and broadcast (rdd, shuffleDep). // For ResultTask, serialize and broadcast (rdd, func). - var taskBinaryBytes: Array[Byte] = null - // taskBinaryBytes and partitions are both effected by the checkpoint status. We need - // this synchronization in case another concurrent job is checkpointing this RDD, so we get a - // consistent view of both variables. - RDDCheckpointData.synchronized { - taskBinaryBytes = stage match { - case stage: ShuffleMapStage => - JavaUtils.bufferToArray( - closureSerializer.serialize((stage.rdd, stage.shuffleDep): AnyRef)) - case stage: ResultStage => - JavaUtils.bufferToArray(closureSerializer.serialize((stage.rdd, stage.func): AnyRef)) - } - - partitions = stage.rdd.partitions + val taskBinaryBytes: Array[Byte] = stage match { + case stage: ShuffleMapStage => + JavaUtils.bufferToArray( + closureSerializer.serialize((stage.rdd, stage.shuffleDep): AnyRef)) + case stage: ResultStage => + JavaUtils.bufferToArray(closureSerializer.serialize((stage.rdd, stage.func): AnyRef)) } - // use direct byte shipping for small size or if number of partitions is small - val taskBytesLen = taskBinaryBytes.length - if (taskBytesLen <= DAGScheduler.TASK_INLINE_LIMIT || - partitionsToCompute.length == 1 || - (taskBytesLen < math.min(maxRpcMessageSize, DAGScheduler.TASK_INLINE_UPPER_LIMIT) && - partitionsToCompute.length <= DAGScheduler.TASK_INLINE_PARTITION_LIMIT)) { - if (stage.taskData.uncompressedLen > 0) { - taskData = stage.taskData - } else { - // compress inline task data (broadcast compresses as per conf) - taskData = new TaskData(env.createCompressionCodec.compress( - taskBinaryBytes, taskBytesLen), taskBytesLen) - stage.taskData = taskData - } - } else { - taskBinary = Some(sc.broadcast(taskBinaryBytes)) - } + taskBinary = sc.broadcast(taskBinaryBytes) } catch { // In the case of a failure during serialization, abort the stage. case e: NotSerializableException => @@ -1076,9 +1217,9 @@ class DAGScheduler( stage.pendingPartitions.clear() partitionsToCompute.map { id => val locs = taskIdToLocations(id) - val part = partitions(id) + val part = stage.rdd.partitions(id) stage.pendingPartitions += id - new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, taskData, + new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, _taskData = TaskData.EMPTY, taskBinary, part, locs, properties, serializedTaskMetrics, Option(jobId), Option(sc.applicationId), sc.applicationAttemptId) } @@ -1088,9 +1229,9 @@ class DAGScheduler( val p: Int = stage.partitions(id) val part = stage.rdd.partitions(p) val locs = taskIdToLocations(id) - new ResultTask(stage.id, stage.latestInfo.attemptId, taskData, - taskBinary, part, locs, id, properties, serializedTaskMetrics, Option(jobId), - Option(sc.applicationId), sc.applicationAttemptId) + new ResultTask(stage.id, stage.latestInfo.attemptId, _taskData = TaskData.EMPTY, + taskBinary, part, locs, id, properties, serializedTaskMetrics, + Option(jobId), Option(sc.applicationId), sc.applicationAttemptId) } } } catch { diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index 6709b7d5b2f15..d08fbd73ed9b0 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -38,7 +38,7 @@ import org.apache.spark.rdd.RDD * @param stageAttemptId attempt id of the stage this task belongs to * @param _taskData if serialized RDD and function are small, then it is compressed * and sent with its original decompressed size - * @param _taskBinary broadcasted version of the serialized RDD and the function to apply on each + * @param taskBinary broadcasted version of the serialized RDD and the function to apply on each * partition of the given RDD. Once deserialized, the type should be * (RDD[T], (TaskContext, Iterator[T]) => U). * @param partition partition of the RDD this task is associated with @@ -58,7 +58,7 @@ private[spark] class ResultTask[T, U]( stageId: Int, stageAttemptId: Int, _taskData: TaskData, - _taskBinary: Option[Broadcast[Array[Byte]]], + taskBinary: Broadcast[Array[Byte]], private var partition: Partition, locs: Seq[TaskLocation], private var _outputId: Int, @@ -68,7 +68,7 @@ private[spark] class ResultTask[T, U]( appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[U](stageId, stageAttemptId, partition.index, _taskData, - _taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) + taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) with Serializable with KryoSerializable { final def outputId: Int = _outputId @@ -86,7 +86,7 @@ private[spark] class ResultTask[T, U]( } else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, func) = ser.deserialize[(RDD[T], (TaskContext, Iterator[T]) => U)]( - ByteBuffer.wrap(getTaskBytes), Thread.currentThread.getContextClassLoader) + ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = math.max(System.nanoTime() - deserializeStartTime, 0L) _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { math.max(threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime, 0L) diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index 83215edaedfaf..67619b1e889eb 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -42,7 +42,7 @@ import org.apache.spark.shuffle.ShuffleWriter * @param stageAttemptId attempt id of the stage this task belongs to * @param _taskData if serialized RDD and function are small, then it is compressed * and sent with its original decompressed size - * @param _taskBinary broadcast version of the RDD and the ShuffleDependency. Once deserialized, + * @param taskBinary broadcast version of the RDD and the ShuffleDependency. Once deserialized, * the type should be (RDD[_], ShuffleDependency[_, _, _]). * @param partition partition of the RDD this task is associated with * @param locs preferred task execution locations for locality scheduling @@ -56,8 +56,8 @@ import org.apache.spark.shuffle.ShuffleWriter private[spark] class ShuffleMapTask( stageId: Int, stageAttemptId: Int, - _taskData: TaskData, - _taskBinary: Option[Broadcast[Array[Byte]]], + _taskData: TaskData = TaskData.EMPTY, + taskBinary: Broadcast[Array[Byte]], private var partition: Partition, @transient private var locs: Seq[TaskLocation], localProperties: Properties, @@ -66,7 +66,7 @@ private[spark] class ShuffleMapTask( appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[MapStatus](stageId, stageAttemptId, partition.index, _taskData, - _taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) + taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) with KryoSerializable with Logging { /** A constructor used only in test suites. This does not require passing in an RDD. */ diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index cf31304935a21..936c7ae434def 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -63,7 +63,7 @@ private[spark] abstract class Task[T]( private var _partitionId: Int, @transient private[spark] var taskData: TaskData = TaskData.EMPTY, // The default value is only used in tests. - protected var taskBinary: Option[Broadcast[Array[Byte]]] = None, + protected var taskBinary: Broadcast[Array[Byte]], @transient var localProperties: Properties = new Properties, // The default value is only used in tests. serializedTaskMetrics: Array[Byte] = @@ -93,7 +93,7 @@ private[spark] abstract class Task[T]( protected final def getTaskBytes: Array[Byte] = { val bytes = taskDataBytes - if ((bytes ne null) && bytes.length > 0) bytes else taskBinary.get.value + if ((bytes ne null) && bytes.length > 0) bytes else taskBinary.value } /** @@ -261,7 +261,7 @@ private[spark] abstract class Task[T]( output.writeBoolean(true) } else { output.writeBoolean(false) - kryo.writeClassAndObject(output, taskBinary.get) + kryo.writeClassAndObject(output, taskBinary) } output.writeString(appId) output.writeString(appAttemptId) @@ -278,10 +278,10 @@ private[spark] abstract class Task[T]( // actual bytes are shipped in TaskDescription taskData = TaskData.EMPTY if (input.readBoolean()) { - taskBinary = None + taskBinary = null } else { - taskBinary = Some(kryo.readClassAndObject(input) - .asInstanceOf[Broadcast[Array[Byte]]]) + taskBinary = kryo.readClassAndObject(input) + .asInstanceOf[Broadcast[Array[Byte]]] } _appId = Option(input.readString()) _appAttemptId = Option(input.readString()) diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala index 07942abf9e48e..6f83a59631e07 100644 --- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala +++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala @@ -144,7 +144,7 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - _taskBinary = Some(taskBinary), + taskBinary = taskBinary, partition = secondRDD.partitions(0), locs = Seq(), _outputId = 0, @@ -188,7 +188,7 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - _taskBinary = Some(taskBinary), + taskBinary = taskBinary, partition = secondRDD.partitions(0), locs = Seq(), _outputId = 0, diff --git a/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala b/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala index ddb04c94e1ed4..841eeff228b7b 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala @@ -71,7 +71,7 @@ object FakeTask { new ShuffleMapTask(stageId, stageAttemptId, TaskData.EMPTY, - None, + null, new Partition {override def index: Int = i}, prefLocs(i), new Properties, diff --git a/core/src/test/scala/org/apache/spark/scheduler/NotSerializableFakeTask.scala b/core/src/test/scala/org/apache/spark/scheduler/NotSerializableFakeTask.scala index 1af7b853261a8..6776148a5a671 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/NotSerializableFakeTask.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/NotSerializableFakeTask.scala @@ -25,7 +25,7 @@ import org.apache.spark.TaskContext * A Task implementation that fails to serialize. */ private[spark] class NotSerializableFakeTask(myId: Int, stageId: Int) - extends Task[Array[Byte]](stageId, 0, 0) { + extends Task[Array[Byte]](stageId, 0, 0, taskBinary = null) { override def runTask(context: TaskContext): Array[Byte] = Array.empty[Byte] override def preferredLocations: Seq[TaskLocation] = Seq[TaskLocation]() diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala index 1d96e7cf7bc4e..e63bba6d983d0 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala @@ -69,7 +69,7 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - _taskBinary = Some(taskBinary), + taskBinary = taskBinary, partition = rdd.partitions(0), locs = Seq(), _outputId = 0, @@ -98,7 +98,7 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - _taskBinary = Some(taskBinary), + taskBinary = taskBinary, partition = rdd.partitions(0), locs = Seq(), _outputId = 0, @@ -228,7 +228,7 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark // Create a dummy task. We won't end up running this; we just want to collect // accumulator updates from it. val taskMetrics = TaskMetrics.empty - val task = new Task[Int](0, 0, 0) { + val task = new Task[Int](0, 0, 0, taskBinary = null) { context = new TaskContextImpl(0, 0, 0, 0L, 0, new TaskMemoryManager(SparkEnv.get.memoryManager, 0L), new Properties, @@ -251,7 +251,7 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark // Create a dummy task. We won't end up running this; we just want to collect // accumulator updates from it. val taskMetrics = TaskMetrics.registered - val task = new Task[Int](0, 0, 0) { + val task = new Task[Int](0, 0, 0, taskBinary = null) { context = new TaskContextImpl(0, 0, 0, 0L, 0, new TaskMemoryManager(SparkEnv.get.memoryManager, 0L), new Properties, diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala index 5a0f9bdf680eb..20aa012b1f438 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala @@ -154,7 +154,7 @@ class FakeTaskScheduler(sc: SparkContext, liveExecutors: (String, String)* /* ex /** * A Task implementation that results in a large serialized task. */ -class LargeTask(stageId: Int) extends Task[Array[Byte]](stageId, 0, 0) { +class LargeTask(stageId: Int) extends Task[Array[Byte]](stageId, 0, 0, taskBinary = null) { val randomBuffer = new Array[Byte](TaskSetManager.TASK_SIZE_TO_WARN_KB * 1024) val random = new Random(0) diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReader.java index 5e7cad470e1d1..ff40c6e8a28bf 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReader.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReader.java @@ -148,7 +148,7 @@ public void initBatch( StructField[] requiredFields, StructType partitionSchema, InternalRow partitionValues) { - batch = orcSchema.createRowBatch(CAPACITY); + // batch = orcSchema.createRowBatch(CAPACITY); assert(!batch.selectedInUse); // `selectedInUse` should be initialized with `false`. this.requiredFields = requiredFields; @@ -222,7 +222,7 @@ public void initBatch( * by copying from ORC VectorizedRowBatch columns to Spark ColumnarBatch columns. */ private boolean nextBatch() throws IOException { - recordReader.nextBatch(batch); + // recordReader.nextBatch(batch); int batchSize = batch.size; if (batchSize == 0) { return false; diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala index 237ed9bc05988..6eec1e99747e6 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala @@ -124,7 +124,7 @@ class OrcFileFormat extends FileFormat with DataSourceRegister with Serializable if (sparkSession.sessionState.conf.orcFilterPushDown) { // Sets pushed predicates OrcFilters.createFilter(requiredSchema, filters.toArray).foreach { f => - hadoopConf.set(OrcFileFormat.SARG_PUSHDOWN, f.toKryo) + // hadoopConf.set(OrcFileFormat.SARG_PUSHDOWN, f.toKryo) hadoopConf.setBoolean(ConfVars.HIVEOPTINDEXFILTER.varname, true) } } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala index d9efd0cb457cd..7a90b67a07066 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala @@ -123,31 +123,31 @@ private[orc] object OrcFilters extends Logging { // wrapped by a "parent" predicate (`And`, `Or`, or `Not`). case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) => - Some(builder.startAnd().equals(attribute, value).end()) + None // Some(builder.startAnd().equals(attribute, value).end()) case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) => - Some(builder.startAnd().nullSafeEquals(attribute, value).end()) + None // Some(builder.startAnd().nullSafeEquals(attribute, value).end()) case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) => - Some(builder.startAnd().lessThan(attribute, value).end()) + None // Some(builder.startAnd().lessThan(attribute, value).end()) case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) => - Some(builder.startAnd().lessThanEquals(attribute, value).end()) + None // Some(builder.startAnd().lessThanEquals(attribute, value).end()) case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) => - Some(builder.startNot().lessThanEquals(attribute, value).end()) + None // Some(builder.startNot().lessThanEquals(attribute, value).end()) case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) => - Some(builder.startNot().lessThan(attribute, value).end()) + None // Some(builder.startNot().lessThan(attribute, value).end()) case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) => - Some(builder.startAnd().isNull(attribute).end()) + None // Some(builder.startAnd().isNull(attribute).end()) case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) => - Some(builder.startNot().isNull(attribute).end()) + None // Some(builder.startNot().isNull(attribute).end()) case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) => - Some(builder.startAnd().in(attribute, values.map(_.asInstanceOf[AnyRef]): _*).end()) + None // Some(builder.startAnd().in(attribute, values.map(_.asInstanceOf[AnyRef]): _*).end()) case _ => None } From 5ddf8e8b9d59b429f8ea4db6c6b437158c18cd79 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Sun, 8 Apr 2018 20:54:48 -0700 Subject: [PATCH 130/142] Addressing precheckin failures --- assembly/build.gradle | 18 +-- mesos/build.gradle | 36 ----- .../cluster/YarnClusterScheduler.scala | 74 ++++----- yarn/build.gradle | 140 ------------------ ...che.spark.scheduler.ExternalClusterManager | 1 - .../cluster/YarnClusterManager.scala | 56 ------- 6 files changed, 42 insertions(+), 283 deletions(-) delete mode 100644 mesos/build.gradle delete mode 100644 yarn/build.gradle delete mode 100644 yarn/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager delete mode 100644 yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala diff --git a/assembly/build.gradle b/assembly/build.gradle index 301819d47a3f2..2a95c8e30145b 100644 --- a/assembly/build.gradle +++ b/assembly/build.gradle @@ -28,22 +28,14 @@ dependencies { compile project(subprojectBase + 'snappy-spark-sql-kafka-0.10_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-mllib_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-graphx_' + scalaBinaryVersion) - if (rootProject.hasProperty('yarn')) { - compile project(subprojectBase + 'snappy-spark-yarn_' + scalaBinaryVersion) - } - if (rootProject.hasProperty('mesos')) { - compile project(subprojectBase + 'snappy-spark-mesos_' + scalaBinaryVersion) - } + compile project(subprojectBase + 'snappy-spark-yarn_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-mesos_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-hive_' + scalaBinaryVersion) + compile project(subprojectBase + 'snappy-spark-hive-thriftserver_' + scalaBinaryVersion) if (rootProject.hasProperty('kubernetes')) { compile project(subprojectBase + 'snappy-spark-kubernetes_' + scalaBinaryVersion) } - if (rootProject.hasProperty('hive')) { - compile project(subprojectBase + 'snappy-spark-hive_' + scalaBinaryVersion) - } - if (rootProject.hasProperty('hive-thriftserver')) { - compile project(subprojectBase + 'snappy-spark-hive-thriftserver_' + scalaBinaryVersion) - } - if (rootProject.hasProperty('spark-ganglia-lgpl')) { + if (rootProject.hasProperty('spark-ganglia-lgpl')) { compile project(subprojectBase + 'snappy-spark-ganglia-lgpl_' + scalaBinaryVersion) } } diff --git a/mesos/build.gradle b/mesos/build.gradle deleted file mode 100644 index 6309c04f0af3e..0000000000000 --- a/mesos/build.gradle +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2017 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -description = 'Spark Project Mesos' - -dependencies { - compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) - - compile(group: 'org.apache.mesos', name: 'mesos', version: '1.0.0', classifier: 'shaded-protobuf') { - exclude(group: 'com.google.protobuf', module: 'protobuf-java') - } - compile group: 'com.google.guava', name: 'guava', version: guavaVersion - compile group: 'org.eclipse.jetty', name: 'jetty-server', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-plus', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-util', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-http', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-servlets', version: jettyVersion - - testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput') - testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') -} diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala index 96c9151fc351d..494c8ab7fd343 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala @@ -1,37 +1,37 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.scheduler.cluster - -import org.apache.spark._ -import org.apache.spark.deploy.yarn.ApplicationMaster - -/** - * This is a simple extension to ClusterScheduler - to ensure that appropriate initialization of - * ApplicationMaster, etc is done - */ -private[spark] class YarnClusterScheduler(sc: SparkContext) extends YarnScheduler(sc) { - - logInfo("Created YarnClusterScheduler") - - override def postStartHook() { - ApplicationMaster.sparkContextInitialized(sc) - super.postStartHook() - logInfo("YarnClusterScheduler.postStartHook done") - } - -} +///* +// * Licensed to the Apache Software Foundation (ASF) under one or more +// * contributor license agreements. See the NOTICE file distributed with +// * this work for additional information regarding copyright ownership. +// * The ASF licenses this file to You under the Apache License, Version 2.0 +// * (the "License"); you may not use this file except in compliance with +// * the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package org.apache.spark.scheduler.cluster +// +//import org.apache.spark._ +//import org.apache.spark.deploy.yarn.ApplicationMaster +// +///** +// * This is a simple extension to ClusterScheduler - to ensure that appropriate initialization of +// * ApplicationMaster, etc is done +// */ +//private[spark] class YarnClusterScheduler(sc: SparkContext) extends YarnScheduler(sc) { +// +// logInfo("Created YarnClusterScheduler") +// +// override def postStartHook() { +// ApplicationMaster.sparkContextInitialized(sc) +// super.postStartHook() +// logInfo("YarnClusterScheduler.postStartHook done") +// } +// +//} diff --git a/yarn/build.gradle b/yarn/build.gradle deleted file mode 100644 index 13424cda13bd2..0000000000000 --- a/yarn/build.gradle +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2017 SnappyData, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -description = 'Spark Project YARN' - -dependencies { - compile project(subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion) - compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) - - compile(group: 'org.apache.hadoop', name: 'hadoop-yarn-api', version: hadoopVersion) { - exclude(group: 'javax.servlet', module: 'servlet-api') - exclude(group: 'asm', module: 'asm') - exclude(group: 'org.ow2.asm', module: 'asm') - exclude(group: 'org.jboss.netty', module: 'netty') - exclude(group: 'commons-logging', module: 'commons-logging') - exclude(group: 'com.sun.jersey') - exclude(group: 'com.sun.jersey.jersey-test-framework') - exclude(group: 'com.sun.jersey.contribs') - } - compile(group: 'org.apache.hadoop', name: 'hadoop-yarn-common', version: hadoopVersion) { - exclude(group: 'asm', module: 'asm') - exclude(group: 'org.ow2.asm', module: 'asm') - exclude(group: 'org.jboss.netty', module: 'netty') - exclude(group: 'javax.servlet', module: 'servlet-api') - exclude(group: 'commons-logging', module: 'commons-logging') - exclude(group: 'com.sun.jersey') - exclude(group: 'com.sun.jersey.jersey-test-framework') - exclude(group: 'com.sun.jersey.contribs') - } - compile(group: 'org.apache.hadoop', name: 'hadoop-yarn-server-web-proxy', version: hadoopVersion) { - exclude(group: 'asm', module: 'asm') - exclude(group: 'org.ow2.asm', module: 'asm') - exclude(group: 'org.jboss.netty', module: 'netty') - exclude(group: 'javax.servlet', module: 'servlet-api') - exclude(group: 'commons-logging', module: 'commons-logging') - exclude(group: 'com.sun.jersey') - exclude(group: 'com.sun.jersey.jersey-test-framework') - exclude(group: 'com.sun.jersey.contribs') - } - compile(group: 'org.apache.hadoop', name: 'hadoop-yarn-client', version: hadoopVersion) { - exclude(group: 'asm', module: 'asm') - exclude(group: 'org.ow2.asm', module: 'asm') - exclude(group: 'org.jboss.netty', module: 'netty') - exclude(group: 'javax.servlet', module: 'servlet-api') - exclude(group: 'commons-logging', module: 'commons-logging') - exclude(group: 'com.sun.jersey') - exclude(group: 'com.sun.jersey.jersey-test-framework') - exclude(group: 'com.sun.jersey.contribs') - } - compile group: 'com.google.guava', name: 'guava', version: guavaVersion - compile group: 'org.eclipse.jetty', name: 'jetty-server', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-plus', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-util', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-http', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion - compile group: 'org.eclipse.jetty', name: 'jetty-servlets', version: jettyVersion - compile group: 'org.apache.derby', name: 'derby', version: derbyVersion - compile(group: 'org.spark-project.hive', name: 'hive-exec', version: hiveVersion) { - exclude(group: 'org.spark-project.hive', module: 'hive-metastore') - exclude(group: 'org.spark-project.hive', module: 'hive-shims') - exclude(group: 'org.spark-project.hive', module: 'hive-ant') - exclude(group: 'org.spark-project.hive', module: 'spark-client') - exclude(group: 'org.apache.ant', module: 'ant') - exclude(group: 'com.esotericsoftware.kryo', module: 'kryo') - exclude(group: 'commons-codec', module: 'commons-codec') - exclude(group: 'commons-httpclient', module: 'commons-httpclient') - exclude(group: 'org.apache.avro', module: 'avro-mapred') - exclude(group: 'org.apache.calcite', module: 'calcite-core') - exclude(group: 'org.apache.curator', module: 'apache-curator') - exclude(group: 'org.apache.curator', module: 'curator-client') - exclude(group: 'org.apache.curator', module: 'curator-framework') - exclude(group: 'org.apache.thrift', module: 'libthrift') - exclude(group: 'org.apache.thrift', module: 'libfb303') - exclude(group: 'org.apache.zookeeper', module: 'zookeeper') - exclude(group: 'org.slf4j', module: 'slf4j-api') - exclude(group: 'org.slf4j', module: 'slf4j-log4j12') - exclude(group: 'log4j', module: 'log4j') - exclude(group: 'commons-logging', module: 'commons-logging') - exclude(group: 'org.codehaus.groovy', module: 'groovy-all') - exclude(group: 'jline', module: 'jline') - exclude(group: 'org.json', module: 'json') - } - compile(group: 'org.spark-project.hive', name: 'hive-metastore', version: hiveVersion) { - exclude(group: 'org.spark-project.hive', module: 'hive-serde') - exclude(group: 'org.spark-project.hive', module: 'hive-shims') - exclude(group: 'org.apache.thrift', module: 'libfb303') - exclude(group: 'org.apache.thrift', module: 'libthrift') - exclude(group: 'javax.servlet', module: 'servlet-api') - exclude(group: 'com.google.guava', module: 'guava') - exclude(group: 'org.slf4j', module: 'slf4j-api') - exclude(group: 'org.slf4j', module: 'slf4j-log4j12') - exclude(group: 'log4j', module: 'log4j') - exclude(group: 'org.apache.derby', module: 'derby') - } - compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) { - exclude(group: 'org.slf4j', module: 'slf4j-api') - } - compile(group: 'org.apache.thrift', name: 'libfb303', version: thriftVersion) { - exclude(group: 'org.slf4j', module: 'slf4j-api') - } - - testCompile project(subprojectBase + 'snappy-spark-network-yarn_' + scalaBinaryVersion) - testCompile project(path: subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion, configuration: 'testOutput') - - testCompile group: 'org.eclipse.jetty.orbit', name: 'javax.servlet.jsp', version: '2.2.0.v201112011158' - testCompile group: 'org.eclipse.jetty.orbit', name: 'javax.servlet.jsp.jstl', version: '1.2.0.v201105211821' - testCompile(group: 'org.apache.hadoop', name: 'hadoop-yarn-server-tests', version: hadoopVersion, classifier:'tests') { - exclude(group: 'asm', module: 'asm') - exclude(group: 'org.ow2.asm', module: 'asm') - exclude(group: 'org.jboss.netty', module: 'netty') - exclude(group: 'javax.servlet', module: 'servlet-api') - exclude(group: 'commons-logging', module: 'commons-logging') - exclude(group: 'com.sun.jersey') - exclude(group: 'com.sun.jersey.jersey-test-framework') - exclude(group: 'com.sun.jersey.contribs') - } - testCompile(group: 'org.mortbay.jetty', name: 'jetty', version: '6.1.26') { - exclude(group: 'org.mortbay.jetty', module: 'servlet-api') - } - testCompile group: 'com.sun.jersey', name: 'jersey-core', version: sunJerseyVersion - testCompile group: 'com.sun.jersey', name: 'jersey-server', version: sunJerseyVersion - testCompile(group: 'com.sun.jersey', name: 'jersey-json', version: sunJerseyVersion) { - exclude(group: 'stax', module: 'stax-api') - } - testCompile group: 'com.sun.jersey.contribs', name: 'jersey-guice', version: sunJerseyVersion -} diff --git a/yarn/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager b/yarn/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager deleted file mode 100644 index 6e8a1ebfc61da..0000000000000 --- a/yarn/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager +++ /dev/null @@ -1 +0,0 @@ -org.apache.spark.scheduler.cluster.YarnClusterManager diff --git a/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala deleted file mode 100644 index 55465334c79db..0000000000000 --- a/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterManager.scala +++ /dev/null @@ -1,56 +0,0 @@ -///* -// * Licensed to the Apache Software Foundation (ASF) under one or more -// * contributor license agreements. See the NOTICE file distributed with -// * this work for additional information regarding copyright ownership. -// * The ASF licenses this file to You under the Apache License, Version 2.0 -// * (the "License"); you may not use this file except in compliance with -// * the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package org.apache.spark.scheduler.cluster -// -//import org.apache.spark.{SparkContext, SparkException} -//import org.apache.spark.scheduler.{ExternalClusterManager, SchedulerBackend, TaskScheduler, TaskSchedulerImpl} -// -///** -// * Cluster Manager for creation of Yarn scheduler and backend -// */ -//private[spark] class YarnClusterManager extends ExternalClusterManager { -// -// override def canCreate(masterURL: String): Boolean = { -// masterURL == "yarn" -// } -// -// override def createTaskScheduler(sc: SparkContext, masterURL: String): TaskScheduler = { -// sc.deployMode match { -// case "cluster" => new YarnClusterScheduler(sc) -// case "client" => new YarnScheduler(sc) -// case _ => throw new SparkException(s"Unknown deploy mode '${sc.deployMode}' for Yarn") -// } -// } -// -// override def createSchedulerBackend(sc: SparkContext, -// masterURL: String, -// scheduler: TaskScheduler): SchedulerBackend = { -// sc.deployMode match { -// case "cluster" => -// new YarnClusterSchedulerBackend(scheduler.asInstanceOf[TaskSchedulerImpl], sc) -// case "client" => -// new YarnClientSchedulerBackend(scheduler.asInstanceOf[TaskSchedulerImpl], sc) -// case _ => -// throw new SparkException(s"Unknown deploy mode '${sc.deployMode}' for Yarn") -// } -// } -// -// override def initialize(scheduler: TaskScheduler, backend: SchedulerBackend): Unit = { -// scheduler.asInstanceOf[TaskSchedulerImpl].initialize(backend) -// } -//} From b1a7664c888e5c7ed42a1d28ece7535573230e7d Mon Sep 17 00:00:00 2001 From: ymahajan Date: Sun, 22 Apr 2018 22:02:55 -0700 Subject: [PATCH 131/142] Addressing prechekin failures --- .../org/apache/spark/executor/Executor.scala | 14 +-- .../org/apache/spark/rpc/RpcEndpointRef.scala | 18 +-- .../scala/org/apache/spark/rpc/RpcEnv.scala | 10 -- .../apache/spark/rpc/netty/NettyRpcEnv.scala | 117 +++++------------- .../scala/org/apache/spark/util/Utils.scala | 2 +- 5 files changed, 50 insertions(+), 111 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 4f541272c48c1..f2b0c93acfa19 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -39,7 +39,7 @@ import org.apache.spark.util._ import org.apache.spark.util.io.ChunkedByteBuffer import scala.collection.JavaConverters._ -import scala.collection.mutable.{ArrayBuffer, HashMap} +import scala.collection.mutable.{ArrayBuffer, HashMap, Map} import scala.util.control.NonFatal /** @@ -306,13 +306,14 @@ private[spark] class Executor( try { // Must be set before updateDependencies() is called, in case fetching dependencies // requires access to properties contained within (e.g. for access control). -// Executor.taskDeserializationProps.set(taskDescription.properties) + Executor.taskDeserializationProps.set(taskDescription.properties) -// updateDependencies(taskDescription.addedFiles, taskDescription.addedJars) + updateDependencies(taskDescription.addedFiles, taskDescription.addedJars) task = ser.deserialize[Task[Any]]( taskDescription.serializedTask, Thread.currentThread.getContextClassLoader) -// task.localProperties = taskDescription.properties + task.localProperties = taskDescription.properties task.setTaskMemoryManager(taskMemoryManager) + // If this task has been killed before we deserialized it, let's quit now. Otherwise, // continue executing the task. val killReason = reasonIfKilled @@ -570,7 +571,6 @@ private[spark] class Executor( // SparkUncaughtExceptionHandler.uncaughtException(t) } } - } finally { runningTasks.remove(taskId) } @@ -748,8 +748,8 @@ private[spark] class Executor( * Download any missing dependencies if we receive a new set of files and JARs from the * SparkContext. Also adds any new JARs we fetched to the class loader. */ - protected def updateDependencies(newFiles: HashMap[String, Long], - newJars: HashMap[String, Long]) { + protected def updateDependencies(newFiles: Map[String, Long], + newJars: Map[String, Long]) { lazy val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf) synchronized { // Fetch missing dependencies diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala index 4788230bbd000..fe4d2ee9da8fe 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala @@ -17,22 +17,22 @@ package org.apache.spark.rpc -import scala.concurrent.Future -import scala.reflect.ClassTag - -import org.apache.spark.{SparkConf, SparkException} +import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.util.RpcUtils +import scala.concurrent.Future +import scala.reflect.ClassTag + /** * A reference for a remote [[RpcEndpoint]]. [[RpcEndpointRef]] is thread-safe. */ -private[spark] abstract class RpcEndpointRef(conf: SparkConf, - _env: RpcEnv) extends Serializable with Logging { +private[spark] abstract class RpcEndpointRef(conf: SparkConf) + extends Serializable with Logging { - @transient protected var maxRetries = _env.maxRetries - @transient protected var retryWaitMs = _env.retryWaitMs - @transient protected var defaultAskTimeout = _env.defaultAskTimeout + private[this] val maxRetries = RpcUtils.numRetries(conf) + private[this] val retryWaitMs = RpcUtils.retryWaitMs(conf) + private[this] val defaultAskTimeout = RpcUtils.askRpcTimeout(conf) /** * return the address for the [[RpcEndpointRef]] diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala index e55ba295aa0d5..59fce6a4e731b 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala @@ -151,16 +151,6 @@ private[spark] abstract class RpcEnv(conf: SparkConf) { * @param uri URI with location of the file. */ def openChannel(uri: String): ReadableByteChannel - - /** - * Open a channel to download a file from the given URI. If the URIs returned by the - * RpcEnvFileServer use the "spark" scheme, this method will be called by the Utils class to - * retrieve the files. - * - * @param uri URI with location of the file. - * @param readTimeoutMs timeout in reading in millisecond - */ - def openChannel(uri: String, readTimeoutMs: Long): ReadableByteChannel } /** diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala index 18c2c725f0b7c..6186b21e7a892 100644 --- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala +++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala @@ -24,8 +24,12 @@ import java.util.concurrent._ import java.util.concurrent.atomic.AtomicBoolean import javax.annotation.Nullable -import com.esotericsoftware.kryo.io.{Input, Output} -import com.esotericsoftware.kryo.{Kryo, KryoSerializable} +import scala.concurrent.{Future, Promise} +import scala.reflect.ClassTag +import scala.util.{DynamicVariable, Failure, Success, Try} +import scala.util.control.NonFatal + +import org.apache.spark.{SecurityManager, SparkConf} import org.apache.spark.internal.Logging import org.apache.spark.network.TransportContext import org.apache.spark.network.client._ @@ -33,7 +37,7 @@ import org.apache.spark.network.crypto.{AuthClientBootstrap, AuthServerBootstrap import org.apache.spark.network.netty.SparkTransportConf import org.apache.spark.network.server._ import org.apache.spark.rpc._ -import org.apache.spark.serializer.{SerializationStream, Serializer, SerializerInstance} +import org.apache.spark.serializer.{JavaSerializer, JavaSerializerInstance, SerializationStream} import org.apache.spark.util.{ByteBufferInputStream, ByteBufferOutputStream, ThreadUtils, Utils} import org.apache.spark.{SecurityManager, SparkConf, SparkEnv} @@ -45,7 +49,7 @@ import scala.util.{DynamicVariable, Failure, Success, Try} private[netty] class NettyRpcEnv( val conf: SparkConf, - serializer: Serializer, + javaSerializerInstance: JavaSerializerInstance, host: String, securityManager: SecurityManager, numUsableCores: Int) extends RpcEnv(conf) with Logging { @@ -55,10 +59,6 @@ private[netty] class NettyRpcEnv( "rpc", conf.getInt("spark.rpc.io.threads", 0)) - private val serializerInstance = new ThreadLocal[SerializerInstance] { - override def initialValue(): SerializerInstance = serializer.newInstance() - } - private val dispatcher: Dispatcher = new Dispatcher(this, numUsableCores) private val streamManager = new NettyStreamManager(this) @@ -262,20 +262,20 @@ private[netty] class NettyRpcEnv( } private[netty] def serialize(content: Any): ByteBuffer = { - serializerInstance.get().serialize(content) + javaSerializerInstance.serialize(content) } /** * Returns [[SerializationStream]] that forwards the serialized bytes to `out`. */ private[netty] def serializeStream(out: OutputStream): SerializationStream = { - serializerInstance.get().serializeStream(out) + javaSerializerInstance.serializeStream(out) } private[netty] def deserialize[T: ClassTag](client: TransportClient, bytes: ByteBuffer): T = { NettyRpcEnv.currentClient.withValue(client) { deserialize { () => - serializerInstance.get().deserialize[T](bytes) + javaSerializerInstance.deserialize[T](bytes) } } } @@ -339,23 +339,17 @@ private[netty] class NettyRpcEnv( val pipe = Pipe.open() val source = new FileDownloadChannel(pipe.source()) - Utils.tryWithSafeFinallyAndFailureCallbacks(block = { + try { val client = downloadClient(parsedUri.getHost(), parsedUri.getPort()) val callback = new FileDownloadCallback(pipe.sink(), source, client) client.stream(parsedUri.getPath(), callback) - })(catchBlock = { - pipe.sink().close() - source.close() - }) - - source - } - - override def openChannel(uri: String, readTimeoutMs: Long): ReadableByteChannel = { - val source = openChannel(uri) - if (readTimeoutMs > 0) { - source.asInstanceOf[FileDownloadChannel].setTimeoutMs(readTimeoutMs) + } catch { + case e: Exception => + pipe.sink().close() + source.close() + throw e } + source } @@ -383,37 +377,24 @@ private[netty] class NettyRpcEnv( fileDownloadFactory.createClient(host, port) } - private class FileDownloadChannel(source: Pipe.SourceChannel) extends ReadableByteChannel { + private class FileDownloadChannel(source: ReadableByteChannel) extends ReadableByteChannel { @volatile private var error: Throwable = _ - private var timeoutMs: Long = _ def setError(e: Throwable): Unit = { - // This setError callback is invoked by internal RPC threads in order to propagate remote - // exceptions to application-level threads which are reading from this channel. When an - // RPC error occurs, the RPC system will call setError() and then will close the - // Pipe.SinkChannel corresponding to the other end of the `source` pipe. Closing of the pipe - // sink will cause `source.read()` operations to return EOF, unblocking the application-level - // reading thread. Thus there is no need to actually call `source.close()` here in the - // onError() callback and, in fact, calling it here would be dangerous because the close() - // would be asynchronous with respect to the read() call and could trigger race-conditions - // that lead to data corruption. See the PR for SPARK-22982 for more details on this topic. error = e - } - - def setTimeoutMs(millis: Long): Unit = { - timeoutMs = millis + source.close() } override def read(dst: ByteBuffer): Int = { - def readBuffer: Int = if (timeoutMs > 0) { - val context = ExecutionContext.fromExecutorService(clientConnectionExecutor) - val future = Future(source.read(dst))(context) - ThreadUtils.awaitResult(future, Duration(timeoutMs, TimeUnit.MILLISECONDS)) - } else source.read(dst) - Try(readBuffer) match { + Try(source.read(dst)) match { case Success(bytesRead) => bytesRead - case Failure(readErr) => throw readErr + case Failure(readErr) => + if (error != null) { + throw error + } else { + throw readErr + } } } @@ -472,9 +453,12 @@ private[rpc] class NettyRpcEnvFactory extends RpcEnvFactory with Logging { def create(config: RpcEnvConfig): RpcEnv = { val sparkConf = config.conf - val serializer = SparkEnv.getClosureSerializer(sparkConf) + // Use JavaSerializerInstance in multiple threads is safe. However, if we plan to support + // KryoSerializer in future, we have to use ThreadLocal to store SerializerInstance + val javaSerializerInstance = + new JavaSerializer(sparkConf).newInstance().asInstanceOf[JavaSerializerInstance] val nettyEnv = - new NettyRpcEnv(sparkConf, serializer, config.advertiseAddress, + new NettyRpcEnv(sparkConf, javaSerializerInstance, config.advertiseAddress, config.securityManager, config.numUsableCores) if (!config.clientMode) { val startNettyRpcEnv: Int => (NettyRpcEnv, Int) = { actualPort => @@ -516,13 +500,9 @@ private[rpc] class NettyRpcEnvFactory extends RpcEnvFactory with Logging { private[netty] class NettyRpcEndpointRef( @transient private val conf: SparkConf, private val endpointAddress: RpcEndpointAddress, - @transient @volatile private var nettyEnv: NettyRpcEnv) - extends RpcEndpointRef(conf, nettyEnv) with Serializable with KryoSerializable with Logging { + @transient @volatile private var nettyEnv: NettyRpcEnv) extends RpcEndpointRef(conf) { @transient @volatile var client: TransportClient = _ - - private var _address = if (endpointAddress.rpcAddress != null) endpointAddress else null - private var _name = endpointAddress.name override def address: RpcAddress = if (endpointAddress.rpcAddress != null) endpointAddress.rpcAddress else null @@ -531,10 +511,6 @@ private[netty] class NettyRpcEndpointRef( in.defaultReadObject() nettyEnv = NettyRpcEnv.currentEnv.value client = NettyRpcEnv.currentClient.value - - maxRetries = nettyEnv.maxRetries - retryWaitMs = nettyEnv.retryWaitMs - defaultAskTimeout = nettyEnv.defaultAskTimeout } private def writeObject(out: ObjectOutputStream): Unit = { @@ -543,33 +519,6 @@ private[netty] class NettyRpcEndpointRef( override def name: String = endpointAddress.name - override def write(kryo: Kryo, output: Output): Unit = { - val addr = address - output.writeString(_name) - if (addr != null && addr.host != null) { - output.writeString(addr.host) - output.writeInt(addr.port) - } else { - output.writeString(null) - } - } - - override def read(kryo: Kryo, input: Input): Unit = { - _name = input.readString() - _address = null - val host = input.readString() - if (host != null) { - val port = input.readInt() - _address = RpcEndpointAddress(host, port, _name) - } - nettyEnv = NettyRpcEnv.currentEnv.value - client = NettyRpcEnv.currentClient.value - - maxRetries = nettyEnv.maxRetries - retryWaitMs = nettyEnv.retryWaitMs - defaultAskTimeout = nettyEnv.defaultAskTimeout - } - override def ask[T: ClassTag](message: Any, timeout: RpcTimeout): Future[T] = { nettyEnv.ask(new RequestMessage(nettyEnv.address, this, message), timeout) } diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index f80ebbad28ce0..956028b8fddae 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -716,7 +716,7 @@ private[spark] object Utils extends Logging { } // wait for max double the configured time (connect + read time) val timeoutMs = conf.getTimeAsSeconds("spark.files.fetchTimeout", "60s") * 2000L - val source = SparkEnv.get.rpcEnv.openChannel(url, timeoutMs) + val source = SparkEnv.get.rpcEnv.openChannel(url) val is = Channels.newInputStream(source) downloadFile(url, is, targetFile, fileOverwrite) case "http" | "https" | "ftp" => From 4b275d87d0469362c5d490545fa802f42fe1052d Mon Sep 17 00:00:00 2001 From: ymahajan Date: Mon, 30 Apr 2018 10:27:42 -0700 Subject: [PATCH 132/142] Addressing precheckin failures --- .../spark/sql/catalyst/expressions/literals.scala | 14 +++----------- .../catalyst/expressions/stringExpressions.scala | 12 +++++++----- .../spark/sql/catalyst/plans/QueryPlan.scala | 11 ++++++----- .../spark/sql/execution/datasources/rules.scala | 11 +++++++++-- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 2c16579aacf23..cd176d941819f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -308,18 +308,10 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression { case ByteType | ShortType => ev.copy(code = "", value = s"($javaType)$value") case TimestampType | LongType => - ev.isNull = "false" - ev.value = s"${value}L" - ev.copy("") + ev.copy(code = "", value = s"${value}L") case _ => - if (value == null) { - ev.isNull = "true" - ev.value = "null" - } else { - ev.isNull = "false" - ev.value = ctx.addReferenceObj("value", value, ctx.javaType(dataType)) - } - ev.copy("") + ev.copy(code = "", value = ctx.addReferenceObj("literal", value, + ctx.javaType(dataType))) } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index dd96ee11b9a97..d7612e30b4c57 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -1808,9 +1808,9 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp override def inputTypes: Seq[DataType] = Seq(StringType) protected override def nullSafeEval(string: Any): Any = { - val str = string.asInstanceOf[UTF8String] - if (str.numBytes() > 0) { - str.getByte(0).asInstanceOf[Int] + val bytes = string.asInstanceOf[UTF8String].getBytes + if (bytes.length > 0) { + bytes(0).asInstanceOf[Int] } else { 0 } @@ -1818,9 +1818,11 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { nullSafeCodeGen(ctx, ev, (child) => { + val bytes = ctx.freshName("bytes") s""" - if ($child.numBytes() > 0) { - ${ev.value} = (int)$child.getByte(0); + byte[] $bytes = $child.getBytes(); + if ($bytes.length > 0) { + ${ev.value} = (int) $bytes[0]; } else { ${ev.value} = 0; } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala index a9f9ffac795b8..60c9c74872f4a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala @@ -36,7 +36,7 @@ package org.apache.spark.sql.catalyst.plans import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.trees.TreeNode +import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StructType} @@ -121,7 +121,9 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT var changed = false @inline def transformExpression(e: Expression): Expression = { - val newE = f(e) + val newE = CurrentOrigin.withOrigin(e.origin) { + f(e) + } if (newE.fastEquals(e)) { e } else { @@ -132,8 +134,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT def recursiveTransform(arg: Any): AnyRef = arg match { case e: Expression => transformExpression(e) - case Some(e: Expression) => Some(transformExpression(e)) - case Some(seq: Traversable[_]) => Some(seq.map(recursiveTransform)) + case Some(value) => Some(recursiveTransform(value)) case m: Map[_, _] => m case d: DataType => d // Avoid unpacking Structs case seq: Traversable[_] => seq.map(recursiveTransform) @@ -167,7 +168,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT productIterator.flatMap { case e: Expression => e :: Nil - case Some(seq: Traversable[_] ) => seqToExpressions(seq) + case s: Some[_] => seqToExpressions(s.toSeq) case seq: Traversable[_] => seqToExpressions(seq) case other => Nil }.toSeq diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala index d37bc302725f2..98ed2de8ae538 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala @@ -123,6 +123,14 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi s"`${existingProvider.getSimpleName}`. It doesn't match the specified format " + s"`${specifiedProvider.getSimpleName}`.") } + tableDesc.storage.locationUri match { + case Some(location) if location.getPath != existingTable.location.getPath => + throw new AnalysisException( + s"The location of the existing table ${tableIdentWithDB.quotedString} is " + + s"`${existingTable.location}`. It doesn't match the specified location " + + s"`${tableDesc.location}`.") + case _ => + } if (query.schema.length != existingTable.schema.length) { throw new AnalysisException( @@ -322,8 +330,7 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi * table. It also does data type casting and field renaming, to make sure that the columns to be * inserted have the correct data type and fields have the correct names. */ -case class -PreprocessTableInsertion(conf: SQLConf) extends Rule[LogicalPlan] { +case class PreprocessTableInsertion(conf: SQLConf) extends Rule[LogicalPlan] { private def preprocess( insert: InsertIntoTable, tblName: String, From 7c936df5d7ea687e06ce041c305fd440a64d367e Mon Sep 17 00:00:00 2001 From: ymahajan Date: Tue, 1 May 2018 17:57:33 -0700 Subject: [PATCH 133/142] Addressing precheckin failures --- .../expressions/codegen/GenerateUnsafeProjection.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala index c5390839d53d1..36ffa8dcdd2b6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala @@ -111,6 +111,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro s""" // Remember the current cursor so that we can calculate how many bytes are // written later. + final int $tmpCursor = $bufferHolder.cursor; ${writeStructToBuffer(ctx, input.value, t.map(_.dataType), bufferHolder)} $rowWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); """ @@ -217,8 +218,6 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro final int $tmpCursor = $bufferHolder.cursor; ${writeArrayToBuffer(ctx, element, et, bufferHolder)} $arrayWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); - $arrayWriter.setOffset($index); - ${writeArrayToBuffer(ctx, element, et, bufferHolder)} """ case m @ MapType(kt, vt, _) => From 11a26a65a7fceab23c508c02a56e0887b2166491 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Thu, 3 May 2018 17:39:14 -0700 Subject: [PATCH 134/142] Addressing precheckin failures --- .../CoarseGrainedExecutorBackend.scala | 19 +- .../org/apache/spark/executor/Executor.scala | 131 ++++--- .../apache/spark/scheduler/DAGScheduler.scala | 209 ++--------- .../apache/spark/scheduler/ResultTask.scala | 16 +- .../spark/scheduler/ShuffleMapTask.scala | 16 +- .../org/apache/spark/scheduler/Task.scala | 84 +++-- .../spark/scheduler/TaskSetManager.scala | 24 +- .../local/LocalSchedulerBackend.scala | 3 +- .../apache/spark/executor/ExecutorSuite.scala | 14 +- .../org/apache/spark/scheduler/FakeTask.scala | 13 +- .../spark/scheduler/TaskContextSuite.scala | 12 +- .../spark/scheduler/TaskSetManagerSuite.scala | 2 +- .../codegen/GenerateUnsafeProjection.scala | 1 - .../plans/physical/partitioning.scala | 347 +++++++----------- .../v2/DataSourcePartitioning.scala | 2 - .../exchange/EnsureRequirements.scala | 137 +++---- 16 files changed, 385 insertions(+), 645 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala index 591112c84e6ec..1a0a797ad2f44 100644 --- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala +++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala @@ -84,34 +84,34 @@ private[spark] class CoarseGrainedExecutorBackend( case RegisteredExecutor => logInfo("Successfully registered with driver") try { - executor = new Executor(executorId, hostname, env, userClassPath, isLocal = false) + executor = registerExecutor } catch { case NonFatal(e) => exitExecutor(1, "Unable to create executor due to " + e.getMessage, e) } case RegisterExecutorFailed(message) => - logError("Slave registration failed: " + message) - exitExecutor(1, "Slave registration failed") + exitExecutor(1, "Slave registration failed: " + message) case LaunchTask(taskDesc) => if (executor == null) { - logError("Received LaunchTask command but executor was null") exitExecutor(1, "Received LaunchTask command but executor was null") } else { - // val taskDesc = TaskDescription.decode(data) - logInfo("Got assigned task " + taskDesc.taskId) - executor.launchTask(this, taskDesc) + env.taskLogger.logInfo("Got assigned task " + taskDesc.taskId) + executor.launchTask(this, taskId = taskDesc.taskId, attemptNumber = taskDesc.attemptNumber, + taskDesc.name, taskDesc.serializedTask, taskDesc.taskData.decompress(env)) } case LaunchTasks(tasks, taskDataList) => if (executor ne null) { logDebug("Got assigned tasks " + tasks.map(_.taskId).mkString(",")) for (task <- tasks) { - logInfo("Got assigned task " + task.taskId) + env.taskLogger.logInfo("Got assigned task " + task.taskId) val ref = task.taskData.reference val taskData = if (ref >= 0) taskDataList(ref) else task.taskData - executor.launchTask(this, task) + executor.launchTask(this, taskId = task.taskId, + attemptNumber = task.attemptNumber, task.name, task.serializedTask, + taskData.decompress(env)) } } else { exitExecutor(1, "Received LaunchTasks command but executor was null") @@ -119,7 +119,6 @@ private[spark] class CoarseGrainedExecutorBackend( case KillTask(taskId, _, interruptThread, reason) => if (executor == null) { - logError("Received KillTask command but executor was null") exitExecutor(1, "Received KillTask command but executor was null") } else { executor.killTask(taskId, interruptThread, reason) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index f2b0c93acfa19..40a92b9424400 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -30,16 +30,16 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder import org.apache.spark._ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging -import org.apache.spark.memory.TaskMemoryManager +import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager} import org.apache.spark.rpc.RpcTimeout -import org.apache.spark.scheduler.{DirectTaskResult, IndirectTaskResult, Task, TaskDescription} +import org.apache.spark.scheduler.{DirectTaskResult, IndirectTaskResult, Task} import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.storage.{StorageLevel, TaskResultBlockId} import org.apache.spark.util._ import org.apache.spark.util.io.ChunkedByteBuffer import scala.collection.JavaConverters._ -import scala.collection.mutable.{ArrayBuffer, HashMap, Map} +import scala.collection.mutable.{ArrayBuffer, HashMap} import scala.util.control.NonFatal /** @@ -170,9 +170,16 @@ private[spark] class Executor( private[executor] def numRunningTasks: Int = runningTasks.size() - def launchTask(context: ExecutorBackend, taskDescription: TaskDescription): Unit = { - val tr = new TaskRunner(context, taskDescription) - runningTasks.put(taskDescription.taskId, tr) + def launchTask( + context: ExecutorBackend, + taskId: Long, + attemptNumber: Int, + taskName: String, + serializedTask: ByteBuffer, + taskData: (Array[Byte], Long)): Unit = { + val tr = new TaskRunner(context, taskId = taskId, attemptNumber = attemptNumber, + taskName, serializedTask, taskData._1, taskData._2) + runningTasks.put(taskId, tr) threadPool.execute(tr) } @@ -230,12 +237,15 @@ private[spark] class Executor( class TaskRunner( execBackend: ExecutorBackend, - private val taskDescription: TaskDescription) + val taskId: Long, + val attemptNumber: Int, + taskName: String, + serializedTask: ByteBuffer, + taskDataBytes: Array[Byte], + taskDecompressTime: Long) extends Runnable { - val taskId = taskDescription.taskId val threadName = s"Executor task launch worker for task $taskId" - private val taskName = taskDescription.name /** If specified, this task has been killed and this option contains the reason. */ @volatile private var reasonIfKilled: Option[String] = None @@ -304,16 +314,18 @@ private[spark] class Executor( startGCTime = computeTotalGcTime() try { + val (taskFiles, taskJars, taskProps, taskBytes) = + Task.deserializeWithDependencies(serializedTask) + // Must be set before updateDependencies() is called, in case fetching dependencies // requires access to properties contained within (e.g. for access control). - Executor.taskDeserializationProps.set(taskDescription.properties) + Executor.taskDeserializationProps.set(taskProps) - updateDependencies(taskDescription.addedFiles, taskDescription.addedJars) - task = ser.deserialize[Task[Any]]( - taskDescription.serializedTask, Thread.currentThread.getContextClassLoader) - task.localProperties = taskDescription.properties + updateDependencies(taskFiles, taskJars) + task = ser.deserialize[Task[Any]](taskBytes, Thread.currentThread.getContextClassLoader) + task.taskDataBytes = taskDataBytes + task.localProperties = taskProps task.setTaskMemoryManager(taskMemoryManager) - // If this task has been killed before we deserialized it, let's quit now. Otherwise, // continue executing the task. val killReason = reasonIfKilled @@ -343,7 +355,7 @@ private[spark] class Executor( val value = try { val res = task.run( taskAttemptId = taskId, - attemptNumber = taskDescription.attemptNumber, + attemptNumber = attemptNumber, metricsSystem = env.metricsSystem) threwException = false res @@ -388,10 +400,15 @@ private[spark] class Executor( // If the task has been killed, let's fail it. task.context.killTaskIfInterrupted() + val resultSer = env.serializer.newInstance() + val beforeSerialization = System.currentTimeMillis() + val valueBytes = resultSer.serialize(value) + val afterSerialization = System.currentTimeMillis() + // Deserialization happens in two parts: first, we deserialize a Task object, which // includes the Partition. Second, Task.run() deserializes the RDD and function to be run. task.metrics.setExecutorDeserializeTime(math.max(taskStart - deserializeStartTime + - task.executorDeserializeTime /* + taskDecompressTime */, 0L).toDouble / 1000000.0) + task.executorDeserializeTime + taskDecompressTime, 0L).toDouble / 1000000.0) task.metrics.setExecutorDeserializeCpuTime(math.max(taskStartCpu - deserializeStartCpuTime + task.executorDeserializeCpuTime, 0L).toDouble / 1000000.0) // We need to subtract Task.run()'s deserialization time to avoid double-counting @@ -400,6 +417,7 @@ private[spark] class Executor( task.metrics.setExecutorCpuTime(math.max(taskFinishCpu - taskStartCpu - task.executorDeserializeCpuTime, 0L).toDouble / 1000000.0) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) + task.metrics.setResultSerializationTime(afterSerialization - beforeSerialization) // Expose task metrics using the Dropwizard metrics system. // Update task metrics counters @@ -444,10 +462,11 @@ private[spark] class Executor( // Note: accumulator updates must be collected after TaskMetrics is updated val accumUpdates = task.collectAccumulatorUpdates() - // TODO: do not serialize value twice - val directResult = new DirectTaskResult(value, accumUpdates) - val serializedDirectResult = ser.serialize(directResult) - val resultSize = serializedDirectResult.limit() + val directResult = new DirectTaskResult(value, accumUpdates, + Some(task.metrics.resultSerializationTimeMetric)) + val taskSer = env.serializer.newInstance() + val serializedDirectResult = taskSer.serialize(directResult) + val resultSize = serializedDirectResult.limit // directSend = sending directly back to the driver val serializedResult: ByteBuffer = { @@ -455,7 +474,7 @@ private[spark] class Executor( logWarning(s"Finished $taskName (TID $taskId). Result is larger than maxResultSize " + s"(${Utils.bytesToString(resultSize)} > ${Utils.bytesToString(maxResultSize)}), " + s"dropping it.") - ser.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize)) + taskSer.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize)) } else if (resultSize > maxDirectResultSize) { val blockId = TaskResultBlockId(taskId) env.blockManager.putBytes( @@ -464,7 +483,7 @@ private[spark] class Executor( StorageLevel.MEMORY_AND_DISK_SER) logInfo( s"Finished $taskName (TID $taskId). $resultSize bytes result sent via BlockManager)") - ser.serialize(new IndirectTaskResult[Any](blockId, resultSize)) + taskSer.serialize(new IndirectTaskResult[Any](blockId, resultSize)) } else { logInfo(s"Finished $taskName (TID $taskId). $resultSize bytes result sent to driver") serializedDirectResult @@ -531,45 +550,43 @@ private[spark] class Executor( // the default uncaught exception handler, which will terminate the Executor. logError(s"Exception in $taskName (TID $taskId)", t) - // Collect latest accumulator values to report back to the driver - val accums: Seq[AccumulatorV2[_, _]] = - if (task != null) { - task.metrics.setExecutorRunTime( - math.max(System.nanoTime() - taskStart, 0L).toDouble / 1000000.0) - val taskEndCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { - threadMXBean.getCurrentThreadCpuTime - } else 0L - task.metrics.setExecutorCpuTime( - math.max(taskEndCpu - taskStartCpu, 0L).toDouble / 1000000.0) - task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) - task.collectAccumulatorUpdates(taskFailed = true) - } else { - Seq.empty - } + // SPARK-20904: Do not report failure to driver if if happened during shut down. Because + // libraries may set up shutdown hooks that race with running tasks during shutdown, + // spurious failures may occur and can result in improper accounting in the driver (e.g. + // the task failure would not be ignored if the shutdown happened because of premption, + // instead of an app issue). + if (!ShutdownHookManager.inShutdown()) { + // Collect latest accumulator values to report back to the driver + val accums: Seq[AccumulatorV2[_, _]] = + if (task != null) { + task.metrics.setExecutorRunTime(System.currentTimeMillis() - taskStart) + task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) + task.collectAccumulatorUpdates(taskFailed = true) + } else { + Seq.empty + } - val accUpdates = accums.map(acc => acc.toInfo(Some(acc.value), None)) + val accUpdates = accums.map(acc => acc.toInfo(Some(acc.value), None)) - val serializedTaskEndReason = { - try { - ser.serialize(new ExceptionFailure(t, accUpdates).withAccums(accums)) - } catch { - case _: NotSerializableException => - // t is not serializable so just send the stacktrace - ser.serialize(new ExceptionFailure(t, accUpdates, false).withAccums(accums)) + val serializedTaskEndReason = { + try { + ser.serialize(new ExceptionFailure(t, accUpdates).withAccums(accums)) + } catch { + case _: NotSerializableException => + // t is not serializable so just send the stacktrace + ser.serialize(new ExceptionFailure(t, accUpdates, false).withAccums(accums)) + } } + setTaskFinishedAndClearInterruptStatus() + execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskEndReason) + } else { + logInfo("Not reporting error to driver during JVM shutdown.") } - setTaskFinishedAndClearInterruptStatus() - execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskEndReason) // Don't forcibly exit unless the exception was inherently fatal, to avoid // stopping other tasks unnecessarily. - if (isFatalError(t)) { - if (!isLocal) { - Thread.getDefaultUncaughtExceptionHandler. - uncaughtException(Thread.currentThread(), t) - } else { - // SparkUncaughtExceptionHandler.uncaughtException(t) - } + if (!t.isInstanceOf[SparkOutOfMemoryError] && Utils.isFatalError(t)) { + uncaughtExceptionHandler.uncaughtException(Thread.currentThread(), t) } } finally { runningTasks.remove(taskId) @@ -748,8 +765,8 @@ private[spark] class Executor( * Download any missing dependencies if we receive a new set of files and JARs from the * SparkContext. Also adds any new JARs we fetched to the class loader. */ - protected def updateDependencies(newFiles: Map[String, Long], - newJars: Map[String, Long]) { + protected def updateDependencies(newFiles: HashMap[String, Long], + newJars: HashMap[String, Long]) { lazy val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf) synchronized { // Fetch missing dependencies diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 62bfa05addaf0..c6124c12ebef8 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -958,173 +958,6 @@ class DAGScheduler( } } -// /** Called when stage's parents are available and we can now do its task. */ -// private def submitMissingTasks(stage: Stage, jobId: Int) { -// logDebug("submitMissingTasks(" + stage + ")") -// -// // First figure out the indexes of partition ids to compute. -// val partitionsToCompute: Seq[Int] = stage.findMissingPartitions() -// -// // Use the scheduling pool, job group, description, etc. from an ActiveJob associated -// // with this Stage -// val properties = jobIdToActiveJob(jobId).properties -// -// runningStages += stage -// // SparkListenerStageSubmitted should be posted before testing whether tasks are -// // serializable. If tasks are not serializable, a SparkListenerStageCompleted event -// // will be posted, which should always come after a corresponding SparkListenerStageSubmitted -// // event. -// stage match { -// case s: ShuffleMapStage => -// outputCommitCoordinator.stageStart(stage = s.id, maxPartitionId = s.numPartitions - 1) -// case s: ResultStage => -// outputCommitCoordinator.stageStart( -// stage = s.id, maxPartitionId = s.rdd.partitions.length - 1) -// } -// val taskIdToLocations: Map[Int, Seq[TaskLocation]] = try { -// stage match { -// case s: ShuffleMapStage => -// partitionsToCompute.map { id => (id, getPreferredLocs(stage.rdd, id))}.toMap -// case s: ResultStage => -// partitionsToCompute.map { id => -// val p = s.partitions(id) -// (id, getPreferredLocs(stage.rdd, p)) -// }.toMap -// } -// } catch { -// case NonFatal(e) => -// stage.makeNewStageAttempt(partitionsToCompute.size) -// listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo, properties)) -// abortStage(stage, s"Task creation failed: $e\n${Utils.exceptionString(e)}", Some(e)) -// runningStages -= stage -// return -// } -// -// stage.makeNewStageAttempt(partitionsToCompute.size, taskIdToLocations.values.toSeq) -// -// // If there are tasks to execute, record the submission time of the stage. Otherwise, -// // post the even without the submission time, which indicates that this stage was -// // skipped. -// if (partitionsToCompute.nonEmpty) { -// stage.latestInfo.submissionTime = Some(clock.getTimeMillis()) -// } -// listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo, properties)) -// -// // TODO: Maybe we can keep the taskBinary in Stage to avoid serializing it multiple times. -// // Broadcasted binary for the task, used to dispatch tasks to executors. Note that we broadcast -// // the serialized copy of the RDD and for each task we will deserialize it, which means each -// // task gets a different copy of the RDD. This provides stronger isolation between tasks that -// // might modify state of objects referenced in their closures. This is necessary in Hadoop -// // where the JobConf/Configuration object is not thread-safe. -// var taskBinary: Broadcast[Array[Byte]] = null -// var taskData: TaskData = TaskData.EMPTY -// var partitions: Array[Partition] = null -// try { -// // For ShuffleMapTask, serialize and broadcast (rdd, shuffleDep). -// // For ResultTask, serialize and broadcast (rdd, func). -// var taskBinaryBytes: Array[Byte] = null -// // taskBinaryBytes and partitions are both effected by the checkpoint status. We need -// // this synchronization in case another concurrent job is checkpointing this RDD, so we get a -// // consistent view of both variables. -// RDDCheckpointData.synchronized { -// taskBinaryBytes = stage match { -// case stage: ShuffleMapStage => -// JavaUtils.bufferToArray( -// closureSerializer.serialize((stage.rdd, stage.shuffleDep): AnyRef)) -// case stage: ResultStage => -// JavaUtils.bufferToArray(closureSerializer.serialize((stage.rdd, stage.func): AnyRef)) -// } -// -// partitions = stage.rdd.partitions -// } -// -// // use direct byte shipping for small size or if number of partitions is small -// val taskBytesLen = taskBinaryBytes.length -// if (taskBytesLen <= DAGScheduler.TASK_INLINE_LIMIT || -// partitionsToCompute.length == 1 || -// (taskBytesLen < math.min(maxRpcMessageSize, DAGScheduler.TASK_INLINE_UPPER_LIMIT) && -// partitionsToCompute.length <= DAGScheduler.TASK_INLINE_PARTITION_LIMIT)) { -// if (stage.taskData.uncompressedLen > 0) { -// taskData = stage.taskData -// } else { -// // compress inline task data (broadcast compresses as per conf) -// taskData = new TaskData(env.createCompressionCodec.compress( -// taskBinaryBytes, taskBytesLen), taskBytesLen) -// stage.taskData = taskData -// } -// } else { -// taskBinary = sc.broadcast(taskBinaryBytes) -// } -// } catch { -// // In the case of a failure during serialization, abort the stage. -// case e: NotSerializableException => -// abortStage(stage, "Task not serializable: " + e.toString, Some(e)) -// runningStages -= stage -// -// // Abort execution -// return -// case NonFatal(e) => -// abortStage(stage, s"Task serialization failed: $e\n${Utils.exceptionString(e)}", Some(e)) -// runningStages -= stage -// return -// } -// -// val tasks: Seq[Task[_]] = try { -// val serializedTaskMetrics = closureSerializer.serialize(stage.latestInfo.taskMetrics).array() -// stage match { -// case stage: ShuffleMapStage => -// stage.pendingPartitions.clear() -// partitionsToCompute.map { id => -// val locs = taskIdToLocations(id) -// val part = partitions(id) -// stage.pendingPartitions += id -// new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, taskData, -// taskBinary, part, locs, properties, serializedTaskMetrics, Option(jobId), -// Option(sc.applicationId), sc.applicationAttemptId) -// } -// -// case stage: ResultStage => -// partitionsToCompute.map { id => -// val p: Int = stage.partitions(id) -// val part = stage.rdd.partitions(p) -// val locs = taskIdToLocations(id) -// new ResultTask(stage.id, stage.latestInfo.attemptId, taskData, -// taskBinary, part, locs, id, properties, serializedTaskMetrics, Option(jobId), -// Option(sc.applicationId), sc.applicationAttemptId) -// } -// } -// } catch { -// case NonFatal(e) => -// abortStage(stage, s"Task creation failed: $e\n${Utils.exceptionString(e)}", Some(e)) -// runningStages -= stage -// return -// } -// -// if (tasks.size > 0) { -// logInfo(s"Submitting ${tasks.size} missing tasks from $stage (${stage.rdd}) (first 15 " + -// s"tasks are for partitions ${tasks.take(15).map(_.partitionId)})") -// taskScheduler.submitTasks(new TaskSet( -// tasks.toArray, stage.id, stage.latestInfo.attemptId, jobId, properties)) -// } else { -// // Because we posted SparkListenerStageSubmitted earlier, we should mark -// // the stage as completed here in case there are no tasks to run -// markStageAsFinished(stage, None) -// -// val debugString = stage match { -// case stage: ShuffleMapStage => -// s"Stage ${stage} is actually done; " + -// s"(available: ${stage.isAvailable}," + -// s"available outputs: ${stage.numAvailableOutputs}," + -// s"partitions: ${stage.numPartitions})" -// case stage : ResultStage => -// s"Stage ${stage} is actually done; (partitions: ${stage.numPartitions})" -// } -// logDebug(debugString) -// -// submitWaitingChildStages(stage) -// } -// } - /** Called when stage's parents are available and we can now do its task. */ private def submitMissingTasks(stage: Stage, jobId: Int) { logDebug("submitMissingTasks(" + stage + ")") @@ -1183,19 +1016,38 @@ class DAGScheduler( // task gets a different copy of the RDD. This provides stronger isolation between tasks that // might modify state of objects referenced in their closures. This is necessary in Hadoop // where the JobConf/Configuration object is not thread-safe. - var taskBinary: Broadcast[Array[Byte]] = null + var taskBinary: Option[Broadcast[Array[Byte]]] = None + var taskData: TaskData = TaskData.EMPTY try { // For ShuffleMapTask, serialize and broadcast (rdd, shuffleDep). // For ResultTask, serialize and broadcast (rdd, func). - val taskBinaryBytes: Array[Byte] = stage match { + val bytes = stage.taskBinaryBytes + val taskBinaryBytes: Array[Byte] = if (bytes != null) bytes else stage match { case stage: ShuffleMapStage => JavaUtils.bufferToArray( closureSerializer.serialize((stage.rdd, stage.shuffleDep): AnyRef)) case stage: ResultStage => JavaUtils.bufferToArray(closureSerializer.serialize((stage.rdd, stage.func): AnyRef)) } - - taskBinary = sc.broadcast(taskBinaryBytes) + if (bytes == null) stage.taskBinaryBytes = taskBinaryBytes + + // use direct byte shipping for small size or if number of partitions is small + val taskBytesLen = taskBinaryBytes.length + if (taskBytesLen <= DAGScheduler.TASK_INLINE_LIMIT || + partitionsToCompute.length == 1 || + (taskBytesLen < math.min(maxRpcMessageSize, DAGScheduler.TASK_INLINE_UPPER_LIMIT) && + partitionsToCompute.length <= DAGScheduler.TASK_INLINE_PARTITION_LIMIT)) { + if (stage.taskData.uncompressedLen > 0) { + taskData = stage.taskData + } else { + // compress inline task data (broadcast compresses as per conf) + taskData = new TaskData(env.createCompressionCodec.compress( + taskBinaryBytes, taskBytesLen), taskBytesLen) + stage.taskData = taskData + } + } else { + taskBinary = Some(sc.broadcast(taskBinaryBytes)) + } } catch { // In the case of a failure during serialization, abort the stage. case e: NotSerializableException => @@ -1211,17 +1063,14 @@ class DAGScheduler( } val tasks: Seq[Task[_]] = try { - val serializedTaskMetrics = closureSerializer.serialize(stage.latestInfo.taskMetrics).array() stage match { case stage: ShuffleMapStage => - stage.pendingPartitions.clear() partitionsToCompute.map { id => val locs = taskIdToLocations(id) val part = stage.rdd.partitions(id) - stage.pendingPartitions += id - new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, _taskData = TaskData.EMPTY, - taskBinary, part, locs, properties, serializedTaskMetrics, Option(jobId), - Option(sc.applicationId), sc.applicationAttemptId) + new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, taskData, + taskBinary, part, locs, stage.latestInfo.taskMetrics, properties, + jobId, Option(sc.applicationId), Option(sc.applicationId)) } case stage: ResultStage => @@ -1229,9 +1078,9 @@ class DAGScheduler( val p: Int = stage.partitions(id) val part = stage.rdd.partitions(p) val locs = taskIdToLocations(id) - new ResultTask(stage.id, stage.latestInfo.attemptId, _taskData = TaskData.EMPTY, - taskBinary, part, locs, id, properties, serializedTaskMetrics, - Option(jobId), Option(sc.applicationId), sc.applicationAttemptId) + new ResultTask(stage.id, stage.latestInfo.attemptId, taskData, + taskBinary, part, locs, id, properties, stage.latestInfo.taskMetrics, jobId, + Option(sc.applicationId), sc.applicationAttemptId) } } } catch { diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index d08fbd73ed9b0..d3355aa95fe56 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -27,6 +27,7 @@ import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.spark._ import org.apache.spark.broadcast.Broadcast +import org.apache.spark.executor.TaskMetrics import org.apache.spark.rdd.RDD /** @@ -38,7 +39,7 @@ import org.apache.spark.rdd.RDD * @param stageAttemptId attempt id of the stage this task belongs to * @param _taskData if serialized RDD and function are small, then it is compressed * and sent with its original decompressed size - * @param taskBinary broadcasted version of the serialized RDD and the function to apply on each + * @param _taskBinary broadcasted version of the serialized RDD and the function to apply on each * partition of the given RDD. Once deserialized, the type should be * (RDD[T], (TaskContext, Iterator[T]) => U). * @param partition partition of the RDD this task is associated with @@ -46,8 +47,7 @@ import org.apache.spark.rdd.RDD * @param _outputId index of the task in this job (a job can launch tasks on only a subset of the * input RDD's partitions). * @param localProperties copy of thread-local properties set by the user on the driver side. - * @param serializedTaskMetrics a `TaskMetrics` that is created and serialized on the driver side - * and sent to executor side. + * @param metrics a `TaskMetrics` that is created at driver side and sent to executor side. * * The parameters below are optional: * @param jobId id of the job this task belongs to @@ -58,17 +58,17 @@ private[spark] class ResultTask[T, U]( stageId: Int, stageAttemptId: Int, _taskData: TaskData, - taskBinary: Broadcast[Array[Byte]], + _taskBinary: Option[Broadcast[Array[Byte]]], private var partition: Partition, locs: Seq[TaskLocation], private var _outputId: Int, localProperties: Properties, - serializedTaskMetrics: Array[Byte], - jobId: Option[Int] = None, + metrics: TaskMetrics, + jobId: Int = -1, appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[U](stageId, stageAttemptId, partition.index, _taskData, - taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) + _taskBinary, metrics, localProperties, jobId, appId, appAttemptId) with Serializable with KryoSerializable { final def outputId: Int = _outputId @@ -86,7 +86,7 @@ private[spark] class ResultTask[T, U]( } else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, func) = ser.deserialize[(RDD[T], (TaskContext, Iterator[T]) => U)]( - ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) + ByteBuffer.wrap(getTaskBytes), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = math.max(System.nanoTime() - deserializeStartTime, 0L) _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { math.max(threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime, 0L) diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index 67619b1e889eb..46d3f521f5438 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -28,6 +28,7 @@ import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.spark._ import org.apache.spark.broadcast.Broadcast +import org.apache.spark.executor.TaskMetrics import org.apache.spark.internal.Logging import org.apache.spark.rdd.RDD import org.apache.spark.shuffle.ShuffleWriter @@ -42,10 +43,11 @@ import org.apache.spark.shuffle.ShuffleWriter * @param stageAttemptId attempt id of the stage this task belongs to * @param _taskData if serialized RDD and function are small, then it is compressed * and sent with its original decompressed size - * @param taskBinary broadcast version of the RDD and the ShuffleDependency. Once deserialized, + * @param _taskBinary broadcast version of the RDD and the ShuffleDependency. Once deserialized, * the type should be (RDD[_], ShuffleDependency[_, _, _]). * @param partition partition of the RDD this task is associated with * @param locs preferred task execution locations for locality scheduling + * @param metrics a `TaskMetrics` that is created at driver side and sent to executor side. * @param localProperties copy of thread-local properties set by the user on the driver side. * * The parameters below are optional: @@ -56,23 +58,23 @@ import org.apache.spark.shuffle.ShuffleWriter private[spark] class ShuffleMapTask( stageId: Int, stageAttemptId: Int, - _taskData: TaskData = TaskData.EMPTY, - taskBinary: Broadcast[Array[Byte]], + _taskData: TaskData, + _taskBinary: Option[Broadcast[Array[Byte]]], private var partition: Partition, @transient private var locs: Seq[TaskLocation], + metrics: TaskMetrics, localProperties: Properties, - serializedTaskMetrics: Array[Byte], - jobId: Option[Int] = None, + jobId: Int = -1, appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[MapStatus](stageId, stageAttemptId, partition.index, _taskData, - taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) + _taskBinary, metrics, localProperties, jobId, appId, appAttemptId) with KryoSerializable with Logging { /** A constructor used only in test suites. This does not require passing in an RDD. */ def this(partitionId: Int) { this(0, 0, TaskData.EMPTY, null, new Partition { override def index: Int = 0 }, - null, new Properties, null) + null, null, new Properties) } @transient private val preferredLocs: Seq[TaskLocation] = { diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index 936c7ae434def..099c70243b33a 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -23,12 +23,13 @@ import java.util.Properties import scala.collection.mutable import scala.collection.mutable.HashMap + import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{Input, Output} + import org.apache.spark._ import org.apache.spark.broadcast.Broadcast import org.apache.spark.executor.TaskMetrics -import org.apache.spark.internal.config.APP_CALLER_CONTEXT import org.apache.spark.memory.{MemoryMode, TaskMemoryManager} import org.apache.spark.metrics.MetricsSystem import org.apache.spark.serializer.SerializerInstance @@ -48,10 +49,8 @@ import org.apache.spark.util._ * @param _stageId id of the stage this task belongs to * @param _stageAttemptId attempt id of the stage this task belongs to * @param _partitionId index of the number in the RDD + * @param _metrics a [[TaskMetrics]] that is created at driver side and sent to executor side. * @param localProperties copy of thread-local properties set by the user on the driver side. - * @param serializedTaskMetrics a `TaskMetrics` that is created and serialized on the driver side - * and sent to executor side. - * * The parameters below are optional: * @param _jobId id of the job this task belongs to * @param _appId id of the app this task belongs to @@ -63,12 +62,10 @@ private[spark] abstract class Task[T]( private var _partitionId: Int, @transient private[spark] var taskData: TaskData = TaskData.EMPTY, // The default value is only used in tests. - protected var taskBinary: Broadcast[Array[Byte]], + protected var taskBinary: Option[Broadcast[Array[Byte]]] = None, + private var _metrics: TaskMetrics = TaskMetrics.registered, @transient var localProperties: Properties = new Properties, - // The default value is only used in tests. - serializedTaskMetrics: Array[Byte] = - SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array(), - private var _jobId: Option[Int] = None, + private var _jobId: Int = -1, private var _appId: Option[String] = None, private var _appAttemptId: Option[String] = None) extends Serializable { @@ -78,12 +75,9 @@ private[spark] abstract class Task[T]( final def partitionId: Int = _partitionId - @transient lazy val metrics: TaskMetrics = - SparkEnv.get.closureSerializer.newInstance().deserialize(ByteBuffer.wrap(serializedTaskMetrics)) + final def jobId: Int = _jobId - final var jobId: Int = if (_jobId.isDefined) _jobId.get else -1 - - // final def metrics: TaskMetrics = _metrics + final def metrics: TaskMetrics = _metrics final def appId: String = if (_appId.isDefined) _appId.get else null @@ -93,7 +87,7 @@ private[spark] abstract class Task[T]( protected final def getTaskBytes: Array[Byte] = { val bytes = taskDataBytes - if ((bytes ne null) && bytes.length > 0) bytes else taskBinary.value + if ((bytes ne null) && bytes.length > 0) bytes else taskBinary.get.value } /** @@ -125,16 +119,15 @@ private[spark] abstract class Task[T]( kill(interruptThread = false, _reasonIfKilled) } - new CallerContext( - "TASK", - SparkEnv.get.conf.get(APP_CALLER_CONTEXT), - _appId, - _appAttemptId, - Option(jobId), - Option(stageId), - Option(stageAttemptId), - Option(taskAttemptId), - Option(attemptNumber)).setCurrentContext() + new CallerContext("TASK", + appId = _appId, + appAttemptId = _appAttemptId, + jobId = Some(jobId), + stageId = Some(stageId), + stageAttemptId = Some(stageAttemptId), + taskId = Some(taskAttemptId), + taskAttemptNumber = Some(attemptNumber)) + .setCurrentContext() try { runTask(context) @@ -252,7 +245,7 @@ private[spark] abstract class Task[T]( output.writeInt(_stageId) output.writeVarInt(_stageAttemptId, true) output.writeVarInt(_partitionId, true) - output.writeVarInt(jobId, true) + output.writeVarInt(_jobId, true) output.writeLong(epoch) output.writeLong(_executorDeserializeTime) output.writeLong(_executorDeserializeCpuTime) @@ -261,8 +254,9 @@ private[spark] abstract class Task[T]( output.writeBoolean(true) } else { output.writeBoolean(false) - kryo.writeClassAndObject(output, taskBinary) + kryo.writeClassAndObject(output, taskBinary.get) } + _metrics.write(kryo, output) output.writeString(appId) output.writeString(appAttemptId) } @@ -271,18 +265,20 @@ private[spark] abstract class Task[T]( _stageId = input.readInt() _stageAttemptId = input.readVarInt(true) _partitionId = input.readVarInt(true) - jobId = input.readVarInt(true) + _jobId = input.readVarInt(true) epoch = input.readLong() _executorDeserializeTime = input.readLong() _executorDeserializeCpuTime = input.readLong() // actual bytes are shipped in TaskDescription taskData = TaskData.EMPTY if (input.readBoolean()) { - taskBinary = null + taskBinary = None } else { - taskBinary = kryo.readClassAndObject(input) - .asInstanceOf[Broadcast[Array[Byte]]] + taskBinary = Some(kryo.readClassAndObject(input) + .asInstanceOf[Broadcast[Array[Byte]]]) } + _metrics = new TaskMetrics + _metrics.read(kryo, input) _appId = Option(input.readString()) _appAttemptId = Option(input.readString()) } @@ -330,9 +326,18 @@ private[spark] object Task { } // Write the task properties separately so it is available before full task deserialization. - val propBytes = Utils.serialize(task.localProperties) - dataOut.writeInt(propBytes.length) - dataOut.write(propBytes) + val props = task.localProperties + val numProps = props.size() + + dataOut.writeInt(numProps) + if (numProps > 0) { + val keys = props.keys() + while (keys.hasMoreElements) { + val key = keys.nextElement().asInstanceOf[String] + dataOut.writeUTF(key) + dataOut.writeUTF(props.getProperty(key)) + } + } // Write the task itself and finish dataOut.flush() @@ -368,11 +373,14 @@ private[spark] object Task { for (i <- 0 until numJars) { taskJars(dataIn.readUTF()) = dataIn.readLong() } + val taskProps = new Properties + var numProps = dataIn.readInt() - val propLength = dataIn.readInt() - val propBytes = new Array[Byte](propLength) - dataIn.readFully(propBytes, 0, propLength) - val taskProps = Utils.deserialize[Properties](propBytes) + while (numProps > 0) { + val key = dataIn.readUTF() + taskProps.setProperty(key, dataIn.readUTF()) + numProps -= 1 + } // Create a sub-buffer for the rest of the data, which is the serialized Task object val subBuffer = serializedTask.slice() // ByteBufferInputStream will have read just up to task diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 91f8b806ae892..36ef79540bb60 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -496,7 +496,7 @@ private[spark] class TaskSetManager( } // Serialize and return the task val serializedTask: ByteBuffer = try { - ser.serialize(task) + Task.serializeWithDependencies(task, sched.sc.addedFiles, sched.sc.addedJars, ser) } catch { // If the task cannot be serialized, then there's no point to re-attempt the task, // as it will always fail. So just abort the whole task-set. @@ -506,11 +506,11 @@ private[spark] class TaskSetManager( abort(s"$msg Exception during serialization: $e") throw new TaskNotSerializableException(e) } - if (serializedTask.limit() > TaskSetManager.TASK_SIZE_TO_WARN_KB * 1024 && + if (serializedTask.limit > TaskSetManager.TASK_SIZE_TO_WARN_KB * 1024 && !emittedTaskSizeWarning) { emittedTaskSizeWarning = true logWarning(s"Stage ${task.stageId} contains a task of very large size " + - s"(${serializedTask.limit() / 1024} KB). The maximum recommended task size is " + + s"(${serializedTask.limit / 1024} KB). The maximum recommended task size is " + s"${TaskSetManager.TASK_SIZE_TO_WARN_KB} KB.") } addRunningTask(taskId) @@ -520,19 +520,12 @@ private[spark] class TaskSetManager( // val timeTaken = clock.getTime() - startTime val taskName = s"task ${info.id} in stage ${taskSet.id}" logInfo(s"Starting $taskName (TID $taskId, $host, executor ${info.executorId}, " + - s"partition ${task.partitionId}, $taskLocality, ${serializedTask.limit()} bytes)") + s"partition ${task.partitionId}, $taskLocality, ${serializedTask.limit} bytes)") sched.dagScheduler.taskStarted(task, info) - new TaskDescription( - taskId, - attemptNum, - execId, - taskName, - index, - addedFiles, - addedJars, - task.localProperties, - serializedTask) + new TaskDescription(_taskId = taskId, _attemptNumber = attemptNum, execId, + taskName, index, addedFiles, addedJars, + task.localProperties, serializedTask, task.taskData) } } else { None @@ -1004,8 +997,7 @@ private[spark] class TaskSetManager( } private def getLocalityWait(level: TaskLocality.TaskLocality): Long = { - val defaultWait = conf.get(config.LOCALITY_WAIT) - // val defaultWait = conf.get("spark.locality.wait", "3s") + val defaultWait = conf.get("spark.locality.wait", "3s") val localityWaitKey = level match { case TaskLocality.PROCESS_LOCAL => "spark.locality.wait.process" case TaskLocality.NODE_LOCAL => "spark.locality.wait.node" diff --git a/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala index 4c614c5c0f602..83198c0d7c10d 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala @@ -84,7 +84,8 @@ private[spark] class LocalEndpoint( val offers = IndexedSeq(new WorkerOffer(localExecutorId, localExecutorHostname, freeCores)) for (task <- scheduler.resourceOffers(offers).flatten) { freeCores -= scheduler.CPUS_PER_TASK - executor.launchTask(executorBackend, task) + executor.launchTask(executorBackend, taskId = task.taskId, attemptNumber = task.attemptNumber, + task.name, task.serializedTask, task.taskData.decompress()) } } } diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala index 6f83a59631e07..ba3af24750456 100644 --- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala +++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala @@ -103,7 +103,7 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug try { executor = new Executor("id", "localhost", env, userClassPath = Nil, isLocal = true) // the task will be launched in a dedicated worker thread - executor.launchTask(mockExecutorBackend, taskDescription) + executor.launchTask(mockExecutorBackend, 0, 0, "", serializedTask, (null, 0L)) if (!executorSuiteHelper.latch1.await(5, TimeUnit.SECONDS)) { fail("executor did not send first status update in time") @@ -144,13 +144,12 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - taskBinary = taskBinary, + _taskBinary = Some(taskBinary), partition = secondRDD.partitions(0), locs = Seq(), _outputId = 0, localProperties = new Properties(), - serializedTaskMetrics = serializedTaskMetrics) - + metrics = null) val serTask = serializer.serialize(task) val taskDescription = createFakeTaskDescription(serTask) @@ -188,12 +187,12 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - taskBinary = taskBinary, + _taskBinary = Some(taskBinary), partition = secondRDD.partitions(0), locs = Seq(), _outputId = 0, localProperties = new Properties(), - serializedTaskMetrics = serializedTaskMetrics + metrics = null ) val serTask = serializer.serialize(task) @@ -268,7 +267,8 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug executor = new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true, uncaughtExceptionHandler = mockUncaughtExceptionHandler) // the task will be launched in a dedicated worker thread - executor.launchTask(mockBackend, taskDescription) + executor.launchTask(mockBackend, 0, 0, "", taskDescription.serializedTask, (null, 0L)) + eventually(timeout(5.seconds), interval(10.milliseconds)) { assert(executor.numRunningTasks === 0) } diff --git a/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala b/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala index 841eeff228b7b..9d4a53f6ad571 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala @@ -19,17 +19,14 @@ package org.apache.spark.scheduler import java.util.Properties -import org.apache.spark.{Partition, SparkEnv, TaskContext} -import org.apache.spark.executor.TaskMetrics +import org.apache.spark.{Partition, TaskContext} class FakeTask( stageId: Int, partitionId: Int, - prefLocs: Seq[TaskLocation] = Nil, - serializedTaskMetrics: Array[Byte] = - SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array()) + prefLocs: Seq[TaskLocation] = Nil) extends Task[Int](stageId, 0, partitionId, TaskData.EMPTY, - null, new Properties, serializedTaskMetrics) { + null, null, new Properties) { override def runTask(context: TaskContext): Int = 0 override def preferredLocations: Seq[TaskLocation] = prefLocs @@ -74,8 +71,8 @@ object FakeTask { null, new Partition {override def index: Int = i}, prefLocs(i), - new Properties, - SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array()) + null, + new Properties) } new TaskSet(tasks, stageId, stageAttemptId, priority = 0, null) } diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala index e63bba6d983d0..24001da9d67ed 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala @@ -69,12 +69,12 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - taskBinary = taskBinary, + _taskBinary = Some(taskBinary), partition = rdd.partitions(0), locs = Seq(), _outputId = 0, - localProperties = new Properties(), - serializedTaskMetrics = null) + metrics = null, + localProperties = new Properties()) intercept[RuntimeException] { task.run(0, 0, null) } @@ -98,12 +98,12 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - taskBinary = taskBinary, + _taskBinary = Some(taskBinary), partition = rdd.partitions(0), locs = Seq(), _outputId = 0, - localProperties = new Properties(), - serializedTaskMetrics = null) + metrics = null, + localProperties = new Properties()) intercept[RuntimeException] { task.run(0, 0, null) } diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala index 20aa012b1f438..6f4fdd5bb2039 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala @@ -716,7 +716,7 @@ class TaskSetManagerSuite extends SparkFunSuite with LocalSparkContext with Logg val singleTask = new ShuffleMapTask(0, 0, TaskData.EMPTY, null, new Partition { override def index: Int = 0 - }, Seq(TaskLocation("host1", "execA")), new Properties, null) + }, Seq(TaskLocation("host1", "execA")), null, new Properties) val taskSet = new TaskSet(Array(singleTask), 0, 0, 0, null) val manager = new TaskSetManager(sched, taskSet, MAX_TASK_FAILURES) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala index 36ffa8dcdd2b6..a3e1f10c202f5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala @@ -111,7 +111,6 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro s""" // Remember the current cursor so that we can calculate how many bytes are // written later. - final int $tmpCursor = $bufferHolder.cursor; ${writeStructToBuffer(ctx, input.value, t.map(_.dataType), bufferHolder)} $rowWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index b3f50fc56bdbc..fa1a824ff88fe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -33,6 +33,7 @@ * LICENSE file. */ + package org.apache.spark.sql.catalyst.plans.physical import org.apache.spark.sql.catalyst.expressions._ @@ -48,18 +49,43 @@ import org.apache.spark.sql.types.{DataType, IntegerType} * - Intra-partition ordering of data: In this case the distribution describes guarantees made * about how tuples are distributed within a single partition. */ -sealed trait Distribution +sealed trait Distribution { + /** + * The required number of partitions for this distribution. If it's None, then any number of + * partitions is allowed for this distribution. + */ + def requiredNumPartitions: Option[Int] + + /** + * Creates a default partitioning for this distribution, which can satisfy this distribution while + * matching the given number of partitions. + */ + def createPartitioning(numPartitions: Int): Partitioning +} /** * Represents a distribution where no promises are made about co-location of data. */ -case object UnspecifiedDistribution extends Distribution +case object UnspecifiedDistribution extends Distribution { + override def requiredNumPartitions: Option[Int] = None + + override def createPartitioning(numPartitions: Int): Partitioning = { + throw new IllegalStateException("UnspecifiedDistribution does not have default partitioning.") + } +} /** * Represents a distribution that only has a single partition and all tuples of the dataset * are co-located. */ -case object AllTuples extends Distribution +case object AllTuples extends Distribution { + override def requiredNumPartitions: Option[Int] = Some(1) + + override def createPartitioning(numPartitions: Int): Partitioning = { + assert(numPartitions == 1, "The default partitioning of AllTuples can only have 1 partition.") + SinglePartition + } +} /** * Represents data where tuples that share the same values for the `clustering` @@ -69,12 +95,41 @@ case object AllTuples extends Distribution */ case class ClusteredDistribution( clustering: Seq[Expression], - numPartitions: Option[Int] = None) extends Distribution { + requiredNumPartitions: Option[Int] = None) extends Distribution { require( clustering != Nil, "The clustering expressions of a ClusteredDistribution should not be Nil. " + "An AllTuples should be used to represent a distribution that only has " + "a single partition.") + + override def createPartitioning(numPartitions: Int): Partitioning = { + assert(requiredNumPartitions.isEmpty || requiredNumPartitions.get == numPartitions, + s"This ClusteredDistribution requires ${requiredNumPartitions.get} partitions, but " + + s"the actual number of partitions is $numPartitions.") + HashPartitioning(clustering, numPartitions) + } +} + +/** + * Represents data where tuples have been clustered according to the hash of the given + * `expressions`. The hash function is defined as `HashPartitioning.partitionIdExpression`, so only + * [[HashPartitioning]] can satisfy this distribution. + * + * This is a strictly stronger guarantee than [[ClusteredDistribution]]. Given a tuple and the + * number of partitions, this distribution strictly requires which partition the tuple should be in. + */ +case class HashClusteredDistribution(expressions: Seq[Expression]) extends Distribution { + require( + expressions != Nil, + "The expressions for hash of a HashPartitionedDistribution should not be Nil. " + + "An AllTuples should be used to represent a distribution that only has " + + "a single partition.") + + override def requiredNumPartitions: Option[Int] = None + + override def createPartitioning(numPartitions: Int): Partitioning = { + HashPartitioning(expressions, numPartitions) + } } /** @@ -91,46 +146,31 @@ case class OrderedDistribution(ordering: Seq[SortOrder]) extends Distribution { "An AllTuples should be used to represent a distribution that only has " + "a single partition.") - // TODO: This is not really valid... - def clustering: Set[Expression] = ordering.map(_.child).toSet + override def requiredNumPartitions: Option[Int] = None + + override def createPartitioning(numPartitions: Int): Partitioning = { + RangePartitioning(ordering, numPartitions) + } } /** * Represents data where tuples are broadcasted to every node. It is quite common that the * entire set of tuples is transformed into different data structure. */ -case class BroadcastDistribution(mode: BroadcastMode) extends Distribution +case class BroadcastDistribution(mode: BroadcastMode) extends Distribution { + override def requiredNumPartitions: Option[Int] = Some(1) + + override def createPartitioning(numPartitions: Int): Partitioning = { + assert(numPartitions == 1, + "The default partitioning of BroadcastDistribution can only have 1 partition.") + BroadcastPartitioning(mode) + } +} /** - * Describes how an operator's output is split across partitions. The `compatibleWith`, - * `guarantees`, and `satisfies` methods describe relationships between child partitionings, - * target partitionings, and [[Distribution]]s. These relations are described more precisely in - * their individual method docs, but at a high level: - * - * - `satisfies` is a relationship between partitionings and distributions. - * - `compatibleWith` is relationships between an operator's child output partitionings. - * - `guarantees` is a relationship between a child's existing output partitioning and a target - * output partitioning. - * - * Diagrammatically: - * - * +--------------+ - * | Distribution | - * +--------------+ - * ^ - * | - * satisfies - * | - * +--------------+ +--------------+ - * | Child | | Target | - * +----| Partitioning |----guarantees--->| Partitioning | - * | +--------------+ +--------------+ - * | ^ - * | | - * | compatibleWith - * | | - * +------------+ - * + * Describes how an operator's output is split across partitions. It has 2 major properties: + * 1. number of partitions. + * 2. if it can satisfy a given distribution. */ trait Partitioning { /** Returns the number of partitions that the data is split across */ @@ -141,146 +181,70 @@ trait Partitioning { * to satisfy the partitioning scheme mandated by the `required` [[Distribution]], * i.e. the current dataset does not need to be re-partitioned for the `required` * Distribution (it is possible that tuples within a partition need to be reorganized). - */ - def satisfies(required: Distribution): Boolean - - def compatibleWith(other: Partitioning): Boolean - - /** - * Returns true iff we can say that the partitioning scheme of this [[Partitioning]] guarantees - * the same partitioning scheme described by `other`. If a `A.guarantees(B)`, then repartitioning - * the child's output according to `B` will be unnecessary. `guarantees` is used as a performance - * optimization to allow the exchange planner to avoid redundant repartitionings. By default, - * a partitioning only guarantees partitionings that are equal to itself (i.e. the same number - * of partitions, same strategy (range or hash), etc). * - * In order to enable more aggressive optimization, this strict equality check can be relaxed. - * For example, say that the planner needs to repartition all of an operator's children so that - * they satisfy the [[AllTuples]] distribution. One way to do this is to repartition all children - * to have the [[SinglePartition]] partitioning. If one of the operator's children already happens - * to be hash-partitioned with a single partition then we do not need to re-shuffle this child; - * this repartitioning can be avoided if a single-partition [[HashPartitioning]] `guarantees` - * [[SinglePartition]]. - * - * The SinglePartition example given above is not particularly interesting; guarantees' real - * value occurs for more advanced partitioning strategies. SPARK-7871 will introduce a notion - * of null-safe partitionings, under which partitionings can specify whether rows whosespark.sql.autoBroadcastJoinThreshold - * partitioning keys contain null values will be grouped into the same partition or whether they - * will have an unknown / random distribution. If a partitioning does not require nulls to be - * clustered then a partitioning which _does_ cluster nulls will guarantee the null clustered - * partitioning. The converse is not true, however: a partitioning which clusters nulls cannot - * be guaranteed by one which does not cluster them. Thus, in general `guarantees` is not a - * symmetric relation. - * - * Another way to think about `guarantees`: if `A.guarantees(B)`, then any partitioning of rows - * produced by `A` could have also been produced by `B`. + * By default a [[Partitioning]] can satisfy [[UnspecifiedDistribution]], and [[AllTuples]] if + * the [[Partitioning]] only have one partition. Implementations can overwrite this method with + * special logic. */ - def guarantees(other: Partitioning): Boolean = this == other -} - -object Partitioning { - def allCompatible(partitionings: Seq[Partitioning]): Boolean = { - // Note: this assumes transitivity - partitionings.sliding(2).map { - case Seq(a) => true - case Seq(a, b) => - if (a.numPartitions != b.numPartitions) { - assert(!a.compatibleWith(b) && !b.compatibleWith(a)) - false - } else { - a.compatibleWith(b) && b.compatibleWith(a) - } - }.forall(_ == true) - } -} - -case class UnknownPartitioning(numPartitions: Int) extends Partitioning { - override def satisfies(required: Distribution): Boolean = required match { + def satisfies(required: Distribution): Boolean = required match { case UnspecifiedDistribution => true + case AllTuples => numPartitions == 1 case _ => false } - - override def compatibleWith(other: Partitioning): Boolean = false - - override def guarantees(other: Partitioning): Boolean = false } +case class UnknownPartitioning(numPartitions: Int) extends Partitioning + /** * Represents a partitioning where rows are distributed evenly across output partitions * by starting from a random target partition number and distributing rows in a round-robin * fashion. This partitioning is used when implementing the DataFrame.repartition() operator. */ -case class RoundRobinPartitioning(numPartitions: Int) extends Partitioning { - override def satisfies(required: Distribution): Boolean = required match { - case UnspecifiedDistribution => true - case _ => false - } - - override def compatibleWith(other: Partitioning): Boolean = false - - override def guarantees(other: Partitioning): Boolean = false -} +case class RoundRobinPartitioning(numPartitions: Int) extends Partitioning case object SinglePartition extends Partitioning { val numPartitions = 1 override def satisfies(required: Distribution): Boolean = required match { case _: BroadcastDistribution => false - case ClusteredDistribution(_, desiredPartitions) => desiredPartitions.forall(_ == 1) + case ClusteredDistribution(_, Some(requiredNumPartitions)) => requiredNumPartitions == 1 case _ => true } - - override def compatibleWith(other: Partitioning): Boolean = other.numPartitions == 1 - - override def guarantees(other: Partitioning): Boolean = other.numPartitions == 1 } + /** - * Represents a partitioning where rows are split up across partitions based on the hash - * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be - * in the same partition. Moreover while evaluating expressions if they are given in different order - * than this partitioning then also it is considered equal. - */ + * Represents a partitioning where rows are split up across partitions based on the hash + * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be + * in the same partition. Moreover while evaluating expressions if they are given in different order + * than this partitioning then also it is considered equal. + */ case class OrderlessHashPartitioning(expressions: Seq[Expression], - aliases: Seq[Seq[Attribute]], numPartitions: Int, numBuckets: Int, tableBuckets: Int) - extends Expression with Partitioning with Unevaluable { + aliases: Seq[Seq[Attribute]], + numPartitions: Int, + numBuckets: Int, + tableBuckets: Int) + extends Expression with Partitioning with Unevaluable { override def children: Seq[Expression] = expressions override def nullable: Boolean = false override def dataType: DataType = IntegerType - private def matchExpressions(otherExpression: Seq[Expression]): Boolean = { - expressions.length == otherExpression.length && - expressions.zipWithIndex.forall { case (a, i) => - otherExpression.exists(e => e.semanticEquals(a) || - (aliases.nonEmpty && aliases(i).exists(a2 => e.semanticEquals(a2)))) - } - } - - override def satisfies(required: Distribution): Boolean = required match { - case UnspecifiedDistribution => true - case ClusteredDistribution(requiredClustering, _) => - matchExpressions(requiredClustering) - case _ => false - } - - private def anyOrderEquals(other: HashPartitioning) : Boolean = { - other.numBuckets == this.numBuckets && - other.numPartitions == this.numPartitions && - matchExpressions(other.expressions) - } - - override def compatibleWith(other: Partitioning): Boolean = other match { - case p: HashPartitioning => anyOrderEquals(p) - case _ => false - } - - override def guarantees(other: Partitioning): Boolean = other match { - case p: HashPartitioning => anyOrderEquals(p) - case _ => false + override def satisfies(required: Distribution): Boolean = { + super.satisfies(required) || { + required match { + case h: HashClusteredDistribution => + expressions.length == h.expressions.length && expressions.zip(h.expressions).forall { + case (l, r) => l.semanticEquals(r) + } + case ClusteredDistribution(requiredClustering, requiredNumPartitions) => + expressions.forall(x => requiredClustering.exists(_.semanticEquals(x))) && + (requiredNumPartitions.isEmpty || requiredNumPartitions.get == numPartitions) + case _ => false + } + } } } - /** * Represents a partitioning where rows are split up across partitions based on the hash * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be @@ -289,30 +253,23 @@ case class OrderlessHashPartitioning(expressions: Seq[Expression], case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int) extends Expression with Partitioning with Unevaluable { - private[sql] var numBuckets: Int = 0 - override def children: Seq[Expression] = expressions override def nullable: Boolean = false override def dataType: DataType = IntegerType - override def satisfies(required: Distribution): Boolean = required match { - case UnspecifiedDistribution => true - case ClusteredDistribution(requiredClustering, desiredPartitions) => - expressions.forall(x => requiredClustering.exists(_.semanticEquals(x))) && - desiredPartitions.forall(_ == numPartitions) // if desiredPartitions = None, returns true - case _ => false - } - - override def compatibleWith(other: Partitioning): Boolean = other match { - case o: HashPartitioning => - this.numBuckets == o.numBuckets && this.semanticEquals(o) - case _ => false - } - - override def guarantees(other: Partitioning): Boolean = other match { - case o: HashPartitioning => - this.numBuckets == o.numBuckets && this.semanticEquals(o) - case _ => false + override def satisfies(required: Distribution): Boolean = { + super.satisfies(required) || { + required match { + case h: HashClusteredDistribution => + expressions.length == h.expressions.length && expressions.zip(h.expressions).forall { + case (l, r) => l.semanticEquals(r) + } + case ClusteredDistribution(requiredClustering, requiredNumPartitions) => + expressions.forall(x => requiredClustering.exists(_.semanticEquals(x))) && + (requiredNumPartitions.isEmpty || requiredNumPartitions.get == numPartitions) + case _ => false + } + } } /** @@ -322,16 +279,6 @@ case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int) def partitionIdExpression: Expression = Pmod(new Murmur3Hash(expressions), Literal(numPartitions)) } -object HashPartitioning { - - def apply(expressions: Seq[Expression], numPartitions: Int, - numBuckets: Int): HashPartitioning = { - val partitioning = HashPartitioning(expressions, numPartitions) - partitioning.numBuckets = numBuckets - partitioning - } -} - /** * Represents a partitioning where rows are split across partitions based on some total ordering of * the expressions specified in `ordering`. When data is partitioned in this manner the following @@ -351,25 +298,18 @@ case class RangePartitioning(ordering: Seq[SortOrder], numPartitions: Int) override def nullable: Boolean = false override def dataType: DataType = IntegerType - override def satisfies(required: Distribution): Boolean = required match { - case UnspecifiedDistribution => true - case OrderedDistribution(requiredOrdering) => - val minSize = Seq(requiredOrdering.size, ordering.size).min - requiredOrdering.take(minSize) == ordering.take(minSize) - case ClusteredDistribution(requiredClustering, desiredPartitions) => - ordering.map(_.child).forall(x => requiredClustering.exists(_.semanticEquals(x))) && - desiredPartitions.forall(_ == numPartitions) // if desiredPartitions = None, returns true - case _ => false - } - - override def compatibleWith(other: Partitioning): Boolean = other match { - case o: RangePartitioning => this.semanticEquals(o) - case _ => false - } - - override def guarantees(other: Partitioning): Boolean = other match { - case o: RangePartitioning => this.semanticEquals(o) - case _ => false + override def satisfies(required: Distribution): Boolean = { + super.satisfies(required) || { + required match { + case OrderedDistribution(requiredOrdering) => + val minSize = Seq(requiredOrdering.size, ordering.size).min + requiredOrdering.take(minSize) == ordering.take(minSize) + case ClusteredDistribution(requiredClustering, requiredNumPartitions) => + ordering.map(_.child).forall(x => requiredClustering.exists(_.semanticEquals(x))) && + (requiredNumPartitions.isEmpty || requiredNumPartitions.get == numPartitions) + case _ => false + } + } } } @@ -410,20 +350,6 @@ case class PartitioningCollection(partitionings: Seq[Partitioning]) override def satisfies(required: Distribution): Boolean = partitionings.exists(_.satisfies(required)) - /** - * Returns true if any `partitioning` of this collection is compatible with - * the given [[Partitioning]]. - */ - override def compatibleWith(other: Partitioning): Boolean = - partitionings.exists(_.compatibleWith(other)) - - /** - * Returns true if any `partitioning` of this collection guarantees - * the given [[Partitioning]]. - */ - override def guarantees(other: Partitioning): Boolean = - partitionings.exists(_.guarantees(other)) - override def toString: String = { partitionings.map(_.toString).mkString("(", " or ", ")") } @@ -440,9 +366,4 @@ case class BroadcastPartitioning(mode: BroadcastMode) extends Partitioning { case BroadcastDistribution(m) if m == mode => true case _ => false } - - override def compatibleWith(other: Partitioning): Boolean = other match { - case BroadcastPartitioning(m) if m == mode => true - case _ => false - } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourcePartitioning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourcePartitioning.scala index 6a31b1c8ed4a8..aae074cd0d754 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourcePartitioning.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourcePartitioning.scala @@ -53,6 +53,4 @@ class DataSourcePartitioning( case _ => false } } - - override def compatibleWith(other: physical.Partitioning): Boolean = false } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala index 20146cfad4610..e3d28388c5470 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala @@ -46,23 +46,6 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { if (minNumPostShufflePartitions > 0) Some(minNumPostShufflePartitions) else None } - /** - * Given a required distribution, returns a partitioning that satisfies that distribution. - * @param requiredDistribution The distribution that is required by the operator - * @param numPartitions Used when the distribution doesn't require a specific number of partitions - */ - private def createPartitioning( - requiredDistribution: Distribution, - numPartitions: Int, numBuckets: Int = 0): Partitioning = { - requiredDistribution match { - case AllTuples => SinglePartition - case ClusteredDistribution(clustering, desiredPartitions) => - HashPartitioning(clustering, desiredPartitions.getOrElse(numPartitions), numBuckets) - case OrderedDistribution(ordering) => RangePartitioning(ordering, numPartitions) - case dist => sys.error(s"Do not know how to satisfy distribution $dist") - } - } - /** * Adds [[ExchangeCoordinator]] to [[ShuffleExchangeExec]]s if adaptive query execution is enabled * and partitioning schemes of these [[ShuffleExchangeExec]]s support [[ExchangeCoordinator]]. @@ -88,8 +71,9 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { // shuffle data when we have more than one children because data generated by // these children may not be partitioned in the same way. // Please see the comment in withCoordinator for more details. - val supportsDistribution = - requiredChildDistributions.forall(_.isInstanceOf[ClusteredDistribution]) + val supportsDistribution = requiredChildDistributions.forall { dist => + dist.isInstanceOf[ClusteredDistribution] || dist.isInstanceOf[HashClusteredDistribution] + } children.length > 1 && supportsDistribution } @@ -142,8 +126,7 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { // // It will be great to introduce a new Partitioning to represent the post-shuffle // partitions when one post-shuffle partition includes multiple pre-shuffle partitions. - val targetPartitioning = - createPartitioning(distribution, defaultNumPreShufflePartitions) + val targetPartitioning = distribution.createPartitioning(defaultNumPreShufflePartitions) assert(targetPartitioning.isInstanceOf[HashPartitioning]) ShuffleExchangeExec(targetPartitioning, child, Some(coordinator)) } @@ -162,82 +145,56 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { assert(requiredChildDistributions.length == children.length) assert(requiredChildOrderings.length == children.length) - // Ensure that the operator's children satisfy their output distribution requirements: - // The second boolean parameter in the result is true when a ShuffleExchange - // was introduced to satisfy the output distribution. - val newChildren = children.zip(requiredChildDistributions).map { + // Ensure that the operator's children satisfy their output distribution requirements. + children = children.zip(requiredChildDistributions).map { case (child, distribution) if child.outputPartitioning.satisfies(distribution) => - (child, false) + child case (child, BroadcastDistribution(mode)) => - (BroadcastExchangeExec(mode, child), false) + BroadcastExchangeExec(mode, child) case (child, distribution) => - (ShuffleExchangeExec(createPartitioning(distribution, - defaultNumPreShufflePartitions), child), true) + val numPartitions = distribution.requiredNumPartitions + .getOrElse(defaultNumPreShufflePartitions) + ShuffleExchangeExec(distribution.createPartitioning(numPartitions), child) } - children = newChildren.map(_._1) - // If the operator has multiple children and specifies child output distributions (e.g. join), - // then the children's output partitionings must be compatible: - def requireCompatiblePartitioning(distribution: Distribution): Boolean = distribution match { - case UnspecifiedDistribution => false - case BroadcastDistribution(_) => false + // Get the indexes of children which have specified distribution requirements and need to have + // same number of partitions. + val childrenIndexes = requiredChildDistributions.zipWithIndex.filter { + case (UnspecifiedDistribution, _) => false + case (_: BroadcastDistribution, _) => false case _ => true - } - if (children.length > 1 - && requiredChildDistributions.exists(requireCompatiblePartitioning) - && !Partitioning.allCompatible(children.map(_.outputPartitioning))) { + }.map(_._2) + + val childrenNumPartitions = + childrenIndexes.map(children(_).outputPartitioning.numPartitions).toSet - // First check if the existing partitions of the children all match. This means they are - // partitioned by the same partitioning into the same number of partitions. In that case, - // don't try to make them match `defaultPartitions`, just use the existing partitioning. - val maxChildrenNumPartitions = math.abs(newChildren.map { - case (child, false) => child.outputPartitioning.numPartitions - case (child, true) => -child.outputPartitioning.numPartitions - }.max) - val numBuckets = children.map(_.outputPartitioning match { - case p: OrderlessHashPartitioning => p.numBuckets - case _ => 0 - }).max - val useExistingPartitioning = children.zip(requiredChildDistributions).forall { - case (child, distribution) => - child.outputPartitioning.guarantees( - createPartitioning(distribution, maxChildrenNumPartitions, numBuckets)) + if (childrenNumPartitions.size > 1) { + // Get the number of partitions which is explicitly required by the distributions. + val requiredNumPartitions = { + val numPartitionsSet = childrenIndexes.flatMap { + index => requiredChildDistributions(index).requiredNumPartitions + }.toSet + assert(numPartitionsSet.size <= 1, + s"$operator have incompatible requirements of the number of partitions for its children") + numPartitionsSet.headOption } - children = if (useExistingPartitioning) { - // We do not need to shuffle any child's output. - children - } else { - // We need to shuffle at least one child's output. - // Now, we will determine the number of partitions that will be used by created - // partitioning schemes. - val numPartitions = { - // Let's see if we need to shuffle all child's outputs when we use - // maxChildrenNumPartitions. - val shufflesAllChildren = children.zip(requiredChildDistributions).forall { - case (child, distribution) => - !child.outputPartitioning.guarantees( - createPartitioning(distribution, maxChildrenNumPartitions)) - } - // If we need to shuffle all children, we use defaultNumPreShufflePartitions as the - // number of partitions. Otherwise, we use maxChildrenNumPartitions. - if (shufflesAllChildren) defaultNumPreShufflePartitions else maxChildrenNumPartitions - } + val targetNumPartitions = requiredNumPartitions.getOrElse(childrenNumPartitions.max) - children.zip(requiredChildDistributions).map { - case (child, distribution) => - val targetPartitioning = createPartitioning(distribution, numPartitions) - if (child.outputPartitioning.guarantees(targetPartitioning)) { - child - } else { - child match { - // If child is an exchange, we replace it with - // a new one having targetPartitioning. - case ShuffleExchangeExec(_, c, _) => ShuffleExchangeExec(targetPartitioning, c) - case _ => ShuffleExchangeExec(targetPartitioning, child) - } + children = children.zip(requiredChildDistributions).zipWithIndex.map { + case ((child, distribution), index) if childrenIndexes.contains(index) => + if (child.outputPartitioning.numPartitions == targetNumPartitions) { + child + } else { + val defaultPartitioning = distribution.createPartitioning(targetNumPartitions) + child match { + // If child is an exchange, we replace it with a new one having defaultPartitioning. + case ShuffleExchangeExec(_, c, _) => ShuffleExchangeExec(defaultPartitioning, c) + case _ => ShuffleExchangeExec(defaultPartitioning, child) + } } - } + + case ((child, _), _) => child } } @@ -335,10 +292,10 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { } def apply(plan: SparkPlan): SparkPlan = plan.transformUp { - case operator @ ShuffleExchangeExec(partitioning, child, _) => - child.children match { - case ShuffleExchangeExec(childPartitioning, baseChild, _)::Nil => - if (childPartitioning.guarantees(partitioning)) child else operator + // TODO: remove this after we create a physical operator for `RepartitionByExpression`. + case operator @ ShuffleExchangeExec(upper: HashPartitioning, child, _) => + child.outputPartitioning match { + case lower: HashPartitioning if upper.semanticEquals(lower) => child case _ => operator } case operator: SparkPlan => From 9552470d18d5ed3b3a51dbc7349dff61fb9ffad1 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Mon, 7 May 2018 17:29:37 -0700 Subject: [PATCH 135/142] Addressing precheckin failures --- .../CoarseGrainedExecutorBackend.scala | 19 +-- .../org/apache/spark/executor/Executor.scala | 131 ++++++++---------- .../apache/spark/scheduler/DAGScheduler.scala | 42 ++---- .../apache/spark/scheduler/ResultTask.scala | 16 +-- .../spark/scheduler/ShuffleMapTask.scala | 16 +-- .../org/apache/spark/scheduler/Task.scala | 84 +++++------ .../spark/scheduler/TaskSetManager.scala | 24 ++-- .../local/LocalSchedulerBackend.scala | 3 +- .../apache/spark/executor/ExecutorSuite.scala | 14 +- .../org/apache/spark/scheduler/FakeTask.scala | 13 +- .../spark/scheduler/TaskContextSuite.scala | 12 +- .../spark/scheduler/TaskSetManagerSuite.scala | 2 +- 12 files changed, 172 insertions(+), 204 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala index 1a0a797ad2f44..591112c84e6ec 100644 --- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala +++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala @@ -84,34 +84,34 @@ private[spark] class CoarseGrainedExecutorBackend( case RegisteredExecutor => logInfo("Successfully registered with driver") try { - executor = registerExecutor + executor = new Executor(executorId, hostname, env, userClassPath, isLocal = false) } catch { case NonFatal(e) => exitExecutor(1, "Unable to create executor due to " + e.getMessage, e) } case RegisterExecutorFailed(message) => - exitExecutor(1, "Slave registration failed: " + message) + logError("Slave registration failed: " + message) + exitExecutor(1, "Slave registration failed") case LaunchTask(taskDesc) => if (executor == null) { + logError("Received LaunchTask command but executor was null") exitExecutor(1, "Received LaunchTask command but executor was null") } else { - env.taskLogger.logInfo("Got assigned task " + taskDesc.taskId) - executor.launchTask(this, taskId = taskDesc.taskId, attemptNumber = taskDesc.attemptNumber, - taskDesc.name, taskDesc.serializedTask, taskDesc.taskData.decompress(env)) + // val taskDesc = TaskDescription.decode(data) + logInfo("Got assigned task " + taskDesc.taskId) + executor.launchTask(this, taskDesc) } case LaunchTasks(tasks, taskDataList) => if (executor ne null) { logDebug("Got assigned tasks " + tasks.map(_.taskId).mkString(",")) for (task <- tasks) { - env.taskLogger.logInfo("Got assigned task " + task.taskId) + logInfo("Got assigned task " + task.taskId) val ref = task.taskData.reference val taskData = if (ref >= 0) taskDataList(ref) else task.taskData - executor.launchTask(this, taskId = task.taskId, - attemptNumber = task.attemptNumber, task.name, task.serializedTask, - taskData.decompress(env)) + executor.launchTask(this, task) } } else { exitExecutor(1, "Received LaunchTasks command but executor was null") @@ -119,6 +119,7 @@ private[spark] class CoarseGrainedExecutorBackend( case KillTask(taskId, _, interruptThread, reason) => if (executor == null) { + logError("Received KillTask command but executor was null") exitExecutor(1, "Received KillTask command but executor was null") } else { executor.killTask(taskId, interruptThread, reason) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 40a92b9424400..f2b0c93acfa19 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -30,16 +30,16 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder import org.apache.spark._ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging -import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager} +import org.apache.spark.memory.TaskMemoryManager import org.apache.spark.rpc.RpcTimeout -import org.apache.spark.scheduler.{DirectTaskResult, IndirectTaskResult, Task} +import org.apache.spark.scheduler.{DirectTaskResult, IndirectTaskResult, Task, TaskDescription} import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.storage.{StorageLevel, TaskResultBlockId} import org.apache.spark.util._ import org.apache.spark.util.io.ChunkedByteBuffer import scala.collection.JavaConverters._ -import scala.collection.mutable.{ArrayBuffer, HashMap} +import scala.collection.mutable.{ArrayBuffer, HashMap, Map} import scala.util.control.NonFatal /** @@ -170,16 +170,9 @@ private[spark] class Executor( private[executor] def numRunningTasks: Int = runningTasks.size() - def launchTask( - context: ExecutorBackend, - taskId: Long, - attemptNumber: Int, - taskName: String, - serializedTask: ByteBuffer, - taskData: (Array[Byte], Long)): Unit = { - val tr = new TaskRunner(context, taskId = taskId, attemptNumber = attemptNumber, - taskName, serializedTask, taskData._1, taskData._2) - runningTasks.put(taskId, tr) + def launchTask(context: ExecutorBackend, taskDescription: TaskDescription): Unit = { + val tr = new TaskRunner(context, taskDescription) + runningTasks.put(taskDescription.taskId, tr) threadPool.execute(tr) } @@ -237,15 +230,12 @@ private[spark] class Executor( class TaskRunner( execBackend: ExecutorBackend, - val taskId: Long, - val attemptNumber: Int, - taskName: String, - serializedTask: ByteBuffer, - taskDataBytes: Array[Byte], - taskDecompressTime: Long) + private val taskDescription: TaskDescription) extends Runnable { + val taskId = taskDescription.taskId val threadName = s"Executor task launch worker for task $taskId" + private val taskName = taskDescription.name /** If specified, this task has been killed and this option contains the reason. */ @volatile private var reasonIfKilled: Option[String] = None @@ -314,18 +304,16 @@ private[spark] class Executor( startGCTime = computeTotalGcTime() try { - val (taskFiles, taskJars, taskProps, taskBytes) = - Task.deserializeWithDependencies(serializedTask) - // Must be set before updateDependencies() is called, in case fetching dependencies // requires access to properties contained within (e.g. for access control). - Executor.taskDeserializationProps.set(taskProps) + Executor.taskDeserializationProps.set(taskDescription.properties) - updateDependencies(taskFiles, taskJars) - task = ser.deserialize[Task[Any]](taskBytes, Thread.currentThread.getContextClassLoader) - task.taskDataBytes = taskDataBytes - task.localProperties = taskProps + updateDependencies(taskDescription.addedFiles, taskDescription.addedJars) + task = ser.deserialize[Task[Any]]( + taskDescription.serializedTask, Thread.currentThread.getContextClassLoader) + task.localProperties = taskDescription.properties task.setTaskMemoryManager(taskMemoryManager) + // If this task has been killed before we deserialized it, let's quit now. Otherwise, // continue executing the task. val killReason = reasonIfKilled @@ -355,7 +343,7 @@ private[spark] class Executor( val value = try { val res = task.run( taskAttemptId = taskId, - attemptNumber = attemptNumber, + attemptNumber = taskDescription.attemptNumber, metricsSystem = env.metricsSystem) threwException = false res @@ -400,15 +388,10 @@ private[spark] class Executor( // If the task has been killed, let's fail it. task.context.killTaskIfInterrupted() - val resultSer = env.serializer.newInstance() - val beforeSerialization = System.currentTimeMillis() - val valueBytes = resultSer.serialize(value) - val afterSerialization = System.currentTimeMillis() - // Deserialization happens in two parts: first, we deserialize a Task object, which // includes the Partition. Second, Task.run() deserializes the RDD and function to be run. task.metrics.setExecutorDeserializeTime(math.max(taskStart - deserializeStartTime + - task.executorDeserializeTime + taskDecompressTime, 0L).toDouble / 1000000.0) + task.executorDeserializeTime /* + taskDecompressTime */, 0L).toDouble / 1000000.0) task.metrics.setExecutorDeserializeCpuTime(math.max(taskStartCpu - deserializeStartCpuTime + task.executorDeserializeCpuTime, 0L).toDouble / 1000000.0) // We need to subtract Task.run()'s deserialization time to avoid double-counting @@ -417,7 +400,6 @@ private[spark] class Executor( task.metrics.setExecutorCpuTime(math.max(taskFinishCpu - taskStartCpu - task.executorDeserializeCpuTime, 0L).toDouble / 1000000.0) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) - task.metrics.setResultSerializationTime(afterSerialization - beforeSerialization) // Expose task metrics using the Dropwizard metrics system. // Update task metrics counters @@ -462,11 +444,10 @@ private[spark] class Executor( // Note: accumulator updates must be collected after TaskMetrics is updated val accumUpdates = task.collectAccumulatorUpdates() - val directResult = new DirectTaskResult(value, accumUpdates, - Some(task.metrics.resultSerializationTimeMetric)) - val taskSer = env.serializer.newInstance() - val serializedDirectResult = taskSer.serialize(directResult) - val resultSize = serializedDirectResult.limit + // TODO: do not serialize value twice + val directResult = new DirectTaskResult(value, accumUpdates) + val serializedDirectResult = ser.serialize(directResult) + val resultSize = serializedDirectResult.limit() // directSend = sending directly back to the driver val serializedResult: ByteBuffer = { @@ -474,7 +455,7 @@ private[spark] class Executor( logWarning(s"Finished $taskName (TID $taskId). Result is larger than maxResultSize " + s"(${Utils.bytesToString(resultSize)} > ${Utils.bytesToString(maxResultSize)}), " + s"dropping it.") - taskSer.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize)) + ser.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize)) } else if (resultSize > maxDirectResultSize) { val blockId = TaskResultBlockId(taskId) env.blockManager.putBytes( @@ -483,7 +464,7 @@ private[spark] class Executor( StorageLevel.MEMORY_AND_DISK_SER) logInfo( s"Finished $taskName (TID $taskId). $resultSize bytes result sent via BlockManager)") - taskSer.serialize(new IndirectTaskResult[Any](blockId, resultSize)) + ser.serialize(new IndirectTaskResult[Any](blockId, resultSize)) } else { logInfo(s"Finished $taskName (TID $taskId). $resultSize bytes result sent to driver") serializedDirectResult @@ -550,43 +531,45 @@ private[spark] class Executor( // the default uncaught exception handler, which will terminate the Executor. logError(s"Exception in $taskName (TID $taskId)", t) - // SPARK-20904: Do not report failure to driver if if happened during shut down. Because - // libraries may set up shutdown hooks that race with running tasks during shutdown, - // spurious failures may occur and can result in improper accounting in the driver (e.g. - // the task failure would not be ignored if the shutdown happened because of premption, - // instead of an app issue). - if (!ShutdownHookManager.inShutdown()) { - // Collect latest accumulator values to report back to the driver - val accums: Seq[AccumulatorV2[_, _]] = - if (task != null) { - task.metrics.setExecutorRunTime(System.currentTimeMillis() - taskStart) - task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) - task.collectAccumulatorUpdates(taskFailed = true) - } else { - Seq.empty - } + // Collect latest accumulator values to report back to the driver + val accums: Seq[AccumulatorV2[_, _]] = + if (task != null) { + task.metrics.setExecutorRunTime( + math.max(System.nanoTime() - taskStart, 0L).toDouble / 1000000.0) + val taskEndCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { + threadMXBean.getCurrentThreadCpuTime + } else 0L + task.metrics.setExecutorCpuTime( + math.max(taskEndCpu - taskStartCpu, 0L).toDouble / 1000000.0) + task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) + task.collectAccumulatorUpdates(taskFailed = true) + } else { + Seq.empty + } - val accUpdates = accums.map(acc => acc.toInfo(Some(acc.value), None)) + val accUpdates = accums.map(acc => acc.toInfo(Some(acc.value), None)) - val serializedTaskEndReason = { - try { - ser.serialize(new ExceptionFailure(t, accUpdates).withAccums(accums)) - } catch { - case _: NotSerializableException => - // t is not serializable so just send the stacktrace - ser.serialize(new ExceptionFailure(t, accUpdates, false).withAccums(accums)) - } + val serializedTaskEndReason = { + try { + ser.serialize(new ExceptionFailure(t, accUpdates).withAccums(accums)) + } catch { + case _: NotSerializableException => + // t is not serializable so just send the stacktrace + ser.serialize(new ExceptionFailure(t, accUpdates, false).withAccums(accums)) } - setTaskFinishedAndClearInterruptStatus() - execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskEndReason) - } else { - logInfo("Not reporting error to driver during JVM shutdown.") } + setTaskFinishedAndClearInterruptStatus() + execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskEndReason) // Don't forcibly exit unless the exception was inherently fatal, to avoid // stopping other tasks unnecessarily. - if (!t.isInstanceOf[SparkOutOfMemoryError] && Utils.isFatalError(t)) { - uncaughtExceptionHandler.uncaughtException(Thread.currentThread(), t) + if (isFatalError(t)) { + if (!isLocal) { + Thread.getDefaultUncaughtExceptionHandler. + uncaughtException(Thread.currentThread(), t) + } else { + // SparkUncaughtExceptionHandler.uncaughtException(t) + } } } finally { runningTasks.remove(taskId) @@ -765,8 +748,8 @@ private[spark] class Executor( * Download any missing dependencies if we receive a new set of files and JARs from the * SparkContext. Also adds any new JARs we fetched to the class loader. */ - protected def updateDependencies(newFiles: HashMap[String, Long], - newJars: HashMap[String, Long]) { + protected def updateDependencies(newFiles: Map[String, Long], + newJars: Map[String, Long]) { lazy val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf) synchronized { // Fetch missing dependencies diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index c6124c12ebef8..e48d874086b2c 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -1016,38 +1016,19 @@ class DAGScheduler( // task gets a different copy of the RDD. This provides stronger isolation between tasks that // might modify state of objects referenced in their closures. This is necessary in Hadoop // where the JobConf/Configuration object is not thread-safe. - var taskBinary: Option[Broadcast[Array[Byte]]] = None - var taskData: TaskData = TaskData.EMPTY + var taskBinary: Broadcast[Array[Byte]] = null try { // For ShuffleMapTask, serialize and broadcast (rdd, shuffleDep). // For ResultTask, serialize and broadcast (rdd, func). - val bytes = stage.taskBinaryBytes - val taskBinaryBytes: Array[Byte] = if (bytes != null) bytes else stage match { + val taskBinaryBytes: Array[Byte] = stage match { case stage: ShuffleMapStage => JavaUtils.bufferToArray( closureSerializer.serialize((stage.rdd, stage.shuffleDep): AnyRef)) case stage: ResultStage => JavaUtils.bufferToArray(closureSerializer.serialize((stage.rdd, stage.func): AnyRef)) } - if (bytes == null) stage.taskBinaryBytes = taskBinaryBytes - - // use direct byte shipping for small size or if number of partitions is small - val taskBytesLen = taskBinaryBytes.length - if (taskBytesLen <= DAGScheduler.TASK_INLINE_LIMIT || - partitionsToCompute.length == 1 || - (taskBytesLen < math.min(maxRpcMessageSize, DAGScheduler.TASK_INLINE_UPPER_LIMIT) && - partitionsToCompute.length <= DAGScheduler.TASK_INLINE_PARTITION_LIMIT)) { - if (stage.taskData.uncompressedLen > 0) { - taskData = stage.taskData - } else { - // compress inline task data (broadcast compresses as per conf) - taskData = new TaskData(env.createCompressionCodec.compress( - taskBinaryBytes, taskBytesLen), taskBytesLen) - stage.taskData = taskData - } - } else { - taskBinary = Some(sc.broadcast(taskBinaryBytes)) - } + + taskBinary = sc.broadcast(taskBinaryBytes) } catch { // In the case of a failure during serialization, abort the stage. case e: NotSerializableException => @@ -1063,14 +1044,17 @@ class DAGScheduler( } val tasks: Seq[Task[_]] = try { + val serializedTaskMetrics = closureSerializer.serialize(stage.latestInfo.taskMetrics).array() stage match { case stage: ShuffleMapStage => + stage.pendingPartitions.clear() partitionsToCompute.map { id => val locs = taskIdToLocations(id) val part = stage.rdd.partitions(id) - new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, taskData, - taskBinary, part, locs, stage.latestInfo.taskMetrics, properties, - jobId, Option(sc.applicationId), Option(sc.applicationId)) + stage.pendingPartitions += id + new ShuffleMapTask(stage.id, stage.latestInfo.attemptId, _taskData = TaskData.EMPTY, + taskBinary, part, locs, properties, serializedTaskMetrics, Option(jobId), + Option(sc.applicationId), sc.applicationAttemptId) } case stage: ResultStage => @@ -1078,9 +1062,9 @@ class DAGScheduler( val p: Int = stage.partitions(id) val part = stage.rdd.partitions(p) val locs = taskIdToLocations(id) - new ResultTask(stage.id, stage.latestInfo.attemptId, taskData, - taskBinary, part, locs, id, properties, stage.latestInfo.taskMetrics, jobId, - Option(sc.applicationId), sc.applicationAttemptId) + new ResultTask(stage.id, stage.latestInfo.attemptId, _taskData = TaskData.EMPTY, + taskBinary, part, locs, id, properties, serializedTaskMetrics, + Option(jobId), Option(sc.applicationId), sc.applicationAttemptId) } } } catch { diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index d3355aa95fe56..d08fbd73ed9b0 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -27,7 +27,6 @@ import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.spark._ import org.apache.spark.broadcast.Broadcast -import org.apache.spark.executor.TaskMetrics import org.apache.spark.rdd.RDD /** @@ -39,7 +38,7 @@ import org.apache.spark.rdd.RDD * @param stageAttemptId attempt id of the stage this task belongs to * @param _taskData if serialized RDD and function are small, then it is compressed * and sent with its original decompressed size - * @param _taskBinary broadcasted version of the serialized RDD and the function to apply on each + * @param taskBinary broadcasted version of the serialized RDD and the function to apply on each * partition of the given RDD. Once deserialized, the type should be * (RDD[T], (TaskContext, Iterator[T]) => U). * @param partition partition of the RDD this task is associated with @@ -47,7 +46,8 @@ import org.apache.spark.rdd.RDD * @param _outputId index of the task in this job (a job can launch tasks on only a subset of the * input RDD's partitions). * @param localProperties copy of thread-local properties set by the user on the driver side. - * @param metrics a `TaskMetrics` that is created at driver side and sent to executor side. + * @param serializedTaskMetrics a `TaskMetrics` that is created and serialized on the driver side + * and sent to executor side. * * The parameters below are optional: * @param jobId id of the job this task belongs to @@ -58,17 +58,17 @@ private[spark] class ResultTask[T, U]( stageId: Int, stageAttemptId: Int, _taskData: TaskData, - _taskBinary: Option[Broadcast[Array[Byte]]], + taskBinary: Broadcast[Array[Byte]], private var partition: Partition, locs: Seq[TaskLocation], private var _outputId: Int, localProperties: Properties, - metrics: TaskMetrics, - jobId: Int = -1, + serializedTaskMetrics: Array[Byte], + jobId: Option[Int] = None, appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[U](stageId, stageAttemptId, partition.index, _taskData, - _taskBinary, metrics, localProperties, jobId, appId, appAttemptId) + taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) with Serializable with KryoSerializable { final def outputId: Int = _outputId @@ -86,7 +86,7 @@ private[spark] class ResultTask[T, U]( } else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, func) = ser.deserialize[(RDD[T], (TaskContext, Iterator[T]) => U)]( - ByteBuffer.wrap(getTaskBytes), Thread.currentThread.getContextClassLoader) + ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = math.max(System.nanoTime() - deserializeStartTime, 0L) _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { math.max(threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime, 0L) diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index 46d3f521f5438..67619b1e889eb 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -28,7 +28,6 @@ import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.spark._ import org.apache.spark.broadcast.Broadcast -import org.apache.spark.executor.TaskMetrics import org.apache.spark.internal.Logging import org.apache.spark.rdd.RDD import org.apache.spark.shuffle.ShuffleWriter @@ -43,11 +42,10 @@ import org.apache.spark.shuffle.ShuffleWriter * @param stageAttemptId attempt id of the stage this task belongs to * @param _taskData if serialized RDD and function are small, then it is compressed * and sent with its original decompressed size - * @param _taskBinary broadcast version of the RDD and the ShuffleDependency. Once deserialized, + * @param taskBinary broadcast version of the RDD and the ShuffleDependency. Once deserialized, * the type should be (RDD[_], ShuffleDependency[_, _, _]). * @param partition partition of the RDD this task is associated with * @param locs preferred task execution locations for locality scheduling - * @param metrics a `TaskMetrics` that is created at driver side and sent to executor side. * @param localProperties copy of thread-local properties set by the user on the driver side. * * The parameters below are optional: @@ -58,23 +56,23 @@ import org.apache.spark.shuffle.ShuffleWriter private[spark] class ShuffleMapTask( stageId: Int, stageAttemptId: Int, - _taskData: TaskData, - _taskBinary: Option[Broadcast[Array[Byte]]], + _taskData: TaskData = TaskData.EMPTY, + taskBinary: Broadcast[Array[Byte]], private var partition: Partition, @transient private var locs: Seq[TaskLocation], - metrics: TaskMetrics, localProperties: Properties, - jobId: Int = -1, + serializedTaskMetrics: Array[Byte], + jobId: Option[Int] = None, appId: Option[String] = None, appAttemptId: Option[String] = None) extends Task[MapStatus](stageId, stageAttemptId, partition.index, _taskData, - _taskBinary, metrics, localProperties, jobId, appId, appAttemptId) + taskBinary, localProperties, serializedTaskMetrics, jobId, appId, appAttemptId) with KryoSerializable with Logging { /** A constructor used only in test suites. This does not require passing in an RDD. */ def this(partitionId: Int) { this(0, 0, TaskData.EMPTY, null, new Partition { override def index: Int = 0 }, - null, null, new Properties) + null, new Properties, null) } @transient private val preferredLocs: Seq[TaskLocation] = { diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index 099c70243b33a..936c7ae434def 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -23,13 +23,12 @@ import java.util.Properties import scala.collection.mutable import scala.collection.mutable.HashMap - import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{Input, Output} - import org.apache.spark._ import org.apache.spark.broadcast.Broadcast import org.apache.spark.executor.TaskMetrics +import org.apache.spark.internal.config.APP_CALLER_CONTEXT import org.apache.spark.memory.{MemoryMode, TaskMemoryManager} import org.apache.spark.metrics.MetricsSystem import org.apache.spark.serializer.SerializerInstance @@ -49,8 +48,10 @@ import org.apache.spark.util._ * @param _stageId id of the stage this task belongs to * @param _stageAttemptId attempt id of the stage this task belongs to * @param _partitionId index of the number in the RDD - * @param _metrics a [[TaskMetrics]] that is created at driver side and sent to executor side. * @param localProperties copy of thread-local properties set by the user on the driver side. + * @param serializedTaskMetrics a `TaskMetrics` that is created and serialized on the driver side + * and sent to executor side. + * * The parameters below are optional: * @param _jobId id of the job this task belongs to * @param _appId id of the app this task belongs to @@ -62,10 +63,12 @@ private[spark] abstract class Task[T]( private var _partitionId: Int, @transient private[spark] var taskData: TaskData = TaskData.EMPTY, // The default value is only used in tests. - protected var taskBinary: Option[Broadcast[Array[Byte]]] = None, - private var _metrics: TaskMetrics = TaskMetrics.registered, + protected var taskBinary: Broadcast[Array[Byte]], @transient var localProperties: Properties = new Properties, - private var _jobId: Int = -1, + // The default value is only used in tests. + serializedTaskMetrics: Array[Byte] = + SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array(), + private var _jobId: Option[Int] = None, private var _appId: Option[String] = None, private var _appAttemptId: Option[String] = None) extends Serializable { @@ -75,9 +78,12 @@ private[spark] abstract class Task[T]( final def partitionId: Int = _partitionId - final def jobId: Int = _jobId + @transient lazy val metrics: TaskMetrics = + SparkEnv.get.closureSerializer.newInstance().deserialize(ByteBuffer.wrap(serializedTaskMetrics)) - final def metrics: TaskMetrics = _metrics + final var jobId: Int = if (_jobId.isDefined) _jobId.get else -1 + + // final def metrics: TaskMetrics = _metrics final def appId: String = if (_appId.isDefined) _appId.get else null @@ -87,7 +93,7 @@ private[spark] abstract class Task[T]( protected final def getTaskBytes: Array[Byte] = { val bytes = taskDataBytes - if ((bytes ne null) && bytes.length > 0) bytes else taskBinary.get.value + if ((bytes ne null) && bytes.length > 0) bytes else taskBinary.value } /** @@ -119,15 +125,16 @@ private[spark] abstract class Task[T]( kill(interruptThread = false, _reasonIfKilled) } - new CallerContext("TASK", - appId = _appId, - appAttemptId = _appAttemptId, - jobId = Some(jobId), - stageId = Some(stageId), - stageAttemptId = Some(stageAttemptId), - taskId = Some(taskAttemptId), - taskAttemptNumber = Some(attemptNumber)) - .setCurrentContext() + new CallerContext( + "TASK", + SparkEnv.get.conf.get(APP_CALLER_CONTEXT), + _appId, + _appAttemptId, + Option(jobId), + Option(stageId), + Option(stageAttemptId), + Option(taskAttemptId), + Option(attemptNumber)).setCurrentContext() try { runTask(context) @@ -245,7 +252,7 @@ private[spark] abstract class Task[T]( output.writeInt(_stageId) output.writeVarInt(_stageAttemptId, true) output.writeVarInt(_partitionId, true) - output.writeVarInt(_jobId, true) + output.writeVarInt(jobId, true) output.writeLong(epoch) output.writeLong(_executorDeserializeTime) output.writeLong(_executorDeserializeCpuTime) @@ -254,9 +261,8 @@ private[spark] abstract class Task[T]( output.writeBoolean(true) } else { output.writeBoolean(false) - kryo.writeClassAndObject(output, taskBinary.get) + kryo.writeClassAndObject(output, taskBinary) } - _metrics.write(kryo, output) output.writeString(appId) output.writeString(appAttemptId) } @@ -265,20 +271,18 @@ private[spark] abstract class Task[T]( _stageId = input.readInt() _stageAttemptId = input.readVarInt(true) _partitionId = input.readVarInt(true) - _jobId = input.readVarInt(true) + jobId = input.readVarInt(true) epoch = input.readLong() _executorDeserializeTime = input.readLong() _executorDeserializeCpuTime = input.readLong() // actual bytes are shipped in TaskDescription taskData = TaskData.EMPTY if (input.readBoolean()) { - taskBinary = None + taskBinary = null } else { - taskBinary = Some(kryo.readClassAndObject(input) - .asInstanceOf[Broadcast[Array[Byte]]]) + taskBinary = kryo.readClassAndObject(input) + .asInstanceOf[Broadcast[Array[Byte]]] } - _metrics = new TaskMetrics - _metrics.read(kryo, input) _appId = Option(input.readString()) _appAttemptId = Option(input.readString()) } @@ -326,18 +330,9 @@ private[spark] object Task { } // Write the task properties separately so it is available before full task deserialization. - val props = task.localProperties - val numProps = props.size() - - dataOut.writeInt(numProps) - if (numProps > 0) { - val keys = props.keys() - while (keys.hasMoreElements) { - val key = keys.nextElement().asInstanceOf[String] - dataOut.writeUTF(key) - dataOut.writeUTF(props.getProperty(key)) - } - } + val propBytes = Utils.serialize(task.localProperties) + dataOut.writeInt(propBytes.length) + dataOut.write(propBytes) // Write the task itself and finish dataOut.flush() @@ -373,14 +368,11 @@ private[spark] object Task { for (i <- 0 until numJars) { taskJars(dataIn.readUTF()) = dataIn.readLong() } - val taskProps = new Properties - var numProps = dataIn.readInt() - while (numProps > 0) { - val key = dataIn.readUTF() - taskProps.setProperty(key, dataIn.readUTF()) - numProps -= 1 - } + val propLength = dataIn.readInt() + val propBytes = new Array[Byte](propLength) + dataIn.readFully(propBytes, 0, propLength) + val taskProps = Utils.deserialize[Properties](propBytes) // Create a sub-buffer for the rest of the data, which is the serialized Task object val subBuffer = serializedTask.slice() // ByteBufferInputStream will have read just up to task diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 36ef79540bb60..91f8b806ae892 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -496,7 +496,7 @@ private[spark] class TaskSetManager( } // Serialize and return the task val serializedTask: ByteBuffer = try { - Task.serializeWithDependencies(task, sched.sc.addedFiles, sched.sc.addedJars, ser) + ser.serialize(task) } catch { // If the task cannot be serialized, then there's no point to re-attempt the task, // as it will always fail. So just abort the whole task-set. @@ -506,11 +506,11 @@ private[spark] class TaskSetManager( abort(s"$msg Exception during serialization: $e") throw new TaskNotSerializableException(e) } - if (serializedTask.limit > TaskSetManager.TASK_SIZE_TO_WARN_KB * 1024 && + if (serializedTask.limit() > TaskSetManager.TASK_SIZE_TO_WARN_KB * 1024 && !emittedTaskSizeWarning) { emittedTaskSizeWarning = true logWarning(s"Stage ${task.stageId} contains a task of very large size " + - s"(${serializedTask.limit / 1024} KB). The maximum recommended task size is " + + s"(${serializedTask.limit() / 1024} KB). The maximum recommended task size is " + s"${TaskSetManager.TASK_SIZE_TO_WARN_KB} KB.") } addRunningTask(taskId) @@ -520,12 +520,19 @@ private[spark] class TaskSetManager( // val timeTaken = clock.getTime() - startTime val taskName = s"task ${info.id} in stage ${taskSet.id}" logInfo(s"Starting $taskName (TID $taskId, $host, executor ${info.executorId}, " + - s"partition ${task.partitionId}, $taskLocality, ${serializedTask.limit} bytes)") + s"partition ${task.partitionId}, $taskLocality, ${serializedTask.limit()} bytes)") sched.dagScheduler.taskStarted(task, info) - new TaskDescription(_taskId = taskId, _attemptNumber = attemptNum, execId, - taskName, index, addedFiles, addedJars, - task.localProperties, serializedTask, task.taskData) + new TaskDescription( + taskId, + attemptNum, + execId, + taskName, + index, + addedFiles, + addedJars, + task.localProperties, + serializedTask) } } else { None @@ -997,7 +1004,8 @@ private[spark] class TaskSetManager( } private def getLocalityWait(level: TaskLocality.TaskLocality): Long = { - val defaultWait = conf.get("spark.locality.wait", "3s") + val defaultWait = conf.get(config.LOCALITY_WAIT) + // val defaultWait = conf.get("spark.locality.wait", "3s") val localityWaitKey = level match { case TaskLocality.PROCESS_LOCAL => "spark.locality.wait.process" case TaskLocality.NODE_LOCAL => "spark.locality.wait.node" diff --git a/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala index 83198c0d7c10d..4c614c5c0f602 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/local/LocalSchedulerBackend.scala @@ -84,8 +84,7 @@ private[spark] class LocalEndpoint( val offers = IndexedSeq(new WorkerOffer(localExecutorId, localExecutorHostname, freeCores)) for (task <- scheduler.resourceOffers(offers).flatten) { freeCores -= scheduler.CPUS_PER_TASK - executor.launchTask(executorBackend, taskId = task.taskId, attemptNumber = task.attemptNumber, - task.name, task.serializedTask, task.taskData.decompress()) + executor.launchTask(executorBackend, task) } } } diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala index ba3af24750456..6f83a59631e07 100644 --- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala +++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala @@ -103,7 +103,7 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug try { executor = new Executor("id", "localhost", env, userClassPath = Nil, isLocal = true) // the task will be launched in a dedicated worker thread - executor.launchTask(mockExecutorBackend, 0, 0, "", serializedTask, (null, 0L)) + executor.launchTask(mockExecutorBackend, taskDescription) if (!executorSuiteHelper.latch1.await(5, TimeUnit.SECONDS)) { fail("executor did not send first status update in time") @@ -144,12 +144,13 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - _taskBinary = Some(taskBinary), + taskBinary = taskBinary, partition = secondRDD.partitions(0), locs = Seq(), _outputId = 0, localProperties = new Properties(), - metrics = null) + serializedTaskMetrics = serializedTaskMetrics) + val serTask = serializer.serialize(task) val taskDescription = createFakeTaskDescription(serTask) @@ -187,12 +188,12 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - _taskBinary = Some(taskBinary), + taskBinary = taskBinary, partition = secondRDD.partitions(0), locs = Seq(), _outputId = 0, localProperties = new Properties(), - metrics = null + serializedTaskMetrics = serializedTaskMetrics ) val serTask = serializer.serialize(task) @@ -267,8 +268,7 @@ class ExecutorSuite extends SparkFunSuite with LocalSparkContext with MockitoSug executor = new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true, uncaughtExceptionHandler = mockUncaughtExceptionHandler) // the task will be launched in a dedicated worker thread - executor.launchTask(mockBackend, 0, 0, "", taskDescription.serializedTask, (null, 0L)) - + executor.launchTask(mockBackend, taskDescription) eventually(timeout(5.seconds), interval(10.milliseconds)) { assert(executor.numRunningTasks === 0) } diff --git a/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala b/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala index 9d4a53f6ad571..841eeff228b7b 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/FakeTask.scala @@ -19,14 +19,17 @@ package org.apache.spark.scheduler import java.util.Properties -import org.apache.spark.{Partition, TaskContext} +import org.apache.spark.{Partition, SparkEnv, TaskContext} +import org.apache.spark.executor.TaskMetrics class FakeTask( stageId: Int, partitionId: Int, - prefLocs: Seq[TaskLocation] = Nil) + prefLocs: Seq[TaskLocation] = Nil, + serializedTaskMetrics: Array[Byte] = + SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array()) extends Task[Int](stageId, 0, partitionId, TaskData.EMPTY, - null, null, new Properties) { + null, new Properties, serializedTaskMetrics) { override def runTask(context: TaskContext): Int = 0 override def preferredLocations: Seq[TaskLocation] = prefLocs @@ -71,8 +74,8 @@ object FakeTask { null, new Partition {override def index: Int = i}, prefLocs(i), - null, - new Properties) + new Properties, + SparkEnv.get.closureSerializer.newInstance().serialize(TaskMetrics.registered).array()) } new TaskSet(tasks, stageId, stageAttemptId, priority = 0, null) } diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala index 24001da9d67ed..e63bba6d983d0 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala @@ -69,12 +69,12 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - _taskBinary = Some(taskBinary), + taskBinary = taskBinary, partition = rdd.partitions(0), locs = Seq(), _outputId = 0, - metrics = null, - localProperties = new Properties()) + localProperties = new Properties(), + serializedTaskMetrics = null) intercept[RuntimeException] { task.run(0, 0, null) } @@ -98,12 +98,12 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark stageId = 1, stageAttemptId = 0, _taskData = TaskData.EMPTY, - _taskBinary = Some(taskBinary), + taskBinary = taskBinary, partition = rdd.partitions(0), locs = Seq(), _outputId = 0, - metrics = null, - localProperties = new Properties()) + localProperties = new Properties(), + serializedTaskMetrics = null) intercept[RuntimeException] { task.run(0, 0, null) } diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala index 6f4fdd5bb2039..20aa012b1f438 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala @@ -716,7 +716,7 @@ class TaskSetManagerSuite extends SparkFunSuite with LocalSparkContext with Logg val singleTask = new ShuffleMapTask(0, 0, TaskData.EMPTY, null, new Partition { override def index: Int = 0 - }, Seq(TaskLocation("host1", "execA")), null, new Properties) + }, Seq(TaskLocation("host1", "execA")), new Properties, null) val taskSet = new TaskSet(Array(singleTask), 0, 0, 0, null) val manager = new TaskSetManager(sched, taskSet, MAX_TASK_FAILURES) From c7d6ed10bcf8779794f9215a2f7e175063674684 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 1 Mar 2018 11:22:07 +0530 Subject: [PATCH 136/142] [SNAPPYDATA] revert changes in Logging to upstream reverting flag check optimization in Logging to be compatible with upstream Spark Conflicts: core/src/main/scala/org/apache/spark/MapOutputTracker.scala core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala core/src/main/scala/org/apache/spark/internal/Logging.scala core/src/main/scala/org/apache/spark/storage/BlockManager.scala core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala --- .../spark/broadcast/TorrentBroadcast.scala | 1 + .../org/apache/spark/internal/Logging.scala | 65 ++++--------------- .../apache/spark/storage/BlockManager.scala | 19 +++--- .../org/apache/spark/storage/DiskStore.scala | 2 +- .../storage/ShuffleBlockFetcherIterator.scala | 5 +- 5 files changed, 26 insertions(+), 66 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala index c393c82f54e22..e3ca126a33906 100644 --- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala +++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala @@ -227,6 +227,7 @@ private[spark] class TorrentBroadcast[T: ClassTag](obj: T, id: Long) } case None => logInfo("Started reading broadcast variable " + id) + val isDebugEnabled = log.isDebugEnabled val startTimeMs = if (isDebugEnabled) System.currentTimeMillis() else 0L val blocks = readBlocks() if (isDebugEnabled) logDebug("Reading broadcast variable " + id + " took" + diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala b/core/src/main/scala/org/apache/spark/internal/Logging.scala index a1105b5ce1a8f..0b77e6acaf820 100644 --- a/core/src/main/scala/org/apache/spark/internal/Logging.scala +++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala @@ -34,8 +34,6 @@ trait Logging { // be serialized and used on another machine @transient private var log_ : Logger = null - @transient private[this] var levelFlags: Int = _ - // Method to get the logger name for this object protected def logName = { // Ignore trailing $'s in the class names for Scala objects @@ -51,54 +49,17 @@ trait Logging { log_ } - private def setLevel(value: Boolean, enabled: Int, disabled: Int): Unit = { - if (value) levelFlags |= enabled else levelFlags |= disabled - } - - protected final def isInfoEnabled: Boolean = { - val levelFlags = this.levelFlags - if ((levelFlags & Logging.INFO_ENABLED) != 0) true - else if ((levelFlags & Logging.INFO_DISABLED) != 0) false - else { - val value = log.isInfoEnabled - setLevel(value, Logging.INFO_ENABLED, Logging.INFO_DISABLED) - value - } - } - - protected final def isDebugEnabled: Boolean = { - val levelFlags = this.levelFlags - if ((levelFlags & Logging.DEBUG_DISABLED) != 0) false - else if ((levelFlags & Logging.DEBUG_ENABLED) != 0) true - else { - val value = log.isDebugEnabled - setLevel(value, Logging.DEBUG_ENABLED, Logging.DEBUG_DISABLED) - value - } - } - - protected final def isTraceEnabled: Boolean = { - val levelFlags = this.levelFlags - if ((levelFlags & Logging.TRACE_DISABLED) != 0) false - else if ((levelFlags & Logging.TRACE_ENABLED) != 0) true - else { - val value = log.isTraceEnabled - setLevel(value, Logging.TRACE_ENABLED, Logging.TRACE_DISABLED) - value - } - } - // Log methods that take only a String protected def logInfo(msg: => String) { - if (isInfoEnabled) log.info(msg) + if (log.isInfoEnabled) log.info(msg) } protected def logDebug(msg: => String) { - if (isDebugEnabled) log.debug(msg) + if (log.isDebugEnabled) log.debug(msg) } protected def logTrace(msg: => String) { - if (isTraceEnabled) log.trace(msg) + if (log.isTraceEnabled) log.trace(msg) } protected def logWarning(msg: => String) { @@ -111,15 +72,15 @@ trait Logging { // Log methods that take Throwables (Exceptions/Errors) too protected def logInfo(msg: => String, throwable: Throwable) { - if (isInfoEnabled) log.info(msg, throwable) + if (log.isInfoEnabled) log.info(msg, throwable) } protected def logDebug(msg: => String, throwable: Throwable) { - if (isDebugEnabled) log.debug(msg, throwable) + if (log.isDebugEnabled) log.debug(msg, throwable) } protected def logTrace(msg: => String, throwable: Throwable) { - if (isTraceEnabled) log.trace(msg, throwable) + if (log.isTraceEnabled) log.trace(msg, throwable) } protected def logWarning(msg: => String, throwable: Throwable) { @@ -130,6 +91,10 @@ trait Logging { if (log.isErrorEnabled) log.error(msg, throwable) } + protected def isTraceEnabled(): Boolean = { + log.isTraceEnabled + } + protected def initializeLogIfNecessary(isInterpreter: Boolean): Unit = { initializeLogIfNecessary(isInterpreter, silent = false) } @@ -197,15 +162,7 @@ trait Logging { } } -private[spark] object Logging { - - private val INFO_ENABLED = 0x1 - private val INFO_DISABLED = 0x2 - private val DEBUG_ENABLED = 0x4 - private val DEBUG_DISABLED = 0x8 - private val TRACE_ENABLED = 0x10 - private val TRACE_DISABLED = 0x20 - +private object Logging { @volatile private var initialized = false @volatile private var defaultRootLevel: Level = null @volatile private var defaultSparkLog4jConfig = false diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index d6a5e78713e08..49764c5093c42 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -498,7 +498,7 @@ private[spark] class BlockManager( * Get locations of an array of blocks. */ private def getLocationBlockIds(blockIds: Array[BlockId]): Array[Seq[BlockManagerId]] = { - val startTimeMs = if (isDebugEnabled) System.currentTimeMillis else 0L + val startTimeMs = if (log.isDebugEnabled) System.currentTimeMillis else 0L val locations = master.getLocations(blockIds).toArray logDebug("Got multiple block location in %s".format(Utils.getUsedTimeMs(startTimeMs))) locations @@ -907,7 +907,7 @@ private[spark] class BlockManager( tellMaster: Boolean = true, keepReadLock: Boolean = false): Boolean = { doPut(blockId, level, classTag, tellMaster = tellMaster, keepReadLock = keepReadLock) { info => - val startTimeMs = if (isDebugEnabled) System.currentTimeMillis else 0L + val startTimeMs = if (log.isDebugEnabled) System.currentTimeMillis else 0L // Since we're storing bytes, initiate the replication before storing them locally. // This is faster as data is already serialized and ready to send. val replicationFuture = if (level.replication > 1) { @@ -1015,7 +1015,7 @@ private[spark] class BlockManager( } } - val startTimeMs = if (isDebugEnabled) System.currentTimeMillis else 0L + val startTimeMs = if (log.isDebugEnabled) System.currentTimeMillis else 0L var exceptionWasThrown: Boolean = true val result: Option[T] = try { val res = putBody(putBlockInfo) @@ -1083,6 +1083,7 @@ private[spark] class BlockManager( classTag: ClassTag[T], tellMaster: Boolean = true, keepReadLock: Boolean = false): Option[PartiallyUnrolledIterator[T]] = { + val isDebugEnabled = log.isDebugEnabled doPut(blockId, level, classTag, tellMaster = tellMaster, keepReadLock = keepReadLock) { info => val startTimeMs = if (isDebugEnabled) System.currentTimeMillis else 0L var iteratorFromFailedMemoryStorePut: Option[PartiallyUnrolledIterator[T]] = None @@ -1323,18 +1324,18 @@ private[spark] class BlockManager( replication = 1) val numPeersToReplicateTo = level.replication - 1 + + val isDebugEnabled = log.isDebugEnabled val startTime = if (isDebugEnabled) System.nanoTime else 0L - val peersReplicatedTo = mutable.HashSet.empty ++ existingReplicas - val peersFailedToReplicateTo = mutable.HashSet.empty[BlockManagerId] + var peersReplicatedTo = mutable.HashSet.empty[BlockManagerId] + var peersFailedToReplicateTo = mutable.HashSet.empty[BlockManagerId] var numFailures = 0 - val initialPeers = getPeers(false).filterNot(existingReplicas.contains) - var peersForReplication = blockReplicationPolicy.prioritize( blockManagerId, - initialPeers, - peersReplicatedTo, + getPeers(false), + mutable.HashSet.empty, blockId, numPeersToReplicateTo) diff --git a/core/src/main/scala/org/apache/spark/storage/DiskStore.scala b/core/src/main/scala/org/apache/spark/storage/DiskStore.scala index e4c11555bbe09..c43792f2bb870 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskStore.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskStore.scala @@ -60,7 +60,7 @@ private[spark] class DiskStore( throw new IllegalStateException(s"Block $blockId is already present in the disk store") } logDebug(s"Attempting to put block $blockId") - val startTime = if (isDebugEnabled) System.currentTimeMillis else 0L + val startTime = if (log.isDebugEnabled) System.currentTimeMillis else 0L val file = diskManager.getFile(blockId) val out = new CountingWritableChannel(openForWrite(file)) var threwException: Boolean = true diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala index 20bec146edc79..87ae34ab12728 100644 --- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala +++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala @@ -88,7 +88,7 @@ final class ShuffleBlockFetcherIterator( private[this] var numBlocksProcessed = 0 private[this] val startTime = - if (isDebugEnabled || isTraceEnabled) System.currentTimeMillis else 0L + if (log.isDebugEnabled || isTraceEnabled) System.currentTimeMillis else 0L /** Local blocks to fetch, excluding zero-sized blocks. */ private[this] val localBlocks = new ArrayBuffer[BlockId]() @@ -232,7 +232,7 @@ final class ShuffleBlockFetcherIterator( remainingBlocks -= blockId results.put(new SuccessFetchResult(BlockId(blockId), address, sizeMap(blockId), buf, remainingBlocks.isEmpty)) - if (isDebugEnabled) { + if (log.isDebugEnabled) { logDebug("remainingBlocks: " + remainingBlocks) } } @@ -356,6 +356,7 @@ final class ShuffleBlockFetcherIterator( fetchUpToMaxBytes() val numFetches = remoteRequests.size - fetchRequests.size + val isDebugEnabled = log.isDebugEnabled if (isDebugEnabled) logDebug("Started " + numFetches + " remote fetches in" + Utils.getUsedTimeMs(startTime)) From 150f5ad7d9064c2f5a2ab12a9ae256e13900155b Mon Sep 17 00:00:00 2001 From: hemanth meka Date: Wed, 7 Mar 2018 18:48:09 +0530 Subject: [PATCH 137/142] [SNAPPYDATA] Changed TestSparkSession in test class APIs to base SparkSession This is to allow override by SnappySession extensions. Conflicts: external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala --- .../spark/sql/kafka010/KafkaSourceSuite.scala | 6 +- .../DatasetSerializerRegistratorSuite.scala | 8 +- .../spark/sql/test/SharedSQLContext.scala | 80 ++++++++++++++++++- 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala index 02c87643568bd..9ccf2fd0a524b 100644 --- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala +++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala @@ -26,15 +26,13 @@ import java.util.concurrent.atomic.AtomicInteger import scala.collection.mutable import scala.util.Random - import org.apache.kafka.clients.producer.RecordMetadata import org.apache.kafka.common.TopicPartition import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.time.SpanSugar._ - import org.apache.spark.SparkContext -import org.apache.spark.sql.{Dataset, ForeachWriter} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.{Dataset, ForeachWriter, SparkSession} import org.apache.spark.sql.execution.streaming._ import org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution import org.apache.spark.sql.functions.{count, window} @@ -1002,7 +1000,7 @@ class KafkaSourceStressForDontFailOnDataLossSuite extends StreamTest with Shared private def newTopic(): String = s"failOnDataLoss-${topicId.getAndIncrement()}" - override def createSparkSession(): TestSparkSession = { + override def createSparkSession(): SparkSession = { // Set maxRetries to 3 to handle NPE from `poll` when deleting a topic new TestSparkSession(new SparkContext("local[2,3]", "test-sql-context", sparkConf)) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala index 68f7de047b392..c09a5e50cd17e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala @@ -30,11 +30,17 @@ import org.apache.spark.sql.test.SharedSQLContext class DatasetSerializerRegistratorSuite extends QueryTest with SharedSQLContext { import testImplicits._ - override protected def sparkConf: SparkConf = { // Make sure we use the KryoRegistrator super.sparkConf.set("spark.kryo.registrator", TestRegistrator().getClass.getCanonicalName) } + /** + * Initialize the [[SparkSession]] with a [[KryoRegistrator]]. + */ + protected override def beforeAll(): Unit = { + sparkConf.set("spark.kryo.registrator", TestRegistrator().getClass.getCanonicalName) + super.beforeAll() + } test("Kryo registrator") { implicit val kryoEncoder = Encoders.kryo[KryoData] diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala index 4d578e21f5494..18f939b8d8397 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala @@ -17,4 +17,82 @@ package org.apache.spark.sql.test -trait SharedSQLContext extends SQLTestUtils with SharedSparkSession +import scala.concurrent.duration._ + +import org.scalatest.BeforeAndAfterEach +import org.scalatest.concurrent.Eventually + +import org.apache.spark.{DebugFilesystem, SparkConf} +import org.apache.spark.sql.{SparkSession, SQLContext} + + +/** + * Helper trait for SQL test suites where all tests share a single [[SparkSession]]. + */ +trait SharedSQLContext extends SQLTestUtils with BeforeAndAfterEach with Eventually { + + protected val sparkConf = new SparkConf() + + /** + * The [[SparkSession]] to use for all tests in this suite. + * + * By default, the underlying [[org.apache.spark.SparkContext]] will be run in local + * mode with the default test configurations. + */ + private var _spark: SparkSession = null + + /** + * The [[SparkSession]] to use for all tests in this suite. + */ + protected implicit def spark: SparkSession = _spark + + /** + * The [[TestSQLContext]] to use for all tests in this suite. + */ + protected implicit def sqlContext: SQLContext = _spark.sqlContext + + protected def createSparkSession: SparkSession = { + new TestSparkSession( + sparkConf.set("spark.hadoop.fs.file.impl", classOf[DebugFilesystem].getName)) + } + + /** + * Initialize the [[TestSparkSession]]. + */ + protected override def beforeAll(): Unit = { + SparkSession.sqlListener.set(null) + if (_spark == null) { + _spark = createSparkSession + } + // Ensure we have initialized the context before calling parent code + super.beforeAll() + } + + /** + * Stop the underlying [[org.apache.spark.SparkContext]], if any. + */ + protected override def afterAll(): Unit = { + try { + if (_spark != null) { + _spark.stop() + _spark = null + } + } finally { + super.afterAll() + } + } + + protected override def beforeEach(): Unit = { + super.beforeEach() + DebugFilesystem.clearOpenStreams() + } + + protected override def afterEach(): Unit = { + super.afterEach() + // files can be closed from other threads, so wait a bit + // normally this doesn't take more than 1s + eventually(timeout(10.seconds)) { + DebugFilesystem.assertNoOpenStreams() + } + } +} From 38c4f1697d3a846a7fdce4f6d99deef9829b1b59 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Fri, 9 Mar 2018 04:04:54 +0530 Subject: [PATCH 138/142] [SNAPPYDATA] increased default codegen cache size to 2K also added MemoryMode in MemoryPool warning message Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala --- .../org/apache/spark/memory/StorageMemoryPool.scala | 2 +- .../catalyst/expressions/codegen/CodeGenerator.scala | 12 ++++++++---- .../spark/sql/execution/WholeStageCodegenExec.scala | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala b/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala index 4c6b639015a90..fa66f957242e2 100644 --- a/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala +++ b/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala @@ -104,7 +104,7 @@ private[memory] class StorageMemoryPool( def releaseMemory(size: Long): Unit = lock.synchronized { if (size > _memoryUsed) { logWarning(s"Attempted to release $size bytes of storage " + - s"memory when we only have ${_memoryUsed} bytes") + s"memory ($memoryMode) when we only have ${_memoryUsed} bytes") _memoryUsed = 0 } else { _memoryUsed -= size diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 34015c390e8dc..00d36c8645a46 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1503,9 +1503,12 @@ object CodeGenerator extends Logging { * automatically, in order to constrain its memory footprint. Note that this cache does not use * weak keys/values and thus does not respond to memory pressure. */ - private val cache = CacheBuilder.newBuilder() - .maximumSize(1000) - .build( + private lazy val cache = { + val env = SparkEnv.get + val cacheSize = if (env ne null) { + env.conf.getInt("spark.sql.codegen.cacheSize", 2000) + } else 2000 + CacheBuilder.newBuilder().maximumSize(cacheSize).build( new CacheLoader[CodeAndComment, (GeneratedClass, Int)]() { override def load(code: CodeAndComment): (GeneratedClass, Int) = { val startTime = System.nanoTime() @@ -1518,4 +1521,5 @@ object CodeGenerator extends Logging { result } }) - } \ No newline at end of file + } +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala index 4514c37f5832c..53376fb56804d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala @@ -607,7 +607,7 @@ case class WholeStageCodegenExec(child: SparkPlan)(val codegenStageId: Int) val durationMs = longMetric("pipelineTime") val rdds = child.asInstanceOf[CodegenSupport].inputRDDs() - new WholeStageCodegenRDD(sqlContext.sparkContext, cleanedSource, + WholeStageCodegenRDD(sqlContext.sparkContext, cleanedSource, references, durationMs, rdds) } From 6863e4407c9224ca8a95f4cce23c56c99d4677ee Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 22 Mar 2018 13:29:37 +0530 Subject: [PATCH 139/142] [SNAPPYDATA] make Dataset.boundEnc as lazy val avoid materializing it immediately (for point queries that won't use it) Conflicts: sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala --- .../sql/catalyst/expressions/namedExpressions.scala | 4 +++- .../main/scala/org/apache/spark/sql/Dataset.scala | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala index 9dc8f7e143425..eb83ab364fa54 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala @@ -62,7 +62,9 @@ object NamedExpression { case class ExprId(id: Long, jvmId: UUID) object ExprId { - def apply(id: Long): ExprId = ExprId(id, NamedExpression.jvmId) + private val INVALID = apply(-1, NamedExpression.jvmId) + + def apply(id: Long): ExprId = if (id == -1) INVALID else ExprId(id, NamedExpression.jvmId) } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala index 0aee1d7be5788..010a9ddb36387 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala @@ -211,6 +211,19 @@ class Dataset[T] private[sql]( private lazy val deserializer = exprEnc.resolveAndBind(logicalPlan.output, sparkSession.sessionState.analyzer).deserializer + /** + * Encoder is used mostly as a container of serde expressions in Dataset. We build logical + * plans by these serde expressions and execute it within the query framework. However, for + * performance reasons we may want to use encoder as a function to deserialize internal rows to + * custom objects, e.g. collect. Here we resolve and bind the encoder so that we can call its + * `fromRow` method later. + */ + private lazy val boundEnc = + exprEnc.resolveAndBind(logicalPlan.output, sparkSession.sessionState.analyzer) + + // materialize boundEnc immediately if T is not a Row to throw any analysis exception + if (!classTag.runtimeClass.isAssignableFrom(classOf[Row])) boundEnc + private implicit def classTag = exprEnc.clsTag // sqlContext must be val because a stable identifier is expected when you import implicits From 2049882314524a74b954ddcb4fbe06b55dabf715 Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Wed, 21 Mar 2018 10:52:46 +0530 Subject: [PATCH 140/142] [SNAP-2242] Unique application names & kill app by names (#98) The standalone cluster should support unique application names. As they are user visible and easy to track user can write scripts to kill applications by names. Also, added support to kill Spark applications by names(case insensitive). Conflicts: core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala --- .../apache/spark/deploy/master/Master.scala | 10 +++++++- .../spark/deploy/master/ui/MasterPage.scala | 25 +++++++++++++++---- .../spark/deploy/master/ui/MasterWebUI.scala | 2 ++ .../deploy/master/ui/MasterWebUISuite.scala | 18 +++++++++++++ 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index 2c78c15773af2..d7871f5369077 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -63,6 +63,7 @@ private[deploy] class Master( val workers = new HashSet[WorkerInfo] val idToApp = new HashMap[String, ApplicationInfo] + val nameToApp = new HashMap[String, ApplicationInfo] private val waitingApps = new ArrayBuffer[ApplicationInfo] val apps = new HashSet[ApplicationInfo] @@ -263,6 +264,11 @@ private[deploy] class Master( } else { logInfo("Registering app " + description.name) val app = createApplication(description, driver) + if (nameToApp.get(app.desc.name.toLowerCase).isDefined) { + val msg = s"An application with name ${app.desc.name} is already running" + logError(msg) + driver.send(ApplicationRemoved(msg)) + } registerApplication(app) logInfo("Registered app " + description.name + " with ID " + app.id) persistenceEngine.addApplication(app) @@ -843,6 +849,7 @@ private[deploy] class Master( applicationMetricsSystem.registerSource(app.appSource) apps += app idToApp(app.id) = app + nameToApp(app.desc.name.toLowerCase) = app endpointToApp(app.driver) = app addressToApp(appAddress) = app waitingApps += app @@ -854,9 +861,10 @@ private[deploy] class Master( def removeApplication(app: ApplicationInfo, state: ApplicationState.Value) { if (apps.contains(app)) { - logInfo("Removing app " + app.id) + logInfo(s"Removing application ${app.desc.name} with app.id=${app.id} ") apps -= app idToApp -= app.id + nameToApp -= app.desc.name.toLowerCase endpointToApp -= app.driver addressToApp -= app.driver.address diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala index bc0bf6a1d9700..4f0d18b65bfc3 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala @@ -26,10 +26,11 @@ import org.json4s.JValue import org.apache.spark.deploy.DeployMessages.{KillDriverResponse, MasterStateResponse, RequestKillDriver, RequestMasterState} import org.apache.spark.deploy.JsonProtocol import org.apache.spark.deploy.master._ +import org.apache.spark.internal.Logging import org.apache.spark.ui.{UIUtils, WebUIPage} import org.apache.spark.util.Utils -private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") { +private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") with Logging { private val master = parent.masterEndpointRef def getMasterState: MasterStateResponse = { @@ -48,21 +49,35 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") { }) } + def handleAppKillByNameRequest(request: HttpServletRequest): Unit = { + handleKillRequest(request, name => { + parent.master.nameToApp.get(name.toLowerCase).foreach { app => + parent.master.removeApplication(app, ApplicationState.KILLED) + } + }, killByName = true) + } + def handleDriverKillRequest(request: HttpServletRequest): Unit = { handleKillRequest(request, id => { master.ask[KillDriverResponse](RequestKillDriver(id)) }) } - private def handleKillRequest(request: HttpServletRequest, action: String => Unit): Unit = { + private def handleKillRequest(request: HttpServletRequest, + action: String => Unit, + killByName: Boolean = false): Unit = { if (parent.killEnabled && parent.master.securityMgr.checkModifyPermissions(request.getRemoteUser)) { // stripXSS is called first to remove suspicious characters used in XSS attacks val killFlag = Option(UIUtils.stripXSS(request.getParameter("terminate"))).getOrElse("false").toBoolean - val id = Option(UIUtils.stripXSS(request.getParameter("id"))) - if (id.isDefined && killFlag) { - action(id.get) + val idOrName = if (!killByName) { + Option(UIUtils.stripXSS(request.getParameter("id")) + } else { + Option(UIUtils.stripXSS(request.getParameter("name")) + } + if (idOrName.isDefined && killFlag) { + action(idOrName.get) } Thread.sleep(100) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala index 35b7ddd46e4db..d04f45e079e41 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala @@ -46,6 +46,8 @@ class MasterWebUI( attachHandler(createStaticHandler(MasterWebUI.STATIC_RESOURCE_DIR, "/static")) attachHandler(createRedirectHandler( "/app/kill", "/", masterPage.handleAppKillRequest, httpMethods = Set("POST"))) + attachHandler(createRedirectHandler( + "/app/killByName", "/", masterPage.handleAppKillByNameRequest, httpMethods = Set("POST"))) attachHandler(createRedirectHandler( "/driver/kill", "/", masterPage.handleDriverKillRequest, httpMethods = Set("POST"))) } diff --git a/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala index 69a460fbc7dba..824ea848ed229 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala @@ -74,6 +74,24 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll { verify(master, times(1)).removeApplication(activeApp, ApplicationState.KILLED) } + test("Kill application by name") { + val appDesc = createAppDesc() + // use new start date so it isn't filtered by UI + val activeApp = new ApplicationInfo( + new Date().getTime, "app-0", appDesc, new Date(), null, Int.MaxValue) + + when(master.nameToApp).thenReturn(HashMap[String, + ApplicationInfo]((activeApp.desc.name, activeApp))) + + val url = s"http://localhost:${masterWebUI.boundPort}/app/killByName/" + val body = convPostDataToString(Map(("name", activeApp.desc.name), ("terminate", "true"))) + val conn = sendHttpRequest(url, "POST", body) + conn.getResponseCode + + // Verify the master was called to remove the active app + verify(master, times(1)).removeApplication(activeApp, ApplicationState.KILLED) + } + test("kill driver") { val activeDriverId = "driver-0" val url = s"http://localhost:${masterWebUI.boundPort}/driver/kill/" From 74b3598f2a0a7edbd34224149b8e585f4a01399b Mon Sep 17 00:00:00 2001 From: Rishitesh Mishra Date: Mon, 19 Mar 2018 11:22:18 +0530 Subject: [PATCH 141/142] [SNAP-2225] Removed OrderlessHashPartitioning. (#95) Handled join order in optimization phase. Also removed custom changes in HashPartition. We won't store bucket information in HashPartitioning. Instead based on the flag "linkPartitionToBucket" we can determine the number of partitions to be either numBuckets or num cores assigned to the executor. Reverted changes related to numBuckets in Snappy Spark. Conflicts: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala --- .../plans/physical/partitioning.scala | 38 ------------------- .../exchange/EnsureRequirements.scala | 17 ++++++++- .../exchange/ShuffleExchangeExec.scala | 14 +++---- 3 files changed, 22 insertions(+), 47 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala index fa1a824ff88fe..6e93534492264 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala @@ -212,44 +212,6 @@ case object SinglePartition extends Partitioning { } } -/** - * Represents a partitioning where rows are split up across partitions based on the hash - * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be - * in the same partition. Moreover while evaluating expressions if they are given in different order - * than this partitioning then also it is considered equal. - */ -case class OrderlessHashPartitioning(expressions: Seq[Expression], - aliases: Seq[Seq[Attribute]], - numPartitions: Int, - numBuckets: Int, - tableBuckets: Int) - extends Expression with Partitioning with Unevaluable { - - override def children: Seq[Expression] = expressions - override def nullable: Boolean = false - override def dataType: DataType = IntegerType - - override def satisfies(required: Distribution): Boolean = { - super.satisfies(required) || { - required match { - case h: HashClusteredDistribution => - expressions.length == h.expressions.length && expressions.zip(h.expressions).forall { - case (l, r) => l.semanticEquals(r) - } - case ClusteredDistribution(requiredClustering, requiredNumPartitions) => - expressions.forall(x => requiredClustering.exists(_.semanticEquals(x))) && - (requiredNumPartitions.isEmpty || requiredNumPartitions.get == numPartitions) - case _ => false - } - } - } - -} -/** - * Represents a partitioning where rows are split up across partitions based on the hash - * of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be - * in the same partition. - */ case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int) extends Expression with Partitioning with Unevaluable { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala index e3d28388c5470..9ead84872db1a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.exchange import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.plans.physical._ +import org.apache.spark.sql.catalyst.plans.physical. _ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, ShuffledHashJoinExec, @@ -46,6 +46,20 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { if (minNumPostShufflePartitions > 0) Some(minNumPostShufflePartitions) else None } + /** + * Given a required distribution, returns a partitioning that satisfies that distribution. + */ + private def createPartitioning( + requiredDistribution: Distribution, + numPartitions: Int): Partitioning = { + requiredDistribution match { + case AllTuples => SinglePartition + case ClusteredDistribution(clustering, _) => HashPartitioning(clustering, numPartitions) + case OrderedDistribution(ordering) => RangePartitioning(ordering, numPartitions) + case dist => sys.error(s"Do not know how to satisfy distribution $dist") + } + } + /** * Adds [[ExchangeCoordinator]] to [[ShuffleExchangeExec]]s if adaptive query execution is enabled * and partitioning schemes of these [[ShuffleExchangeExec]]s support [[ExchangeCoordinator]]. @@ -165,6 +179,7 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { case _ => true }.map(_._2) + val childrenNumPartitions = childrenIndexes.map(children(_).outputPartitioning.numPartitions).toSet diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala index 297f46142f0cb..b154e81e6e114 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala @@ -207,14 +207,12 @@ object ShuffleExchangeExec { serializer: Serializer): ShuffleDependency[Int, InternalRow, InternalRow] = { val part: Partitioner = newPartitioning match { case RoundRobinPartitioning(numPartitions) => new HashPartitioner(numPartitions) - case HashPartitioning(_, n) => - new Partitioner { - override def numPartitions: Int = n - // For HashPartitioning, the partitioning key is already a valid partition ID, as we use - // `HashPartitioning.partitionIdExpression` to produce partitioning key. - override def getPartition(key: Any): Int = key.asInstanceOf[Int] - } - // case p@HashPartitioning(_, n) => new HashPartitioner(n, p.numBuckets) + case HashPartitioning(_, n) => new Partitioner { + override def numPartitions: Int = n + // For HashPartitioning, the partitioning key is already a valid partition ID, as we use + // `HashPartitioning.partitionIdExpression` to produce partitioning key. + override def getPartition(key: Any): Int = key.asInstanceOf[Int] + } case RangePartitioning(sortingExpressions, numPartitions) => // Internally, RangePartitioner runs a job on the RDD that samples keys to compute // partition bounds. To get accurate samples, we need to copy the mutable keys. From 58d93d92e0d4b2b97e90d7e6215cffaa8e78c8a6 Mon Sep 17 00:00:00 2001 From: ymahajan Date: Sat, 12 May 2018 08:37:29 -0700 Subject: [PATCH 142/142] Fixing issues after master down merge --- .../org/apache/spark/deploy/SparkSubmit.scala | 2 +- .../spark/deploy/master/ui/MasterPage.scala | 4 +- .../org/apache/spark/internal/Logging.scala | 1 + .../spark/sql/test/SharedSQLContext.scala | 80 +------------------ .../spark/sql/test/SharedSparkSession.scala | 12 +-- 5 files changed, 11 insertions(+), 88 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 1e381965c52ba..fea5742267b70 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -200,7 +200,7 @@ object SparkSubmit extends CommandLineUtils with Logging { // Let the main class re-initialize the logging system once it starts. if (uninitLog) { - Logging.uninitialize() + // Logging.uninitialize() } // In standalone cluster mode, there are two submission gateways: diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala index 4f0d18b65bfc3..9ead93f3d64f2 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala @@ -72,9 +72,9 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") with Log val killFlag = Option(UIUtils.stripXSS(request.getParameter("terminate"))).getOrElse("false").toBoolean val idOrName = if (!killByName) { - Option(UIUtils.stripXSS(request.getParameter("id")) + Option(UIUtils.stripXSS(request.getParameter("id"))) } else { - Option(UIUtils.stripXSS(request.getParameter("name")) + Option(UIUtils.stripXSS(request.getParameter("name"))) } if (idOrName.isDefined && killFlag) { action(idOrName.get) diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala b/core/src/main/scala/org/apache/spark/internal/Logging.scala index 0b77e6acaf820..7ca8a33d1aaac 100644 --- a/core/src/main/scala/org/apache/spark/internal/Logging.scala +++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala @@ -17,6 +17,7 @@ package org.apache.spark.internal + import org.apache.log4j.{Level, LogManager, PropertyConfigurator} import org.slf4j.{Logger, LoggerFactory} import org.slf4j.impl.StaticLoggerBinder diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala index 18f939b8d8397..4d578e21f5494 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala @@ -17,82 +17,4 @@ package org.apache.spark.sql.test -import scala.concurrent.duration._ - -import org.scalatest.BeforeAndAfterEach -import org.scalatest.concurrent.Eventually - -import org.apache.spark.{DebugFilesystem, SparkConf} -import org.apache.spark.sql.{SparkSession, SQLContext} - - -/** - * Helper trait for SQL test suites where all tests share a single [[SparkSession]]. - */ -trait SharedSQLContext extends SQLTestUtils with BeforeAndAfterEach with Eventually { - - protected val sparkConf = new SparkConf() - - /** - * The [[SparkSession]] to use for all tests in this suite. - * - * By default, the underlying [[org.apache.spark.SparkContext]] will be run in local - * mode with the default test configurations. - */ - private var _spark: SparkSession = null - - /** - * The [[SparkSession]] to use for all tests in this suite. - */ - protected implicit def spark: SparkSession = _spark - - /** - * The [[TestSQLContext]] to use for all tests in this suite. - */ - protected implicit def sqlContext: SQLContext = _spark.sqlContext - - protected def createSparkSession: SparkSession = { - new TestSparkSession( - sparkConf.set("spark.hadoop.fs.file.impl", classOf[DebugFilesystem].getName)) - } - - /** - * Initialize the [[TestSparkSession]]. - */ - protected override def beforeAll(): Unit = { - SparkSession.sqlListener.set(null) - if (_spark == null) { - _spark = createSparkSession - } - // Ensure we have initialized the context before calling parent code - super.beforeAll() - } - - /** - * Stop the underlying [[org.apache.spark.SparkContext]], if any. - */ - protected override def afterAll(): Unit = { - try { - if (_spark != null) { - _spark.stop() - _spark = null - } - } finally { - super.afterAll() - } - } - - protected override def beforeEach(): Unit = { - super.beforeEach() - DebugFilesystem.clearOpenStreams() - } - - protected override def afterEach(): Unit = { - super.afterEach() - // files can be closed from other threads, so wait a bit - // normally this doesn't take more than 1s - eventually(timeout(10.seconds)) { - DebugFilesystem.assertNoOpenStreams() - } - } -} +trait SharedSQLContext extends SQLTestUtils with SharedSparkSession diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala index e758c865b908f..12efde328d1a1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala @@ -42,15 +42,15 @@ trait SharedSparkSession } /** - * The [[TestSparkSession]] to use for all tests in this suite. + * The [[SparkSession]] to use for all tests in this suite. * * By default, the underlying [[org.apache.spark.SparkContext]] will be run in local * mode with the default test configurations. */ - private var _spark: TestSparkSession = null + private var _spark: SparkSession = null /** - * The [[TestSparkSession]] to use for all tests in this suite. + * The [[SparkSession]] to use for all tests in this suite. */ protected implicit def spark: SparkSession = _spark @@ -59,12 +59,12 @@ trait SharedSparkSession */ protected implicit def sqlContext: SQLContext = _spark.sqlContext - protected def createSparkSession: TestSparkSession = { + protected def createSparkSession: SparkSession = { new TestSparkSession(sparkConf) } /** - * Initialize the [[TestSparkSession]]. Generally, this is just called from + * Initialize the [[SparkSession]]. Generally, this is just called from * beforeAll; however, in test using styles other than FunSuite, there is * often code that relies on the session between test group constructs and * the actual tests, which may need this session. It is purely a semantic @@ -79,7 +79,7 @@ trait SharedSparkSession } /** - * Make sure the [[TestSparkSession]] is initialized before any tests are run. + * Make sure the [[SparkSession]] is initialized before any tests are run. */ protected override def beforeAll(): Unit = { initializeSession()