Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -58,32 +58,49 @@ object DefaultCostEstimator {
* is the execution time of its longest-running operator. Otherwise the cost is the number of materialized ports in the
* region.
*/
class DefaultCostEstimator(
class DefaultCostEstimator private (
workflowContext: WorkflowContext,
val resourceAllocator: ResourceAllocator,
val actorId: ActorVirtualIdentity
val actorId: ActorVirtualIdentity,
preloadedStats: Option[Option[Map[String, Double]]]
) extends CostEstimator
with AmberLogging {

def this(
workflowContext: WorkflowContext,
resourceAllocator: ResourceAllocator,
actorId: ActorVirtualIdentity
) =
this(workflowContext, resourceAllocator, actorId, preloadedStats = None)

// Requires mysql database to retrieve execution statistics, otherwise use number of materialized ports as a default.
private val operatorEstimatedTimeOption = Try(
this.getOperatorExecutionTimeInSeconds(
this.workflowContext.workflowId.id
)
) match {
case Failure(_) => None
case Success(result) => result
private val operatorEstimatedTimeOption = preloadedStats.getOrElse {
Try(
this.getOperatorExecutionTimeInSeconds(
this.workflowContext.workflowId.id
)
) match {
case Failure(_) => None
case Success(result) => result
}
}

operatorEstimatedTimeOption match {
case None =>
logger.info(
s"WID: ${workflowContext.workflowId.id}, EID: ${workflowContext.executionId.id}, " +
s"no past execution statistics available. Using number of materialized output ports as the cost. "
)
case Some(_) =>
if (preloadedStats.isEmpty && operatorEstimatedTimeOption.isEmpty) {
logger.info(
s"WID: ${workflowContext.workflowId.id}, EID: ${workflowContext.executionId.id}, " +
s"no past execution statistics available. Using number of materialized output ports as the cost. "
)
}

/** Returns a cost estimator sharing the loaded execution statistics with `newAllocator`. */
def withResourceAllocator(newAllocator: ResourceAllocator): DefaultCostEstimator =
new DefaultCostEstimator(
workflowContext,
newAllocator,
actorId,
preloadedStats = Some(operatorEstimatedTimeOption)
)

override def allocateResourcesAndEstimateCost(
region: Region,
resourceUnits: Int
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,18 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala
case class RegionLink(fromRegionId: RegionIdentity, toRegionId: RegionIdentity)

case class RegionIdentity(id: Long)

/**
* A sub-plan scheduled as a unit. When `skipped` is true, its operators are not launched on
* workers; their saved outputs are carried in `resourceConfig` for reuse by later regions.
*/
case class Region(
id: RegionIdentity,
physicalOps: Set[PhysicalOp],
physicalLinks: Set[PhysicalLink],
ports: Set[GlobalPortIdentity] = Set.empty,
resourceConfig: Option[ResourceConfig] = None
resourceConfig: Option[ResourceConfig] = None,
skipped: Boolean = false
) {

private val operators: Map[PhysicalOpIdentity, PhysicalOp] =
Expand Down Expand Up @@ -73,9 +79,8 @@ case class Region(
* The effective source contains operators that have 0 input links in this region.
*/
def getSourceOperators: Set[PhysicalOp] = {
getOperators.filter(physicalOp =>
physicalOp.getInputLinks().forall(link => !operators.contains(link.fromOpId))
)
val operatorsWithInputLink = getLinks.map(_.toOpId)
getOperators.filterNot(op => operatorsWithInputLink.contains(op.id))
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ import org.jgrapht.traverse.TopologicalOrderIterator

import scala.jdk.CollectionConverters.IteratorHasAsScala

/**
* Region-level execution plan of a workflow. `regions` holds both active regions
* (`skipped = false`) and skip regions (`skipped = true`); `regionLinks` captures materialized
* dependencies among active regions.
*/
case class RegionPlan(
regions: Set[Region],
regionLinks: Set[RegionLink]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,24 +45,40 @@ abstract class ScheduleGenerator(
def generate(): (Schedule, PhysicalPlan)

/**
* A schedule is a ranking on the regions of a region plan.
* Regions are dispatched in batches of up to ApplicationConfig.maxConcurrentRegions, respecting the DAG dependencies.
* When maxConcurrentRegions == 1, this is equivalent to a total order (fully sequential execution).
* A schedule is a ranking on the regions of a region plan. Skip regions (`Region.skipped`)
* share level 0 so they complete together without worker allocation; active regions follow in
* topological batches of up to ApplicationConfig.maxConcurrentRegions. When
* maxConcurrentRegions == 1, the active levels form a total order.
*/
def generateScheduleFromRegionPlan(regionPlan: RegionPlan): Schedule = {
val inDegree = mutable.Map.empty[RegionIdentity, Int]
regionPlan.topologicalIterator().foreach { rid =>
inDegree(rid) = regionPlan.dag.inDegreeOf(rid)
val skippedIds = regionPlan.regions.filter(_.skipped).map(_.id)
require(
regionPlan.regionLinks.forall(link =>
!skippedIds.contains(link.fromRegionId) && !skippedIds.contains(link.toRegionId)
),
"skip regions must not have region links"
)

val tmpLevelSets = mutable.Map.empty[Int, Set[RegionIdentity]]
var level = 0
if (skippedIds.nonEmpty) {
tmpLevelSets(0) = skippedIds
level = 1
}

val inDegree = mutable.Map.empty[RegionIdentity, Int]
regionPlan
.topologicalIterator()
.filterNot(skippedIds.contains)
.foreach { rid =>
inDegree(rid) = regionPlan.dag.inDegreeOf(rid)
}

val readyRegionsQueue = mutable.Queue(
inDegree.collect { case (rid, 0) => rid }.toSeq: _*
)

val tmpLevelSets = mutable.Map.empty[Int, Set[RegionIdentity]]
var level = 0

// While there are ready regions:
// While there are ready active regions:
// 1. Dequeue up to maxConcurrentRegions regions to form the current batch.
// 2. Record this batch under the current level.
// 3. For each region in the batch:
Expand Down
Loading
Loading