diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGenerator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGenerator.scala index be376fa19d2..6d38a9f3cdc 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGenerator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGenerator.scala @@ -23,12 +23,17 @@ import org.apache.texera.common.config.ApplicationConfig import org.apache.texera.amber.core.storage.VFSURIFactory.createPortBaseURI import org.apache.texera.amber.core.virtualidentity.{ActorVirtualIdentity, PhysicalOpIdentity} import org.apache.texera.amber.core.workflow._ +import org.apache.texera.amber.engine.architecture.scheduling.EdgeType._ import org.apache.texera.amber.engine.architecture.scheduling.SchedulingUtils.replaceVertex import org.apache.texera.amber.engine.architecture.scheduling.config.{ IntermediateInputPortConfig, OutputPortConfig, ResourceConfig } +import org.apache.texera.amber.engine.architecture.scheduling.resourcePolicies.{ + DefaultResourceAllocator, + ExecutionClusterInfo +} import org.apache.texera.amber.engine.common.AmberLogging import org.jgrapht.Graph import org.jgrapht.alg.connectivity.BiconnectivityInspector @@ -36,6 +41,7 @@ import org.jgrapht.graph.{DirectedAcyclicGraph, DirectedPseudograph} import java.net.URI import java.util.concurrent.TimeoutException +import scala.collection.concurrent.TrieMap import scala.collection.mutable import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.DurationInt @@ -71,14 +77,29 @@ class CostBasedScheduleGenerator( ) with AmberLogging { - case class SearchResult( - state: Set[PhysicalLink], + private[scheduling] case class SearchResult( + state: SearchState, regionDAG: DirectedAcyclicGraph[Region, RegionLink], cost: Double, searchTimeNanoSeconds: Long = 0, numStatesExplored: Int = 0 ) + private[scheduling] val allowPipelined: Boolean = + CostBasedScheduleGenerator.effectiveExecutionMode( + physicalPlan, + workflowContext.workflowSettings.executionMode + ) == ExecutionMode.PIPELINED + + private val matchedResults: Map[GlobalPortIdentity, CachedResult] = + workflowContext.matchedResults + + private[scheduling] val requiredOutputs: Set[GlobalPortIdentity] = + SearchState.requiredOutputPorts(workflowContext, physicalPlan) + + private val allowedLinkTypes: Map[PhysicalLink, Set[EdgeType]] = + SearchState.allowedLinkTypes(physicalPlan, matchedResults, allowPipelined) + private val costEstimator = new DefaultCostEstimator( workflowContext = workflowContext, @@ -93,18 +114,66 @@ class CostBasedScheduleGenerator( resourceConfig: Option[ResourceConfig] ) - private val costEstimatorMemoization - : mutable.Map[CostEstimatorMemoKey, (ResourceConfig, Double)] = - new mutable.HashMap() + private type SubPlanKey = + (Set[PhysicalOpIdentity], Set[PhysicalLink], Map[GlobalPortIdentity, List[URI]]) + private type ReuseBoundaryKey = (Set[PhysicalOpIdentity], Set[PhysicalLink]) + + private class SubPlanEvaluation(val estimator: DefaultCostEstimator) { + val memoization: TrieMap[CostEstimatorMemoKey, (ResourceConfig, Double)] = + new TrieMap() + } + + private val defaultEvaluation = new SubPlanEvaluation(costEstimator) + + private val evaluations: TrieMap[SubPlanKey, SubPlanEvaluation] = { + val map = new TrieMap[SubPlanKey, SubPlanEvaluation]() + map.put( + (physicalPlan.operators.map(_.id), physicalPlan.links, Map.empty), + defaultEvaluation + ) + map + } + + private val retainedSubPlans + : TrieMap[ReuseBoundaryKey, (PhysicalPlan, Map[GlobalPortIdentity, List[URI]])] = + new TrieMap() + + private def evaluationOf( + targetPlan: PhysicalPlan, + readerUrisByInputPort: Map[GlobalPortIdentity, List[URI]] + ): SubPlanEvaluation = { + if (matchedResults.isEmpty && readerUrisByInputPort.isEmpty && (targetPlan eq physicalPlan)) { + return defaultEvaluation + } + evaluations.getOrElseUpdate( + (targetPlan.operators.map(_.id), targetPlan.links, readerUrisByInputPort), { + val scopedAllocator = new DefaultResourceAllocator( + physicalPlan, + new ExecutionClusterInfo(), + workflowContext.workflowSettings, + cuid = workflowContext.cuid + ) + new SubPlanEvaluation(costEstimator.withResourceAllocator(scopedAllocator)) + } + ) + } + + private def subPlanOf(state: SearchState): (PhysicalPlan, Map[GlobalPortIdentity, List[URI]]) = { + if (matchedResults.isEmpty) return (physicalPlan, Map.empty) + retainedSubPlans.getOrElseUpdate( + (state.skippedOperators(physicalPlan, requiredOutputs), state.linksOfType(CacheRead)), + ( + state.retainedPlan(physicalPlan, requiredOutputs), + state.readerUrisByInputPort(physicalPlan, matchedResults, requiredOutputs) + ) + ) + } def generate(): (Schedule, PhysicalPlan) = { val startTime = System.nanoTime() - val regionDAG = createRegionDAG() + val searchResult = createRegionDAGSearchResult() val totalRPGTime = System.nanoTime() - startTime - val regionPlan = RegionPlan( - regions = regionDAG.iterator().asScala.toSet, - regionLinks = regionDAG.edgeSet().asScala.toSet - ) + val regionPlan = buildRegionPlan(searchResult.state, searchResult.regionDAG) val schedule = generateScheduleFromRegionPlan(regionPlan) logger.info( s"WID: ${workflowContext.workflowId.id}, EID: ${workflowContext.executionId.id}, total RPG time: " + @@ -116,6 +185,32 @@ class CostBasedScheduleGenerator( ) } + /** + * Builds the extended [[RegionPlan]] for `state`: the active regions and links from + * `regionDAG`, plus any skip regions (`Region.skipped = true`) when `state` skips operators, + * with skip region ids starting above the largest active region id. When every operator is + * skipped, `regionDAG` is empty and the `RegionPlan` holds only the skip regions. + */ + private[scheduling] def buildRegionPlan( + state: SearchState, + regionDAG: DirectedAcyclicGraph[Region, RegionLink] + ): RegionPlan = { + val activeRegions = regionDAG.vertexSet().asScala.toSet + val regionLinks = regionDAG.edgeSet().asScala.toSet + if (!state.skipsAnything(physicalPlan, requiredOutputs)) { + RegionPlan(activeRegions, regionLinks) + } else { + val firstSkipRegionId = activeRegions.map(_.id.id).maxOption.map(_ + 1).getOrElse(0L) + val skipRegions = state.skipRegions( + physicalPlan, + matchedResults, + requiredOutputs, + firstSkipRegionId + ) + RegionPlan(activeRegions ++ skipRegions, regionLinks) + } + } + /** * Partitions a physical plan into Regions and assigns storage URIs in two passes. * @@ -128,7 +223,7 @@ class CostBasedScheduleGenerator( * For each Region, allocate storage URIs on every output port of materialized edges. *
  • Pass 2 – Input URIs: * Re-traverse the same Regions and attach reader URIs on input ports using - * the URIs created in Pass 1.
  • + * the URIs created in Pass 1 and any saved-result locations in `readerUrisByInputPort`. * * *

    Why two passes?

    @@ -139,19 +234,25 @@ class CostBasedScheduleGenerator( * and avoiding “reader before writer” holes, two passes are required. * * - * @param physicalPlan the original physical plan (without materializations) - * @param matEdges edges to be materialized (including blocking edges) + * @param targetPlan the original physical plan or retained sub-plan (without materializations) + * @param matEdges edges to be materialized (including blocking edges) + * @param readerUrisByInputPort saved-result locations read by input ports of `targetPlan` * @return a set of `Region`s whose `ResourceConfig` contains only `URI`s for `PortConfig`s * (`Partitioning` to be assigned later in `ResourceAllocator`; see `IntermediateInputPortConfig`.) */ - private def createRegions( - physicalPlan: PhysicalPlan, - matEdges: Set[PhysicalLink] + private[scheduling] def createRegions( + targetPlan: PhysicalPlan, + matEdges: Set[PhysicalLink], + readerUrisByInputPort: Map[GlobalPortIdentity, List[URI]] = Map.empty ): Set[Region] = { + require( + readerUrisByInputPort.keys.forall(_.input), + s"readerUrisByInputPort requires input ports, got ${readerUrisByInputPort.keys.filterNot(_.input)}" + ) // Pass 0 – remove materialized edges and create connected components - val matEdgesRemovedDAG: PhysicalPlan = matEdges.foldLeft(physicalPlan)(_.removeLink(_)) + val matEdgesRemovedDAG: PhysicalPlan = matEdges.foldLeft(targetPlan)(_.removeLink(_)) val connectedComponents: Set[Graph[PhysicalOpIdentity, PhysicalLink]] = new BiconnectivityInspector[PhysicalOpIdentity, PhysicalLink]( @@ -168,13 +269,13 @@ class CostBasedScheduleGenerator( val links: Set[PhysicalLink] = operators .flatMap { opId => - physicalPlan.getUpstreamPhysicalLinks(opId) ++ - physicalPlan.getDownstreamPhysicalLinks(opId) + targetPlan.getUpstreamPhysicalLinks(opId) ++ + targetPlan.getDownstreamPhysicalLinks(opId) } - .filter(link => operators.contains(link.fromOpId)) + .filter(link => operators.contains(link.fromOpId) && operators.contains(link.toOpId)) .diff(matEdges) // keep only pipelined edges - val physicalOps: Set[PhysicalOp] = operators.map(physicalPlan.getOperator) + val physicalOps: Set[PhysicalOp] = operators.map(targetPlan.getOperator) // Frontend-specified ports that need to be materailized (output ports of "eye-icon" physicalOps) val outputPortIdsToViewResult: Set[GlobalPortIdentity] = @@ -238,7 +339,7 @@ class CostBasedScheduleGenerator( matEdges.filter(matEdge => regionOpIds.contains(matEdge.toOpId)) // Assign storage URIs to input ports of each materialized edge (each input port could have more than one URI) - val inputPortConfigs: Map[GlobalPortIdentity, IntermediateInputPortConfig] = + val inputUrisFromMatEdges: Map[GlobalPortIdentity, List[URI]] = relevantMatEdges .foldLeft(Map.empty[GlobalPortIdentity, List[URI]]) { (acc, link) => val globalOutputPortId = GlobalPortIdentity(link.fromOpId, link.fromPortId) @@ -262,10 +363,19 @@ class CostBasedScheduleGenerator( acc.getOrElse(globalInputPortId, List.empty[URI]) :+ inputReaderURI ) } - .map { - case (inputPortId, uris) => - inputPortId -> IntermediateInputPortConfig(uris) - } + + val inputUrisFromCache: Map[GlobalPortIdentity, List[URI]] = + readerUrisByInputPort.filter { + case (inputPortId, _) => existingRegion.getPorts.contains(inputPortId) + } + + val inputPortConfigs: Map[GlobalPortIdentity, IntermediateInputPortConfig] = + (inputUrisFromMatEdges.keySet ++ inputUrisFromCache.keySet).map { inputPortId => + inputPortId -> IntermediateInputPortConfig( + inputUrisFromMatEdges.getOrElse(inputPortId, List.empty[URI]) ++ + inputUrisFromCache.getOrElse(inputPortId, List.empty[URI]) + ) + }.toMap val newResourceConfig: Option[ResourceConfig] = existingRegion.resourceConfig match { case Some(existingConfig) => @@ -282,17 +392,23 @@ class CostBasedScheduleGenerator( /** * Checks a plan for schedulability, and returns a region DAG if the plan is schedulable. * - * @param matEdges Set of edges to materialize (including the original blocking edges). - * @return If the plan is schedulable, a region DAG will be returned. Otherwise a DirectedPseudograph (with directed - * cycles) will be returned to indicate that the plan is unschedulable. + * @param targetPlan the original physical plan or retained sub-plan + * @param matEdges Set of edges to materialize (including the original blocking edges). + * @param readerUrisByInputPort saved-result locations read by input ports of `targetPlan` + * @return If the plan is schedulable, an acyclic region DAG will be returned (empty when + * `targetPlan` has no operators). Otherwise a DirectedPseudograph (with directed cycles) + * will be returned to indicate that the plan is unschedulable. */ - private def tryConnectRegionDAG( - matEdges: Set[PhysicalLink] + private[scheduling] def tryConnectRegionDAG( + targetPlan: PhysicalPlan, + matEdges: Set[PhysicalLink], + readerUrisByInputPort: Map[GlobalPortIdentity, List[URI]] ): Either[DirectedAcyclicGraph[Region, RegionLink], DirectedPseudograph[Region, RegionLink]] = { val regionDAG = new DirectedAcyclicGraph[Region, RegionLink](classOf[RegionLink]) + if (targetPlan.operators.isEmpty) return Left(regionDAG) val regionGraph = new DirectedPseudograph[Region, RegionLink](classOf[RegionLink]) val opToRegionMap = new mutable.HashMap[PhysicalOpIdentity, Region] - createRegions(physicalPlan, matEdges).foreach(region => { + createRegions(targetPlan, matEdges, readerUrisByInputPort).foreach(region => { region.getOperators.foreach(op => opToRegionMap(op.id) = region) regionGraph.addVertex(region) regionDAG.addVertex(region) @@ -313,13 +429,18 @@ class CostBasedScheduleGenerator( else Right(regionGraph) } + private def tryConnectRegionDAG( + state: SearchState + ): Either[DirectedAcyclicGraph[Region, RegionLink], DirectedPseudograph[Region, RegionLink]] = { + val (retainedPlan, readerUrisByInputPort) = subPlanOf(state) + tryConnectRegionDAG(retainedPlan, state.linksOfType(Materialized), readerUrisByInputPort) + } + /** - * Performs a search to generate a region DAG. - * Materializations are added only after the plan is determined to be schedulable. - * - * @return A region DAG. + * Performs a search over 4-edge-state [[SearchState]]s to generate a region DAG for the + * retained part of the workflow, falling back to bottom-up greedy search on timeout. */ - private def createRegionDAG(): DirectedAcyclicGraph[Region, RegionLink] = { + private def createRegionDAGSearchResult(): SearchResult = { val searchResultFuture: Future[SearchResult] = Future { val requestedMode = workflowContext.workflowSettings.executionMode val effectiveMode = @@ -368,73 +489,89 @@ class CostBasedScheduleGenerator( s"WID: ${workflowContext.workflowId.id}, EID: ${workflowContext.executionId.id}, search time: " + s"${searchResult.searchTimeNanoSeconds / 1e6} ms." ) - - val regionDAG = searchResult.regionDAG - regionDAG + searchResult } /** - * The core of the search algorithm. If the input physical plan is already schedulable, no search will be executed. - * Otherwise, depending on the configuration, either a global search or a greedy search will be performed to find - * an optimal plan. The search starts from a plan where all non-blocking edges are pipelined, and leads to a low-cost - * schedulable plan by changing pipelined non-blocking edges to materialized. By default all pruning techniques - * are enabled (chains, clean edges, and early stopping on schedulable states). - * - * @return A SearchResult containing the plan, the region DAG (without materializations added yet), the cost, the - * time to finish search, and the number of states explored. + * The core of the search algorithm over 4-edge-state [[SearchState]]s. Starts from the + * eager-reuse seed where matched saved ports stay CacheRead, links into skipped operators are + * Unused, blocking active links are Materialized, and non-blocking active links are + * Pipelined. Transitions Pipelined active links to Materialized using chain, clean-edge, and + * early-stop pruning on the retained part of the plan. */ - def bottomUpSearch( + private[scheduling] def bottomUpSearch( globalSearch: Boolean = false, oChains: Boolean = true, oCleanEdges: Boolean = true, oEarlyStop: Boolean = true ): SearchResult = { val startTime = System.nanoTime() + val seedState = SearchState.eagerSeed( + physicalPlan, + matchedResults, + requiredOutputs, + allowPipelined, + _ => Pipelined + ) + val (retainedPlan, _) = subPlanOf(seedState) + if (retainedPlan.operators.isEmpty) { + return SearchResult( + state = seedState, + regionDAG = new DirectedAcyclicGraph[Region, RegionLink](classOf[RegionLink]), + cost = 0.0, + searchTimeNanoSeconds = System.nanoTime() - startTime, + numStatesExplored = 1 + ) + } + + val retainedBlockingLinks = retainedPlan.getBlockingAndDependeeLinks val originalNonBlockingEdges = if (oCleanEdges) { - physicalPlan.getNonBridgeNonBlockingLinks + retainedPlan.getNonBridgeNonBlockingLinks } else { - physicalPlan.links.diff( - physicalPlan.getBlockingAndDependeeLinks - ) + retainedPlan.links.diff(retainedBlockingLinks) } - // Queue to hold states to be explored, starting with the empty set - val queue: mutable.Queue[Set[PhysicalLink]] = mutable.Queue(Set.empty[PhysicalLink]) - val inQueue: mutable.Set[Set[PhysicalLink]] = mutable.Set(Set.empty[PhysicalLink]) + + def nonBlockingMatLinks(state: SearchState): Set[PhysicalLink] = + state.linksOfType(Materialized).diff(retainedBlockingLinks) + + // Queue to hold states to be explored, starting with the eager-reuse seed + val queue: mutable.Queue[SearchState] = mutable.Queue(seedState) + val inQueue: mutable.Set[SearchState] = mutable.Set(seedState) // Keep track of visited states to avoid revisiting - val visited: mutable.Set[Set[PhysicalLink]] = mutable.Set.empty[Set[PhysicalLink]] + val visited: mutable.Set[SearchState] = mutable.Set.empty[SearchState] // Used for the Early Stop optimization technique - val schedulableStates: mutable.Set[Set[PhysicalLink]] = mutable.Set.empty[Set[PhysicalLink]] + val schedulableNonBlockingMatSets: mutable.Set[Set[PhysicalLink]] = + mutable.Set.empty[Set[PhysicalLink]] // Initialize the bestResult with an impossible high cost for comparison var bestResult: SearchResult = SearchResult( - state = Set.empty, + state = seedState, regionDAG = new DirectedAcyclicGraph[Region, RegionLink](classOf[RegionLink]), cost = Double.PositiveInfinity ) while (queue.nonEmpty) { - // A state is represented as a set of materialized non-blocking edges. val currentState = queue.dequeue() inQueue.remove(currentState) + val currentNonBlockingMat = nonBlockingMatLinks(currentState) breakable { if ( - oEarlyStop && schedulableStates - .exists(ancestorState => ancestorState.subsetOf(currentState)) + oEarlyStop && schedulableNonBlockingMatSets + .exists(ancestorMat => ancestorMat.subsetOf(currentNonBlockingMat)) ) { // Early stop: stopping exploring states beyond a schedulable state since the cost will only increase. - // A state X is a descendant of an ancestor state Y in the bottom-up search process if Y's set of materialized - // edges is a subset of that of X's (since X is reachable from Y by adding more materialized edges.) + // A state X is a descendant of an ancestor state Y in the bottom-up search process if Y's set of + // non-blocking materialized edges is a subset of that of X's (since X is reachable from Y by adding more + // materialized edges.) break() } visited.add(currentState) - tryConnectRegionDAG( - physicalPlan.getBlockingAndDependeeLinks ++ currentState - ) match { + tryConnectRegionDAG(currentState) match { case Left(regionDAG) => - updateOptimumIfApplicable(regionDAG) - addNeighborStatesToFrontier() + updateOptimumIfApplicable(currentState, currentNonBlockingMat, regionDAG) + addNeighborStatesToFrontier(currentState) case Right(_) => - addNeighborStatesToFrontier() + addNeighborStatesToFrontier(currentState) } } @@ -442,30 +579,34 @@ class CostBasedScheduleGenerator( * An internal method of bottom-up search that updates the current optimum if the examined state is schedulable * and has a lower cost. */ - def updateOptimumIfApplicable(regionDAG: DirectedAcyclicGraph[Region, RegionLink]): Unit = { - if (oEarlyStop) schedulableStates.add(currentState) + def updateOptimumIfApplicable( + state: SearchState, + nonBlockingMat: Set[PhysicalLink], + regionDAG: DirectedAcyclicGraph[Region, RegionLink] + ): Unit = { + if (oEarlyStop) schedulableNonBlockingMatSets.add(nonBlockingMat) // Calculate the current state's cost and update the bestResult if it's lower - val cost = allocateResourcesAndEvaluateCost(regionDAG) + val cost = allocateResourcesAndEvaluateCost(state, regionDAG) if (cost < bestResult.cost) { - bestResult = SearchResult(currentState, regionDAG, cost) + bestResult = SearchResult(state, regionDAG, cost) } } /** - * An internal method of bottom-up search that performs state transitions (changing an pipelined edge to + * An internal method of bottom-up search that performs state transitions (changing a pipelined active edge to * materialized) to include the unvisited neighbor(s) of the current state in the frontier (i.e., the queue). * If using global search, all unvisited neighbors will be included. Otherwise in a greedy search, only the * neighbor with the lowest cost will be included. */ - def addNeighborStatesToFrontier(): Unit = { - val allCurrentMaterializedEdges = - currentState ++ physicalPlan.getBlockingAndDependeeLinks + def addNeighborStatesToFrontier(state: SearchState): Unit = { + val currentMatEdges = state.linksOfType(Materialized) // Generate and enqueue all neighbour states that haven't been visited - var candidateEdges = originalNonBlockingEdges - .diff(currentState) + var candidateEdges = originalNonBlockingEdges.filter { link => + state(link) == Pipelined && allowedLinkTypes(link).contains(Materialized) + } if (oChains) { - val edgesInChainWithMaterializedEdges = physicalPlan.maxChains - .filter(chain => chain.intersect(allCurrentMaterializedEdges).nonEmpty) + val edgesInChainWithMaterializedEdges = retainedPlan.maxChains + .filter(chain => chain.intersect(currentMatEdges).nonEmpty) .flatten candidateEdges = candidateEdges.diff( edgesInChainWithMaterializedEdges @@ -473,16 +614,19 @@ class CostBasedScheduleGenerator( } val unvisitedNeighborStates = candidateEdges - .map(edge => currentState + edge) + .map(edge => state.updated(edge, Materialized)) .filter(neighborState => !visited.contains(neighborState) && !inQueue.contains(neighborState) ) val filteredNeighborStates = if (oEarlyStop) { // Any descendant state of a schedulable state is not worth exploring. - unvisitedNeighborStates.filter(neighborState => - !schedulableStates.exists(ancestorState => ancestorState.subsetOf(neighborState)) - ) + unvisitedNeighborStates.filter { neighborState => + val neighborNonBlockingMat = nonBlockingMatLinks(neighborState) + !schedulableNonBlockingMatSets.exists(ancestorMat => + ancestorMat.subsetOf(neighborNonBlockingMat) + ) + } } else { unvisitedNeighborStates } @@ -499,9 +643,9 @@ class CostBasedScheduleGenerator( } private def enqueueFrontier( - neighbors: Iterable[Set[PhysicalLink]], - queue: mutable.Queue[Set[PhysicalLink]], - inQueue: mutable.Set[Set[PhysicalLink]], + neighbors: Iterable[SearchState], + queue: mutable.Queue[SearchState], + inQueue: mutable.Set[SearchState], globalSearch: Boolean ): Unit = { if (globalSearch) { @@ -510,26 +654,44 @@ class CostBasedScheduleGenerator( queue.enqueue(neighborState) } } else if (neighbors.nonEmpty) { - val minCostNeighborState = neighbors.minBy(neighborState => - tryConnectRegionDAG( - physicalPlan.getBlockingAndDependeeLinks ++ neighborState - ) match { - case Left(regionDAG) => allocateResourcesAndEvaluateCost(regionDAG) + val minCostNeighborState = neighbors.minBy { neighborState => + tryConnectRegionDAG(neighborState) match { + case Left(regionDAG) => allocateResourcesAndEvaluateCost(neighborState, regionDAG) case Right(_) => Double.MaxValue } - ) + } inQueue.add(minCostNeighborState) queue.enqueue(minCostNeighborState) } } - /** Constructs a baseline fully materialized region plan (one operator per region) and evaluates its cost. */ - def getFullyMaterializedSearchState: SearchResult = { + /** + * Constructs a baseline fully materialized region plan (one retained operator per region) + * from the eager-reuse all-Materialized seed and evaluates its cost. + */ + private[scheduling] def getFullyMaterializedSearchState: SearchResult = { val startTime = System.nanoTime() + val seedState = SearchState.eagerSeed( + physicalPlan, + matchedResults, + requiredOutputs, + allowPipelined, + _ => Materialized + ) + val (retainedPlan, _) = subPlanOf(seedState) + if (retainedPlan.operators.isEmpty) { + return SearchResult( + state = seedState, + regionDAG = new DirectedAcyclicGraph[Region, RegionLink](classOf[RegionLink]), + cost = 0.0, + searchTimeNanoSeconds = System.nanoTime() - startTime, + numStatesExplored = 1 + ) + } val (regionDAG, cost) = - tryConnectRegionDAG(physicalPlan.links) match { - case Left(dag) => (dag, allocateResourcesAndEvaluateCost(dag)) + tryConnectRegionDAG(seedState) match { + case Left(dag) => (dag, allocateResourcesAndEvaluateCost(seedState, dag)) case Right(_) => ( new DirectedAcyclicGraph[Region, RegionLink](classOf[RegionLink]), @@ -538,7 +700,7 @@ class CostBasedScheduleGenerator( } SearchResult( - state = Set.empty, + state = seedState, regionDAG = regionDAG, cost = cost, searchTimeNanoSeconds = System.nanoTime() - startTime, @@ -547,50 +709,70 @@ class CostBasedScheduleGenerator( } /** - * Another direction to perform the search. Depending on the configuration, either a global search or a greedy search - * will be performed to find an optimal plan. The search starts from a plan where all edges are materialized, and - * leads to a low-cost schedulable plan by changing materialized non-blocking edges to pipelined. - * By default, all pruning techniques are enabled (chains, clean edges). - * - * @return A SearchResult containing the plan, the region DAG (without materializations added yet), the cost, the - * time to finish search, and the number of states explored. + * Top-down search over 4-edge-state [[SearchState]]s. Starts from the eager-reuse seed where + * matched saved ports stay CacheRead, links into skipped operators are Unused, and active + * retained links start Materialized except those pre-pipelined by chain and clean-edge pruning + * on the retained part. Transitions Materialized non-blocking active links to Pipelined. */ - def topDownSearch( + private[scheduling] def topDownSearch( globalSearch: Boolean = false, oChains: Boolean = true, oCleanEdges: Boolean = true ): SearchResult = { val startTime = System.nanoTime() - // Starting from a state where all non-blocking edges are materialized - val originalSeedState = physicalPlan.links.diff( - physicalPlan.getBlockingAndDependeeLinks + val matSeedState = SearchState.eagerSeed( + physicalPlan, + matchedResults, + requiredOutputs, + allowPipelined, + _ => Materialized ) + val (retainedPlan, _) = subPlanOf(matSeedState) + if (retainedPlan.operators.isEmpty) { + return SearchResult( + state = matSeedState, + regionDAG = new DirectedAcyclicGraph[Region, RegionLink](classOf[RegionLink]), + cost = 0.0, + searchTimeNanoSeconds = System.nanoTime() - startTime, + numStatesExplored = 1 + ) + } + val retainedBlockingLinks = retainedPlan.getBlockingAndDependeeLinks + // Starting from a state where all non-blocking retained edges are materialized + val originalSeedMatLinks = retainedPlan.links.diff(retainedBlockingLinks) // Chain optimization: an edge in the same chain as a blocking edge should not be materialized - val seedStateOptimizedByChainsIfApplicable = if (oChains) { - val edgesInChainWithBlockingEdge = physicalPlan.maxChains - .filter(chain => chain.intersect(physicalPlan.getBlockingAndDependeeLinks).nonEmpty) + val seedMatLinksOptimizedByChains = if (oChains) { + val edgesInChainWithBlockingEdge = retainedPlan.maxChains + .filter(chain => chain.intersect(retainedBlockingLinks).nonEmpty) .flatten - originalSeedState.diff(edgesInChainWithBlockingEdge) + originalSeedMatLinks.diff(edgesInChainWithBlockingEdge) } else { - originalSeedState + originalSeedMatLinks } - // Clean edge optimization: a clean edge should not be materialized - val finalSeedState = if (oCleanEdges) { - seedStateOptimizedByChainsIfApplicable.intersect(physicalPlan.getNonBridgeNonBlockingLinks) + val finalSeedNonBlockingMatLinks = if (oCleanEdges) { + seedMatLinksOptimizedByChains.intersect(retainedPlan.getNonBridgeNonBlockingLinks) } else { - seedStateOptimizedByChainsIfApplicable + seedMatLinksOptimizedByChains } + val seedMatLinks = retainedBlockingLinks ++ finalSeedNonBlockingMatLinks + val seedState = SearchState.eagerSeed( + physicalPlan, + matchedResults, + requiredOutputs, + allowPipelined, + link => if (seedMatLinks.contains(link)) Materialized else Pipelined + ) // Queue to hold states to be explored, starting with the seed state - val queue: mutable.Queue[Set[PhysicalLink]] = mutable.Queue(finalSeedState) - val inQueue: mutable.Set[Set[PhysicalLink]] = mutable.Set(finalSeedState) + val queue: mutable.Queue[SearchState] = mutable.Queue(seedState) + val inQueue: mutable.Set[SearchState] = mutable.Set(seedState) // Keep track of visited states to avoid revisiting - val visited: mutable.Set[Set[PhysicalLink]] = mutable.Set.empty[Set[PhysicalLink]] + val visited: mutable.Set[SearchState] = mutable.Set.empty[SearchState] // Initialize the bestResult with an impossible high cost for comparison var bestResult: SearchResult = SearchResult( - state = Set.empty, + state = seedState, regionDAG = new DirectedAcyclicGraph[Region, RegionLink](classOf[RegionLink]), cost = Double.PositiveInfinity ) @@ -599,38 +781,43 @@ class CostBasedScheduleGenerator( val currentState = queue.dequeue() inQueue.remove(currentState) visited.add(currentState) - tryConnectRegionDAG( - physicalPlan.getBlockingAndDependeeLinks ++ currentState - ) match { + tryConnectRegionDAG(currentState) match { case Left(regionDAG) => - updateOptimumIfApplicable(regionDAG) - addNeighborStatesToFrontier() + updateOptimumIfApplicable(currentState, regionDAG) + addNeighborStatesToFrontier(currentState) // No need to explore further case Right(_) => - addNeighborStatesToFrontier() + addNeighborStatesToFrontier(currentState) } /** * An internal method of top-down search that updates the current optimum if the examined state is schedulable * and has a lower cost. */ - def updateOptimumIfApplicable(regionDAG: DirectedAcyclicGraph[Region, RegionLink]): Unit = { + def updateOptimumIfApplicable( + state: SearchState, + regionDAG: DirectedAcyclicGraph[Region, RegionLink] + ): Unit = { // Calculate the current state's cost and update the bestResult if it's lower - val cost = allocateResourcesAndEvaluateCost(regionDAG) + val cost = allocateResourcesAndEvaluateCost(state, regionDAG) if (cost < bestResult.cost) { - bestResult = SearchResult(currentState, regionDAG, cost) + bestResult = SearchResult(state, regionDAG, cost) } } /** - * An internal method of top-down search that performs state transitions (changing an materialized edge to + * An internal method of top-down search that performs state transitions (changing a materialized active edge to * pipelined) to include the unvisited neighbor(s) of the current state in the frontier (i.e., the queue). * If using global search, all unvisited neighbors will be included. Otherwise in a greedy search, only the * neighbor with the lowest cost will be included. */ - def addNeighborStatesToFrontier(): Unit = { - val unvisitedNeighborStates = currentState - .map(edge => currentState - edge) + def addNeighborStatesToFrontier(state: SearchState): Unit = { + val candidateEdges = + state.linksOfType(Materialized).diff(retainedBlockingLinks).filter { link => + allowedLinkTypes(link).contains(Pipelined) + } + val unvisitedNeighborStates = candidateEdges + .map(edge => state.updated(edge, Pipelined)) .filter(neighborState => !visited.contains(neighborState) && !inQueue.contains(neighborState) ) @@ -647,15 +834,18 @@ class CostBasedScheduleGenerator( } /** - * Takes a region DAG, generates one or more (to be done in the future) schedules based on the region DAG, allocates - * resources to each region in the region DAG, and calculates the cost of the schedule(s) using Cost Estimator. Uses - * the cost of the best schedule (currently only considers one schedule) as the cost of the region DAG. - * - * @return A cost determined by the cost estimator. + * Generates a schedule from `regionDAG`, allocates resources and estimates cost per active + * region using the isolated allocator and memoization map for + * `(targetPlan, readerUrisByInputPort)`, updates `regionDAG` vertices in place with their + * allocated configs, and returns the total cost. */ - private def allocateResourcesAndEvaluateCost( + private[scheduling] def allocateResourcesAndEvaluateCost( + targetPlan: PhysicalPlan, + readerUrisByInputPort: Map[GlobalPortIdentity, List[URI]], regionDAG: DirectedAcyclicGraph[Region, RegionLink] ): Double = { + if (regionDAG.vertexSet().isEmpty) return 0.0 + val evaluation = evaluationOf(targetPlan, readerUrisByInputPort) val regionPlan = RegionPlan(regionDAG.vertexSet().asScala.toSet, regionDAG.edgeSet().asScala.toSet) val schedule = generateScheduleFromRegionPlan(regionPlan) @@ -670,10 +860,10 @@ class CostBasedScheduleGenerator( portIds = region.ports, resourceConfig = region.resourceConfig ) - val (resourceConfig, regionCost) = costEstimatorMemoization + val (resourceConfig, regionCost) = evaluation.memoization .getOrElseUpdate( costEstimatorMemoKey, - costEstimator.allocateResourcesAndEstimateCost(region, 1) + evaluation.estimator.allocateResourcesAndEstimateCost(region, 1) ) // Update the region in the regionDAG to be the new region with resources allocated. val regionWithResourceConfig = region.copy(resourceConfig = Some(resourceConfig)) @@ -685,4 +875,12 @@ class CostBasedScheduleGenerator( .sum } + private def allocateResourcesAndEvaluateCost( + state: SearchState, + regionDAG: DirectedAcyclicGraph[Region, RegionLink] + ): Double = { + val (retainedPlan, readerUrisByInputPort) = subPlanOf(state) + allocateResourcesAndEvaluateCost(retainedPlan, readerUrisByInputPort, regionDAG) + } + } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/CostEstimator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/CostEstimator.scala index d86101a1f06..a3683167d42 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/CostEstimator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/CostEstimator.scala @@ -58,32 +58,49 @@ object DefaultCostEstimator { * is the execution time of its longest-running operator. Otherwise the cost is the number of materialized ports in the * region. */ -class DefaultCostEstimator( +class DefaultCostEstimator private ( workflowContext: WorkflowContext, val resourceAllocator: ResourceAllocator, - val actorId: ActorVirtualIdentity + val actorId: ActorVirtualIdentity, + preloadedStats: Option[Option[Map[String, Double]]] ) extends CostEstimator with AmberLogging { + def this( + workflowContext: WorkflowContext, + resourceAllocator: ResourceAllocator, + actorId: ActorVirtualIdentity + ) = + this(workflowContext, resourceAllocator, actorId, preloadedStats = None) + // Requires mysql database to retrieve execution statistics, otherwise use number of materialized ports as a default. - private val operatorEstimatedTimeOption = Try( - this.getOperatorExecutionTimeInSeconds( - this.workflowContext.workflowId.id - ) - ) match { - case Failure(_) => None - case Success(result) => result + private val operatorEstimatedTimeOption = preloadedStats.getOrElse { + Try( + this.getOperatorExecutionTimeInSeconds( + this.workflowContext.workflowId.id + ) + ) match { + case Failure(_) => None + case Success(result) => result + } } - operatorEstimatedTimeOption match { - case None => - logger.info( - s"WID: ${workflowContext.workflowId.id}, EID: ${workflowContext.executionId.id}, " + - s"no past execution statistics available. Using number of materialized output ports as the cost. " - ) - case Some(_) => + if (preloadedStats.isEmpty && operatorEstimatedTimeOption.isEmpty) { + logger.info( + s"WID: ${workflowContext.workflowId.id}, EID: ${workflowContext.executionId.id}, " + + s"no past execution statistics available. Using number of materialized output ports as the cost. " + ) } + /** Returns a cost estimator sharing the loaded execution statistics with `newAllocator`. */ + def withResourceAllocator(newAllocator: ResourceAllocator): DefaultCostEstimator = + new DefaultCostEstimator( + workflowContext, + newAllocator, + actorId, + preloadedStats = Some(operatorEstimatedTimeOption) + ) + override def allocateResourcesAndEstimateCost( region: Region, resourceUnits: Int diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Region.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Region.scala index bbb4035e22f..535fd2c0a0d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Region.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Region.scala @@ -30,12 +30,18 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala case class RegionLink(fromRegionId: RegionIdentity, toRegionId: RegionIdentity) case class RegionIdentity(id: Long) + +/** + * A sub-plan scheduled as a unit. When `skipped` is true, its operators are not launched on + * workers; their saved outputs are carried in `resourceConfig` for reuse by later regions. + */ case class Region( id: RegionIdentity, physicalOps: Set[PhysicalOp], physicalLinks: Set[PhysicalLink], ports: Set[GlobalPortIdentity] = Set.empty, - resourceConfig: Option[ResourceConfig] = None + resourceConfig: Option[ResourceConfig] = None, + skipped: Boolean = false ) { private val operators: Map[PhysicalOpIdentity, PhysicalOp] = @@ -73,9 +79,8 @@ case class Region( * The effective source contains operators that have 0 input links in this region. */ def getSourceOperators: Set[PhysicalOp] = { - getOperators.filter(physicalOp => - physicalOp.getInputLinks().forall(link => !operators.contains(link.fromOpId)) - ) + val operatorsWithInputLink = getLinks.map(_.toOpId) + getOperators.filterNot(op => operatorsWithInputLink.contains(op.id)) } /** diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlan.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlan.scala index 95937517de2..006781cc6ea 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlan.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlan.scala @@ -25,6 +25,11 @@ import org.jgrapht.traverse.TopologicalOrderIterator import scala.jdk.CollectionConverters.IteratorHasAsScala +/** + * Region-level execution plan of a workflow. `regions` holds both active regions + * (`skipped = false`) and skip regions (`skipped = true`); `regionLinks` captures materialized + * dependencies among active regions. + */ case class RegionPlan( regions: Set[Region], regionLinks: Set[RegionLink] diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleGenerator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleGenerator.scala index 13f1acebdcf..e4eaa8832aa 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleGenerator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleGenerator.scala @@ -45,24 +45,40 @@ abstract class ScheduleGenerator( def generate(): (Schedule, PhysicalPlan) /** - * A schedule is a ranking on the regions of a region plan. - * Regions are dispatched in batches of up to ApplicationConfig.maxConcurrentRegions, respecting the DAG dependencies. - * When maxConcurrentRegions == 1, this is equivalent to a total order (fully sequential execution). + * A schedule is a ranking on the regions of a region plan. Skip regions (`Region.skipped`) + * share level 0 so they complete together without worker allocation; active regions follow in + * topological batches of up to ApplicationConfig.maxConcurrentRegions. When + * maxConcurrentRegions == 1, the active levels form a total order. */ def generateScheduleFromRegionPlan(regionPlan: RegionPlan): Schedule = { - val inDegree = mutable.Map.empty[RegionIdentity, Int] - regionPlan.topologicalIterator().foreach { rid => - inDegree(rid) = regionPlan.dag.inDegreeOf(rid) + val skippedIds = regionPlan.regions.filter(_.skipped).map(_.id) + require( + regionPlan.regionLinks.forall(link => + !skippedIds.contains(link.fromRegionId) && !skippedIds.contains(link.toRegionId) + ), + "skip regions must not have region links" + ) + + val tmpLevelSets = mutable.Map.empty[Int, Set[RegionIdentity]] + var level = 0 + if (skippedIds.nonEmpty) { + tmpLevelSets(0) = skippedIds + level = 1 } + val inDegree = mutable.Map.empty[RegionIdentity, Int] + regionPlan + .topologicalIterator() + .filterNot(skippedIds.contains) + .foreach { rid => + inDegree(rid) = regionPlan.dag.inDegreeOf(rid) + } + val readyRegionsQueue = mutable.Queue( inDegree.collect { case (rid, 0) => rid }.toSeq: _* ) - val tmpLevelSets = mutable.Map.empty[Int, Set[RegionIdentity]] - var level = 0 - - // While there are ready regions: + // While there are ready active regions: // 1. Dequeue up to maxConcurrentRegions regions to form the current batch. // 2. Record this batch under the current level. // 3. For each region in the batch: diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/SearchState.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/SearchState.scala new file mode 100644 index 00000000000..a0384123c4e --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/SearchState.scala @@ -0,0 +1,399 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.engine.architecture.scheduling + +import org.apache.texera.amber.core.virtualidentity.PhysicalOpIdentity +import org.apache.texera.amber.core.workflow.{ + CachedResult, + GlobalPortIdentity, + PhysicalLink, + PhysicalPlan, + WorkflowContext +} +import org.apache.texera.amber.engine.architecture.scheduling.config.{ + OutputPortConfig, + ResourceConfig +} +import org.jgrapht.alg.connectivity.BiconnectivityInspector + +import java.net.URI +import scala.collection.mutable +import scala.jdk.CollectionConverters._ + +/** + * Internal search choice for one edge during [[CostBasedScheduleGenerator]] planning: + * + * - Materialized: the source operator runs and writes the port's output; the target reads the + * written output once the source's region has finished. + * - Pipelined: the source operator runs and sends each tuple to the target as it is produced. + * - CacheRead: the target reads the port's saved result instead of output produced in this run. + * - Unused: no data crosses the edge in this run because its target operator does not run. + */ +private[scheduling] sealed trait EdgeType +private[scheduling] object EdgeType { + case object Materialized extends EdgeType + case object Pipelined extends EdgeType + case object CacheRead extends EdgeType + case object Unused extends EdgeType +} + +/** + * Candidate 4-edge-state assignment searched by [[CostBasedScheduleGenerator]]. Holds the + * [[EdgeType]] of each [[PhysicalLink]] in the plan and the set of required output ports satisfied + * by reading saved results rather than recomputing. + */ +private[scheduling] final case class SearchState( + linkTypes: Map[PhysicalLink, EdgeType], + reusedOutputs: Set[GlobalPortIdentity] = Set.empty +) { + import EdgeType._ + + require( + reusedOutputs.forall(!_.input), + s"reusedOutputs requires output ports, got ${reusedOutputs.filter(_.input)}" + ) + + def apply(link: PhysicalLink): EdgeType = linkTypes(link) + + def updated(link: PhysicalLink, edgeType: EdgeType): SearchState = + copy(linkTypes = linkTypes.updated(link, edgeType)) + + def linksOfType(edgeType: EdgeType): Set[PhysicalLink] = + linkTypes.collect { case (link, t) if t == edgeType => link }.toSet + + /** + * The operators of `physicalPlan` that run under this state: those with no output ports, those + * with a required output not satisfied by `reusedOutputs`, and those that are the source of a + * `Materialized` or `Pipelined` link. + */ + def retainedOperators( + physicalPlan: PhysicalPlan, + requiredOutputs: Set[GlobalPortIdentity] + ): Set[PhysicalOpIdentity] = { + val sinkOpsWithoutOutputPorts = physicalPlan.operators.filter(_.outputPorts.isEmpty).map(_.id) + val linkSources = linkTypes.collect { + case (link, edgeType) if edgeType == Materialized || edgeType == Pipelined => link.fromOpId + } + val requiredSources = requiredOutputs.diff(reusedOutputs).map(_.opId) + sinkOpsWithoutOutputPorts ++ linkSources ++ requiredSources + } + + /** The operators of `physicalPlan` that do not run under this state. */ + def skippedOperators( + physicalPlan: PhysicalPlan, + requiredOutputs: Set[GlobalPortIdentity] + ): Set[PhysicalOpIdentity] = + physicalPlan.operators.map(_.id).diff(retainedOperators(physicalPlan, requiredOutputs)) + + /** Whether any operator of `physicalPlan` is skipped under this state. */ + def skipsAnything( + physicalPlan: PhysicalPlan, + requiredOutputs: Set[GlobalPortIdentity] + ): Boolean = + skippedOperators(physicalPlan, requiredOutputs).nonEmpty + + /** + * The consumption condition: every plan link is typed, every reused output belongs to + * `requiredOutputs`, and a link is `Unused` if and only if its target operator does not run. + */ + def satisfiesConsumptionCondition( + physicalPlan: PhysicalPlan, + requiredOutputs: Set[GlobalPortIdentity] + ): Boolean = + linkTypes.keySet == physicalPlan.links && + reusedOutputs.subsetOf(requiredOutputs) && { + val retainedOps = retainedOperators(physicalPlan, requiredOutputs) + linkTypes.forall { + case (link, edgeType) => (edgeType == Unused) == !retainedOps.contains(link.toOpId) + } + } + + /** + * Resolves which operators run and which incoming links carry data so every running operator + * receives all of its inputs (`CacheRead`, `Materialized`, or `Pipelined`) and every skipped + * operator receives none (`Unused`). + * + * Why this is needed: `SearchState` assigns an `EdgeType` per link independently, so seeding + * matched outputs or changing one link during search leaves upstream links out of sync with + * which operators must run: + * - Skipping unneeded upstream work (input necessity): when `CacheRead` or `reusedOutputs` + * covers every needed output of an operator, that operator skips and its incoming links must + * become `Unused`, cascading upstream unless another running branch (such as a shared diamond + * or multi-output `Split`) still requires an upstream operator. + * - Supplying needed upstream inputs (input satisfiability): when an operator runs (no output + * ports, un-reused required output, or active `Materialized`/`Pipelined` outgoing link, + * including when a `CacheRead` link is switched back to recompute), any `Unused` incoming + * link becomes required and resolves to `CacheRead` if its source port is in `matchedResults` + * or to `defaultActiveType` (falling back to `Materialized`) otherwise, requiring active + * upstream producers to run. + * + * How it works: one reverse-topological pass (`physicalPlan.topologicalIterator().toList.reverse`) + * propagates data-input requirements backward from sinks and `requiredOutputs` toward sources; + * every downstream consumer of an operator is already resolved before that operator is visited. + */ + def resolveInputRequirements( + physicalPlan: PhysicalPlan, + requiredOutputs: Set[GlobalPortIdentity], + matchedResults: Map[GlobalPortIdentity, CachedResult] = Map.empty, + allowPipelined: Boolean = true, + defaultActiveType: PhysicalLink => EdgeType = _ => Materialized + ): SearchState = { + val allowedLinks = SearchState.allowedLinkTypes(physicalPlan, matchedResults, allowPipelined) + require( + linkTypes.keySet == physicalPlan.links, + "linkTypes keys must match physicalPlan.links" + ) + require( + reusedOutputs.subsetOf(requiredOutputs), + s"reusedOutputs must be a subset of requiredOutputs, got ${reusedOutputs.diff(requiredOutputs)}" + ) + val updated = mutable.Map(linkTypes.toSeq: _*) + val outgoingByOp = physicalPlan.links.groupBy(_.fromOpId).withDefaultValue(Set.empty) + val incomingByOp = physicalPlan.links.groupBy(_.toOpId).withDefaultValue(Set.empty) + val sinkOpsWithoutOutputPorts = physicalPlan.operators.filter(_.outputPorts.isEmpty).map(_.id) + val requiredNotReusedOps = requiredOutputs.diff(reusedOutputs).map(_.opId) + physicalPlan.topologicalIterator().toList.reverse.foreach { opId => + val runs = sinkOpsWithoutOutputPorts.contains(opId) || + requiredNotReusedOps.contains(opId) || + outgoingByOp(opId).exists { link => + val t = updated(link) + t == Materialized || t == Pipelined + } + if (runs) { + incomingByOp(opId).foreach { inLink => + if (updated(inLink) == Unused) { + val sourcePort = GlobalPortIdentity(inLink.fromOpId, inLink.fromPortId) + updated(inLink) = + if (matchedResults.contains(sourcePort)) CacheRead + else { + val candidate = defaultActiveType(inLink) + if ( + candidate != Unused && candidate != CacheRead && + allowedLinks(inLink).contains(candidate) + ) candidate + else Materialized + } + } + } + } else { + incomingByOp(opId).foreach { inLink => + if (updated(inLink) != Unused) { + updated(inLink) = Unused + } + } + } + } + copy(linkTypes = updated.toMap) + } + + /** + * The output ports whose saved results this state reads, with each saved location and row + * count: the source port of every `CacheRead` link, plus every reused required output of a + * skipped operator. When a required output belongs to a retained operator, this run stores and + * shows its own result for that port while retained operators reading it use its saved copy. + */ + def cacheReadPorts( + physicalPlan: PhysicalPlan, + matchedResults: Map[GlobalPortIdentity, CachedResult], + requiredOutputs: Set[GlobalPortIdentity] + ): Map[GlobalPortIdentity, CachedResult] = { + SearchState.requireOutputPortsOfPlan(physicalPlan, matchedResults) + require( + satisfiesConsumptionCondition(physicalPlan, requiredOutputs), + "only a search state satisfying the consumption condition has cache-read ports" + ) + val skippedOps = skippedOperators(physicalPlan, requiredOutputs) + val readPorts = + linksOfType(CacheRead).map(link => GlobalPortIdentity(link.fromOpId, link.fromPortId)) ++ + reusedOutputs.filter(port => skippedOps.contains(port.opId)) + readPorts.map(port => port -> matchedResults(port)).toMap + } + + /** + * The sub-plan of `physicalPlan` that runs under this state: the retained operators, keeping the + * plan's own operator objects, and the links between them except `CacheRead` links. + */ + def retainedPlan( + physicalPlan: PhysicalPlan, + requiredOutputs: Set[GlobalPortIdentity] + ): PhysicalPlan = { + require( + satisfiesConsumptionCondition(physicalPlan, requiredOutputs), + "only a search state satisfying the consumption condition has a retained plan" + ) + val retained = physicalPlan.getSubPlan(retainedOperators(physicalPlan, requiredOutputs)) + retained.copy(links = retained.links.diff(linksOfType(CacheRead))) + } + + /** + * Saved-result locations keyed by each consumer input port over `CacheRead` links, sorted by + * link so multi-writer ports (such as Union) keep a deterministic order. + */ + def readerUrisByInputPort( + physicalPlan: PhysicalPlan, + matchedResults: Map[GlobalPortIdentity, CachedResult], + requiredOutputs: Set[GlobalPortIdentity] + ): Map[GlobalPortIdentity, List[URI]] = { + val readPorts = cacheReadPorts(physicalPlan, matchedResults, requiredOutputs) + linksOfType(CacheRead).toList + .sortBy(_.toString) + .foldLeft(Map.empty[GlobalPortIdentity, List[URI]]) { (acc, link) => + val sourcePort = GlobalPortIdentity(link.fromOpId, link.fromPortId) + val inputPort = GlobalPortIdentity(link.toOpId, link.toPortId, input = true) + val savedLocation = readPorts(sourcePort).storageUri + acc.updated(inputPort, acc.getOrElse(inputPort, List.empty[URI]) :+ savedLocation) + } + } + + /** + * The skipped operators of `physicalPlan` under this state as skip regions (`Region.skipped = + * true`), one per connected component, with ids `firstRegionId`, `firstRegionId + 1`, ... in + * deterministic order. + */ + def skipRegions( + physicalPlan: PhysicalPlan, + matchedResults: Map[GlobalPortIdentity, CachedResult], + requiredOutputs: Set[GlobalPortIdentity], + firstRegionId: Long + ): Set[Region] = { + val skippedOps = skippedOperators(physicalPlan, requiredOutputs) + val skippedLinks = linksOfType(Unused) + val readPorts = cacheReadPorts(physicalPlan, matchedResults, requiredOutputs) + val groups: List[Set[PhysicalOpIdentity]] = + new BiconnectivityInspector[PhysicalOpIdentity, PhysicalLink]( + physicalPlan.getSubPlan(skippedOps).dag + ).getConnectedComponents.asScala + .map(_.vertexSet().asScala.toSet) + .toList + .sortBy(_.map(_.toString).min) + groups.zipWithIndex.map { + case (group, index) => + val physicalOps = group.map(physicalPlan.getOperator) + Region( + id = RegionIdentity(firstRegionId + index), + physicalOps = physicalOps, + physicalLinks = skippedLinks.filter(link => + group.contains(link.fromOpId) && group.contains(link.toOpId) + ), + ports = physicalOps.flatMap(op => + op.inputPorts.keys.map(portId => GlobalPortIdentity(op.id, portId, input = true)) ++ + op.outputPorts.keys.map(portId => GlobalPortIdentity(op.id, portId)) + ), + resourceConfig = Some( + ResourceConfig(portConfigs = readPorts.collect { + case (port, saved) if group.contains(port.opId) => + port -> OutputPortConfig(saved.storageUri, saved.tupleCount) + }) + ), + skipped = true + ) + }.toSet + } +} + +/** + * Validation rules and seeds for [[SearchState]], internal to the scheduling package: + * + * - Every key in `matchedResults` must be an output port of `physicalPlan` + * ([[SearchState.requireOutputPortsOfPlan]]). + * - Required outputs are `outputPortsNeedingStorage` plus every output port with no outgoing link. + * - An operator with no output ports always runs: nothing in the workflow reads it. + * - Under eager reuse (`eagerSeed`), every matched result is reused and a single reverse- + * topological pass marks links into skipped operators `Unused`. + */ +private[scheduling] object SearchState { + import EdgeType._ + + /** Verifies that every port in `matchedResults` is an output port of `physicalPlan`. */ + def requireOutputPortsOfPlan( + physicalPlan: PhysicalPlan, + matchedResults: Map[GlobalPortIdentity, CachedResult] + ): Unit = { + if (matchedResults.isEmpty) return + val planOutputPorts = physicalPlan.operators.flatMap(op => + op.outputPorts.keys.map(portId => GlobalPortIdentity(op.id, portId)) + ) + val invalidPorts = matchedResults.keySet.diff(planOutputPorts) + require( + invalidPorts.isEmpty, + s"matchedResults requires output ports of physicalPlan, got $invalidPorts" + ) + } + + /** + * The output ports the run must have results for by its end: `outputPortsNeedingStorage` and + * every output port with no outgoing link. + */ + def requiredOutputPorts( + workflowContext: WorkflowContext, + physicalPlan: PhysicalPlan + ): Set[GlobalPortIdentity] = { + val portsWithLinks = + physicalPlan.links.map(link => GlobalPortIdentity(link.fromOpId, link.fromPortId)) + physicalPlan.operators + .flatMap(op => op.outputPorts.keys.map(portId => GlobalPortIdentity(op.id, portId))) + .filter(port => + workflowContext.workflowSettings.outputPortsNeedingStorage.contains(port) || + !portsWithLinks.contains(port) + ) + } + + def allowedLinkTypes( + physicalPlan: PhysicalPlan, + matchedResults: Map[GlobalPortIdentity, CachedResult], + allowPipelined: Boolean + ): Map[PhysicalLink, Set[EdgeType]] = { + requireOutputPortsOfPlan(physicalPlan, matchedResults) + val blockingAndDependee = physicalPlan.getBlockingAndDependeeLinks + physicalPlan.links.map { link => + val sourcePort = GlobalPortIdentity(link.fromOpId, link.fromPortId) + val types = Set[EdgeType](Materialized, Unused) ++ + (if (allowPipelined && !blockingAndDependee.contains(link)) Set(Pipelined) + else Set.empty) ++ + (if (matchedResults.contains(sourcePort)) Set(CacheRead) else Set.empty) + link -> types + }.toMap + } + + /** + * Builds the eager-reuse seed directly via [[SearchState.resolveInputRequirements]]: starts all + * links `Unused`, sets required end-of-run outputs present in `matchedResults` in `reusedOutputs`, + * and propagates input requirements backward from running operators so matched inputs become + * `CacheRead`, required unmatched inputs take `defaultActiveType(link)` (falling back to + * `Materialized`), and unreachable upstream links stay `Unused`. + */ + def eagerSeed( + physicalPlan: PhysicalPlan, + matchedResults: Map[GlobalPortIdentity, CachedResult], + requiredOutputs: Set[GlobalPortIdentity], + allowPipelined: Boolean = true, + defaultActiveType: PhysicalLink => EdgeType = _ => Materialized + ): SearchState = { + val initialLinkTypes = physicalPlan.links.map(_ -> (Unused: EdgeType)).toMap + val initialReusedOutputs = requiredOutputs.filter(matchedResults.contains) + SearchState(initialLinkTypes, initialReusedOutputs).resolveInputRequirements( + physicalPlan, + requiredOutputs, + matchedResults, + allowPipelined, + defaultActiveType + ) + } +} diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/PortConfig.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/PortConfig.scala index 56743ae0956..14c2c3eb3d3 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/PortConfig.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/PortConfig.scala @@ -32,11 +32,16 @@ sealed trait PortConfig { } /** - * An output port requires exactly one materialization base URI. Result and - * state URIs hang off it via `VFSURIFactory.resultURI` / `stateURI`; this - * field is *not* a URI you can pass straight to `DocumentFactory`. + * An output port requires one storage base URI. Result and state URIs hang + * off it via `VFSURIFactory.resultURI` / `stateURI`; this field is *not* a + * URI you can pass straight to `DocumentFactory`. For a skipped operator, + * `cachedTupleCount` holds the saved output count so runtime statistics + * match the run that wrote it. */ -final case class OutputPortConfig(storageURIBase: URI) extends PortConfig { +final case class OutputPortConfig( + storageURIBase: URI, + cachedTupleCount: Option[Long] = None +) extends PortConfig { override val storageURIs: List[URI] = List(storageURIBase) } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowSchedulerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowSchedulerSpec.scala index 8cf601e2d62..0a3d63c0f9b 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowSchedulerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowSchedulerSpec.scala @@ -19,16 +19,44 @@ package org.apache.texera.amber.engine.architecture.coordinator -import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} +import org.apache.texera.amber.core.storage.VFSURIFactory +import org.apache.texera.amber.core.virtualidentity.{ + ExecutionIdentity, + OperatorIdentity, + PhysicalOpIdentity, + WorkflowIdentity +} +import org.apache.texera.amber.core.workflow.{ + CachedResult, + ExecutionMode, + GlobalPortIdentity, + HashPartition, + PhysicalLink, + PortIdentity, + WorkflowContext, + WorkflowSettings +} +import org.apache.texera.amber.engine.architecture.scheduling.config.{ + InputPortConfig, + OutputPortConfig +} +import org.apache.texera.amber.engine.architecture.scheduling.{CostBasedScheduleGenerator, Region} +import org.apache.texera.amber.engine.architecture.sendsemantics.partitionings.RoundRobinPartitioning import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.engine.e2e.TestUtils.buildWorkflow -import org.apache.texera.amber.operator.TestOperators +import org.apache.texera.amber.operator.split.SplitOpDesc +import org.apache.texera.amber.operator.union.UnionOpDesc +import org.apache.texera.amber.operator.{LogicalOp, TestOperators} import org.apache.texera.common.compiler.model.LogicalLink import org.scalatest.flatspec.AnyFlatSpec +import java.net.URI + class WorkflowSchedulerSpec extends AnyFlatSpec { - private def buildHeaderlessCsvKeywordWorkflow() = { + private def buildHeaderlessCsvKeywordWorkflow( + context: WorkflowContext = new WorkflowContext() + ) = { val csvOpDesc = TestOperators.headerlessSmallCsvScanOpDesc() val keywordOpDesc = TestOperators.keywordSearchOpDesc("column-1", "Asia") buildWorkflow( @@ -41,10 +69,104 @@ class WorkflowSchedulerSpec extends AnyFlatSpec { PortIdentity(0) ) ), - new WorkflowContext() + context ) } + /** csv1 -> join (build, port 0), csv2 -> join (probe, port 1). */ + private def buildJoinWorkflow(context: WorkflowContext = new WorkflowContext()) = { + val csv1 = TestOperators.headerlessSmallCsvScanOpDesc() + val csv2 = TestOperators.headerlessSmallCsvScanOpDesc() + val join = TestOperators.joinOpDesc("column-1", "column-1") + buildWorkflow( + List(csv1, csv2, join), + List( + LogicalLink( + csv1.operatorIdentifier, + PortIdentity(0), + join.operatorIdentifier, + PortIdentity(0) + ), + LogicalLink( + csv2.operatorIdentifier, + PortIdentity(0), + join.operatorIdentifier, + PortIdentity(1) + ) + ), + context + ) + } + + /** + * csv -> split; split's port 0 -> keyword A and port 1 -> keyword B. With `withUnion`, A and + * B both feed one union. Split's port 0 gets a saved result of execution 1; port 1 has none, + * so Split runs for B. Returns the workflow, the link from split's port 0 to A, which reads + * the saved result, and the saved location. + */ + private def buildSplitWorkflowWithSavedPort0( + context: WorkflowContext, + withUnion: Boolean = false + ): (Workflow, PhysicalLink, URI) = { + val csv = TestOperators.headerlessSmallCsvScanOpDesc() + val split = new SplitOpDesc() + val keywordA = TestOperators.keywordSearchOpDesc("column-1", "Asia") + val keywordB = TestOperators.keywordSearchOpDesc("column-1", "Asia") + val union = new UnionOpDesc() + def logicalLink(from: LogicalOp, fromPort: Int, to: LogicalOp) = + LogicalLink( + from.operatorIdentifier, + PortIdentity(fromPort), + to.operatorIdentifier, + PortIdentity(0) + ) + val splitLinks = + List( + logicalLink(csv, 0, split), + logicalLink(split, 0, keywordA), + logicalLink(split, 1, keywordB) + ) + val unionLinks = List(logicalLink(keywordA, 0, union), logicalLink(keywordB, 0, union)) + val workflow = + if (withUnion) + buildWorkflow( + List(csv, split, keywordA, keywordB, union), + splitLinks ++ unionLinks, + context + ) + else buildWorkflow(List(csv, split, keywordA, keywordB), splitLinks, context) + val splitId = workflow.physicalPlan.getPhysicalOpsOfLogicalOp(split.operatorIdentifier).head.id + val splitOut0 = GlobalPortIdentity(splitId, PortIdentity(0)) + val savedUri = VFSURIFactory.createPortBaseURI( + context.workflowId, + ExecutionIdentity(1L), + splitOut0, + context.warehouse + ) + context.matchedResults = Map(splitOut0 -> CachedResult(savedUri, Some(4L))) + val splitToA = workflow.physicalPlan.links + .find(link => link.fromOpId == splitId && link.fromPortId == PortIdentity(0)) + .get + (workflow, splitToA, savedUri) + } + + private def pipelinedContext() = + new WorkflowContext( + WorkflowIdentity(7L), + ExecutionIdentity(2L), + WorkflowSettings(executionMode = ExecutionMode.PIPELINED) + ) + + /** The locations an input port reads from storage, from the one region that configures it. */ + private def readerUris(regions: Iterable[Region], inputPort: GlobalPortIdentity): List[URI] = { + val configs = regions.flatMap(_.resourceConfig.get.portConfigs.get(inputPort)).toList + assert(configs.size == 1, s"expected one region to configure $inputPort, got $configs") + configs.head match { + case InputPortConfig(pairs) => pairs.map(_._1) + case other => fail(s"expected a reader for $inputPort, got $other") + } + } + "WorkflowScheduler.updateSchedule" should "populate the schedule and physicalPlan fields" in { val workflow = buildHeaderlessCsvKeywordWorkflow() val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) @@ -100,4 +222,468 @@ class WorkflowSchedulerSpec extends AnyFlatSpec { assert(pulledRegions == expectedRegions) } + + // --------------------------------------------------------------------------- + // Matched results on the context + // --------------------------------------------------------------------------- + + /** A region without its id, which the generator does not assign deterministically. */ + private def shape(regions: Set[Region]) = + regions.map(r => + (r.physicalOps.map(_.id), r.physicalLinks, r.resourceConfig.map(_.portConfigs), r.skipped) + ) + + /** Baseline levels when `matchedResults` is empty on an otherwise identical context. */ + private def baselineEmptyCacheLevels(workflow: Workflow) = { + val uncachedContext = new WorkflowContext( + workflowId = workflow.context.workflowId, + executionId = workflow.context.executionId, + workflowSettings = workflow.context.workflowSettings, + cuid = workflow.context.cuid, + warehouse = workflow.context.warehouse, + matchedResults = Map.empty + ) + new CostBasedScheduleGenerator(uncachedContext, workflow.physicalPlan, COORDINATOR) + .generate() + ._1 + .levelSets + } + + private def assertSameLevelsAsEmptyCacheRun(workflow: Workflow): Unit = { + val expected = baselineEmptyCacheLevels(workflow) + val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) + scheduler.updateSchedule(workflow.physicalPlan) + val actual = scheduler.getSchedule.levelSets + assert(actual.keySet == expected.keySet) + actual.foreach { case (level, regions) => assert(shape(regions) == shape(expected(level))) } + assert(actual.values.flatten.forall(!_.skipped)) + assert(scheduler.physicalPlan eq workflow.physicalPlan) + } + + private def outputPortOf( + workflow: Workflow, + logicalOpId: org.apache.texera.amber.core.virtualidentity.OperatorIdentity + ) = { + val op = workflow.physicalPlan.operators.find(_.id.logicalOpId == logicalOpId).get + GlobalPortIdentity(op.id, PortIdentity(0)) + } + + "WorkflowScheduler.updateSchedule" should "produce the same levels as a direct generator run when the cache is empty" in { + assertSameLevelsAsEmptyCacheRun(buildHeaderlessCsvKeywordWorkflow()) + assertSameLevelsAsEmptyCacheRun(buildJoinWorkflow()) + } + + it should "reject matchedResults naming a port outside the physical plan" in { + val workflow = buildHeaderlessCsvKeywordWorkflow() + val keywordOp = workflow.physicalPlan + .getPhysicalOpsOfLogicalOp( + workflow.logicalPlan.getTerminalOperatorIds.head + ) + .head + val sampleUri = URI.create("vfs:///tmp/unused") + + workflow.context.matchedResults = Map( + GlobalPortIdentity(PhysicalOpIdentity(OperatorIdentity("ghost"), "main"), PortIdentity(0)) -> + CachedResult(sampleUri, Some(1L)) + ) + assertThrows[IllegalArgumentException] { + new WorkflowScheduler(workflow.context, COORDINATOR).updateSchedule(workflow.physicalPlan) + } + + workflow.context.matchedResults = Map( + GlobalPortIdentity(keywordOp.id, PortIdentity(0), input = true) -> + CachedResult(sampleUri, Some(1L)) + ) + assertThrows[IllegalArgumentException] { + new WorkflowScheduler(workflow.context, COORDINATOR).updateSchedule(workflow.physicalPlan) + } + } + + it should "keep cuid and warehouse, and put skip regions first, on a run with a match" in { + val context = new WorkflowContext( + workflowId = WorkflowIdentity(7L), + executionId = ExecutionIdentity(2L), + cuid = Some(7), + warehouse = Some("wh1") + ) + val workflow = buildHeaderlessCsvKeywordWorkflow(context) + val csvOp = workflow.physicalPlan.operators.find(_.isSourceOperator).get + val keywordOp = workflow.physicalPlan.operators.find(!_.isSourceOperator).get + val csvOut = GlobalPortIdentity(csvOp.id, PortIdentity(0)) + val cachedUri = + VFSURIFactory.createPortBaseURI( + WorkflowIdentity(7L), + ExecutionIdentity(1L), + csvOut, + Some("wh1") + ) + context.matchedResults = Map(csvOut -> CachedResult(cachedUri, Some(5L))) + + val scheduler = new WorkflowScheduler(context, COORDINATOR) + scheduler.updateSchedule(workflow.physicalPlan) + val levels = scheduler.getSchedule.levelSets + assert(levels.keySet == Set(0, 1)) + + // Level 0: the skipped csv operator, with its stored output. + val skipRegions = levels(0) + assert(skipRegions.size == 1) + val skipRegion = skipRegions.head + assert(skipRegion.skipped) + assert(skipRegion.physicalOps.map(_.id) == Set(csvOp.id)) + assert( + skipRegion.resourceConfig.get.portConfigs == Map( + csvOut -> OutputPortConfig(cachedUri, Some(5L)) + ) + ) + + // Level 1: the keyword operator, planned with the original context. + val generated = levels(1) + assert(generated.size == 1) + val region = generated.head + assert(!region.skipped) + assert(region.physicalOps.map(_.id) == Set(keywordOp.id)) + val config = region.resourceConfig.get + assert(config.operatorConfigs.values.flatMap(_.workerConfigs).forall(_.cuid.contains(7))) + config.portConfigs(GlobalPortIdentity(keywordOp.id, PortIdentity(0), input = true)) match { + case InputPortConfig(pairs) => assert(pairs.map(_._1) == List(cachedUri)) + case other => fail(s"expected a reader for the saved csv output, got $other") + } + config.portConfigs(GlobalPortIdentity(keywordOp.id, PortIdentity(0))) match { + case OutputPortConfig(uri, count) => + assert(uri.toString.contains("/wh/wh1/")) + assert(uri.toString.contains("/eid/2/")) + assert(count.isEmpty) + case other => fail(s"expected a new output URI, got $other") + } + + // Skip region ids sit above the generated ids; the full plan stays on the scheduler. + assert(skipRegion.id.id > generated.map(_.id.id).max) + assert(scheduler.physicalPlan eq workflow.physicalPlan) + } + + it should "give a run that skips every operator exactly one level of skip regions" in { + val workflow = buildHeaderlessCsvKeywordWorkflow() + val keywordOut = outputPortOf(workflow, workflow.logicalPlan.getTerminalOperatorIds.head) + val cachedUri = VFSURIFactory.createPortBaseURI( + workflow.context.workflowId, + ExecutionIdentity(1L), + keywordOut + ) + workflow.context.matchedResults = Map(keywordOut -> CachedResult(cachedUri, None)) + + val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) + scheduler.updateSchedule(workflow.physicalPlan) + val levels = scheduler.getSchedule.levelSets + assert(levels.keySet == Set(0)) + val regions = levels(0) + assert(regions.forall(_.skipped)) + assert(regions.flatMap(_.physicalOps.map(_.id)) == workflow.physicalPlan.operators.map(_.id)) + assert( + regions.flatMap(_.resourceConfig.get.portConfigs).toMap == + Map(keywordOut -> OutputPortConfig(cachedUri, None)) + ) + assert(scheduler.hasPendingRegions) + assert(scheduler.getNextRegions == regions) + assert(!scheduler.hasPendingRegions) + } + + it should "read the build side's input from the cache" in { + // The saved result is the output of the scan that feeds the build operator: the scan is + // skipped, and the build operator runs and reads its input from storage. + val workflow = buildJoinWorkflow() + val plan = workflow.physicalPlan + val buildLink = plan.links.find(_.toPortId == PortIdentity(0)).get + val buildSourceOut = GlobalPortIdentity(buildLink.fromOpId, buildLink.fromPortId) + val cachedUri = VFSURIFactory.createPortBaseURI( + workflow.context.workflowId, + ExecutionIdentity(1L), + buildSourceOut + ) + workflow.context.matchedResults = Map(buildSourceOut -> CachedResult(cachedUri, Some(9L))) + + val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) + scheduler.updateSchedule(plan) + val levels = scheduler.getSchedule.levelSets + assert(levels.size >= 2) + val skipRegions = levels(0) + assert(skipRegions.forall(_.skipped)) + assert(skipRegions.flatMap(_.physicalOps.map(_.id)) == Set(buildLink.fromOpId)) + val generated = (1 until levels.size).flatMap(levels).toSet + assert(generated.forall(!_.skipped)) + assert( + generated.flatMap(_.physicalOps.map(_.id)) == plan.operators.map(_.id) - buildLink.fromOpId + ) + val buildInput = GlobalPortIdentity(buildLink.toOpId, buildLink.toPortId, input = true) + val readers = generated.flatMap(_.resourceConfig.get.portConfigs.get(buildInput)) + assert(readers.size == 1) + readers.head match { + case InputPortConfig(pairs) => assert(pairs.map(_._1) == List(cachedUri)) + case other => fail(s"expected a reader for the saved scan output, got $other") + } + assert(skipRegions.map(_.id.id).min > generated.map(_.id.id).max) + } + + it should "skip the scan and the build operator when the build operator's output is saved" in { + // The build operator sends the rows of its hash table to the probe through an internal + // output port. That link is blocking, so every run stores the port. With its saved + // result, neither the build operator nor the scan feeding it runs, and the probe reads + // its build input from storage while the other scan still sends its rows to the probe. + val workflow = buildJoinWorkflow() + val plan = workflow.physicalPlan + val buildToProbe = plan.links.find(_.fromPortId == PortIdentity(0, internal = true)).get + val buildOpId = buildToProbe.fromOpId + val buildScanId = plan.getUpstreamPhysicalOpIds(buildOpId).head + val buildOut = GlobalPortIdentity(buildOpId, buildToProbe.fromPortId) + val otherScanToProbe = + plan.links.find(link => link.toOpId == buildToProbe.toOpId && link != buildToProbe).get + val cachedUri = VFSURIFactory.createPortBaseURI( + workflow.context.workflowId, + ExecutionIdentity(1L), + buildOut + ) + workflow.context.matchedResults = Map(buildOut -> CachedResult(cachedUri, Some(9L))) + + val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) + scheduler.updateSchedule(plan) + val levels = scheduler.getSchedule.levelSets + assert(levels.keySet == Set(0, 1)) + + // Level 0: the scan and the build operator in one skip region, with the build output's + // saved result. + val skipRegions = levels(0) + assert(skipRegions.size == 1) + val skipRegion = skipRegions.head + assert(skipRegion.skipped) + assert(skipRegion.physicalOps.map(_.id) == Set(buildScanId, buildOpId)) + assert( + skipRegion.resourceConfig.get.portConfigs == Map( + buildOut -> OutputPortConfig(cachedUri, Some(9L)) + ) + ) + + // Level 1: the probe and the scan that feeds it, with the probe's build input read from + // the saved result. + val generated = levels(1) + assert(generated.size == 1) + val region = generated.head + assert(!region.skipped) + assert(region.physicalOps.map(_.id) == plan.operators.map(_.id) - buildScanId - buildOpId) + assert(region.physicalLinks == Set(otherScanToProbe)) + val probeBuildInput = + GlobalPortIdentity(buildToProbe.toOpId, buildToProbe.toPortId, input = true) + region.resourceConfig.get.portConfigs(probeBuildInput) match { + case InputPortConfig(pairs) => assert(pairs.map(_._1) == List(cachedUri)) + case other => fail(s"expected a reader for the saved build output, got $other") + } + + // Skip region ids sit above the generated ids; the full plan stays on the scheduler. + assert(skipRegion.id.id > region.id.id) + assert(scheduler.physicalPlan eq plan) + } + + it should "hold the plan's own operator objects in every region" in { + // The scan and the build operator are skipped, and the probe reads the build output from + // storage. Neither the probe nor the build operator is a copy with their shared link + // removed: the regions hold the plan's own operators, whose link lists still name it. + val workflow = buildJoinWorkflow() + val plan = workflow.physicalPlan + val buildToProbe = plan.links.find(_.fromPortId == PortIdentity(0, internal = true)).get + val buildOut = GlobalPortIdentity(buildToProbe.fromOpId, buildToProbe.fromPortId) + val cachedUri = VFSURIFactory.createPortBaseURI( + workflow.context.workflowId, + ExecutionIdentity(1L), + buildOut + ) + workflow.context.matchedResults = Map(buildOut -> CachedResult(cachedUri, Some(9L))) + + val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) + scheduler.updateSchedule(plan) + val regions = scheduler.getSchedule.getRegions + assert(regions.exists(_.skipped) && regions.exists(!_.skipped)) + assert(regions.flatMap(_.getOperators).map(_.id).toSet == plan.operators.map(_.id)) + regions.flatMap(_.getOperators).foreach(op => assert(op eq plan.getOperator(op.id))) + // The link between them is in no region, while both operators still list it. + assert(regions.forall(!_.getLinks.contains(buildToProbe))) + val probeRegion = regions.find(_.getOperators.exists(_.id == buildToProbe.toOpId)).get + assert(probeRegion.getOperator(buildToProbe.toOpId).getInputLinks().contains(buildToProbe)) + val buildRegion = regions.find(_.getOperators.exists(_.id == buildToProbe.fromOpId)).get + assert(buildRegion.skipped) + assert( + buildRegion + .getOperator(buildToProbe.fromOpId) + .getOutputLinks(buildToProbe.fromPortId) + .contains(buildToProbe) + ) + } + + it should "read a saved port from storage even when its operator runs" in { + // Split runs for B, whose input has no saved result, and A reads split's port 0 from its + // saved result instead of waiting for Split. Nothing is skipped, so there is no skip level. + val context = pipelinedContext() + val (workflow, splitToA, savedUri) = buildSplitWorkflowWithSavedPort0(context) + val plan = workflow.physicalPlan + val scheduler = new WorkflowScheduler(context, COORDINATOR) + scheduler.updateSchedule(plan) + val regions = scheduler.getSchedule.getRegions + assert(regions.forall(!_.skipped)) + assert(scheduler.getSchedule.levelSets.values.forall(_.nonEmpty)) + assert(regions.flatMap(_.getOperators).map(_.id).toSet == plan.operators.map(_.id)) + // The link from split's port 0 to A carries nothing: no region holds it, and A reads only + // the saved location. + assert(regions.forall(!_.getLinks.contains(splitToA))) + val aInput = GlobalPortIdentity(splitToA.toOpId, splitToA.toPortId, input = true) + assert(readerUris(regions, aInput) == List(savedUri)) + // Split's port 1 has no saved result, so B reads it over its link, as without a cache. + val splitToB = plan.links + .find(link => link.fromOpId == splitToA.fromOpId && link.fromPortId == PortIdentity(1)) + .get + assert(regions.exists(_.getLinks.contains(splitToB))) + val bInput = GlobalPortIdentity(splitToB.toOpId, splitToB.toPortId, input = true) + assert(regions.forall(!_.resourceConfig.get.portConfigs.contains(bInput))) + assert(scheduler.physicalPlan eq plan) + } + + it should "store a port the run must store from this run while its reader uses the saved copy" in { + // The user views Split's results, so the run must store both of Split's ports. Split runs + // for B, so this run stores its own port 0 result, which the user sees; A still reads the + // saved copy of port 0. + val context = pipelinedContext() + val (workflow, splitToA, savedUri) = buildSplitWorkflowWithSavedPort0(context) + val splitOut0 = GlobalPortIdentity(splitToA.fromOpId, splitToA.fromPortId) + context.workflowSettings = context.workflowSettings.copy( + outputPortsNeedingStorage = context.workflowSettings.outputPortsNeedingStorage ++ + Set(splitOut0, GlobalPortIdentity(splitToA.fromOpId, PortIdentity(1))) + ) + val scheduler = new WorkflowScheduler(context, COORDINATOR) + scheduler.updateSchedule(workflow.physicalPlan) + val regions = scheduler.getSchedule.getRegions + val splitRegion = regions.find(_.getOperators.exists(_.id == splitOut0.opId)).get + splitRegion.resourceConfig.get.portConfigs(splitOut0) match { + case OutputPortConfig(uri, count) => + assert( + uri == VFSURIFactory.createPortBaseURI( + WorkflowIdentity(7L), + ExecutionIdentity(2L), + splitOut0, + None + ) + ) + assert(uri != savedUri) + assert(count.isEmpty) + case other => fail(s"expected this run's location for split's port 0, got $other") + } + val aInput = GlobalPortIdentity(splitToA.toOpId, splitToA.toPortId, input = true) + assert(readerUris(regions, aInput) == List(savedUri)) + } + + it should "read a saved result inside the region of the port's retained operator" in { + // A and B both feed a union, so in pipelined mode every operator shares one region, Split + // and A included. The link from split's port 0 to A is not one of the region's links, yet + // A's own link list still names it: A has no input link in the region, so it counts as one + // of the region's sources, and it starts first because its input port reads the saved + // result. + val context = pipelinedContext() + val (workflow, splitToA, savedUri) = buildSplitWorkflowWithSavedPort0(context, withUnion = true) + val plan = workflow.physicalPlan.setOperator( + workflow.physicalPlan + .getOperator(splitToA.toOpId) + .withPartitionRequirement(List(Some(HashPartition(List("column-1"))))) + ) + val scheduler = new WorkflowScheduler(context, COORDINATOR) + scheduler.updateSchedule(plan) + val regions = scheduler.getSchedule.getRegions + assert(regions.size == 1) + val region = regions.head + assert(region.getOperators.map(_.id) == plan.operators.map(_.id)) + assert(!region.getLinks.contains(splitToA)) + assert(region.getLinks == plan.links - splitToA) + val a = region.getOperator(splitToA.toOpId) + assert(a.getInputLinks().contains(splitToA)) + assert(region.getSourceOperators.map(_.id).contains(a.id)) + assert(region.getStarterOperators.map(_.id).contains(a.id)) + val aInput = GlobalPortIdentity(a.id, splitToA.toPortId, input = true) + assert(readerUris(Seq(region), aInput) == List(savedUri)) + // B reads split's port 1 over its link in the region: neither a source nor a starter. + val splitToB = plan.links + .find(link => link.fromOpId == splitToA.fromOpId && link.fromPortId == PortIdentity(1)) + .get + assert(!region.getSourceOperators.map(_.id).contains(splitToB.toOpId)) + assert(!region.getStarterOperators.map(_.id).contains(splitToB.toOpId)) + // A reads a saved result, so its output link to the union is partitioned as behind a + // materialized link: round robin, since A's output partitioning is unknown. + val aToUnion = plan.links.find(_.fromOpId == a.id).get + assert( + region.resourceConfig.get + .linkConfigs(aToUnion) + .partitioning + .isInstanceOf[RoundRobinPartitioning] + ) + } + + it should "skip an upstream cached scan and break a downstream HashJoin cycle" in { + val upstreamScan = TestOperators.headerlessSmallCsvScanOpDesc() + val upstreamFilter = TestOperators.keywordSearchOpDesc("column-1", "Asia") + val probeFilter = TestOperators.keywordSearchOpDesc("column-1", "Asia") + val join = TestOperators.joinOpDesc("column-1", "column-1") + val context = pipelinedContext() + val workflow = buildWorkflow( + List(upstreamScan, upstreamFilter, probeFilter, join), + List( + LogicalLink( + upstreamScan.operatorIdentifier, + PortIdentity(0), + upstreamFilter.operatorIdentifier, + PortIdentity(0) + ), + LogicalLink( + upstreamFilter.operatorIdentifier, + PortIdentity(0), + join.operatorIdentifier, + PortIdentity(0) + ), + LogicalLink( + upstreamFilter.operatorIdentifier, + PortIdentity(0), + probeFilter.operatorIdentifier, + PortIdentity(0) + ), + LogicalLink( + probeFilter.operatorIdentifier, + PortIdentity(0), + join.operatorIdentifier, + PortIdentity(1) + ) + ), + context + ) + val plan = workflow.physicalPlan + val scanOp = plan.getPhysicalOpsOfLogicalOp(upstreamScan.operatorIdentifier).head + val upstreamFilterOp = plan.getPhysicalOpsOfLogicalOp(upstreamFilter.operatorIdentifier).head + val scanOut = GlobalPortIdentity(scanOp.id, PortIdentity(0)) + val savedUri = VFSURIFactory.createPortBaseURI( + context.workflowId, + ExecutionIdentity(1L), + scanOut + ) + context.matchedResults = Map(scanOut -> CachedResult(savedUri, Some(6L))) + + val scheduler = new WorkflowScheduler(context, COORDINATOR) + scheduler.updateSchedule(plan) + val levels = scheduler.getSchedule.levelSets + assert(levels.size >= 3) + val skipRegions = levels(0) + assert(skipRegions.forall(_.skipped)) + assert(skipRegions.flatMap(_.getOperators.map(_.id)) == Set(scanOp.id)) + val generated = (1 until levels.size).flatMap(levels).toSet + assert(generated.forall(!_.skipped)) + assert(generated.flatMap(_.getOperators.map(_.id)) == plan.operators.map(_.id) - scanOp.id) + assert( + readerUris( + generated, + GlobalPortIdentity(upstreamFilterOp.id, PortIdentity(0), input = true) + ) == + List(savedUri) + ) + assert(skipRegions.map(_.id.id).min > generated.map(_.id.id).max) + } } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGeneratorSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGeneratorSpec.scala index 0b4aabe3b19..9e8a70bf8b2 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGeneratorSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGeneratorSpec.scala @@ -19,20 +19,46 @@ package org.apache.texera.amber.engine.architecture.scheduling +import org.apache.texera.amber.core.executor.OpExecInitInfo +import org.apache.texera.amber.core.storage.VFSURIFactory +import org.apache.texera.amber.core.virtualidentity.{ + ExecutionIdentity, + OperatorIdentity, + PhysicalOpIdentity, + WorkflowIdentity +} import org.apache.texera.amber.core.workflow.{ + CachedResult, ExecutionMode, + GlobalPortIdentity, + HashPartition, + InputPort, + OutputPort, + PartitionInfo, + PhysicalLink, + PhysicalOp, PhysicalPlan, PortIdentity, WorkflowContext, WorkflowSettings } +import org.apache.texera.amber.engine.architecture.scheduling.config.{ + InputPortConfig, + OutputPortConfig +} +import org.apache.texera.amber.engine.architecture.sendsemantics.partitionings.{ + HashBasedShufflePartitioning, + RoundRobinPartitioning +} import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.engine.e2e.TestUtils.buildWorkflow -import org.apache.texera.amber.operator.TestOperators +import org.apache.texera.amber.operator.union.UnionOpDesc +import org.apache.texera.amber.operator.{LogicalOp, TestOperators} import org.apache.texera.common.compiler.model.LogicalLink import org.scalamock.scalatest.MockFactory import org.scalatest.flatspec.AnyFlatSpec +import java.net.URI import scala.jdk.CollectionConverters._ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { @@ -613,8 +639,19 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { // early-stop pruning and queue ordering. assert(greedyResult.numStatesExplored <= scheduleGenerator.physicalPlan.links.size + 1) - // The chosen state is a set of materialized non-blocking edges, all of which must be links of the physical plan. - assert(greedyResult.state.subsetOf(scheduleGenerator.physicalPlan.links)) + // The chosen state is a 4-edge-state SearchState satisfying the consumption condition whose + // materialized links are links of the physical plan. + assert( + greedyResult.state.satisfiesConsumptionCondition( + scheduleGenerator.physicalPlan, + scheduleGenerator.requiredOutputs + ) + ) + assert( + greedyResult.state + .linksOfType(EdgeType.Materialized) + .subsetOf(scheduleGenerator.physicalPlan.links) + ) } "CostBasedRegionPlanGenerator" should "finish top-down greedy search (globalSearch=false) in csv->->filter->join->filter2 workflow" in { @@ -673,8 +710,488 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { assert(greedyResult.regionDAG.vertexSet().asScala.nonEmpty) assert(greedyResult.cost < Double.PositiveInfinity) - // The chosen state is a set of materialized non-blocking edges, all of which must be links of the physical plan. - assert(greedyResult.state.subsetOf(scheduleGenerator.physicalPlan.links)) + // The chosen state is a 4-edge-state SearchState satisfying the consumption condition whose + // materialized links are links of the physical plan. + assert( + greedyResult.state.satisfiesConsumptionCondition( + scheduleGenerator.physicalPlan, + scheduleGenerator.requiredOutputs + ) + ) + assert( + greedyResult.state + .linksOfType(EdgeType.Materialized) + .subsetOf(scheduleGenerator.physicalPlan.links) + ) + } + + // --------------------------------------------------------------------------- + // Saved-result reads and extended RegionPlan in CostBasedScheduleGenerator + // --------------------------------------------------------------------------- + + private def csvKeywordWorkflow(context: WorkflowContext) = { + val csv = TestOperators.headerlessSmallCsvScanOpDesc() + val keyword = TestOperators.keywordSearchOpDesc("column-1", "Asia") + buildWorkflow( + List(csv, keyword), + List( + LogicalLink( + csv.operatorIdentifier, + PortIdentity(0), + keyword.operatorIdentifier, + PortIdentity(0) + ) + ), + context + ) + } + + private def scheduleSubPlan( + generator: CostBasedScheduleGenerator, + retainedPlan: PhysicalPlan, + matEdges: Set[PhysicalLink], + readerUrisByInputPort: Map[GlobalPortIdentity, List[URI]] + ): Schedule = { + val regionDAG = + generator.tryConnectRegionDAG(retainedPlan, matEdges, readerUrisByInputPort) match { + case Left(dag) => dag + case Right(graph) => fail(s"expected acyclic region DAG, got cycle: $graph") + } + generator.allocateResourcesAndEvaluateCost(retainedPlan, readerUrisByInputPort, regionDAG) + generator.generateScheduleFromRegionPlan( + RegionPlan(regionDAG.vertexSet().asScala.toSet, regionDAG.edgeSet().asScala.toSet) + ) + } + + "CostBasedScheduleGenerator" should "add the reader URIs of cache read inputs to the input port configs" in { + val context = new WorkflowContext(WorkflowIdentity(3L), ExecutionIdentity(2L)) + val workflow = csvKeywordWorkflow(context) + val fullPlan = workflow.physicalPlan + val link = fullPlan.links.head + val keywordOp = fullPlan.getOperator(link.toOpId) + // The part of the plan that runs when the csv operator is skipped: the keyword operator + // alone. Its upstream link leaves the sub-plan, while its own link list still names it. + val retainedPart = fullPlan.getSubPlan(Set(keywordOp.id)) + assert(retainedPart.links.isEmpty) + assert(retainedPart.getOperator(keywordOp.id).getInputLinks() == List(link)) + val inputPort = GlobalPortIdentity(keywordOp.id, PortIdentity(0), input = true) + val readerUri = VFSURIFactory.createPortBaseURI( + WorkflowIdentity(3L), + ExecutionIdentity(1L), + GlobalPortIdentity(link.fromOpId, link.fromPortId) + ) + val generator = new CostBasedScheduleGenerator(context, fullPlan, COORDINATOR) + + val withCacheRead = scheduleSubPlan( + generator, + retainedPart, + Set.empty, + Map(inputPort -> List(readerUri)) + ) + val regions = withCacheRead.getRegions + assert(regions.size == 1) + regions.head.resourceConfig.get.portConfigs(inputPort) match { + case InputPortConfig(pairs) => assert(pairs.map(_._1) == List(readerUri)) + case other => fail(s"expected a reader for the saved result, got $other") + } + // The operator reads from storage, so it starts first, as after a materialized link. + assert(regions.head.getStarterOperators.map(_.id) == Set(keywordOp.id)) + + val withoutCacheRead = scheduleSubPlan(generator, retainedPart, Set.empty, Map.empty) + assert(!withoutCacheRead.getRegions.head.resourceConfig.get.portConfigs.contains(inputPort)) + } + + /** + * Two csv scans into the one input port of a union, with the first scan skipped: the part + * of the plan that runs is the second scan and the union, and the union's input port reads + * the first scan's saved result besides the second scan's link. The union's own link list + * still names the first scan's link. + */ + private def scheduleUnionReadingOneInputFromCache(materializeRemainingLink: Boolean) = { + val context = new WorkflowContext(WorkflowIdentity(3L), ExecutionIdentity(2L)) + val csv1 = TestOperators.headerlessSmallCsvScanOpDesc() + val csv2 = TestOperators.headerlessSmallCsvScanOpDesc() + val union = new UnionOpDesc() + val workflow = buildWorkflow( + List(csv1, csv2, union), + List(csv1, csv2).map(csv => + LogicalLink( + csv.operatorIdentifier, + PortIdentity(0), + union.operatorIdentifier, + PortIdentity(0) + ) + ), + context + ) + val fullPlan = workflow.physicalPlan + def idOf(op: LogicalOp) = fullPlan.getPhysicalOpsOfLogicalOp(op.operatorIdentifier).head.id + val skippedLink = fullPlan.links.find(_.fromOpId == idOf(csv1)).get + val retainedPart = fullPlan.getSubPlan(Set(idOf(csv2), idOf(union))) + val matEdges = if (materializeRemainingLink) retainedPart.links else Set.empty[PhysicalLink] + val unionInput = GlobalPortIdentity(idOf(union), PortIdentity(0), input = true) + val savedUri = VFSURIFactory.createPortBaseURI( + WorkflowIdentity(3L), + ExecutionIdentity(1L), + GlobalPortIdentity(skippedLink.fromOpId, skippedLink.fromPortId) + ) + val generator = new CostBasedScheduleGenerator(context, fullPlan, COORDINATOR) + val schedule = scheduleSubPlan( + generator, + retainedPart, + matEdges, + Map(unionInput -> List(savedUri)) + ) + (schedule, idOf(csv2), unionInput, savedUri) + } + + it should "give an input port both the location of a materialized link and a saved result" in { + val (schedule, csv2, unionInput, savedUri) = + scheduleUnionReadingOneInputFromCache(materializeRemainingLink = true) + val regionOf = schedule.levelSets.toList.flatMap { + case (level, regions) => regions.map(region => region -> level) + } + val (csv2Region, csv2Level) = regionOf.find(_._1.getOperators.exists(_.id == csv2)).get + val (unionRegion, unionLevel) = + regionOf.find(_._1.getOperators.exists(_.id == unionInput.opId)).get + assert(csv2Level < unionLevel) + val csv2OutputUri = + csv2Region.resourceConfig.get.portConfigs(GlobalPortIdentity(csv2, PortIdentity(0))) match { + case config: OutputPortConfig => config.storageURIBase + case other => fail(s"expected an output URI for the scan, got $other") + } + unionRegion.resourceConfig.get.portConfigs(unionInput) match { + case InputPortConfig(pairs) => + assert(pairs.size == 2) + assert(pairs.map(_._1).toSet == Set(csv2OutputUri, savedUri)) + case other => fail(s"expected a reader for both locations, got $other") + } + } + + it should "give an input port with a pipelined link only the location of the saved result" in { + val (schedule, csv2, unionInput, savedUri) = + scheduleUnionReadingOneInputFromCache(materializeRemainingLink = false) + val regions = schedule.getRegions + assert(regions.size == 1) + val region = regions.head + assert(region.getOperators.map(_.id) == Set(csv2, unionInput.opId)) + assert( + region.getLinks.map(link => (link.fromOpId, link.toOpId)) == Set((csv2, unionInput.opId)) + ) + region.resourceConfig.get.portConfigs(unionInput) match { + case InputPortConfig(pairs) => assert(pairs.map(_._1) == List(savedUri)) + case other => fail(s"expected a reader for the saved result, got $other") + } + } + + it should "put the context's warehouse on every new output URI" in { + val context = + new WorkflowContext(WorkflowIdentity(3L), ExecutionIdentity(2L), warehouse = Some("wh1")) + val workflow = csvKeywordWorkflow(context) + val (schedule, _) = + new CostBasedScheduleGenerator(context, workflow.physicalPlan, COORDINATOR).generate() + val outputUris = schedule.getRegions.flatMap(_.resourceConfig.get.portConfigs.values).collect { + case config: OutputPortConfig => config.storageURIBase.toString + } + assert(outputUris.nonEmpty) + assert(outputUris.forall(_.contains("/wh/wh1/"))) + } + + it should "partition the output of an operator reading a saved result as behind a materialized link and isolate distinct sub-plans" in { + val context = new WorkflowContext(WorkflowIdentity(3L), ExecutionIdentity(2L)) + def id(name: String) = PhysicalOpIdentity(OperatorIdentity(name), "main") + def parallelOp(name: String, inputs: Int, requirement: Option[PartitionInfo]): PhysicalOp = + PhysicalOp + .oneToOnePhysicalOp(id(name), context.workflowId, context.executionId, OpExecInitInfo.Empty) + .withInputPorts((0 until inputs).map(i => InputPort(PortIdentity(i))).toList) + .withOutputPorts(List(OutputPort(PortIdentity(0)))) + .withParallelizable(true) + .withSuggestedWorkerNum(2) + .withPartitionRequirement(if (inputs == 0) List.empty else List(requirement)) + val xy = PhysicalLink(id("x"), PortIdentity(0), id("y"), PortIdentity(0)) + val yz = PhysicalLink(id("y"), PortIdentity(0), id("z"), PortIdentity(0)) + val x = parallelOp("x", inputs = 0, None).addOutputLink(xy) + val y = + parallelOp("y", inputs = 1, Some(HashPartition(List("k")))).addInputLink(xy).addOutputLink(yz) + val z = parallelOp("z", inputs = 1, None).addInputLink(yz) + val fullPlan = PhysicalPlan(Set(x, y, z), Set(xy, yz)) + val retainedPart = fullPlan.getSubPlan(Set(id("y"), id("z"))) + val yInput = GlobalPortIdentity(id("y"), PortIdentity(0), input = true) + val savedLocation = VFSURIFactory.createPortBaseURI( + context.workflowId, + ExecutionIdentity(1L), + GlobalPortIdentity(id("x"), PortIdentity(0)) + ) + + val generator = new CostBasedScheduleGenerator(context, fullPlan, COORDINATOR) + // First evaluate the full pipelined plan on the same evaluator: x -> y hash-shuffles into y, + // so y passes HashPartition onto y -> z. + val fullSchedule = scheduleSubPlan(generator, fullPlan, Set.empty, Map.empty) + assert( + fullSchedule.getRegions.head.resourceConfig.get + .linkConfigs(yz) + .partitioning + .isInstanceOf[HashBasedShufflePartitioning] + ) + + // Next evaluate the sub-plan {y, z} with y reading x's saved result on the SAME evaluator: + // isolated per-sub-plan allocator state + fullPlan source check must give RoundRobinPartitioning, + // identical to materializing x -> y on the full plan. + val cacheReadSchedule = scheduleSubPlan( + generator, + retainedPart, + Set.empty, + Map(yInput -> List(savedLocation)) + ) + val materializedFullSchedule = scheduleSubPlan( + new CostBasedScheduleGenerator(context, fullPlan, COORDINATOR), + fullPlan, + Set(xy), + Map.empty + ) + val cacheReadPartitioning = cacheReadSchedule.getRegions + .flatMap(_.resourceConfig.get.linkConfigs.get(yz)) + .map(_.partitioning.getClass) + val materializedPartitioning = materializedFullSchedule.getRegions + .flatMap(_.resourceConfig.get.linkConfigs.get(yz)) + .map(_.partitioning.getClass) + assert(cacheReadPartitioning == List(classOf[RoundRobinPartitioning])) + assert(cacheReadPartitioning == materializedPartitioning) + } + + it should "return an empty region DAG with zero cost when the sub-plan has no operators" in { + val context = new WorkflowContext(WorkflowIdentity(3L), ExecutionIdentity(2L)) + val workflow = csvKeywordWorkflow(context) + val generator = + new CostBasedScheduleGenerator(context, workflow.physicalPlan, COORDINATOR) + val emptyPlan = PhysicalPlan(Set.empty, Set.empty) + val regionDAG = generator.tryConnectRegionDAG(emptyPlan, Set.empty, Map.empty) match { + case Left(dag) => dag + case Right(graph) => fail(s"expected empty acyclic DAG, got $graph") + } + assert(regionDAG.vertexSet().isEmpty) + assert(generator.allocateResourcesAndEvaluateCost(emptyPlan, Map.empty, regionDAG) == 0.0) + } + + it should "refuse cache reader URIs keyed by an output port in createRegions" in { + val context = new WorkflowContext(WorkflowIdentity(3L), ExecutionIdentity(2L)) + val workflow = csvKeywordWorkflow(context) + val generator = + new CostBasedScheduleGenerator(context, workflow.physicalPlan, COORDINATOR) + val outputPort = + GlobalPortIdentity(PhysicalOpIdentity(OperatorIdentity("x"), "main"), PortIdentity(0)) + assertThrows[IllegalArgumentException]( + generator.createRegions( + workflow.physicalPlan, + Set.empty, + Map(outputPort -> List(new URI("vfs:///x"))) + ) + ) + } + + // --------------------------------------------------------------------------- + // 4-edge-state CostBasedScheduleGenerator search and extended RegionPlan + // --------------------------------------------------------------------------- + + it should "evaluate a single zero-cost empty region DAG state when every operator is skipped" in { + val context = new WorkflowContext(WorkflowIdentity(7L), ExecutionIdentity(2L)) + val workflow = csvKeywordWorkflow(context) + val terminalId = workflow.logicalPlan.getTerminalOperatorIds.head + val terminalOp = workflow.physicalPlan.operators.find(_.id.logicalOpId == terminalId).get + val terminalOut = GlobalPortIdentity(terminalOp.id, PortIdentity(0)) + val savedUri = VFSURIFactory.createPortBaseURI( + context.workflowId, + ExecutionIdentity(1L), + terminalOut + ) + context.matchedResults = Map(terminalOut -> CachedResult(savedUri, Some(12L))) + + val generator = new CostBasedScheduleGenerator(context, workflow.physicalPlan, COORDINATOR) + List( + generator.getFullyMaterializedSearchState, + generator.bottomUpSearch(globalSearch = true), + generator.topDownSearch(globalSearch = true) + ).foreach { result => + assert(result.numStatesExplored == 1) + assert(result.regionDAG.vertexSet().isEmpty) + assert(result.cost == 0.0) + assert( + result.state.skippedOperators(workflow.physicalPlan, generator.requiredOutputs) == + workflow.physicalPlan.operators.map(_.id) + ) + } + + val (schedule, returnedPlan) = generator.generate() + assert(returnedPlan eq workflow.physicalPlan) + assert(schedule.levelSets.keySet == Set(0)) + assert(schedule.levelSets(0).forall(_.skipped)) + } + + it should "skip an upstream cached branch, extend RegionPlan with skip regions, and break a downstream HashJoin cycle" in { + // upstreamScan -> upstreamFilter -> join (build, port 0) + // \-> probeFilter -> join (probe, port 1) -> tailFilter + // Saving upstreamScan skips upstreamScan while keeping the downstream HashJoin cycle intact. + val upstreamScan = TestOperators.headerlessSmallCsvScanOpDesc() + val upstreamFilter = TestOperators.keywordSearchOpDesc("column-1", "Asia") + val probeFilter = TestOperators.keywordSearchOpDesc("column-1", "Asia") + val join = TestOperators.joinOpDesc("column-1", "column-1") + val tailFilter = TestOperators.keywordSearchOpDesc("column-1", "Asia") + val context = new WorkflowContext(WorkflowIdentity(7L), ExecutionIdentity(2L)) + val workflow = buildWorkflow( + List(upstreamScan, upstreamFilter, probeFilter, join, tailFilter), + List( + LogicalLink( + upstreamScan.operatorIdentifier, + PortIdentity(0), + upstreamFilter.operatorIdentifier, + PortIdentity(0) + ), + LogicalLink( + upstreamFilter.operatorIdentifier, + PortIdentity(0), + join.operatorIdentifier, + PortIdentity(0) + ), + LogicalLink( + upstreamFilter.operatorIdentifier, + PortIdentity(0), + probeFilter.operatorIdentifier, + PortIdentity(0) + ), + LogicalLink( + probeFilter.operatorIdentifier, + PortIdentity(0), + join.operatorIdentifier, + PortIdentity(1) + ), + LogicalLink( + join.operatorIdentifier, + PortIdentity(0), + tailFilter.operatorIdentifier, + PortIdentity(0) + ) + ), + context + ) + val plan = workflow.physicalPlan + val scanOp = plan.getPhysicalOpsOfLogicalOp(upstreamScan.operatorIdentifier).head + val upstreamFilterOp = plan.getPhysicalOpsOfLogicalOp(upstreamFilter.operatorIdentifier).head + val scanOut = GlobalPortIdentity(scanOp.id, PortIdentity(0)) + val savedUri = VFSURIFactory.createPortBaseURI( + context.workflowId, + ExecutionIdentity(1L), + scanOut + ) + context.matchedResults = Map(scanOut -> CachedResult(savedUri, Some(8L))) + + val generator = new CostBasedScheduleGenerator(context, plan, COORDINATOR) + val results = List( + generator.bottomUpSearch(globalSearch = true), + generator.bottomUpSearch(globalSearch = false), + generator.topDownSearch(globalSearch = true), + generator.topDownSearch(globalSearch = false) + ) + results.foreach { result => + assert(result.cost < Double.PositiveInfinity) + assert(result.state.skippedOperators(plan, generator.requiredOutputs) == Set(scanOp.id)) + assert( + result.regionDAG.vertexSet().asScala.flatMap(_.getOperators.map(_.id)).toSet == + plan.operators.map(_.id) - scanOp.id + ) + val upstreamFilterInput = + GlobalPortIdentity(upstreamFilterOp.id, PortIdentity(0), input = true) + val readers = result.regionDAG + .vertexSet() + .asScala + .flatMap(_.resourceConfig.get.portConfigs.get(upstreamFilterInput)) + assert(readers.size == 1) + readers.head match { + case InputPortConfig(pairs) => assert(pairs.map(_._1) == List(savedUri)) + case other => fail(s"expected saved reader on upstreamFilter, got $other") + } + val regionPlan = generator.buildRegionPlan(result.state, result.regionDAG) + val skipRegions = regionPlan.regions.filter(_.skipped) + val activeRegions = regionPlan.regions.filterNot(_.skipped) + assert(skipRegions.size == 1) + assert(skipRegions.flatMap(_.getOperators.map(_.id)) == Set(scanOp.id)) + assert(activeRegions == result.regionDAG.vertexSet().asScala.toSet) + assert(regionPlan.regionLinks == result.regionDAG.edgeSet().asScala.toSet) + assert(regionPlan.getRegionOfPortId(scanOut).contains(skipRegions.head)) + + val fullSchedule = generator.generateScheduleFromRegionPlan(regionPlan) + assert(fullSchedule.levelSets(0).forall(_.skipped)) + assert(fullSchedule.levelSets(0).flatMap(_.getOperators.map(_.id)) == Set(scanOp.id)) + assert(fullSchedule.levelSets.size >= 3) + } + } + + it should "schedule partial cache reuse under ExecutionMode.MATERIALIZED and keep retained links materialized on fallback" in { + val csv = TestOperators.headerlessSmallCsvScanOpDesc() + val filter1 = TestOperators.keywordSearchOpDesc("column-1", "Asia") + val filter2 = TestOperators.keywordSearchOpDesc("column-1", "Asia") + val context = new WorkflowContext( + workflowId = WorkflowIdentity(7L), + executionId = ExecutionIdentity(2L), + workflowSettings = WorkflowSettings(executionMode = ExecutionMode.MATERIALIZED) + ) + val workflow = buildWorkflow( + List(csv, filter1, filter2), + List( + LogicalLink( + csv.operatorIdentifier, + PortIdentity(0), + filter1.operatorIdentifier, + PortIdentity(0) + ), + LogicalLink( + filter1.operatorIdentifier, + PortIdentity(0), + filter2.operatorIdentifier, + PortIdentity(0) + ) + ), + context + ) + val plan = workflow.physicalPlan + val csvOp = plan.getPhysicalOpsOfLogicalOp(csv.operatorIdentifier).head + val filter1Op = plan.getPhysicalOpsOfLogicalOp(filter1.operatorIdentifier).head + val filter2Op = plan.getPhysicalOpsOfLogicalOp(filter2.operatorIdentifier).head + val filter1ToFilter2 = plan.links.find(_.fromOpId == filter1Op.id).get + val csvOut = GlobalPortIdentity(csvOp.id, PortIdentity(0)) + val savedUri = VFSURIFactory.createPortBaseURI( + context.workflowId, + ExecutionIdentity(1L), + csvOut + ) + context.matchedResults = Map(csvOut -> CachedResult(savedUri, Some(7L))) + + val generator = new CostBasedScheduleGenerator(context, plan, COORDINATOR) + assert(!generator.allowPipelined) + + val (schedule, returnedPlan) = generator.generate() + assert(returnedPlan eq plan) + assert(schedule.levelSets.keySet == Set(0, 1, 2)) + assert(schedule.levelSets(0).forall(_.skipped)) + assert(schedule.levelSets(0).flatMap(_.getOperators.map(_.id)) == Set(csvOp.id)) + + val filter1Region = schedule.levelSets(1).head + assert(!filter1Region.skipped) + assert(filter1Region.getOperators.map(_.id) == Set(filter1Op.id)) + val filter1Input = GlobalPortIdentity(filter1Op.id, PortIdentity(0), input = true) + filter1Region.resourceConfig.get.portConfigs(filter1Input) match { + case InputPortConfig(pairs) => assert(pairs.map(_._1) == List(savedUri)) + case other => fail(s"expected saved reader URI on filter1 in MATERIALIZED mode, got $other") + } + + val filter2Region = schedule.levelSets(2).head + assert(!filter2Region.skipped) + assert(filter2Region.getOperators.map(_.id) == Set(filter2Op.id)) + + // Timeout fallback path: bottomUpSearch under !allowPipelined keeps all retained links Materialized. + val fallbackResult = generator.bottomUpSearch() + assert(fallbackResult.numStatesExplored == 1) + assert(fallbackResult.state.skippedOperators(plan, generator.requiredOutputs) == Set(csvOp.id)) + assert(fallbackResult.state.linksOfType(EdgeType.Materialized) == Set(filter1ToFilter2)) } } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/DefaultCostEstimatorSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/DefaultCostEstimatorSpec.scala index c3438d5d387..fce5c2e5a72 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/DefaultCostEstimatorSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/DefaultCostEstimatorSpec.scala @@ -364,4 +364,152 @@ class DefaultCostEstimatorSpec // The cost of the region plan should be the sum of region costs assert(searchResult.cost == groupByRegionCost + keywordRegionCost) } + + "DefaultCostEstimator.withResourceAllocator" should "share loaded execution statistics without re-querying DB or storage" in { + val workflow = buildWorkflow( + List(headerlessCsvOpDesc, keywordOpDesc), + List( + LogicalLink( + headerlessCsvOpDesc.operatorIdentifier, + PortIdentity(0), + keywordOpDesc.operatorIdentifier, + PortIdentity(0) + ) + ), + new WorkflowContext() + ) + + seedExecutionRecord() + + val headerlessCsvOpRuntimeStatistics = new Tuple( + ResultSchema.runtimeStatisticsSchema, + Array( + headerlessCsvOpDesc.operatorIdentifier.id, + new Timestamp(System.currentTimeMillis()), + 0L, + 0L, + 0L, + 0L, + 100L, + 100L, + 0L, + 1, + 0 + ) + ) + val keywordOpRuntimeStatistics = new Tuple( + ResultSchema.runtimeStatisticsSchema, + Array( + keywordOpDesc.operatorIdentifier.id, + new Timestamp(System.currentTimeMillis()), + 0L, + 0L, + 0L, + 0L, + 300L, + 300L, + 0L, + 1, + 0 + ) + ) + + writer.putOne(headerlessCsvOpRuntimeStatistics) + writer.putOne(keywordOpRuntimeStatistics) + writer.close() + + val firstAllocator = + new DefaultResourceAllocator( + workflow.physicalPlan, + new ExecutionClusterInfo(), + workflow.context.workflowSettings + ) + val costEstimator = new DefaultCostEstimator( + workflow.context, + firstAllocator, + COORDINATOR + ) + + // Delete the DB execution row so any re-query would fall back to 0. + workflowExecutionsDao.deleteById(Integer.valueOf(1)) + + val secondAllocator = + new DefaultResourceAllocator( + workflow.physicalPlan, + new ExecutionClusterInfo(), + workflow.context.workflowSettings + ) + val scopedEstimator = costEstimator.withResourceAllocator(secondAllocator) + assert(scopedEstimator.resourceAllocator eq secondAllocator) + + val ports = workflow.physicalPlan.operators.flatMap(op => + op.inputPorts.keys + .map(inputPortId => GlobalPortIdentity(op.id, inputPortId, input = true)) + .toSet ++ op.outputPorts.keys + .map(outputPortId => GlobalPortIdentity(op.id, outputPortId)) + .toSet + ) + val region = Region( + id = RegionIdentity(0), + physicalOps = workflow.physicalPlan.operators, + physicalLinks = workflow.physicalPlan.links, + ports = ports + ) + + val (_, costOfRegion) = scopedEstimator.allocateResourcesAndEstimateCost(region, 1) + val expectedKeywordCost = (300L + 300L) / 1e9 + assert(costOfRegion == expectedKeywordCost) + } + + it should "keep fallback mode on withResourceAllocator when no past statistics existed at construction" in { + val workflow = buildWorkflow( + List(headerlessCsvOpDesc, keywordOpDesc), + List( + LogicalLink( + headerlessCsvOpDesc.operatorIdentifier, + PortIdentity(0), + keywordOpDesc.operatorIdentifier, + PortIdentity(0) + ) + ), + new WorkflowContext() + ) + val firstAllocator = + new DefaultResourceAllocator( + workflow.physicalPlan, + new ExecutionClusterInfo(), + workflow.context.workflowSettings + ) + val costEstimator = new DefaultCostEstimator( + workflow.context, + firstAllocator, + COORDINATOR + ) + + val secondAllocator = + new DefaultResourceAllocator( + workflow.physicalPlan, + new ExecutionClusterInfo(), + workflow.context.workflowSettings + ) + val scopedEstimator = costEstimator.withResourceAllocator(secondAllocator) + assert(scopedEstimator.resourceAllocator eq secondAllocator) + + val ports = workflow.physicalPlan.operators.flatMap(op => + op.inputPorts.keys + .map(inputPortId => GlobalPortIdentity(op.id, inputPortId, input = true)) + .toSet ++ op.outputPorts.keys + .map(outputPortId => GlobalPortIdentity(op.id, outputPortId)) + .toSet + ) + val region = Region( + id = RegionIdentity(0), + physicalOps = workflow.physicalPlan.operators, + physicalLinks = workflow.physicalPlan.links, + ports = ports + ) + + val (_, costOfRegion) = scopedEstimator.allocateResourcesAndEstimateCost(region, 1) + assert(costOfRegion == 0) + } } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlanSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlanSpec.scala index f8d71ea91de..e258decd60c 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlanSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlanSpec.scala @@ -236,4 +236,31 @@ class RegionPlanSpec extends AnyFlatSpec with BarePhysicalPlanFixture { assert(order.head == ids(0)) assert(order.last == ids(8)) } + + "RegionPlan with skip regions" should "hold active and skipped regions together and resolve lookups across both" in { + val active = buildDiamondPlan() + val skippedInternalLink = link("skip-a", "skip-b") + val skippedPort = globalPort("skip-b") + val skipRegion = Region( + RegionIdentity(10), + physicalOps = Set(op("skip-a"), op("skip-b")), + physicalLinks = Set(skippedInternalLink), + ports = Set(skippedPort), + skipped = true + ) + val plan = RegionPlan( + regions = active.regions + skipRegion, + regionLinks = active.regionLinks + ) + + assert(plan.getRegion(RegionIdentity(10)) == skipRegion) + assert(plan.getRegionOfLink(skippedInternalLink) == skipRegion) + assert(plan.getRegionOfPortId(skippedPort).contains(skipRegion)) + val order = plan.topologicalIterator().toList + assert(order.toSet == active.regions.map(_.id) + skipRegion.id) + val pos = order.zipWithIndex.toMap + active.regionLinks.foreach { edge => + assert(pos(edge.fromRegionId) < pos(edge.toRegionId)) + } + } } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionSpec.scala index 8974a33cf88..f28632d8bb5 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionSpec.scala @@ -71,6 +71,11 @@ class RegionSpec extends AnyFlatSpec with BarePhysicalPlanFixture { assert(region.getPorts == Set(portId)) } + it should "default skipped to false and accept true" in { + assert(!Region(RegionIdentity(1), Set(op("a")), Set.empty).skipped) + assert(Region(RegionIdentity(1), Set(op("a")), Set.empty, skipped = true).skipped) + } + "Region.getOperator" should "look up a physical operator by id" in { val a = op("a") val b = op("b") @@ -118,6 +123,17 @@ class RegionSpec extends AnyFlatSpec with BarePhysicalPlanFixture { assert(region.getSourceOperators == Set(a, c)) } + it should "count only the region's own links, not every link an operator lists" in { + // b lists an input link from a, and both are in the region, yet the link is not one of the + // region's links (for instance when b reads a's saved result): b is a source of the region. + val ab = link("a", "b") + val a = wiredOp("a").addOutputLink(ab) + val b = wiredOp("b").addInputLink(ab) + assert(Region(RegionIdentity(1), Set(a, b), Set.empty).getSourceOperators == Set(a, b)) + // With the link among the region's links, b is not a source. + assert(Region(RegionIdentity(1), Set(a, b), Set(ab)).getSourceOperators == Set(a)) + } + "Region.getStarterOperators" should "match getSourceOperators when no resource config is provided" in { val a = op("a") val b = op("b") diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleSpec.scala index b4709fb2d1b..2747446d2f6 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleSpec.scala @@ -115,4 +115,76 @@ class ScheduleSpec extends AnyFlatSpec with BarePhysicalPlanFixture { assert(order.indexOf(r2) < order.indexOf(r3)) } + it should "place all skip regions at level 0 with no extra empty level when nothing else runs" in { + val skip0 = region(10, "s0").copy(skipped = true) + val skip1 = region(11, "s1").copy(skipped = true) + val plan = RegionPlan(Set(skip0, skip1), Set.empty) + val generator = new ScheduleGenerator( + new WorkflowContext(), + PhysicalPlan(Set.empty, Set.empty) + ) { + override def generate(): (Schedule, PhysicalPlan) = (Schedule(Map.empty), physicalPlan) + } + + val schedule = generator.generateScheduleFromRegionPlan(plan) + assert(schedule.levelSets == Map(0 -> Set(skip0, skip1))) + } + + it should "place all skip regions at level 0 ahead of topological active batches" in { + val r0 = region(0, "a") + val r1 = region(1, "b") + val r2 = region(2, "c") + val r3 = region(3, "d") + val skip0 = region(4, "s0").copy(skipped = true) + val skip1 = region(5, "s1").copy(skipped = true) + val plan = RegionPlan( + Set(r0, r1, r2, r3, skip0, skip1), + Set( + RegionLink(r0.id, r1.id), + RegionLink(r0.id, r2.id), + RegionLink(r1.id, r3.id), + RegionLink(r2.id, r3.id) + ) + ) + val generator = new ScheduleGenerator( + new WorkflowContext(), + PhysicalPlan(Set.empty, Set.empty) + ) { + override def generate(): (Schedule, PhysicalPlan) = (Schedule(Map.empty), physicalPlan) + } + + val levels = generator.generateScheduleFromRegionPlan(plan).toList + assert(levels.head == Set(skip0, skip1)) + val activeLevels = levels.tail + assert(activeLevels.forall(_.size <= ApplicationConfig.maxConcurrentRegions)) + val activeOrder = activeLevels.flatten + assert(activeOrder.toSet == Set(r0, r1, r2, r3)) + assert(activeOrder.indexOf(r0) < activeOrder.indexOf(r1)) + assert(activeOrder.indexOf(r0) < activeOrder.indexOf(r2)) + assert(activeOrder.indexOf(r1) < activeOrder.indexOf(r3)) + assert(activeOrder.indexOf(r2) < activeOrder.indexOf(r3)) + } + + it should "reject a region plan whose regionLinks reference a skipped region" in { + val skipRegion = region(0, "s").copy(skipped = true) + val activeRegion = region(1, "a") + val generator = new ScheduleGenerator( + new WorkflowContext(), + PhysicalPlan(Set.empty, Set.empty) + ) { + override def generate(): (Schedule, PhysicalPlan) = (Schedule(Map.empty), physicalPlan) + } + + assertThrows[IllegalArgumentException] { + generator.generateScheduleFromRegionPlan( + RegionPlan(Set(skipRegion, activeRegion), Set(RegionLink(skipRegion.id, activeRegion.id))) + ) + } + assertThrows[IllegalArgumentException] { + generator.generateScheduleFromRegionPlan( + RegionPlan(Set(skipRegion, activeRegion), Set(RegionLink(activeRegion.id, skipRegion.id))) + ) + } + } + } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/SearchStateSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/SearchStateSpec.scala new file mode 100644 index 00000000000..63daf139090 --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/SearchStateSpec.scala @@ -0,0 +1,989 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.engine.architecture.scheduling + +import org.apache.texera.amber.core.executor.OpExecInitInfo +import org.apache.texera.amber.core.storage.VFSURIFactory +import org.apache.texera.amber.core.virtualidentity.{ + ExecutionIdentity, + OperatorIdentity, + PhysicalOpIdentity, + WorkflowIdentity +} +import org.apache.texera.amber.core.workflow.{ + CachedResult, + GlobalPortIdentity, + InputPort, + OutputPort, + PhysicalLink, + PhysicalOp, + PhysicalPlan, + PortIdentity, + WorkflowContext, + WorkflowSettings +} +import org.apache.texera.amber.engine.architecture.scheduling.EdgeType._ +import org.apache.texera.amber.engine.architecture.scheduling.config.OutputPortConfig +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.net.URI +import scala.util.Random + +/** + * [[SearchState]] is pure, so the plans here are built by hand: no compiler, no engine. As in a + * compiled plan, every operator lists its own links; `SearchState` reads links from the plan, + * leaving the operators unchanged. + */ +class SearchStateSpec extends AnyFlatSpec with Matchers { + + private val workflowId = WorkflowIdentity(7L) + private val executionId = ExecutionIdentity(2L) + // The execution that saved the results. + private val sourceExecutionId = ExecutionIdentity(1L) + + private def opId(name: String): PhysicalOpIdentity = + PhysicalOpIdentity(OperatorIdentity(name), "main") + + private def op(name: String, inputs: Int = 1, outputs: Int = 1): PhysicalOp = + PhysicalOp + .oneToOnePhysicalOp(opId(name), workflowId, executionId, OpExecInitInfo.Empty) + .withInputPorts((0 until inputs).map(i => InputPort(PortIdentity(i))).toList) + .withOutputPorts((0 until outputs).map(i => OutputPort(PortIdentity(i))).toList) + + private def source(name: String, outputs: Int = 1): PhysicalOp = op(name, inputs = 0, outputs) + + private def link(from: String, to: String, fromPort: Int = 0, toPort: Int = 0): PhysicalLink = + PhysicalLink(opId(from), PortIdentity(fromPort), opId(to), PortIdentity(toPort)) + + /** A plan whose operators list their own links, as the compiler builds it. */ + private def planOf(ops: Set[PhysicalOp], links: Set[PhysicalLink]): PhysicalPlan = + links.foldLeft(PhysicalPlan(ops, Set.empty))(_.addLink(_)) + + private def out(name: String, port: Int = 0): GlobalPortIdentity = + GlobalPortIdentity(opId(name), PortIdentity(port)) + + private def in(name: String, port: Int = 0): GlobalPortIdentity = + GlobalPortIdentity(opId(name), PortIdentity(port), input = true) + + private def baseUri( + port: GlobalPortIdentity, + warehouse: Option[String] = None, + wid: WorkflowIdentity = workflowId + ): URI = VFSURIFactory.createPortBaseURI(wid, sourceExecutionId, port, warehouse) + + private def entry( + port: GlobalPortIdentity, + count: Option[Long] = Some(10L), + warehouse: Option[String] = None, + wid: WorkflowIdentity = workflowId + ): (GlobalPortIdentity, CachedResult) = + port -> CachedResult(baseUri(port, warehouse, wid), count) + + private def context( + matchedResults: Map[GlobalPortIdentity, CachedResult], + requiredOutputs: Set[GlobalPortIdentity], + warehouse: Option[String] = None + ): WorkflowContext = + new WorkflowContext( + workflowId, + executionId, + WorkflowSettings(outputPortsNeedingStorage = requiredOutputs), + warehouse = warehouse, + matchedResults = matchedResults + ) + + private def ids(plan: PhysicalPlan): Set[PhysicalOpIdentity] = plan.operators.map(_.id) + + /** a -> b -> c, with c's output the terminal port. */ + private def chain(): PhysicalPlan = + planOf(Set(source("a"), op("b"), op("c")), Set(link("a", "b"), link("b", "c"))) + + /** + * src -> split; split's port 0 -> x and port 1 -> y, with x and y terminal. Split has one + * input and two outputs, like the Split operator. + */ + private def split(): PhysicalPlan = + planOf( + Set(source("src"), op("split", outputs = 2), op("x"), op("y")), + Set(link("src", "split"), link("split", "x", fromPort = 0), link("split", "y", fromPort = 1)) + ) + + private case class EagerSetup( + matched: Map[GlobalPortIdentity, CachedResult], + required: Set[GlobalPortIdentity], + state: SearchState + ) + + private def eagerSetup( + plan: PhysicalPlan, + ctx: WorkflowContext, + allowPipelined: Boolean = true + ): EagerSetup = { + val matched = ctx.matchedResults + val required = SearchState.requiredOutputPorts(ctx, plan) + val state = SearchState.eagerSeed(plan, matched, required, allowPipelined = allowPipelined) + EagerSetup(matched, required, state) + } + + /** Independent backward-reachability oracle for resolving input requirements. */ + private def referenceResolveInputRequirements( + plan: PhysicalPlan, + requiredOutputs: Set[GlobalPortIdentity], + matchedResults: Map[GlobalPortIdentity, CachedResult], + allowPipelined: Boolean, + defaultActiveType: PhysicalLink => EdgeType, + state: SearchState + ): SearchState = { + def sourcePort(l: PhysicalLink): GlobalPortIdentity = + GlobalPortIdentity(l.fromOpId, l.fromPortId) + + val allowedLinks = SearchState.allowedLinkTypes(plan, matchedResults, allowPipelined) + def willPropagateToProducer(l: PhysicalLink): Boolean = + state(l) == Materialized || state(l) == Pipelined || + (state(l) == Unused && !matchedResults.contains(sourcePort(l))) + + var runOps = plan.operators.filter(_.outputPorts.isEmpty).map(_.id) ++ + requiredOutputs.diff(state.reusedOutputs).map(_.opId) + var changed = true + while (changed) { + val next = runOps ++ plan.links.collect { + case l if runOps.contains(l.toOpId) && willPropagateToProducer(l) => l.fromOpId + } + changed = next != runOps + runOps = next + } + + val expectedLinks = plan.links.map { l => + val edgeType = + if (!runOps.contains(l.toOpId)) Unused + else if (state(l) != Unused) state(l) + else if (matchedResults.contains(sourcePort(l))) CacheRead + else { + val candidate = defaultActiveType(l) + if ( + candidate != Unused && candidate != CacheRead && allowedLinks(l).contains(candidate) + ) { + candidate + } else Materialized + } + l -> edgeType + }.toMap + state.copy(linkTypes = expectedLinks) + } + + private def allMaterialized(physicalPlan: PhysicalPlan): SearchState = + SearchState(physicalPlan.links.map(_ -> Materialized).toMap) + + private def withReusedOutput( + state: SearchState, + port: GlobalPortIdentity, + reused: Boolean + ): SearchState = + if (reused) state.copy(reusedOutputs = state.reusedOutputs + port) + else state.copy(reusedOutputs = state.reusedOutputs - port) + + "SearchState.eagerSeed" should "mark nothing when the cache is empty" in { + val plan = chain() + val setup = eagerSetup(plan, context(Map.empty, Set(out("c")))) + val s = setup.state + s.skipsAnything(plan, setup.required) shouldBe false + s.skippedOperators(plan, setup.required) shouldBe empty + s.linksOfType(Unused) shouldBe empty + s.linksOfType(CacheRead) shouldBe empty + s.retainedPlan(plan, setup.required) shouldBe plan + s.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe empty + s.cacheReadPorts(plan, setup.matched, setup.required) shouldBe empty + } + + it should "skip everything when the terminal port is matched" in { + val plan = chain() + val cached = Map(entry(out("c"), count = Some(3L))) + val setup = eagerSetup(plan, context(cached, Set(out("c")))) + val s = setup.state + s.skipsAnything(plan, setup.required) shouldBe true + s.skippedOperators(plan, setup.required) shouldBe ids(plan) + s.linksOfType(Unused) shouldBe plan.links + s.linksOfType(CacheRead) shouldBe empty + s.retainedPlan(plan, setup.required).operators shouldBe empty + s.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe empty + s.cacheReadPorts(plan, setup.matched, setup.required) shouldBe + Map(out("c") -> CachedResult(baseUri(out("c")), Some(3L))) + val regions = s.skipRegions(plan, setup.matched, setup.required, firstRegionId = 5) + regions.map(_.id.id) shouldBe Set(5L) + val region = regions.head + region.skipped shouldBe true + region.physicalOps.map(_.id) shouldBe ids(plan) + region.physicalLinks shouldBe plan.links + region.ports shouldBe Set(out("a"), in("b"), out("b"), in("c"), out("c")) + // The saved result becomes the port's config: its location and row count. + region.resourceConfig.get.portConfigs shouldBe Map( + out("c") -> OutputPortConfig(baseUri(out("c")), Some(3L)) + ) + } + + it should "run only the operators downstream of a match in the middle" in { + val plan = chain() + val cached = Map(entry(out("b"))) + val setup = eagerSetup(plan, context(cached, Set(out("c")))) + val s = setup.state + // a is skipped too: nothing needs its output once b's output is read from the cache. + s.skippedOperators(plan, setup.required) shouldBe Set(opId("a"), opId("b")) + s.linksOfType(Unused) shouldBe Set(link("a", "b")) + s.linksOfType(CacheRead) shouldBe Set(link("b", "c")) + ids(s.retainedPlan(plan, setup.required)) shouldBe Set(opId("c")) + s.retainedPlan(plan, setup.required).links shouldBe empty + s.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe Map( + in("c") -> List(cached(out("b")).storageUri) + ) + s.cacheReadPorts(plan, setup.matched, setup.required) shouldBe cached + val region = s.skipRegions(plan, setup.matched, setup.required, 0).head + region.resourceConfig.get.portConfigs.keySet shouldBe Set(out("b")) + } + + it should "run an operator whose results the user views when its port has no saved result" in { + // a -> b -> c, and the user views b, so the run must store b's port, which has no saved + // result: b runs, and so does a, which b reads. Only c is skipped, on its own saved result. + val plan = chain() + val setup = eagerSetup(plan, context(Map(entry(out("c"))), Set(out("b"), out("c")))) + val s = setup.state + s.skippedOperators(plan, setup.required) shouldBe Set(opId("c")) + ids(s.retainedPlan(plan, setup.required)) shouldBe Set(opId("a"), opId("b")) + s.linksOfType(Unused) shouldBe Set(link("b", "c")) + s.linksOfType(CacheRead) shouldBe empty + s.cacheReadPorts(plan, setup.matched, setup.required).keySet shouldBe Set(out("c")) + } + + it should "read a join's build-side input from the cache and run the rest" in { + // s1 -> j (port 0, build), s2 -> j (port 1, probe), j's output is terminal. + val plan = planOf( + Set(source("s1"), source("s2"), op("j", inputs = 2)), + Set(link("s1", "j", toPort = 0), link("s2", "j", toPort = 1)) + ) + val cached = Map(entry(out("s1"))) + val setup = eagerSetup(plan, context(cached, Set(out("j")))) + val s = setup.state + s.skippedOperators(plan, setup.required) shouldBe Set(opId("s1")) + s.linksOfType(Unused) shouldBe empty + s.linksOfType(CacheRead) shouldBe Set(link("s1", "j", toPort = 0)) + ids(s.retainedPlan(plan, setup.required)) shouldBe Set(opId("s2"), opId("j")) + s.retainedPlan(plan, setup.required).links shouldBe Set(link("s2", "j", toPort = 1)) + s.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe Map( + in("j", 0) -> List(cached(out("s1")).storageUri) + ) + s.cacheReadPorts(plan, setup.matched, setup.required) shouldBe cached + } + + it should "give each input port the saved location of its own link" in { + // s1 -> j's port 0 and s2 -> j's port 1, both from port 0 and both saved: each of j's + // input ports reads the saved result of the link into it. + val plan = planOf( + Set(source("s1"), source("s2"), op("j", inputs = 2)), + Set(link("s1", "j", toPort = 0), link("s2", "j", toPort = 1)) + ) + val cached = Map(entry(out("s1")), entry(out("s2"))) + val setup = eagerSetup(plan, context(cached, Set(out("j")))) + setup.state.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe Map( + in("j", 0) -> List(cached(out("s1")).storageUri), + in("j", 1) -> List(cached(out("s2")).storageUri) + ) + } + + it should "list the saved locations of one input port in a fixed order" in { + // s1, s2 and s3 all feed u's one input port and all have saved results. The locations are + // sorted by link, so the same plan configures the port the same way whatever order its links + // were added in. + val ops = Set(source("s1"), source("s2"), source("s3"), op("u")) + val links = List(link("s1", "u"), link("s2", "u"), link("s3", "u")) + val cached = Map(entry(out("s1")), entry(out("s2")), entry(out("s3"))) + val expected = List("s1", "s2", "s3").map(name => cached(out(name)).storageUri) + List(links, links.reverse).foreach { linkOrder => + val p = planOf(ops, linkOrder.toSet) + val setup = eagerSetup(p, context(cached, Set(out("u")))) + setup.state.readerUrisByInputPort(p, setup.matched, setup.required) shouldBe Map( + in("u") -> expected + ) + } + } + + it should "run a multi-output operator as a whole when one needed output has no saved result" in { + // m has two outputs: m.0 -> x, m.1 -> y; x and y are terminal. + val plan = planOf( + Set(source("m", outputs = 2), op("x"), op("y")), + Set(link("m", "x", fromPort = 0), link("m", "y", fromPort = 1)) + ) + val cached = Map(entry(out("m", 0)), entry(out("x"))) + val setup = eagerSetup(plan, context(cached, Set(out("x"), out("y")))) + val s = setup.state + // y needs m.1, which has no saved result, so m runs and computes m.0 again too. + ids(s.retainedPlan(plan, setup.required)) shouldBe Set(opId("m"), opId("y")) + s.retainedPlan(plan, setup.required).links shouldBe Set(link("m", "y", fromPort = 1)) + // x is skipped on its own saved result, so m.0's saved result is read by nothing. + s.skippedOperators(plan, setup.required) shouldBe Set(opId("x")) + s.linksOfType(CacheRead) shouldBe empty + s.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe empty + s.cacheReadPorts(plan, setup.matched, setup.required).keySet shouldBe Set(out("x")) + } + + it should "skip every link into a skipped operator, from a retained operator too" in { + // m.0 -> x -> w and m.1 -> y, with w and y terminal. w's saved result skips w and x; y has + // none, so y and m run. + val plan = planOf( + Set(source("m", outputs = 2), op("x"), op("w"), op("y")), + Set(link("m", "x", fromPort = 0), link("x", "w"), link("m", "y", fromPort = 1)) + ) + val setup = eagerSetup(plan, context(Map(entry(out("w"))), Set(out("w"), out("y")))) + val s = setup.state + s.skippedOperators(plan, setup.required) shouldBe Set(opId("x"), opId("w")) + // The link from the retained m into the skipped x is Unused, like the one between x and w. + s.linksOfType(Unused) shouldBe Set(link("m", "x", fromPort = 0), link("x", "w")) + // The link between the retained m and y is not. + s.retainedPlan(plan, setup.required).links shouldBe Set(link("m", "y", fromPort = 1)) + s.linksOfType(CacheRead) shouldBe empty + // The skip region keeps only the skipped link inside it. + val regions = s.skipRegions(plan, setup.matched, setup.required, 0) + regions.map(_.physicalOps.map(_.id)) shouldBe Set(Set(opId("x"), opId("w"))) + regions.head.physicalLinks shouldBe Set(link("x", "w")) + } + + it should "read a saved port from storage even when its operator runs" in { + // split.0 has a saved result and split.1 has none. y needs split.1, so split runs, and x + // reads split.0's saved result instead of waiting for split. + val plan = split() + val cached = Map(entry(out("split", 0))) + val setup = eagerSetup(plan, context(cached, Set(out("x"), out("y")))) + val s = setup.state + s.skipsAnything(plan, setup.required) shouldBe false + s.skippedOperators(plan, setup.required) shouldBe empty + s.linksOfType(Unused) shouldBe empty + s.linksOfType(CacheRead) shouldBe Set(link("split", "x", fromPort = 0)) + ids(s.retainedPlan(plan, setup.required)) shouldBe ids(plan) + s.retainedPlan(plan, setup.required).links shouldBe plan.links - link( + "split", + "x", + fromPort = 0 + ) + s.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe Map( + in("x") -> List(cached(out("split", 0)).storageUri) + ) + s.cacheReadPorts(plan, setup.matched, setup.required) shouldBe cached + // split.1 has no saved result, so y still reads it over its link. + s.retainedPlan(plan, setup.required).links should contain(link("split", "y", fromPort = 1)) + } + + it should "change nothing for a matched saved result that no retained operator reads" in { + // m.0 -> x and m.1 has no link. m.1 has a matched saved result while x does not. m runs for + // m.0 -> x, and nothing reads m.1. + val plan = planOf( + Set(source("m", outputs = 2), op("x")), + Set(link("m", "x", fromPort = 0)) + ) + val cached = Map(entry(out("m", 1))) + val setup = eagerSetup(plan, context(cached, Set(out("x")))) + val s = setup.state + s.skippedOperators(plan, setup.required) shouldBe empty + s.linksOfType(CacheRead) shouldBe empty + s.cacheReadPorts(plan, setup.matched, setup.required) shouldBe empty + s.retainedPlan(plan, setup.required) shouldBe plan + } + + it should "leave the plan's own operators in the retained plan and the skip regions" in { + val plan = chain() + val setup = eagerSetup(plan, context(Map(entry(out("b"))), Set(out("c")))) + val s = setup.state + val retained = s.retainedPlan(plan, setup.required) + retained.operators.foreach(op => op should be theSameInstanceAs plan.getOperator(op.id)) + // c still lists its input link from b, which the retained plan no longer holds. + retained.getOperator(opId("c")).getInputLinks() shouldBe List(link("b", "c")) + retained.links should not contain link("b", "c") + val region = s.skipRegions(plan, setup.matched, setup.required, 0).head + region.physicalOps.foreach(op => op should be theSameInstanceAs plan.getOperator(op.id)) + // b still lists its output link to c, which is in no region. + region.getOperator(opId("b")).getOutputLinks(PortIdentity(0)) shouldBe List(link("b", "c")) + region.physicalLinks shouldBe Set(link("a", "b")) + } + + it should "run an operator whose unconnected output port has no saved result" in { + // m.0 -> x (x has its own saved result), m.1 has no link and no saved result: an output + // port with no link is a required output, so m still runs, as it would without the cache. + val plan = planOf( + Set(source("m", outputs = 2), op("x")), + Set(link("m", "x", fromPort = 0)) + ) + val cached = Map(entry(out("x"))) + val setup = eagerSetup(plan, context(cached, Set(out("x")))) + val s = setup.state + ids(s.retainedPlan(plan, setup.required)) shouldBe Set(opId("m")) + s.skippedOperators(plan, setup.required) shouldBe Set(opId("x")) + s.linksOfType(Unused) shouldBe Set(link("m", "x", fromPort = 0)) + s.linksOfType(CacheRead) shouldBe empty + s.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe empty + s.cacheReadPorts(plan, setup.matched, setup.required).keySet shouldBe Set(out("x")) + } + + it should "list a skipped operator's unconnected port among the saved results read" in { + // The same plan with a saved result for m.1 too: m is skipped, and m.1, a required output + // of a skipped operator, is read from its saved result. m.0's saved result is not listed: + // it feeds only the skipped x. + val plan = planOf( + Set(source("m", outputs = 2), op("x")), + Set(link("m", "x", fromPort = 0)) + ) + val cached = Map(entry(out("m", 0)), entry(out("m", 1), count = Some(4L)), entry(out("x"))) + // The compiler stores the ports of operators with no outgoing links: only x's here. + val setup = eagerSetup(plan, context(cached, Set(out("x")))) + val s = setup.state + s.skippedOperators(plan, setup.required) shouldBe Set(opId("m"), opId("x")) + s.cacheReadPorts(plan, setup.matched, setup.required) shouldBe Map( + out("m", 1) -> CachedResult(baseUri(out("m", 1)), Some(4L)), + out("x") -> CachedResult(baseUri(out("x")), Some(10L)) + ) + s.skipRegions(plan, setup.matched, setup.required, 0) + .head + .resourceConfig + .get + .portConfigs shouldBe Map( + out("m", 1) -> OutputPortConfig(baseUri(out("m", 1)), Some(4L)), + out("x") -> OutputPortConfig(baseUri(out("x")), Some(10L)) + ) + } + + it should "keep a /wh/ warehouse on the URIs it hands out" in { + val plan = chain() + val cached = Map(entry(out("b"), warehouse = Some("wh1"))) + val setup = eagerSetup(plan, context(cached, Set(out("c")), warehouse = Some("wh1"))) + val s = setup.state + s.skippedOperators(plan, setup.required) shouldBe Set(opId("a"), opId("b")) + val uri = s.cacheReadPorts(plan, setup.matched, setup.required)(out("b")).storageUri + uri.toString should include("/wh/wh1/") + uri shouldBe cached(out("b")).storageUri + s.readerUrisByInputPort(plan, setup.matched, setup.required)(in("c")) shouldBe List(uri) + } + + it should "reject matchedResults naming an input port or a port outside the plan" in { + val plan = chain() + val required = SearchState.requiredOutputPorts(context(Map.empty, Set(out("c"))), plan) + val state = allMaterialized(plan) + val invalidEntries = Map( + "input port" -> Map(in("b") -> CachedResult(baseUri(out("b")), None)), + "unknown operator" -> Map(out("zz") -> CachedResult(baseUri(out("zz")), None)), + "unknown output port" -> Map(out("b", 9) -> CachedResult(baseUri(out("b", 9)), None)), + "mixed valid and invalid port" -> Map( + entry(out("b")), + out("zz") -> CachedResult(baseUri(out("zz")), None) + ) + ) + invalidEntries.foreach { + case (label, badMatched) => + withClue(s"$label: ") { + an[IllegalArgumentException] should be thrownBy + SearchState.allowedLinkTypes(plan, badMatched, allowPipelined = true) + an[IllegalArgumentException] should be thrownBy + SearchState.eagerSeed(plan, badMatched, required) + an[IllegalArgumentException] should be thrownBy + state.cacheReadPorts(plan, badMatched, required) + an[IllegalArgumentException] should be thrownBy + state.readerUrisByInputPort(plan, badMatched, required) + an[IllegalArgumentException] should be thrownBy + state.skipRegions(plan, badMatched, required, 0) + } + } + } + + it should "run an operator with no output ports even when its input has a saved result" in { + // a -> s, and s has no output ports: nothing in the workflow reads s, so it may be there + // for what it does outside the workflow; it runs and reads a's saved result. + val plan = planOf(Set(source("a"), op("s", outputs = 0)), Set(link("a", "s"))) + val cached = Map(entry(out("a"))) + val setup = eagerSetup(plan, context(cached, Set.empty)) + val s = setup.state + ids(s.retainedPlan(plan, setup.required)) shouldBe Set(opId("s")) + s.skippedOperators(plan, setup.required) shouldBe Set(opId("a")) + s.linksOfType(CacheRead) shouldBe Set(link("a", "s")) + s.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe Map( + in("s") -> List(cached(out("a")).storageUri) + ) + s.cacheReadPorts(plan, setup.matched, setup.required) shouldBe cached + } + + it should "number skip regions from the given id, one per connected component" in { + // Two chains that never meet: a1 -> b1, a2 -> b2; both terminal ports matched. + val plan = planOf( + Set(source("a1"), op("b1"), source("a2"), op("b2")), + Set(link("a1", "b1"), link("a2", "b2")) + ) + val cached = Map(entry(out("b1")), entry(out("b2"))) + val setup = eagerSetup(plan, context(cached, Set(out("b1"), out("b2")))) + val regions = setup.state.skipRegions(plan, setup.matched, setup.required, firstRegionId = 3) + regions.map(_.id.id) shouldBe Set(3L, 4L) + regions.map(_.physicalOps.map(_.id)) shouldBe Set( + Set(opId("a1"), opId("b1")), + Set(opId("a2"), opId("b2")) + ) + regions.forall(_.skipped) shouldBe true + // Deterministic: the same ids for the same input. + setup.state + .skipRegions(plan, setup.matched, setup.required, 3) + .map(r => r.id -> r.physicalOps.map(_.id)) shouldBe + regions.map(r => r.id -> r.physicalOps.map(_.id)) + } + + it should "give each skip region only its own group's saved results" in { + // Two chains that never meet, s1 -> t1 and s2 -> t2, are skipped on t1's and t2's saved + // results. Beside them m runs for y, and x reads m's port 0 from its saved result: that + // port is read too, and its operator runs, so it is in no skip region. + val plan = planOf( + Set( + source("s1"), + op("t1"), + source("s2"), + op("t2"), + source("m", outputs = 2), + op("x"), + op("y") + ), + Set( + link("s1", "t1"), + link("s2", "t2"), + link("m", "x", fromPort = 0), + link("m", "y", fromPort = 1) + ) + ) + val cached = Map(entry(out("t1")), entry(out("t2")), entry(out("m", 0))) + val setup = eagerSetup(plan, context(cached, Set(out("t1"), out("t2"), out("x"), out("y")))) + setup.state.cacheReadPorts(plan, setup.matched, setup.required).keySet shouldBe Set( + out("t1"), + out("t2"), + out("m", 0) + ) + setup.state + .skipRegions(plan, setup.matched, setup.required, 0) + .map(region => + region.physicalOps.map(_.id) -> region.resourceConfig.get.portConfigs.keySet + ) shouldBe + Set( + Set(opId("s1"), opId("t1")) -> Set(out("t1")), + Set(opId("s2"), opId("t2")) -> Set(out("t2")) + ) + } + + // --------------------------------------------------------------------------- + // Four-edge-state SearchState checks + // --------------------------------------------------------------------------- + + "SearchState" should "reject reusedOutputs containing an input port" in { + an[IllegalArgumentException] should be thrownBy SearchState(Map.empty, Set(in("b"))) + } + + "SearchState.allowedLinkTypes" should + "distinguish pipelined, materialized, blocking and saved edges" in { + val blockingSource = PhysicalOp + .oneToOnePhysicalOp(opId("a"), workflowId, executionId, OpExecInitInfo.Empty) + .withOutputPorts(List(OutputPort(PortIdentity(0), blocking = true))) + val plan = planOf(Set(blockingSource, op("b"), op("c")), Set(link("a", "b"), link("b", "c"))) + val cached = Map(entry(out("a")), entry(out("b")), entry(out("c"))) + val ctx = context(cached, Set(out("b"))) + val matched = ctx.matchedResults + + // Pipelined mode: blocking link a->b drops Pipelined; non-blocking b->c allows all four. + val pipelinedLinks = SearchState.allowedLinkTypes(plan, matched, allowPipelined = true) + pipelinedLinks(link("a", "b")) shouldBe Set(Materialized, CacheRead, Unused) + pipelinedLinks(link("b", "c")) shouldBe Set(Materialized, Pipelined, CacheRead, Unused) + + // Without saved results, CacheRead is absent. + val uncachedLinks = SearchState.allowedLinkTypes(plan, Map.empty, allowPipelined = true) + uncachedLinks(link("b", "c")) shouldBe Set(Materialized, Pipelined, Unused) + + // Materialized mode: no link allows Pipelined. + val materializedLinks = SearchState.allowedLinkTypes(plan, matched, allowPipelined = false) + materializedLinks.values.foreach(types => types should not contain Pipelined) + } + + "SearchState" should "check the consumption condition and resolve both skipped and required inputs" in { + val plan = chain() + val cached = Map(entry(out("c"))) + val ctx = context(cached, Set(out("c"))) + val matched = ctx.matchedResults + val required = SearchState.requiredOutputPorts(ctx, plan) + + // Every link Materialized, nothing reused: every operator runs and no link is Unused. + val allMat = allMaterialized(plan) + allMat.satisfiesConsumptionCondition(plan, required) shouldBe true + + // Reuse c's required output without resolving yet: c stops running while b->c is still + // Materialized, so the consumption condition fails until resolveInputRequirements runs. + val beforeResolve = withReusedOutput(allMat, out("c"), reused = true) + beforeResolve.satisfiesConsumptionCondition(plan, required) shouldBe false + + val resolved = beforeResolve.resolveInputRequirements(plan, required, matched) + resolved.satisfiesConsumptionCondition(plan, required) shouldBe true + resolved(link("a", "b")) shouldBe Unused + resolved(link("b", "c")) shouldBe Unused + resolved.reusedOutputs shouldBe Set(out("c")) + SearchState.eagerSeed(plan, matched, required, defaultActiveType = _ => Unused) shouldBe + resolved + + // Making an incoming link of a running operator Unused violates the consumption condition, + // and resolveInputRequirements supplies its required inputs. + val unusedIntoRunning = allMat.updated(link("b", "c"), Unused) + unusedIntoRunning.satisfiesConsumptionCondition(plan, required) shouldBe false + val supplied = unusedIntoRunning.resolveInputRequirements(plan, required, matched) + supplied.satisfiesConsumptionCondition(plan, required) shouldBe true + supplied(link("b", "c")) shouldBe Materialized + + // Naming a non-required output in reusedOutputs fails the precondition. + val unknownReused = withReusedOutput(allMat, out("b"), reused = true) + unknownReused.satisfiesConsumptionCondition(plan, required) shouldBe false + an[IllegalArgumentException] should be thrownBy + unknownReused.resolveInputRequirements(plan, required, matched) + + // Mismatched link keys or invalid matchedResults port fail fast. + val missingLink = allMat.copy(linkTypes = allMat.linkTypes - link("b", "c")) + an[IllegalArgumentException] should be thrownBy + missingLink.resolveInputRequirements(plan, required, matched) + an[IllegalArgumentException] should be thrownBy + allMat.resolveInputRequirements(plan, required, Map(entry(out("missing")))) + } + + it should "cascade Unused backward across a dead chain and resolve upstream inputs when un-reusing" in { + val plan = planOf( + Set(source("a"), op("b"), op("c"), op("d"), op("e")), + Set(link("a", "b"), link("b", "c"), link("c", "d"), link("d", "e")) + ) + val cached = Map(entry(out("b")), entry(out("d"))) + val required = Set(out("e")) + + // Skipping all upstream work from e: cascading Unused clears every link in one pass. + val allMat = allMaterialized(plan).updated(link("d", "e"), CacheRead) + val skippedState = allMat.resolveInputRequirements(plan, required, cached) + skippedState.retainedOperators(plan, required) shouldBe Set(opId("e")) + skippedState(link("a", "b")) shouldBe Unused + skippedState(link("b", "c")) shouldBe Unused + skippedState(link("c", "d")) shouldBe Unused + skippedState(link("d", "e")) shouldBe CacheRead + skippedState.satisfiesConsumptionCondition(plan, required) shouldBe true + + // Switching d->e back to Materialized requires d and c to run, supplying c->d (Pipelined) + // and stopping at matched b->c (CacheRead) while leaving a->b Unused and b skipped. + val unReused = skippedState + .updated(link("d", "e"), Materialized) + .resolveInputRequirements( + plan, + required, + cached, + allowPipelined = true, + defaultActiveType = _ => Pipelined + ) + unReused.retainedOperators(plan, required) shouldBe Set(opId("c"), opId("d"), opId("e")) + unReused(link("d", "e")) shouldBe Materialized + unReused(link("c", "d")) shouldBe Pipelined + unReused(link("b", "c")) shouldBe CacheRead + unReused(link("a", "b")) shouldBe Unused + unReused.satisfiesConsumptionCondition(plan, required) shouldBe true + + // Empty plan and Materialized-only / CacheRead fallback on defaultActiveType. + val empty = PhysicalPlan(Set.empty, Set.empty) + SearchState(Map.empty).resolveInputRequirements(empty, Set.empty) shouldBe SearchState( + Map.empty + ) + + val unReusedMatFallback = skippedState + .updated(link("d", "e"), Materialized) + .resolveInputRequirements( + plan, + required, + cached, + allowPipelined = false, + defaultActiveType = _ => Pipelined + ) + unReusedMatFallback(link("c", "d")) shouldBe Materialized + + val unReusedInvalidDefaultFallback = skippedState + .updated(link("d", "e"), Materialized) + .resolveInputRequirements( + plan, + required, + cached, + allowPipelined = true, + defaultActiveType = _ => CacheRead + ) + unReusedInvalidDefaultFallback(link("c", "d")) shouldBe Materialized + } + + it should "derive retained operators, skipped operators, and cache-read ports from SearchState" in { + // split.0 is saved AND viewed; split.1 is unsaved, so split runs for y. + val plan = split() + val cached = Map(entry(out("split", 0))) + val ctx = context(cached, Set(out("split", 0), out("x"), out("y"))) + val matched = ctx.matchedResults + val required = SearchState.requiredOutputPorts(ctx, plan) + + val mixedState = SearchState.eagerSeed( + plan, + matched, + required, + allowPipelined = true, + defaultActiveType = l => if (l == link("src", "split")) Pipelined else Materialized + ) + mixedState.satisfiesConsumptionCondition(plan, required) shouldBe true + mixedState.retainedOperators(plan, required) shouldBe + Set(opId("src"), opId("split"), opId("x"), opId("y")) + mixedState.skippedOperators(plan, required) shouldBe empty + mixedState.linksOfType(Pipelined) shouldBe Set(link("src", "split")) + mixedState.linksOfType(CacheRead) shouldBe Set(link("split", "x", fromPort = 0)) + mixedState.linksOfType(Materialized) shouldBe Set(link("split", "y", fromPort = 1)) + mixedState.reusedOutputs shouldBe Set(out("split", 0)) + mixedState.cacheReadPorts(plan, matched, required) shouldBe cached + + // Materializing split.0's required output keeps x reading split.0's saved result while split stores split.0. + val storeSplitOut0 = withReusedOutput(mixedState, out("split", 0), reused = false) + storeSplitOut0.satisfiesConsumptionCondition(plan, required) shouldBe true + storeSplitOut0.reusedOutputs shouldBe empty + storeSplitOut0.cacheReadPorts(plan, matched, required) shouldBe cached + + // Materialized mode forbids Pipelined and falls back to Materialized on defaultActiveType. + val matLinks = SearchState.allowedLinkTypes(plan, matched, allowPipelined = false) + matLinks(link("src", "split")) should not contain Pipelined + val matSeed = SearchState.eagerSeed( + plan, + matched, + required, + allowPipelined = false, + defaultActiveType = _ => Pipelined + ) + matSeed(link("src", "split")) shouldBe Materialized + + // Rejects a SearchState violating the consumption condition. + val invalid = allMaterialized(plan).updated(link("src", "split"), Unused) + invalid.satisfiesConsumptionCondition(plan, required) shouldBe false + an[IllegalArgumentException] should be thrownBy invalid.cacheReadPorts(plan, matched, required) + an[IllegalArgumentException] should be thrownBy invalid.retainedPlan(plan, required) + an[IllegalArgumentException] should be thrownBy + invalid.readerUrisByInputPort(plan, matched, required) + an[IllegalArgumentException] should be thrownBy invalid.skipRegions(plan, matched, required, 0) + } + + /** What a reference computation expects of one search state. */ + private case class Expected( + retained: Set[PhysicalOpIdentity], + skippedLinks: Set[PhysicalLink], + cacheReadLinks: Set[PhysicalLink], + retainedLinks: Set[PhysicalLink], + readPorts: Set[GlobalPortIdentity] + ) + + /** + * A slow reference computation of the rules: a fixed point over all operators, in any order. + * + * Every required output gets an operator that always runs and reads it (the user, in + * effect), and an output port with no link is a required output. An operator runs when it + * has a link from a port without a saved result into an operator that runs, or when it has + * no output ports. A link into an operator that runs is a cache-read link when its port has + * a saved result, and a retained link otherwise; every other link is skipped. + */ + private def reference( + plan: PhysicalPlan, + savedPorts: Set[GlobalPortIdentity], + mustStore: Set[GlobalPortIdentity] + ): Expected = { + def portOf(l: PhysicalLink) = GlobalPortIdentity(l.fromOpId, l.fromPortId) + val required = plan.operators + .flatMap(o => o.outputPorts.keys.map(p => GlobalPortIdentity(o.id, p))) + .filter(p => mustStore.contains(p) || !plan.links.exists(l => portOf(l) == p)) + var retained = plan.operators.filter(_.outputPorts.isEmpty).map(_.id) + var changed = true + while (changed) { + val next = retained ++ + required.filterNot(savedPorts.contains).map(_.opId) ++ + plan.links + .filter(l => retained.contains(l.toOpId) && !savedPorts.contains(portOf(l))) + .map(_.fromOpId) + changed = next != retained + retained = next + } + val skipped = plan.operators.map(_.id) -- retained + val intoRetained = plan.links.filter(l => retained.contains(l.toOpId)) + val cacheRead = intoRetained.filter(l => savedPorts.contains(portOf(l))) + Expected( + retained = retained, + skippedLinks = plan.links.filter(l => skipped.contains(l.toOpId)), + cacheReadLinks = cacheRead, + retainedLinks = intoRetained -- cacheRead, + // The saved results read: behind every cache-read link, and every required output of a + // skipped operator. A required output of a retained operator is stored and shown from this + // run, not read from its saved result. + readPorts = cacheRead.map(portOf) ++ required.filter(p => skipped.contains(p.opId)) + ) + } + + /** The connected groups of `ops` over the links between them, the slow way. */ + private def groupsOf( + ops: Set[PhysicalOpIdentity], + links: Set[PhysicalLink] + ): Set[Set[PhysicalOpIdentity]] = { + val inside = links.filter(l => ops.contains(l.fromOpId) && ops.contains(l.toOpId)) + ops.map { start => + var group = Set(start) + var changed = true + while (changed) { + val next = group ++ inside.collect { + case l if group.contains(l.fromOpId) => l.toOpId + case l if group.contains(l.toOpId) => l.fromOpId + } + changed = next != group + group = next + } + group + } + } + + /** Up to six operators with two input ports and zero to two output ports, linked at random. */ + private def randomPlan(random: Random): PhysicalPlan = { + val n = 1 + random.nextInt(6) + val outputs = (0 until n).map(_ => if (random.nextInt(8) == 0) 0 else 1 + random.nextInt(2)) + val ops = (0 until n).map(i => op(s"o$i", inputs = 2, outputs = outputs(i))) + val links = for { + i <- 0 until n + j <- i + 1 until n + if outputs(i) > 0 && random.nextInt(3) == 0 + } yield link(s"o$i", s"o$j", fromPort = random.nextInt(outputs(i)), toPort = random.nextInt(2)) + planOf(ops.toSet, links.toSet) + } + + it should "agree with a slow reference computation on random small plans" in { + val random = new Random(42) + (1 to 400).foreach { _ => + val plan = randomPlan(random) + val allOutputs = + plan.operators.flatMap(o => o.outputPorts.keys.map(p => GlobalPortIdentity(o.id, p))) + // Close to the compiler, which stores the ports of operators with no outgoing links and + // of operators whose results the user views: here most ports with no link, and a few + // others. A port with no link left out of the set is a required output all the same. + val noLink = allOutputs.filter(p => + !plan.links.exists(l => l.fromOpId == p.opId && l.fromPortId == p.portId) + ) + val mustStore = + noLink.filter(_ => random.nextInt(4) != 0) ++ allOutputs.filter(_ => random.nextInt(6) == 0) + val savedPorts = allOutputs.filter(_ => random.nextBoolean()) + val cached = savedPorts.map(p => entry(p)).toMap + val ctx = context(cached, mustStore) + val setup = eagerSetup(plan, ctx) + val s = setup.state + val expected = reference(plan, savedPorts, mustStore) + val expectedSkipped = ids(plan) -- expected.retained + withClue(s"plan ${plan.links} stores $mustStore saved $savedPorts: ") { + s.retainedOperators(plan, setup.required) shouldBe expected.retained + s.skippedOperators(plan, setup.required) shouldBe expectedSkipped + s.skipsAnything(plan, setup.required) shouldBe expectedSkipped.nonEmpty + s.linksOfType(Unused) shouldBe expected.skippedLinks + s.linksOfType(CacheRead) shouldBe expected.cacheReadLinks + val retained = s.retainedPlan(plan, setup.required) + ids(retained) shouldBe expected.retained + retained.links shouldBe expected.retainedLinks + retained.operators.foreach(o => o should be theSameInstanceAs plan.getOperator(o.id)) + expected.retainedLinks.foreach(l => expected.retained should contain(l.fromOpId)) + val readerUris = s.readerUrisByInputPort(plan, setup.matched, setup.required) + readerUris.keySet shouldBe + expected.cacheReadLinks.map(l => GlobalPortIdentity(l.toOpId, l.toPortId, input = true)) + readerUris.foreach { + case (inputPort, uris) => + val links = expected.cacheReadLinks + .filter(l => l.toOpId == inputPort.opId && l.toPortId == inputPort.portId) + uris.size shouldBe links.size + uris.toSet shouldBe links.map(l => + cached(GlobalPortIdentity(l.fromOpId, l.fromPortId)).storageUri + ) + } + s.cacheReadPorts(plan, setup.matched, setup.required).keySet shouldBe expected.readPorts + s.cacheReadPorts(plan, setup.matched, setup.required).foreach { + case (port, saved) => saved shouldBe cached(port) + } + val regions = s.skipRegions(plan, setup.matched, setup.required, 10) + regions.map(_.physicalOps.map(_.id)) shouldBe groupsOf( + expectedSkipped, + expected.skippedLinks + ) + regions.map(_.id.id) shouldBe (10L until 10L + regions.size).toSet + regions.foreach { region => + val group = region.physicalOps.map(_.id) + region.physicalLinks shouldBe expected.skippedLinks.filter(l => + group.contains(l.fromOpId) && group.contains(l.toOpId) + ) + region.resourceConfig.get.portConfigs shouldBe + expected.readPorts + .filter(port => group.contains(port.opId)) + .map { port => + port -> OutputPortConfig(cached(port).storageUri, cached(port).tupleCount) + } + .toMap + region.physicalOps.foreach(o => o should be theSameInstanceAs plan.getOperator(o.id)) + } + + // Verify single-pass reverse-topological resolveInputRequirements against the independent + // backward-reachability oracle across arbitrary starting link states, and confirm both the + // consumption condition and seed equivalence. + val allowedLinks = SearchState.allowedLinkTypes(plan, setup.matched, allowPipelined = true) + val preResolve = SearchState( + linkTypes = plan.links.map { l => + val opts = allowedLinks(l).toVector + l -> opts(random.nextInt(opts.size)) + }.toMap, + reusedOutputs = + setup.required.filter(p => setup.matched.contains(p) && random.nextBoolean()) + ) + val usePipelinedDefault = random.nextBoolean() + val defaultActive = + (_: PhysicalLink) => if (usePipelinedDefault) Pipelined else Materialized + val allowPipe = random.nextBoolean() + val resolved = preResolve.resolveInputRequirements( + plan, + setup.required, + setup.matched, + allowPipelined = allowPipe, + defaultActiveType = defaultActive + ) + resolved shouldBe referenceResolveInputRequirements( + plan, + setup.required, + setup.matched, + allowPipelined = allowPipe, + defaultActiveType = defaultActive, + preResolve + ) + resolved.satisfiesConsumptionCondition(plan, setup.required) shouldBe true + + val seed = SearchState.eagerSeed( + plan, + setup.matched, + setup.required, + defaultActiveType = _ => if (random.nextBoolean()) Pipelined else Materialized + ) + seed.satisfiesConsumptionCondition(plan, setup.required) shouldBe true + seed.skippedOperators(plan, setup.required) shouldBe expectedSkipped + seed.linksOfType(Unused) shouldBe expected.skippedLinks + seed.linksOfType(CacheRead) shouldBe expected.cacheReadLinks + seed.retainedPlan(plan, setup.required) shouldBe retained + seed.readerUrisByInputPort(plan, setup.matched, setup.required) shouldBe readerUris + seed.cacheReadPorts(plan, setup.matched, setup.required) shouldBe + s.cacheReadPorts(plan, setup.matched, setup.required) + } + } + } +} diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/config/SchedulingConfigsSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/config/SchedulingConfigsSpec.scala index f41f55b3656..b6d4e686e98 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/config/SchedulingConfigsSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/config/SchedulingConfigsSpec.scala @@ -228,6 +228,14 @@ class SchedulingConfigsSpec extends AnyFlatSpec { val uri = new URI("vfs:///wid/1/eid/1/result") val cfg = OutputPortConfig(uri) assert(cfg.storageURIs == List(uri)) + assert(cfg.cachedTupleCount.isEmpty) + } + + it should "preserve an optional cachedTupleCount" in { + val uri = new URI("vfs:///wid/1/eid/1/result") + val cfg = OutputPortConfig(uri, cachedTupleCount = Some(42L)) + assert(cfg.storageURIs == List(uri)) + assert(cfg.cachedTupleCount.contains(42L)) } "IntermediateInputPortConfig" should "expose every URI it was constructed with" in { diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/resourcePolicies/ResourcePoliciesSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/resourcePolicies/ResourcePoliciesSpec.scala index 70958ec059d..b8c06beecd5 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/resourcePolicies/ResourcePoliciesSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/resourcePolicies/ResourcePoliciesSpec.scala @@ -19,7 +19,31 @@ package org.apache.texera.amber.engine.architecture.scheduling.resourcePolicies -import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} +import org.apache.texera.amber.core.executor.OpExecInitInfo +import org.apache.texera.amber.core.storage.VFSURIFactory +import org.apache.texera.amber.core.virtualidentity.{ + ExecutionIdentity, + OperatorIdentity, + PhysicalOpIdentity, + WorkflowIdentity +} +import org.apache.texera.amber.core.workflow.{ + GlobalPortIdentity, + HashPartition, + InputPort, + OutputPort, + PartitionInfo, + PhysicalLink, + PhysicalOp, + PhysicalPlan, + PortIdentity, + WorkflowContext, + WorkflowSettings +} +import org.apache.texera.amber.engine.architecture.scheduling.config.{ + IntermediateInputPortConfig, + ResourceConfig +} import org.apache.texera.amber.engine.architecture.scheduling.{Region, RegionIdentity} import org.apache.texera.amber.engine.architecture.sendsemantics.partitionings.{ BroadcastPartitioning, @@ -171,4 +195,60 @@ class ResourcePoliciesSpec extends AnyFlatSpec { val (resourceConfig, _) = allocator.allocate(region) assert(resourceConfig.portConfigs.isEmpty) } + + // --------------------------------------------------------------------------- + // Operators that read saved results under DefaultResourceAllocator(fullPlan) + // --------------------------------------------------------------------------- + + /** + * y requires a hash partition on its input; z requires nothing. When DefaultResourceAllocator + * receives the full plan (Set(x, y, z) with x -> y -> z), y has an upstream link in the full + * plan's DAG even when x is skipped and y reads x's saved result in region Set(y, z). So y is + * not treated as a source, and y -> z gets RoundRobinPartitioning just as behind a + * materialized link. Passing only the sub-plan Set(y, z) would treat y as a source and give + * y -> z a hash shuffle it never gets in the full plan. + */ + "DefaultResourceAllocator" should "partition an operator reading a saved result like one behind a materialized link when given the full plan" in { + val wid = WorkflowIdentity(1L) + val eid = ExecutionIdentity(1L) + def id(name: String) = PhysicalOpIdentity(OperatorIdentity(name), "main") + def parallelOp(name: String, requirement: Option[PartitionInfo]): PhysicalOp = + PhysicalOp + .oneToOnePhysicalOp(id(name), wid, eid, OpExecInitInfo.Empty) + .withInputPorts(List(InputPort(PortIdentity(0)))) + .withOutputPorts(List(OutputPort(PortIdentity(0)))) + .withParallelizable(true) + .withSuggestedWorkerNum(2) + .withPartitionRequirement(List(requirement)) + val xy = PhysicalLink(id("x"), PortIdentity(0), id("y"), PortIdentity(0)) + val yz = PhysicalLink(id("y"), PortIdentity(0), id("z"), PortIdentity(0)) + val x = parallelOp("x", None).addOutputLink(xy) + val y = parallelOp("y", Some(HashPartition(List("k")))).addInputLink(xy).addOutputLink(yz) + val z = parallelOp("z", None).addInputLink(yz) + val fullPlan = PhysicalPlan(Set(x, y, z), Set(xy, yz)) + val retainedPlan = PhysicalPlan(Set(y, z), Set(yz)) + val readerUri = + VFSURIFactory.createPortBaseURI(wid, eid, GlobalPortIdentity(id("x"), PortIdentity(0))) + val yInput = GlobalPortIdentity(id("y"), PortIdentity(0), input = true) + val region = + Region( + RegionIdentity(0), + Set(y, z), + Set(yz), + resourceConfig = Some( + ResourceConfig(portConfigs = Map(yInput -> IntermediateInputPortConfig(List(readerUri)))) + ) + ) + def partitioningOfYZ(allocator: DefaultResourceAllocator) = + allocator.allocate(region)._1.linkConfigs(yz).partitioning.getClass + + val withFullPlan = partitioningOfYZ( + new DefaultResourceAllocator(fullPlan, new ExecutionClusterInfo(), WorkflowSettings()) + ) + val withRetainedPlanOnly = partitioningOfYZ( + new DefaultResourceAllocator(retainedPlan, new ExecutionClusterInfo(), WorkflowSettings()) + ) + assert(withFullPlan == classOf[RoundRobinPartitioning]) + assert(withRetainedPlanOnly == classOf[HashBasedShufflePartitioning]) + } } diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/CachedResult.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/CachedResult.scala new file mode 100644 index 00000000000..ee17fcd2264 --- /dev/null +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/CachedResult.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.core.workflow + +import java.net.URI + +/** + * The saved result of an output port, which the operators that read the port may read + * instead. The port's own operator, if it runs, still computes the port. + * + * @param storageUri the port's base URI, as `VFSURIFactory.createPortBaseURI` builds it; the + * result and state documents hang off it via `resultURI` and `stateURI` + * @param tupleCount the saved row count when the writer recorded one; a skipped operator + * reports it as the port's output count + */ +case class CachedResult(storageUri: URI, tupleCount: Option[Long]) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowContext.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowContext.scala index 71b325cfdd0..c12d9a9c5f6 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowContext.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowContext.scala @@ -38,5 +38,11 @@ class WorkflowContext( var cuid: Option[Int] = None, // The Lakekeeper catalog name this execution writes into; becomes the `/wh/` // URI segment. None routes to the shared default warehouse. - var warehouse: Option[String] = None + var warehouse: Option[String] = None, + // Saved results found for output ports of this run, keyed by port; operators that read + // those ports may read the saved results instead (a port's own operator, if it runs, + // still computes it). Set on the computing unit by the cache lookup before scheduling; no + // request type carries it, so the editor cannot supply it. Empty means the scheduler + // runs exactly as it does without a cache. + var matchedResults: Map[GlobalPortIdentity, CachedResult] = Map.empty ) diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowContextSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowContextSpec.scala index 2407531e99a..66e7b2f0f2d 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowContextSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowContextSpec.scala @@ -19,10 +19,17 @@ package org.apache.texera.amber.core.workflow -import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.virtualidentity.{ + ExecutionIdentity, + OperatorIdentity, + PhysicalOpIdentity, + WorkflowIdentity +} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.net.URI + class WorkflowContextSpec extends AnyFlatSpec with Matchers { "WorkflowContext companion" should "expose pinned defaults" in { @@ -59,4 +66,20 @@ class WorkflowContextSpec extends AnyFlatSpec with Matchers { // Settings argument was not overridden, so the companion default holds. ctx.workflowSettings shouldBe WorkflowContext.DEFAULT_WORKFLOW_SETTINGS } + + "WorkflowContext.matchedResults" should "default to empty and be reassignable" in { + // Empty is the contract the scheduler relies on: no matched results, no cache code path. + val ctx = new WorkflowContext() + ctx.matchedResults shouldBe Map.empty + val port = + GlobalPortIdentity(PhysicalOpIdentity(OperatorIdentity("op"), "main"), PortIdentity(0)) + val cached = CachedResult(new URI("vfs:///wid/1/eid/1/globalportid/x"), Some(3L)) + ctx.matchedResults = Map(port -> cached) + ctx.matchedResults shouldBe Map(port -> cached) + // a context built with the other arguments only still starts empty + new WorkflowContext( + WorkflowIdentity(9L), + ExecutionIdentity(8L) + ).matchedResults shouldBe Map.empty + } }