diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Region.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Region.scala index bbb4035e22f..535fd2c0a0d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Region.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Region.scala @@ -30,12 +30,18 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala case class RegionLink(fromRegionId: RegionIdentity, toRegionId: RegionIdentity) case class RegionIdentity(id: Long) + +/** + * A sub-plan scheduled as a unit. When `skipped` is true, its operators are not launched on + * workers; their saved outputs are carried in `resourceConfig` for reuse by later regions. + */ case class Region( id: RegionIdentity, physicalOps: Set[PhysicalOp], physicalLinks: Set[PhysicalLink], ports: Set[GlobalPortIdentity] = Set.empty, - resourceConfig: Option[ResourceConfig] = None + resourceConfig: Option[ResourceConfig] = None, + skipped: Boolean = false ) { private val operators: Map[PhysicalOpIdentity, PhysicalOp] = @@ -73,9 +79,8 @@ case class Region( * The effective source contains operators that have 0 input links in this region. */ def getSourceOperators: Set[PhysicalOp] = { - getOperators.filter(physicalOp => - physicalOp.getInputLinks().forall(link => !operators.contains(link.fromOpId)) - ) + val operatorsWithInputLink = getLinks.map(_.toOpId) + getOperators.filterNot(op => operatorsWithInputLink.contains(op.id)) } /** diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlan.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlan.scala index 95937517de2..006781cc6ea 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlan.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlan.scala @@ -25,6 +25,11 @@ import org.jgrapht.traverse.TopologicalOrderIterator import scala.jdk.CollectionConverters.IteratorHasAsScala +/** + * Region-level execution plan of a workflow. `regions` holds both active regions + * (`skipped = false`) and skip regions (`skipped = true`); `regionLinks` captures materialized + * dependencies among active regions. + */ case class RegionPlan( regions: Set[Region], regionLinks: Set[RegionLink] diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleGenerator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleGenerator.scala index 13f1acebdcf..e4eaa8832aa 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleGenerator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleGenerator.scala @@ -45,24 +45,40 @@ abstract class ScheduleGenerator( def generate(): (Schedule, PhysicalPlan) /** - * A schedule is a ranking on the regions of a region plan. - * Regions are dispatched in batches of up to ApplicationConfig.maxConcurrentRegions, respecting the DAG dependencies. - * When maxConcurrentRegions == 1, this is equivalent to a total order (fully sequential execution). + * A schedule is a ranking on the regions of a region plan. Skip regions (`Region.skipped`) + * share level 0 so they complete together without worker allocation; active regions follow in + * topological batches of up to ApplicationConfig.maxConcurrentRegions. When + * maxConcurrentRegions == 1, the active levels form a total order. */ def generateScheduleFromRegionPlan(regionPlan: RegionPlan): Schedule = { - val inDegree = mutable.Map.empty[RegionIdentity, Int] - regionPlan.topologicalIterator().foreach { rid => - inDegree(rid) = regionPlan.dag.inDegreeOf(rid) + val skippedIds = regionPlan.regions.filter(_.skipped).map(_.id) + require( + regionPlan.regionLinks.forall(link => + !skippedIds.contains(link.fromRegionId) && !skippedIds.contains(link.toRegionId) + ), + "skip regions must not have region links" + ) + + val tmpLevelSets = mutable.Map.empty[Int, Set[RegionIdentity]] + var level = 0 + if (skippedIds.nonEmpty) { + tmpLevelSets(0) = skippedIds + level = 1 } + val inDegree = mutable.Map.empty[RegionIdentity, Int] + regionPlan + .topologicalIterator() + .filterNot(skippedIds.contains) + .foreach { rid => + inDegree(rid) = regionPlan.dag.inDegreeOf(rid) + } + val readyRegionsQueue = mutable.Queue( inDegree.collect { case (rid, 0) => rid }.toSeq: _* ) - val tmpLevelSets = mutable.Map.empty[Int, Set[RegionIdentity]] - var level = 0 - - // While there are ready regions: + // While there are ready active regions: // 1. Dequeue up to maxConcurrentRegions regions to form the current batch. // 2. Record this batch under the current level. // 3. For each region in the batch: diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/PortConfig.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/PortConfig.scala index 56743ae0956..14c2c3eb3d3 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/PortConfig.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/PortConfig.scala @@ -32,11 +32,16 @@ sealed trait PortConfig { } /** - * An output port requires exactly one materialization base URI. Result and - * state URIs hang off it via `VFSURIFactory.resultURI` / `stateURI`; this - * field is *not* a URI you can pass straight to `DocumentFactory`. + * An output port requires one storage base URI. Result and state URIs hang + * off it via `VFSURIFactory.resultURI` / `stateURI`; this field is *not* a + * URI you can pass straight to `DocumentFactory`. For a skipped operator, + * `cachedTupleCount` holds the saved output count so runtime statistics + * match the run that wrote it. */ -final case class OutputPortConfig(storageURIBase: URI) extends PortConfig { +final case class OutputPortConfig( + storageURIBase: URI, + cachedTupleCount: Option[Long] = None +) extends PortConfig { override val storageURIs: List[URI] = List(storageURIBase) } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlanSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlanSpec.scala index f8d71ea91de..e258decd60c 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlanSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionPlanSpec.scala @@ -236,4 +236,31 @@ class RegionPlanSpec extends AnyFlatSpec with BarePhysicalPlanFixture { assert(order.head == ids(0)) assert(order.last == ids(8)) } + + "RegionPlan with skip regions" should "hold active and skipped regions together and resolve lookups across both" in { + val active = buildDiamondPlan() + val skippedInternalLink = link("skip-a", "skip-b") + val skippedPort = globalPort("skip-b") + val skipRegion = Region( + RegionIdentity(10), + physicalOps = Set(op("skip-a"), op("skip-b")), + physicalLinks = Set(skippedInternalLink), + ports = Set(skippedPort), + skipped = true + ) + val plan = RegionPlan( + regions = active.regions + skipRegion, + regionLinks = active.regionLinks + ) + + assert(plan.getRegion(RegionIdentity(10)) == skipRegion) + assert(plan.getRegionOfLink(skippedInternalLink) == skipRegion) + assert(plan.getRegionOfPortId(skippedPort).contains(skipRegion)) + val order = plan.topologicalIterator().toList + assert(order.toSet == active.regions.map(_.id) + skipRegion.id) + val pos = order.zipWithIndex.toMap + active.regionLinks.foreach { edge => + assert(pos(edge.fromRegionId) < pos(edge.toRegionId)) + } + } } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionSpec.scala index 8974a33cf88..f28632d8bb5 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionSpec.scala @@ -71,6 +71,11 @@ class RegionSpec extends AnyFlatSpec with BarePhysicalPlanFixture { assert(region.getPorts == Set(portId)) } + it should "default skipped to false and accept true" in { + assert(!Region(RegionIdentity(1), Set(op("a")), Set.empty).skipped) + assert(Region(RegionIdentity(1), Set(op("a")), Set.empty, skipped = true).skipped) + } + "Region.getOperator" should "look up a physical operator by id" in { val a = op("a") val b = op("b") @@ -118,6 +123,17 @@ class RegionSpec extends AnyFlatSpec with BarePhysicalPlanFixture { assert(region.getSourceOperators == Set(a, c)) } + it should "count only the region's own links, not every link an operator lists" in { + // b lists an input link from a, and both are in the region, yet the link is not one of the + // region's links (for instance when b reads a's saved result): b is a source of the region. + val ab = link("a", "b") + val a = wiredOp("a").addOutputLink(ab) + val b = wiredOp("b").addInputLink(ab) + assert(Region(RegionIdentity(1), Set(a, b), Set.empty).getSourceOperators == Set(a, b)) + // With the link among the region's links, b is not a source. + assert(Region(RegionIdentity(1), Set(a, b), Set(ab)).getSourceOperators == Set(a)) + } + "Region.getStarterOperators" should "match getSourceOperators when no resource config is provided" in { val a = op("a") val b = op("b") diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleSpec.scala index b4709fb2d1b..2747446d2f6 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/ScheduleSpec.scala @@ -115,4 +115,76 @@ class ScheduleSpec extends AnyFlatSpec with BarePhysicalPlanFixture { assert(order.indexOf(r2) < order.indexOf(r3)) } + it should "place all skip regions at level 0 with no extra empty level when nothing else runs" in { + val skip0 = region(10, "s0").copy(skipped = true) + val skip1 = region(11, "s1").copy(skipped = true) + val plan = RegionPlan(Set(skip0, skip1), Set.empty) + val generator = new ScheduleGenerator( + new WorkflowContext(), + PhysicalPlan(Set.empty, Set.empty) + ) { + override def generate(): (Schedule, PhysicalPlan) = (Schedule(Map.empty), physicalPlan) + } + + val schedule = generator.generateScheduleFromRegionPlan(plan) + assert(schedule.levelSets == Map(0 -> Set(skip0, skip1))) + } + + it should "place all skip regions at level 0 ahead of topological active batches" in { + val r0 = region(0, "a") + val r1 = region(1, "b") + val r2 = region(2, "c") + val r3 = region(3, "d") + val skip0 = region(4, "s0").copy(skipped = true) + val skip1 = region(5, "s1").copy(skipped = true) + val plan = RegionPlan( + Set(r0, r1, r2, r3, skip0, skip1), + Set( + RegionLink(r0.id, r1.id), + RegionLink(r0.id, r2.id), + RegionLink(r1.id, r3.id), + RegionLink(r2.id, r3.id) + ) + ) + val generator = new ScheduleGenerator( + new WorkflowContext(), + PhysicalPlan(Set.empty, Set.empty) + ) { + override def generate(): (Schedule, PhysicalPlan) = (Schedule(Map.empty), physicalPlan) + } + + val levels = generator.generateScheduleFromRegionPlan(plan).toList + assert(levels.head == Set(skip0, skip1)) + val activeLevels = levels.tail + assert(activeLevels.forall(_.size <= ApplicationConfig.maxConcurrentRegions)) + val activeOrder = activeLevels.flatten + assert(activeOrder.toSet == Set(r0, r1, r2, r3)) + assert(activeOrder.indexOf(r0) < activeOrder.indexOf(r1)) + assert(activeOrder.indexOf(r0) < activeOrder.indexOf(r2)) + assert(activeOrder.indexOf(r1) < activeOrder.indexOf(r3)) + assert(activeOrder.indexOf(r2) < activeOrder.indexOf(r3)) + } + + it should "reject a region plan whose regionLinks reference a skipped region" in { + val skipRegion = region(0, "s").copy(skipped = true) + val activeRegion = region(1, "a") + val generator = new ScheduleGenerator( + new WorkflowContext(), + PhysicalPlan(Set.empty, Set.empty) + ) { + override def generate(): (Schedule, PhysicalPlan) = (Schedule(Map.empty), physicalPlan) + } + + assertThrows[IllegalArgumentException] { + generator.generateScheduleFromRegionPlan( + RegionPlan(Set(skipRegion, activeRegion), Set(RegionLink(skipRegion.id, activeRegion.id))) + ) + } + assertThrows[IllegalArgumentException] { + generator.generateScheduleFromRegionPlan( + RegionPlan(Set(skipRegion, activeRegion), Set(RegionLink(activeRegion.id, skipRegion.id))) + ) + } + } + } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/config/SchedulingConfigsSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/config/SchedulingConfigsSpec.scala index f41f55b3656..b6d4e686e98 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/config/SchedulingConfigsSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/config/SchedulingConfigsSpec.scala @@ -228,6 +228,14 @@ class SchedulingConfigsSpec extends AnyFlatSpec { val uri = new URI("vfs:///wid/1/eid/1/result") val cfg = OutputPortConfig(uri) assert(cfg.storageURIs == List(uri)) + assert(cfg.cachedTupleCount.isEmpty) + } + + it should "preserve an optional cachedTupleCount" in { + val uri = new URI("vfs:///wid/1/eid/1/result") + val cfg = OutputPortConfig(uri, cachedTupleCount = Some(42L)) + assert(cfg.storageURIs == List(uri)) + assert(cfg.cachedTupleCount.contains(42L)) } "IntermediateInputPortConfig" should "expose every URI it was constructed with" in { diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/CachedResult.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/CachedResult.scala new file mode 100644 index 00000000000..ee17fcd2264 --- /dev/null +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/CachedResult.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.core.workflow + +import java.net.URI + +/** + * The saved result of an output port, which the operators that read the port may read + * instead. The port's own operator, if it runs, still computes the port. + * + * @param storageUri the port's base URI, as `VFSURIFactory.createPortBaseURI` builds it; the + * result and state documents hang off it via `resultURI` and `stateURI` + * @param tupleCount the saved row count when the writer recorded one; a skipped operator + * reports it as the port's output count + */ +case class CachedResult(storageUri: URI, tupleCount: Option[Long]) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowContext.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowContext.scala index 71b325cfdd0..c12d9a9c5f6 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowContext.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowContext.scala @@ -38,5 +38,11 @@ class WorkflowContext( var cuid: Option[Int] = None, // The Lakekeeper catalog name this execution writes into; becomes the `/wh/` // URI segment. None routes to the shared default warehouse. - var warehouse: Option[String] = None + var warehouse: Option[String] = None, + // Saved results found for output ports of this run, keyed by port; operators that read + // those ports may read the saved results instead (a port's own operator, if it runs, + // still computes it). Set on the computing unit by the cache lookup before scheduling; no + // request type carries it, so the editor cannot supply it. Empty means the scheduler + // runs exactly as it does without a cache. + var matchedResults: Map[GlobalPortIdentity, CachedResult] = Map.empty ) diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowContextSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowContextSpec.scala index 2407531e99a..66e7b2f0f2d 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowContextSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowContextSpec.scala @@ -19,10 +19,17 @@ package org.apache.texera.amber.core.workflow -import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.virtualidentity.{ + ExecutionIdentity, + OperatorIdentity, + PhysicalOpIdentity, + WorkflowIdentity +} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.net.URI + class WorkflowContextSpec extends AnyFlatSpec with Matchers { "WorkflowContext companion" should "expose pinned defaults" in { @@ -59,4 +66,20 @@ class WorkflowContextSpec extends AnyFlatSpec with Matchers { // Settings argument was not overridden, so the companion default holds. ctx.workflowSettings shouldBe WorkflowContext.DEFAULT_WORKFLOW_SETTINGS } + + "WorkflowContext.matchedResults" should "default to empty and be reassignable" in { + // Empty is the contract the scheduler relies on: no matched results, no cache code path. + val ctx = new WorkflowContext() + ctx.matchedResults shouldBe Map.empty + val port = + GlobalPortIdentity(PhysicalOpIdentity(OperatorIdentity("op"), "main"), PortIdentity(0)) + val cached = CachedResult(new URI("vfs:///wid/1/eid/1/globalportid/x"), Some(3L)) + ctx.matchedResults = Map(port -> cached) + ctx.matchedResults shouldBe Map(port -> cached) + // a context built with the other arguments only still starts empty + new WorkflowContext( + WorkflowIdentity(9L), + ExecutionIdentity(8L) + ).matchedResults shouldBe Map.empty + } }