Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,18 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala
case class RegionLink(fromRegionId: RegionIdentity, toRegionId: RegionIdentity)

case class RegionIdentity(id: Long)

/**
* A sub-plan scheduled as a unit. When `skipped` is true, its operators are not launched on
* workers; their saved outputs are carried in `resourceConfig` for reuse by later regions.
*/
case class Region(
id: RegionIdentity,
physicalOps: Set[PhysicalOp],
physicalLinks: Set[PhysicalLink],
ports: Set[GlobalPortIdentity] = Set.empty,
resourceConfig: Option[ResourceConfig] = None
resourceConfig: Option[ResourceConfig] = None,
skipped: Boolean = false
) {

private val operators: Map[PhysicalOpIdentity, PhysicalOp] =
Expand Down Expand Up @@ -73,9 +79,8 @@ case class Region(
* The effective source contains operators that have 0 input links in this region.
*/
def getSourceOperators: Set[PhysicalOp] = {
getOperators.filter(physicalOp =>
physicalOp.getInputLinks().forall(link => !operators.contains(link.fromOpId))
)
val operatorsWithInputLink = getLinks.map(_.toOpId)
getOperators.filterNot(op => operatorsWithInputLink.contains(op.id))
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ import org.jgrapht.traverse.TopologicalOrderIterator

import scala.jdk.CollectionConverters.IteratorHasAsScala

/**
* Region-level execution plan of a workflow. `regions` holds both active regions
* (`skipped = false`) and skip regions (`skipped = true`); `regionLinks` captures materialized
* dependencies among active regions.
*/
case class RegionPlan(
regions: Set[Region],
regionLinks: Set[RegionLink]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,24 +45,40 @@ abstract class ScheduleGenerator(
def generate(): (Schedule, PhysicalPlan)

/**
* A schedule is a ranking on the regions of a region plan.
* Regions are dispatched in batches of up to ApplicationConfig.maxConcurrentRegions, respecting the DAG dependencies.
* When maxConcurrentRegions == 1, this is equivalent to a total order (fully sequential execution).
* A schedule is a ranking on the regions of a region plan. Skip regions (`Region.skipped`)
* share level 0 so they complete together without worker allocation; active regions follow in
* topological batches of up to ApplicationConfig.maxConcurrentRegions. When
* maxConcurrentRegions == 1, the active levels form a total order.
*/
def generateScheduleFromRegionPlan(regionPlan: RegionPlan): Schedule = {
val inDegree = mutable.Map.empty[RegionIdentity, Int]
regionPlan.topologicalIterator().foreach { rid =>
inDegree(rid) = regionPlan.dag.inDegreeOf(rid)
val skippedIds = regionPlan.regions.filter(_.skipped).map(_.id)
require(
regionPlan.regionLinks.forall(link =>
!skippedIds.contains(link.fromRegionId) && !skippedIds.contains(link.toRegionId)
),
"skip regions must not have region links"
)

val tmpLevelSets = mutable.Map.empty[Int, Set[RegionIdentity]]
var level = 0
if (skippedIds.nonEmpty) {
tmpLevelSets(0) = skippedIds
level = 1
}

val inDegree = mutable.Map.empty[RegionIdentity, Int]
regionPlan
.topologicalIterator()
.filterNot(skippedIds.contains)
.foreach { rid =>
inDegree(rid) = regionPlan.dag.inDegreeOf(rid)
}

val readyRegionsQueue = mutable.Queue(
inDegree.collect { case (rid, 0) => rid }.toSeq: _*
)

val tmpLevelSets = mutable.Map.empty[Int, Set[RegionIdentity]]
var level = 0

// While there are ready regions:
// While there are ready active regions:
// 1. Dequeue up to maxConcurrentRegions regions to form the current batch.
// 2. Record this batch under the current level.
// 3. For each region in the batch:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,16 @@ sealed trait PortConfig {
}

/**
* An output port requires exactly one materialization base URI. Result and
* state URIs hang off it via `VFSURIFactory.resultURI` / `stateURI`; this
* field is *not* a URI you can pass straight to `DocumentFactory`.
* An output port requires one storage base URI. Result and state URIs hang
* off it via `VFSURIFactory.resultURI` / `stateURI`; this field is *not* a
* URI you can pass straight to `DocumentFactory`. For a skipped operator,
* `cachedTupleCount` holds the saved output count so runtime statistics
* match the run that wrote it.
*/
final case class OutputPortConfig(storageURIBase: URI) extends PortConfig {
final case class OutputPortConfig(
storageURIBase: URI,
cachedTupleCount: Option[Long] = None
) extends PortConfig {
override val storageURIs: List[URI] = List(storageURIBase)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -236,4 +236,31 @@ class RegionPlanSpec extends AnyFlatSpec with BarePhysicalPlanFixture {
assert(order.head == ids(0))
assert(order.last == ids(8))
}

"RegionPlan with skip regions" should "hold active and skipped regions together and resolve lookups across both" in {
val active = buildDiamondPlan()
val skippedInternalLink = link("skip-a", "skip-b")
val skippedPort = globalPort("skip-b")
val skipRegion = Region(
RegionIdentity(10),
physicalOps = Set(op("skip-a"), op("skip-b")),
physicalLinks = Set(skippedInternalLink),
ports = Set(skippedPort),
skipped = true
)
val plan = RegionPlan(
regions = active.regions + skipRegion,
regionLinks = active.regionLinks
)

assert(plan.getRegion(RegionIdentity(10)) == skipRegion)
assert(plan.getRegionOfLink(skippedInternalLink) == skipRegion)
assert(plan.getRegionOfPortId(skippedPort).contains(skipRegion))
val order = plan.topologicalIterator().toList
assert(order.toSet == active.regions.map(_.id) + skipRegion.id)
val pos = order.zipWithIndex.toMap
active.regionLinks.foreach { edge =>
assert(pos(edge.fromRegionId) < pos(edge.toRegionId))
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@ class RegionSpec extends AnyFlatSpec with BarePhysicalPlanFixture {
assert(region.getPorts == Set(portId))
}

it should "default skipped to false and accept true" in {
assert(!Region(RegionIdentity(1), Set(op("a")), Set.empty).skipped)
assert(Region(RegionIdentity(1), Set(op("a")), Set.empty, skipped = true).skipped)
}

"Region.getOperator" should "look up a physical operator by id" in {
val a = op("a")
val b = op("b")
Expand Down Expand Up @@ -118,6 +123,17 @@ class RegionSpec extends AnyFlatSpec with BarePhysicalPlanFixture {
assert(region.getSourceOperators == Set(a, c))
}

it should "count only the region's own links, not every link an operator lists" in {
// b lists an input link from a, and both are in the region, yet the link is not one of the
// region's links (for instance when b reads a's saved result): b is a source of the region.
val ab = link("a", "b")
val a = wiredOp("a").addOutputLink(ab)
val b = wiredOp("b").addInputLink(ab)
assert(Region(RegionIdentity(1), Set(a, b), Set.empty).getSourceOperators == Set(a, b))
// With the link among the region's links, b is not a source.
assert(Region(RegionIdentity(1), Set(a, b), Set(ab)).getSourceOperators == Set(a))
}

"Region.getStarterOperators" should "match getSourceOperators when no resource config is provided" in {
val a = op("a")
val b = op("b")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,4 +115,76 @@ class ScheduleSpec extends AnyFlatSpec with BarePhysicalPlanFixture {
assert(order.indexOf(r2) < order.indexOf(r3))
}

it should "place all skip regions at level 0 with no extra empty level when nothing else runs" in {
val skip0 = region(10, "s0").copy(skipped = true)
val skip1 = region(11, "s1").copy(skipped = true)
val plan = RegionPlan(Set(skip0, skip1), Set.empty)
val generator = new ScheduleGenerator(
new WorkflowContext(),
PhysicalPlan(Set.empty, Set.empty)
) {
override def generate(): (Schedule, PhysicalPlan) = (Schedule(Map.empty), physicalPlan)
}

val schedule = generator.generateScheduleFromRegionPlan(plan)
assert(schedule.levelSets == Map(0 -> Set(skip0, skip1)))
}

it should "place all skip regions at level 0 ahead of topological active batches" in {
val r0 = region(0, "a")
val r1 = region(1, "b")
val r2 = region(2, "c")
val r3 = region(3, "d")
val skip0 = region(4, "s0").copy(skipped = true)
val skip1 = region(5, "s1").copy(skipped = true)
val plan = RegionPlan(
Set(r0, r1, r2, r3, skip0, skip1),
Set(
RegionLink(r0.id, r1.id),
RegionLink(r0.id, r2.id),
RegionLink(r1.id, r3.id),
RegionLink(r2.id, r3.id)
)
)
val generator = new ScheduleGenerator(
new WorkflowContext(),
PhysicalPlan(Set.empty, Set.empty)
) {
override def generate(): (Schedule, PhysicalPlan) = (Schedule(Map.empty), physicalPlan)
}

val levels = generator.generateScheduleFromRegionPlan(plan).toList
assert(levels.head == Set(skip0, skip1))
val activeLevels = levels.tail
assert(activeLevels.forall(_.size <= ApplicationConfig.maxConcurrentRegions))
val activeOrder = activeLevels.flatten
assert(activeOrder.toSet == Set(r0, r1, r2, r3))
assert(activeOrder.indexOf(r0) < activeOrder.indexOf(r1))
assert(activeOrder.indexOf(r0) < activeOrder.indexOf(r2))
assert(activeOrder.indexOf(r1) < activeOrder.indexOf(r3))
assert(activeOrder.indexOf(r2) < activeOrder.indexOf(r3))
}

it should "reject a region plan whose regionLinks reference a skipped region" in {
val skipRegion = region(0, "s").copy(skipped = true)
val activeRegion = region(1, "a")
val generator = new ScheduleGenerator(
new WorkflowContext(),
PhysicalPlan(Set.empty, Set.empty)
) {
override def generate(): (Schedule, PhysicalPlan) = (Schedule(Map.empty), physicalPlan)
}

assertThrows[IllegalArgumentException] {
generator.generateScheduleFromRegionPlan(
RegionPlan(Set(skipRegion, activeRegion), Set(RegionLink(skipRegion.id, activeRegion.id)))
)
}
assertThrows[IllegalArgumentException] {
generator.generateScheduleFromRegionPlan(
RegionPlan(Set(skipRegion, activeRegion), Set(RegionLink(activeRegion.id, skipRegion.id)))
)
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,14 @@ class SchedulingConfigsSpec extends AnyFlatSpec {
val uri = new URI("vfs:///wid/1/eid/1/result")
val cfg = OutputPortConfig(uri)
assert(cfg.storageURIs == List(uri))
assert(cfg.cachedTupleCount.isEmpty)
}

it should "preserve an optional cachedTupleCount" in {
val uri = new URI("vfs:///wid/1/eid/1/result")
val cfg = OutputPortConfig(uri, cachedTupleCount = Some(42L))
assert(cfg.storageURIs == List(uri))
assert(cfg.cachedTupleCount.contains(42L))
}

"IntermediateInputPortConfig" should "expose every URI it was constructed with" in {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.texera.amber.core.workflow

import java.net.URI

/**
* The saved result of an output port, which the operators that read the port may read
* instead. The port's own operator, if it runs, still computes the port.
*
* @param storageUri the port's base URI, as `VFSURIFactory.createPortBaseURI` builds it; the
* result and state documents hang off it via `resultURI` and `stateURI`
* @param tupleCount the saved row count when the writer recorded one; a skipped operator
* reports it as the port's output count
*/
case class CachedResult(storageUri: URI, tupleCount: Option[Long])
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,11 @@ class WorkflowContext(
var cuid: Option[Int] = None,
// The Lakekeeper catalog name this execution writes into; becomes the `/wh/<name>`
// URI segment. None routes to the shared default warehouse.
var warehouse: Option[String] = None
var warehouse: Option[String] = None,
// Saved results found for output ports of this run, keyed by port; operators that read
// those ports may read the saved results instead (a port's own operator, if it runs,
// still computes it). Set on the computing unit by the cache lookup before scheduling; no
// request type carries it, so the editor cannot supply it. Empty means the scheduler
// runs exactly as it does without a cache.
var matchedResults: Map[GlobalPortIdentity, CachedResult] = Map.empty
)
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,17 @@

package org.apache.texera.amber.core.workflow

import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
import org.apache.texera.amber.core.virtualidentity.{
ExecutionIdentity,
OperatorIdentity,
PhysicalOpIdentity,
WorkflowIdentity
}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

import java.net.URI

class WorkflowContextSpec extends AnyFlatSpec with Matchers {

"WorkflowContext companion" should "expose pinned defaults" in {
Expand Down Expand Up @@ -59,4 +66,20 @@ class WorkflowContextSpec extends AnyFlatSpec with Matchers {
// Settings argument was not overridden, so the companion default holds.
ctx.workflowSettings shouldBe WorkflowContext.DEFAULT_WORKFLOW_SETTINGS
}

"WorkflowContext.matchedResults" should "default to empty and be reassignable" in {
// Empty is the contract the scheduler relies on: no matched results, no cache code path.
val ctx = new WorkflowContext()
ctx.matchedResults shouldBe Map.empty
val port =
GlobalPortIdentity(PhysicalOpIdentity(OperatorIdentity("op"), "main"), PortIdentity(0))
val cached = CachedResult(new URI("vfs:///wid/1/eid/1/globalportid/x"), Some(3L))
ctx.matchedResults = Map(port -> cached)
ctx.matchedResults shouldBe Map(port -> cached)
// a context built with the other arguments only still starts empty
new WorkflowContext(
WorkflowIdentity(9L),
ExecutionIdentity(8L)
).matchedResults shouldBe Map.empty
}
}
Loading