Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,36 +19,14 @@

package org.apache.texera.amber.engine.architecture.coordinator.execution

import org.apache.texera.amber.core.executor.OpExecInitInfo
import org.apache.texera.amber.core.virtualidentity.{
ActorVirtualIdentity,
ExecutionIdentity,
OperatorIdentity,
PhysicalOpIdentity,
WorkflowIdentity
}
import org.apache.texera.amber.core.workflow.{GlobalPortIdentity, PhysicalOp, PortIdentity}
import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState
import org.apache.texera.amber.engine.architecture.scheduling.{Region, RegionIdentity}
import org.apache.texera.amber.engine.architecture.worker.statistics.WorkerState
import org.apache.texera.amber.engine.common.BarePhysicalPlanFixture
import org.scalatest.flatspec.AnyFlatSpec

class WorkflowExecutionSpec extends AnyFlatSpec {

private def physicalOpId(opId: String): PhysicalOpIdentity =
PhysicalOpIdentity(OperatorIdentity(opId), "main")

private def op(opId: String): PhysicalOp =
PhysicalOp(
physicalOpId(opId),
WorkflowIdentity(0),
ExecutionIdentity(0),
OpExecInitInfo.Empty
)

/** A region with no ports — its `RegionExecution.getState` defaults to COMPLETED. */
private def region(regionId: Long, opId: String): Region =
Region(RegionIdentity(regionId), Set(op(opId)), Set.empty)
class WorkflowExecutionSpec extends AnyFlatSpec with BarePhysicalPlanFixture {

/**
* A region carrying a single (input) port on `opId`. Because a freshly created
Expand All @@ -57,18 +35,7 @@ class WorkflowExecutionSpec extends AnyFlatSpec {
* so its operators' states are actually inspected by `WorkflowExecution.getState`.
*/
private def regionWithPort(regionId: Long, opId: String): Region =
Region(
RegionIdentity(regionId),
Set(op(opId)),
Set.empty,
ports = Set(
GlobalPortIdentity(
opId = physicalOpId(opId),
portId = PortIdentity(id = 0, internal = false),
input = true
)
)
)
region(regionId, opId, Set(globalPort(opId)))

"WorkflowExecution.initRegionExecution" should "create a new RegionExecution for the given region" in {
val we = WorkflowExecution()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,31 +19,13 @@

package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers

import com.twitter.util.{Await, Duration}
import org.apache.texera.amber.core.executor.OpExecInitInfo
import com.twitter.util.Await
import org.apache.texera.amber.core.virtualidentity.{
ActorVirtualIdentity,
ChannelIdentity,
EmbeddedControlMessageIdentity,
OperatorIdentity,
PhysicalOpIdentity
}
import org.apache.texera.amber.core.workflow.WorkflowContext.{
DEFAULT_EXECUTION_ID,
DEFAULT_WORKFLOW_ID
}
import org.apache.texera.amber.core.workflow.{
GlobalPortIdentity,
PhysicalLink,
PhysicalOp,
PortIdentity,
WorkflowContext
}
import org.apache.texera.amber.engine.architecture.coordinator.{
CoordinatorAsyncRPCHandlerInitializer,
CoordinatorConfig,
CoordinatorProcessor
EmbeddedControlMessageIdentity
}
import org.apache.texera.amber.core.workflow.{GlobalPortIdentity, PhysicalLink, PortIdentity}
import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer
import org.apache.texera.amber.engine.architecture.logreplay.{ReplayLogManager, ReplayLogRecord}
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.{
NO_ALIGNMENT,
Expand All @@ -62,11 +44,9 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{
}
import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceGrpc.METHOD_RETRIEVE_STATE
import org.apache.texera.amber.engine.architecture.scheduling.{Region, RegionIdentity}
import org.apache.texera.amber.engine.architecture.worker.WorkflowWorker.MainThreadDelegateMessage
import org.apache.texera.amber.engine.common.ambermessage.WorkflowFIFOMessage
import org.apache.texera.amber.engine.common.storage.SequentialRecordStorage.SequentialRecordWriter
import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR
import org.apache.texera.amber.util.VirtualIdentityUtils
import org.scalatest.flatspec.AnyFlatSpec

import scala.collection.mutable.ArrayBuffer
Expand All @@ -89,41 +69,32 @@ import scala.collection.mutable.ArrayBuffer
* propagation is still in flight.
* - dispatching to workers of a region execution that has already completed.
*/
class EmbeddedControlMessageHandlerSpec extends AnyFlatSpec {
class EmbeddedControlMessageHandlerSpec extends AnyFlatSpec with CoordinatorMessageFixture {

private val awaitTimeout = Duration.fromSeconds(1)
private val ecmId = EmbeddedControlMessageIdentity("ecm-under-test")
private val retrieveState = METHOD_RETRIEVE_STATE.getBareMethodName

private val sourceOp = mkPhysicalOp("source", "main")
private val middleOp = mkPhysicalOp("middle", "main")
private val sinkOp = mkPhysicalOp("sink", "main")
private val sourceOp = mkPhysicalOp("source")
private val middleOp = mkPhysicalOp("middle")
private val sinkOp = mkPhysicalOp("sink")
private val sourceWorker = mkWorkerId(sourceOp, 0)
private val middleWorker = mkWorkerId(middleOp, 0)
private val sinkWorker = mkWorkerId(sinkOp, 0)
private val sourceToMiddle =
PhysicalLink(sourceOp.id, PortIdentity(), middleOp.id, PortIdentity())
private val middleToSink = PhysicalLink(middleOp.id, PortIdentity(), sinkOp.id, PortIdentity())

private def mkPhysicalOp(logicalOpId: String, layerName: String): PhysicalOp =
PhysicalOp(
PhysicalOpIdentity(OperatorIdentity(logicalOpId), layerName),
DEFAULT_WORKFLOW_ID,
DEFAULT_EXECUTION_ID,
OpExecInitInfo.Empty
)

private def mkWorkerId(physicalOp: PhysicalOp, index: Int): ActorVirtualIdentity =
VirtualIdentityUtils.createWorkerIdentity(DEFAULT_WORKFLOW_ID, physicalOp.id, index)

/** Records the ECM ids the handler marks, so completion ordering can be asserted. */
private class RecordingLogManager extends ReplayLogManager {
val marked: ArrayBuffer[EmbeddedControlMessageIdentity] = ArrayBuffer()

override def setupWriter(logWriter: SequentialRecordWriter[ReplayLogRecord]): Unit = ()

override def sendCommitted(
msg: Either[MainThreadDelegateMessage, WorkflowFIFOMessage]
msg: Either[
org.apache.texera.amber.engine.architecture.worker.WorkflowWorker.MainThreadDelegateMessage,
WorkflowFIFOMessage
]
): Unit = ()

override def terminate(): Unit = ()
Expand All @@ -138,19 +109,10 @@ class EmbeddedControlMessageHandlerSpec extends AnyFlatSpec {
)

private def newFixture(): Fixture = {
val sent = ArrayBuffer[WorkflowFIFOMessage]()
val cp = new CoordinatorProcessor(
new WorkflowContext(),
CoordinatorConfig(None, None, None, None),
COORDINATOR,
{
case Right(m) => sent += m
case _ => ()
}
)
val (init, sent) = createCoordinatorFixture()
val logManager = new RecordingLogManager()
cp.setupLogManager(logManager)
Fixture(new CoordinatorAsyncRPCHandlerInitializer(cp), sent, logManager)
init.cp.setupLogManager(logManager)
Fixture(init, sent, logManager)
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,8 @@

package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers

import org.apache.texera.amber.core.executor.OpExecInitInfo
import org.apache.texera.amber.core.virtualidentity.{
ExecutionIdentity,
OperatorIdentity,
PhysicalOpIdentity,
WorkflowIdentity
}
import org.apache.texera.amber.core.workflow.{PhysicalOp, WorkflowContext}
import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
import org.apache.texera.amber.core.workflow.WorkflowContext
import org.apache.texera.amber.engine.architecture.coordinator.{
CoordinatorAsyncRPCHandlerInitializer,
CoordinatorConfig,
Expand All @@ -36,24 +30,12 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{
AsyncRPCContext,
JumpToOperatorRegionRequest
}
import org.apache.texera.amber.engine.architecture.scheduling.{Region, RegionIdentity, Schedule}
import org.apache.texera.amber.engine.architecture.scheduling.{Region, Schedule}
import org.apache.texera.amber.engine.common.BarePhysicalPlanFixture
import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR
import org.scalatest.flatspec.AnyFlatSpec

class JumpToOperatorRegionHandlerSpec extends AnyFlatSpec {

// A single-operator Region, matching the construction pattern used by
// WorkflowExecutionManagerSpec. The logical operator id (opId) is what the handler
// matches against JumpToOperatorRegionRequest.targetOperatorId.
private def singleOpRegion(regionId: Long, opId: String): Region = {
val physicalOp = PhysicalOp(
PhysicalOpIdentity(OperatorIdentity(opId), "main"),
WorkflowIdentity(0),
ExecutionIdentity(0),
OpExecInitInfo.Empty
)
Region(RegionIdentity(regionId), Set(physicalOp), Set.empty)
}
class JumpToOperatorRegionHandlerSpec extends AnyFlatSpec with BarePhysicalPlanFixture {

// Build a real CoordinatorProcessor + our own initializer (the processor's own
// initializer is private/inaccessible). Seed a 3-level, single-op-per-region schedule
Expand All @@ -66,9 +48,9 @@ class JumpToOperatorRegionHandlerSpec extends AnyFlatSpec {
COORDINATOR,
_ => ()
)
val first = singleOpRegion(1, "first")
val second = singleOpRegion(2, "second")
val third = singleOpRegion(3, "third")
val first = region(1, "first")
val second = region(2, "second")
val third = region(3, "third")
cp.workflowExecutionManager.schedule = Schedule(
Map(
0 -> Set(first),
Expand Down
Loading
Loading