Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,8 @@ object ArrowFlightActorBench {
// batch_size = cfg.batchSize keeps the Python-side partitioning
// buffer aligned with our send size — one Java DataFrame in maps
// to exactly one Python DataFrame out.
OneToOnePartitioning(batchSize = cfg.batchSize, channels = Seq(dataChannelOut))
OneToOnePartitioning(batchSize = cfg.batchSize, channels = Seq(dataChannelOut)),
columnar = false
),
ctx,
4L
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,10 @@ message AddInputChannelRequest {
message AddPartitioningRequest {
core.PhysicalLink tag = 1 [(scalapb.field).no_box = true];
sendsemantics.Partitioning partitioning = 2 [(scalapb.field).no_box = true];
// Per-edge columnar-wire decision from the planner: ship this link's batches as
// Arrow (ColumnarFrame) rather than rows. True only when the downstream operator
// consumes columns natively and the columnar wire is enabled.
bool columnar = 3;
}

message AssignPortRequest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ trait LinkWorkersHandler {
linkExecution.initChannelExecution(channelId)
Seq(
workerInterface.addPartitioning(
AddPartitioningRequest(msg.link, linkConfig.partitioning),
AddPartitioningRequest(msg.link, linkConfig.partitioning, linkConfig.columnar),
mkContext(channelId.fromWorkerId)
),
workerInterface.addInputChannel(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ import org.apache.texera.amber.engine.architecture.worker.managers.{
PortStorageWriterTerminateSignal
}
import org.apache.texera.amber.engine.common.AmberLogging
import org.apache.texera.amber.util.VirtualIdentityUtils
import org.apache.texera.amber.engine.common.ambermessage.ColumnarFrame
import org.apache.texera.amber.util.{ArrowUtils, VirtualIdentityUtils}
import org.apache.arrow.vector.VectorSchemaRoot

import java.net.URI
import scala.collection.mutable
Expand Down Expand Up @@ -134,12 +136,16 @@ class OutputManager(
*/
def addPartitionerWithPartitioning(
link: PhysicalLink,
partitioning: Partitioning
partitioning: Partitioning,
// Per-edge columnar-wire decision from the planner: ship this edge's batches as
// Arrow (ColumnarFrame) rather than rows. Defaults off so the row wire is unchanged.
columnar: Boolean = false
): Unit = {
val partitioner = toPartitioner(partitioning, actorId)
partitioners.update(link, partitioner)
partitioner.allReceivers.foreach(receiver => {
val buffer = new NetworkOutputBuffer(receiver, outputGateway, getBatchSize(partitioning))
val buffer =
new NetworkOutputBuffer(receiver, outputGateway, getBatchSize(partitioning), columnar)
networkOutputBuffers.update((link, receiver), buffer)
outputGateway.addOutputChannel(ChannelIdentity(actorId, receiver, isControl = false))
})
Expand Down Expand Up @@ -168,6 +174,59 @@ class OutputManager(
}
}

/**
* Columnar output fast path (no round-trip): ship an operator's Arrow result batch
* straight downstream without decoding it to tuples and re-encoding it. Eligible only
* when the output port needs no storage and every edge on it is one-to-one (a single
* receiver gets the whole batch) with its buffer in columnar mode; hash/range/
* round-robin shuffles need per-row partitioning (a later phase) and broadcast/storage
* are deferred, so those fall back to the row path. The root is serialized once and the
* same bytes are sent to each one-to-one receiver. Does NOT close `root` (caller owns it).
*
* @return Some((portId, serializedByteCount)) if it shipped the batch (caller records
* output statistics), or None if the port is not eligible (caller falls back).
*/
/**
* Provisional (phase 1): turn the columnar output path on or off for every current
* edge. Phase 2 replaces this blanket switch with a per-edge decision made by the
* planner (local vs cross-node, and whether the receiver consumes columns).
*/
def setColumnarOutput(enabled: Boolean): Unit =
networkOutputBuffers.values.foreach(_.columnar = enabled)

def tryEmitColumnarBatch(
root: VectorSchemaRoot,
outputPortId: Option[PortIdentity]
): Option[(PortIdentity, Long)] = {
if (outputPortId.isEmpty && ports.size != 1) return None
val portId = outputPortId.getOrElse(ports.head._1)
val links = partitioners.filter(_._1.fromPortId == portId).toSeq
val eligible =
links.nonEmpty &&
!outputPortResultWriterThreads.contains(portId) &&
links.forall(_._2.isInstanceOf[OneToOnePartitioner]) &&
links.forall {
case (link, partitioner) =>
partitioner.allReceivers.forall(r =>
networkOutputBuffers.get((link, r)).exists(_.columnar)
)
}
if (!eligible) return None
val schema = getPort(portId).schema
// The executor's Arrow schema must match the declared output port schema; if not,
// fall back so the row path's enforceSchema can adjust it.
if (ArrowUtils.toTexeraSchema(root.getSchema) != schema) return None
val bytes = ArrowUtils.serializeRoot(root)
val frame = ColumnarFrame(bytes, root.getRowCount, schema)
links.foreach {
case (link, partitioner) =>
partitioner.allReceivers.foreach(r =>
networkOutputBuffers((link, r)).sendColumnarFrame(frame)
)
}
Some((portId, bytes.length.toLong))
}

/**
* Flushes the network output buffers based on the specified set of physical links.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,14 @@ class PythonProxyClient(portNumberPromise: Promise[Int], val actorId: ActorVirtu
dataPayload match {
case DataFrame(frame) =>
writeArrowStream(mutable.Queue(ArraySeq.unsafeWrapArray(frame): _*), from, "Data")
case ColumnarFrame(arrowIpcBytes, _, _) =>
// A columnar-wire batch headed to a Python worker: decode it back to tuples
// and hand it over as a Data batch, same as a DataFrame.
writeArrowStream(
mutable.Queue(ArraySeq.unsafeWrapArray(ArrowUtils.deserializeTuples(arrowIpcBytes)): _*),
from,
"Data"
)
case StateFrame(state, loopCounter, loopStartId) =>
// The Arrow wire format for states IS the State row (content +
// loop_counter + loop_start_id), so the envelope rides its own
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,5 +94,8 @@ case object LinkConfig {

case class LinkConfig(
channelConfigs: List[ChannelConfig],
partitioning: Partitioning
partitioning: Partitioning,
// Per-edge columnar-wire decision: ship this link's batches as Arrow when the
// downstream operator consumes columns natively and the columnar wire is enabled.
columnar: Boolean = false
)
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.texera.amber.engine.architecture.scheduling.resourcePolicies

import org.apache.texera.common.config.ApplicationConfig
import org.apache.texera.amber.core.virtualidentity.PhysicalOpIdentity
import org.apache.texera.amber.core.workflow._
import org.apache.texera.amber.engine.architecture.scheduling.Region
Expand Down Expand Up @@ -89,7 +90,14 @@ class DefaultResourceAllocator(
operatorConfigs(physicalLink.toOpId).workerConfigs.map(_.workerId),
linkPartitionInfos(physicalLink),
workflowSettings.dataTransferBatchSize
)
),
// Per-edge columnar-wire decision: enabled only when the wire is on and the
// downstream operator consumes columns natively (otherwise shipping Arrow
// would force a costly per-tuple decode on the receiver). Locality (same-JVM
// links gain nothing) is a later refinement.
columnar =
ApplicationConfig.enableColumnarWire &&
physicalPlan.getOperator(physicalLink.toOpId).isColumnarCapable
)
}.toMap

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ import org.apache.texera.amber.core.state.State
import org.apache.texera.amber.core.tuple.Tuple
import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
import org.apache.texera.amber.engine.architecture.messaginglayer.NetworkOutputGateway
import org.apache.texera.amber.engine.common.ambermessage.{DataFrame, StateFrame}
import org.apache.texera.amber.engine.common.ambermessage.{ColumnarFrame, DataFrame, StateFrame}
import org.apache.texera.amber.util.ArrowUtils

import scala.collection.mutable.ArrayBuffer

Expand All @@ -37,7 +38,12 @@ trait Partitioner extends Serializable {
class NetworkOutputBuffer(
val to: ActorVirtualIdentity,
val dataOutputPort: NetworkOutputGateway,
val batchSize: Int = ApplicationConfig.defaultDataTransferBatchSize
val batchSize: Int = ApplicationConfig.defaultDataTransferBatchSize,
// Columnar wire: flush as ColumnarFrame (Arrow IPC) instead of DataFrame. The
// planner decides this per edge (it folds in ApplicationConfig.enableColumnarWire
// and the downstream operator's columnar capability) and passes it in; off by
// default so the row wire is unchanged. A var so the output manager can set it.
var columnar: Boolean = false
) {

var buffer = new ArrayBuffer[Tuple]()
Expand All @@ -55,9 +61,27 @@ class NetworkOutputBuffer(
flush()
}

// Columnar fast path: ship an already-serialized Arrow batch straight to the
// receiver, skipping the per-tuple accumulate/re-encode. Any tuples already
// buffered are flushed first so the batch keeps its place in the FIFO stream.
def sendColumnarFrame(frame: ColumnarFrame): Unit = {
flush()
dataOutputPort.sendTo(to, frame)
}

def flush(): Unit = {
if (buffer.nonEmpty) {
dataOutputPort.sendTo(to, DataFrame(buffer.toArray))
val batch = buffer.toArray
if (columnar) {
// Serialize the batch to Arrow once; all tuples on a link share the schema.
val schema = batch(0).getSchema
dataOutputPort.sendTo(
to,
ColumnarFrame(ArrowUtils.serializeTuples(schema, batch), batch.length, schema)
)
} else {
dataOutputPort.sendTo(to, DataFrame(batch))
}
buffer = new ArrayBuffer[Tuple]()
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ package org.apache.texera.amber.engine.architecture.worker

import com.softwaremill.macwire.wire
import io.grpc.MethodDescriptor
import org.apache.texera.amber.core.executor.OperatorExecutor
import org.apache.texera.amber.core.executor.{
ColumnarOperatorExecutor,
ColumnarResult,
OperatorExecutor
}
import org.apache.texera.amber.core.state.State
import org.apache.texera.amber.core.tuple._
import org.apache.texera.amber.core.virtualidentity.{
Expand Down Expand Up @@ -59,6 +63,7 @@ import org.apache.texera.amber.engine.common.ambermessage._
import org.apache.texera.amber.engine.common.statetransition.WorkerStateManager
import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR
import org.apache.texera.amber.error.ErrorUtils.{mkConsoleMessage, mkPrintConsoleMessage, safely}
import org.apache.texera.amber.util.ArrowUtils

import java.util.concurrent.LinkedBlockingQueue

Expand Down Expand Up @@ -114,6 +119,98 @@ class DataProcessor(
* process currentInputTuple through executor logic.
* this function is only called by the DP thread.
*/
// Start processing one data batch: announce the worker is running, load the
// batch, and process its first tuple (the DP loop drains the rest). Shared by the
// DataFrame and ColumnarFrame input paths.
private[this] def startInputBatch(
channelId: ChannelIdentity,
tuples: Array[Tuple]
): Unit = {
transitToRunning()
inputManager.initBatch(channelId, tuples)
processInputTuple(inputManager.getNextTuple)
}

// Announce the worker is running (READY -> RUNNING, once). Shared by the row and
// columnar input paths.
private[this] def transitToRunning(): Unit = {
stateManager.conditionalTransitTo(
READY,
RUNNING,
() => {
val (state, stateVersion) = stateManager.getStateWithVersion
asyncRPCClient.coordinatorInterface.workerStateUpdated(
WorkerStateUpdatedRequest(state, stateVersion),
asyncRPCClient.mkContext(COORDINATOR)
)
}
)
}

// Columnar wire: an Arrow batch arrived. If the executor can consume Arrow
// columns directly (a ColumnarOperatorExecutor), hand it the batch and route the
// result; otherwise, or if the executor declines the batch (Unsupported), decode
// to tuples and process it exactly as a DataFrame.
private[this] def processColumnarFrame(
channelId: ChannelIdentity,
portId: PortIdentity,
arrowIpcBytes: Array[Byte],
rowCount: Int
): Unit = {
val columnarExecutor = executor match {
case c: ColumnarOperatorExecutor => c
case _ => null
}
if (columnarExecutor == null) {
startInputBatch(channelId, ArrowUtils.deserializeTuples(arrowIpcBytes))
return
}
transitToRunning()
inputManager.currentChannelId = channelId
// The executor sees data: bind its loop variables first (parity with the row path).
bindStateReferences()
try {
columnarExecutor.processColumnarBatch(arrowIpcBytes, portId.id) match {
case ColumnarResult.Unsupported =>
// The executor declined this batch: fall back to the row path.
startInputBatch(channelId, ArrowUtils.deserializeTuples(arrowIpcBytes))
case consumed =>
// Consumed natively: no per-tuple input remains for the DP loop to drain.
// Report the whole batch's input statistics in one call.
inputManager.initBatch(channelId, Array.empty)
statisticsManager.increaseInputStatistics(
portId,
rowCount.toLong,
arrowIpcBytes.length.toLong
)
consumed match {
case ColumnarResult.EmitRows(rows) =>
outputManager.outputIterator.setTupleOutput(rows)
case ColumnarResult.Emit(root) =>
try {
val rowCount = root.getRowCount
// No round-trip: on an eligible (one-to-one, no-storage, columnar)
// edge, ship the Arrow batch straight downstream. Otherwise
// materialize it to output tuples (the row output path) and let the
// network buffer handle it.
outputManager.tryEmitColumnarBatch(root, None) match {
case Some((portId, byteCount)) =>
statisticsManager.increaseOutputStatistics(portId, rowCount.toLong, byteCount)
case None =>
val tuples = ArrowUtils.tuplesOfRoot(root)
outputManager.outputIterator.setTupleOutput(
tuples.iterator.map(t => (t: TupleLike, None: Option[PortIdentity]))
)
}
} finally root.close()
case _ => // Consumed: a blocking operator accumulated the batch, nothing to emit now
}
}
} catch safely {
case e => handleExecutorException(e)
}
}

private[this] def processInputTuple(tuple: Tuple): Unit = {
try {
val portIdentity: PortIdentity =
Expand Down Expand Up @@ -243,19 +340,13 @@ class DataProcessor(
val portId = this.inputGateway.getChannel(channelId).getPortId
dataPayload match {
case DataFrame(tuples) =>
stateManager.conditionalTransitTo(
READY,
RUNNING,
() => {
val (state, stateVersion) = stateManager.getStateWithVersion
asyncRPCClient.coordinatorInterface.workerStateUpdated(
WorkerStateUpdatedRequest(state, stateVersion),
asyncRPCClient.mkContext(COORDINATOR)
)
}
)
inputManager.initBatch(channelId, tuples)
processInputTuple(inputManager.getNextTuple)
startInputBatch(channelId, tuples)
case ColumnarFrame(arrowIpcBytes, rowCount, _) =>
// Columnar wire: the batch arrived as Arrow IPC bytes. A
// ColumnarOperatorExecutor consumes the Arrow columns directly (no per-row
// decode); any other operator, or one that declines the batch, decodes it
// back to tuples and processes it exactly as a DataFrame.
processColumnarFrame(channelId, portId, arrowIpcBytes, rowCount)
case StateFrame(state, loopCounter, loopStartId) =>
processInputState(state, portId.id, loopCounter, loopStartId)
}
Expand Down
Loading
Loading