Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,8 @@ object ArrowFlightActorBench {
// batch_size = cfg.batchSize keeps the Python-side partitioning
// buffer aligned with our send size — one Java DataFrame in maps
// to exactly one Python DataFrame out.
OneToOnePartitioning(batchSize = cfg.batchSize, channels = Seq(dataChannelOut))
OneToOnePartitioning(batchSize = cfg.batchSize, channels = Seq(dataChannelOut)),
columnar = false
),
ctx,
4L
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,10 @@ message AddInputChannelRequest {
message AddPartitioningRequest {
core.PhysicalLink tag = 1 [(scalapb.field).no_box = true];
sendsemantics.Partitioning partitioning = 2 [(scalapb.field).no_box = true];
// Per-edge columnar-wire decision from the planner: ship this link's batches as
// Arrow (ColumnarFrame) rather than rows. True only when the downstream operator
// consumes columns natively and the columnar wire is enabled.
bool columnar = 3;
}

message AssignPortRequest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ trait LinkWorkersHandler {
linkExecution.initChannelExecution(channelId)
Seq(
workerInterface.addPartitioning(
AddPartitioningRequest(msg.link, linkConfig.partitioning),
AddPartitioningRequest(msg.link, linkConfig.partitioning, linkConfig.columnar),
mkContext(channelId.fromWorkerId)
),
workerInterface.addInputChannel(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ import org.apache.texera.amber.engine.architecture.worker.managers.{
PortStorageWriterTerminateSignal
}
import org.apache.texera.amber.engine.common.AmberLogging
import org.apache.texera.amber.util.VirtualIdentityUtils
import org.apache.texera.amber.engine.common.ambermessage.ColumnarFrame
import org.apache.texera.amber.util.{ArrowUtils, VirtualIdentityUtils}
import org.apache.arrow.memory.{BufferAllocator, RootAllocator}
import org.apache.arrow.vector.VectorSchemaRoot

import java.net.URI
import scala.collection.mutable
Expand Down Expand Up @@ -134,12 +137,16 @@ class OutputManager(
*/
def addPartitionerWithPartitioning(
link: PhysicalLink,
partitioning: Partitioning
partitioning: Partitioning,
// Per-edge columnar-wire decision from the planner: ship this edge's batches as
// Arrow (ColumnarFrame) rather than rows. Defaults off so the row wire is unchanged.
columnar: Boolean = false
): Unit = {
val partitioner = toPartitioner(partitioning, actorId)
partitioners.update(link, partitioner)
partitioner.allReceivers.foreach(receiver => {
val buffer = new NetworkOutputBuffer(receiver, outputGateway, getBatchSize(partitioning))
val buffer =
new NetworkOutputBuffer(receiver, outputGateway, getBatchSize(partitioning), columnar)
networkOutputBuffers.update((link, receiver), buffer)
outputGateway.addOutputChannel(ChannelIdentity(actorId, receiver, isControl = false))
})
Expand Down Expand Up @@ -168,6 +175,113 @@ class OutputManager(
}
}

/**
* Provisional (phase 1): turn the columnar output path on or off for every current
* edge. Phase 2 replaces this blanket switch with a per-edge decision made by the
* planner (local vs cross-node, and whether the receiver consumes columns).
*/
def setColumnarOutput(enabled: Boolean): Unit =
networkOutputBuffers.values.foreach(_.columnar = enabled)

// Backs the per-bucket roots the columnar shuffle materializes; each is freed right
// after it is serialized, so this allocator's outstanding bytes return to zero.
private lazy val columnarAllocator: BufferAllocator = new RootAllocator()

/**
* Columnar output path (no round-trip): ship an operator's Arrow result batch straight
* downstream without decoding it to tuples and re-encoding it. Eligible only when the
* output port needs no storage and every edge on it is in columnar mode. A
* non-splitting edge (one-to-one, broadcast) sends the whole batch, serialized once, to
* each receiver. A splitting edge (hash/range/round-robin) assigns each row a bucket
* through the same getBucketIndex the row path uses (reading only the key columns into a
* partial tuple), gathers each bucket's rows in Arrow, and sends one ColumnarFrame per
* non-empty bucket. Bucket assignment is therefore identical to the row path. Does NOT
* close `root` (caller owns it).
*
* @return Some((portId, totalSerializedByteCount)) if it shipped the batch (caller
* records output statistics), or None if the port is not eligible (fall back).
*/
def tryEmitColumnarBatch(
root: VectorSchemaRoot,
outputPortId: Option[PortIdentity]
): Option[(PortIdentity, Long)] = {
if (outputPortId.isEmpty && ports.size != 1) return None
val portId = outputPortId.getOrElse(ports.head._1)
val links = partitioners.filter(_._1.fromPortId == portId).toSeq
val eligible =
links.nonEmpty &&
!outputPortResultWriterThreads.contains(portId) &&
links.forall {
case (link, partitioner) =>
partitioner.allReceivers.forall(r =>
networkOutputBuffers.get((link, r)).exists(_.columnar)
)
}
if (!eligible) return None
val schema = getPort(portId).schema
// The executor's Arrow schema must match the declared output port schema; if not,
// fall back so the row path's enforceSchema can adjust it.
if (ArrowUtils.toTexeraSchema(root.getSchema) != schema) return None
Some((portId, links.map { case (link, p) => emitColumnarToLink(root, schema, link, p) }.sum))
}

// Ship `root` to one link's receivers in Arrow, splitting per bucket if the partitioner
// splits. Returns the total serialized bytes put on the wire for this link.
private def emitColumnarToLink(
root: VectorSchemaRoot,
schema: Schema,
link: PhysicalLink,
partitioner: Partitioner
): Long = {
val receivers = partitioner.allReceivers
if (!partitioner.columnarSplits) {
// One-to-one / broadcast: the whole batch goes to each receiver; serialize once.
val bytes = ArrowUtils.serializeRoot(root)
val frame = ColumnarFrame(bytes, root.getRowCount, schema)
receivers.foreach(r => networkOutputBuffers((link, r)).sendColumnarFrame(frame))
bytes.length.toLong
} else {
// Splitting shuffle: assign each row a bucket, then gather and send per bucket.
val n = root.getRowCount
val rowsPerBucket = Array.fill(receivers.length)(new mutable.ArrayBuffer[Int]())
// Build only the columns getBucketIndex needs into a partial tuple per row.
val keyCols =
if (partitioner.requiresFullTuple) schema.getAttributes.map(_.getName)
else partitioner.bucketKeyColumns
val keySchema = Schema(keyCols.map(schema.getAttribute).toList)
val keyIdx =
if (keyCols.isEmpty) Array.empty[Int] else ArrowUtils.projectionIndices(root, keyCols)
val keyTypes = keyCols.map(schema.getAttribute(_).getType).toArray
val vectors = root.getFieldVectors
var i = 0
while (i < n) {
val keyValues = new Array[Any](keyIdx.length)
var c = 0
while (c < keyIdx.length) {
keyValues(c) = ArrowUtils.getCell(vectors.get(keyIdx(c)), i, keyTypes(c)); c += 1
}
partitioner.getBucketIndex(Tuple(keySchema, keyValues)).foreach(b => rowsPerBucket(b) += i)
i += 1
}
var bytes = 0L
var b = 0
while (b < receivers.length) {
val rows = rowsPerBucket(b)
if (rows.nonEmpty) {
val bucketRoot = ArrowUtils.gather(root, rows.toArray, columnarAllocator)
try {
val frameBytes = ArrowUtils.serializeRoot(bucketRoot)
networkOutputBuffers((link, receivers(b)))
.sendColumnarFrame(ColumnarFrame(frameBytes, rows.length, schema))
bytes += frameBytes.length.toLong
} finally bucketRoot.close()
}
b += 1
}
bytes
}
}

/**
* Flushes the network output buffers based on the specified set of physical links.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,14 @@ class PythonProxyClient(portNumberPromise: Promise[Int], val actorId: ActorVirtu
dataPayload match {
case DataFrame(frame) =>
writeArrowStream(mutable.Queue(ArraySeq.unsafeWrapArray(frame): _*), from, "Data")
case ColumnarFrame(arrowIpcBytes, _, _) =>
// A columnar-wire batch headed to a Python worker: decode it back to tuples
// and hand it over as a Data batch, same as a DataFrame.
writeArrowStream(
mutable.Queue(ArraySeq.unsafeWrapArray(ArrowUtils.deserializeTuples(arrowIpcBytes)): _*),
from,
"Data"
)
case StateFrame(state, loopCounter, loopStartId) =>
// The Arrow wire format for states IS the State row (content +
// loop_counter + loop_start_id), so the envelope rides its own
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,5 +94,8 @@ case object LinkConfig {

case class LinkConfig(
channelConfigs: List[ChannelConfig],
partitioning: Partitioning
partitioning: Partitioning,
// Per-edge columnar-wire decision: ship this link's batches as Arrow when the
// downstream operator consumes columns natively and the columnar wire is enabled.
columnar: Boolean = false
)
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.texera.amber.engine.architecture.scheduling.resourcePolicies

import org.apache.texera.common.config.ApplicationConfig
import org.apache.texera.amber.core.virtualidentity.PhysicalOpIdentity
import org.apache.texera.amber.core.workflow._
import org.apache.texera.amber.engine.architecture.scheduling.Region
Expand Down Expand Up @@ -89,7 +90,14 @@ class DefaultResourceAllocator(
operatorConfigs(physicalLink.toOpId).workerConfigs.map(_.workerId),
linkPartitionInfos(physicalLink),
workflowSettings.dataTransferBatchSize
)
),
// Per-edge columnar-wire decision: enabled only when the wire is on and the
// downstream operator consumes columns natively (otherwise shipping Arrow
// would force a costly per-tuple decode on the receiver). Locality (same-JVM
// links gain nothing) is a later refinement.
columnar =
ApplicationConfig.enableColumnarWire &&
physicalPlan.getOperator(physicalLink.toOpId).isColumnarCapable
)
}.toMap

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,7 @@ case class BroadcastPartitioner(partitioning: BroadcastPartitioning) extends Par
}

override def allReceivers: Seq[ActorVirtualIdentity] = receivers

// The whole batch goes to every receiver; no per-row split.
override def columnarSplits: Boolean = false
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,8 @@ case class HashBasedShufflePartitioner(partitioning: HashBasedShufflePartitionin
}

override def allReceivers: Seq[ActorVirtualIdentity] = receivers

// Hash reads the key columns (or the whole tuple when no keys are named).
override def bucketKeyColumns: Seq[String] = partitioning.hashAttributeNames
override def requiresFullTuple: Boolean = partitioning.hashAttributeNames.isEmpty
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,7 @@ case class OneToOnePartitioner(partitioning: OneToOnePartitioning, actorId: Acto

override def allReceivers: Seq[ActorVirtualIdentity] =
Seq(partitioning.channels.filter(_.fromWorkerId == actorId).head.toWorkerId)

// The whole batch goes to the single receiver; no per-row split.
override def columnarSplits: Boolean = false
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,45 @@ import org.apache.texera.amber.core.state.State
import org.apache.texera.amber.core.tuple.Tuple
import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
import org.apache.texera.amber.engine.architecture.messaginglayer.NetworkOutputGateway
import org.apache.texera.amber.engine.common.ambermessage.{DataFrame, StateFrame}
import org.apache.texera.amber.engine.common.ambermessage.{ColumnarFrame, DataFrame, StateFrame}
import org.apache.texera.amber.util.ArrowUtils

import scala.collection.mutable.ArrayBuffer

trait Partitioner extends Serializable {
def getBucketIndex(tuple: Tuple): Iterator[Int]

def allReceivers: Seq[ActorVirtualIdentity]

// ---- Columnar-shuffle support (phase 3) -------------------------------------
// These let the columnar output path assign buckets by reading only the columns a
// partitioner needs, then scatter whole rows in Arrow, without decoding every row
// to a Tuple. Bucket assignment still goes through getBucketIndex, so it is
// identical to the row path by construction.

// Whether this partitioner splits a batch across receivers (hash/range/round-robin)
// or sends the whole batch to each receiver (one-to-one, broadcast).
def columnarSplits: Boolean = true

// The columns getBucketIndex reads from a tuple (hash/range keys). Empty when the
// bucket does not depend on the row's data (round-robin). The columnar path builds a
// partial tuple of just these columns per row to assign the bucket.
def bucketKeyColumns: Seq[String] = Seq.empty

// When true, the bucket depends on the whole row (e.g. hash with no key columns, which
// hashes the entire tuple), so the columnar path must build the full tuple per row.
def requiresFullTuple: Boolean = false
}

class NetworkOutputBuffer(
val to: ActorVirtualIdentity,
val dataOutputPort: NetworkOutputGateway,
val batchSize: Int = ApplicationConfig.defaultDataTransferBatchSize
val batchSize: Int = ApplicationConfig.defaultDataTransferBatchSize,
// Columnar wire: flush as ColumnarFrame (Arrow IPC) instead of DataFrame. The
// planner decides this per edge (it folds in ApplicationConfig.enableColumnarWire
// and the downstream operator's columnar capability) and passes it in; off by
// default so the row wire is unchanged. A var so the output manager can set it.
var columnar: Boolean = false
) {

var buffer = new ArrayBuffer[Tuple]()
Expand All @@ -55,9 +80,27 @@ class NetworkOutputBuffer(
flush()
}

// Columnar fast path: ship an already-serialized Arrow batch straight to the
// receiver, skipping the per-tuple accumulate/re-encode. Any tuples already
// buffered are flushed first so the batch keeps its place in the FIFO stream.
def sendColumnarFrame(frame: ColumnarFrame): Unit = {
flush()
dataOutputPort.sendTo(to, frame)
}

def flush(): Unit = {
if (buffer.nonEmpty) {
dataOutputPort.sendTo(to, DataFrame(buffer.toArray))
val batch = buffer.toArray
if (columnar) {
// Serialize the batch to Arrow once; all tuples on a link share the schema.
val schema = batch(0).getSchema
dataOutputPort.sendTo(
to,
ColumnarFrame(ArrowUtils.serializeTuples(schema, batch), batch.length, schema)
)
} else {
dataOutputPort.sendTo(to, DataFrame(batch))
}
buffer = new ArrayBuffer[Tuple]()
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,6 @@ case class RangeBasedShufflePartitioner(partitioning: RangeBasedShufflePartition

override def allReceivers: Seq[ActorVirtualIdentity] = receivers

// Range partitions on the first range attribute only.
override def bucketKeyColumns: Seq[String] = Seq(partitioning.rangeAttributeNames.head)
}
Loading
Loading