Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
bb184c2
feat(engine): columnar operator contract and Arrow batch paths
Ma77Ball Nov 13, 2026
8c039fa
feat(engine): opt-in columnar wire for data batches
Ma77Ball Nov 13, 2026
73a9450
feat(aggregate): native-Arrow accumulate for the aggregate operator
Ma77Ball Nov 13, 2026
28ee435
perf(wire): faster Arrow-to-tuple decode, and a columnar-wire benchmark
Ma77Ball Nov 14, 2026
9b7735c
feat(engine): vectorized receive for columnar-wire batches
Ma77Ball Nov 14, 2026
1a169f2
feat(engine): columnar output without the round-trip on one-to-one edges
Ma77Ball Nov 14, 2026
ad8fe4f
feat(engine): decide the columnar wire per edge from the downstream o…
Ma77Ball Nov 14, 2026
bf28cb5
feat(engine): columnar shuffle, partition an Arrow batch without deco…
Ma77Ball Nov 14, 2026
5535198
feat(projection): native-Arrow column projection for the projection o…
Ma77Ball Nov 14, 2026
a892fd6
test(bench): end-to-end columnar-region vs row benchmark over a seria…
Ma77Ball Nov 14, 2026
8600cd8
feat(engine): keep local (same-JVM) edges on rows, columnar only for …
Ma77Ball Nov 14, 2026
f6dc5fb
feat(engine): per-receiver columnar on a mixed-locality edge, no deco…
Ma77Ball Nov 14, 2026
3e2aa96
feat(operator): processColumnarRoot, the zero-copy Arrow consumption …
Ma77Ball Nov 14, 2026
8ef3980
feat(engine): local zero-copy wire, hand a live Arrow root to a co-lo…
Ma77Ball Nov 16, 2026
317a935
test(bench): add the local (same-JVM, no-serialization) regime to the…
Ma77Ball Nov 17, 2026
d417d43
test(bench): workflow-matrix benchmark across operator shapes and bot…
Ma77Ball Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,8 @@ object ArrowFlightActorBench {
// batch_size = cfg.batchSize keeps the Python-side partitioning
// buffer aligned with our send size — one Java DataFrame in maps
// to exactly one Python DataFrame out.
OneToOnePartitioning(batchSize = cfg.batchSize, channels = Seq(dataChannelOut))
OneToOnePartitioning(batchSize = cfg.batchSize, channels = Seq(dataChannelOut)),
columnar = false
),
ctx,
4L
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,10 @@ message AddInputChannelRequest {
message AddPartitioningRequest {
core.PhysicalLink tag = 1 [(scalapb.field).no_box = true];
sendsemantics.Partitioning partitioning = 2 [(scalapb.field).no_box = true];
// Per-edge columnar-wire decision from the planner: ship this link's batches as
// Arrow (ColumnarFrame) rather than rows. True only when the downstream operator
// consumes columns natively and the columnar wire is enabled.
bool columnar = 3;
}

message AssignPortRequest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,19 @@ class PekkoActorRefMappingService(actorService: PekkoActorService) extends Amber
actorRefMapping.contains(id)
}

/**
* Whether worker `id` resolves to an actor in this same ActorSystem, i.e. a local,
* by-reference hop that never serializes, as opposed to a remote node. None when the
* ref is not resolved yet (the caller treats unknown as remote). The columnar output
* path uses this to keep a local edge on rows, since serializing it to Arrow would be
* pure overhead for a hop that otherwise copies nothing.
*/
def isLocalWorker(id: ActorVirtualIdentity): Option[Boolean] =
actorRefMapping.get(id).map { ref =>
val address = ref.path.address
address.hasLocalScope || address == self.path.address
}

def forwardToActor(msg: NetworkMessage): Unit = {
val id = msg.internalMessage.channelId.toWorkerId
if (actorRefMapping.contains(id)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ trait LinkWorkersHandler {
linkExecution.initChannelExecution(channelId)
Seq(
workerInterface.addPartitioning(
AddPartitioningRequest(msg.link, linkConfig.partitioning),
AddPartitioningRequest(msg.link, linkConfig.partitioning, linkConfig.columnar),
mkContext(channelId.fromWorkerId)
),
workerInterface.addInputChannel(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ import org.apache.texera.amber.engine.architecture.worker.managers.{
PortStorageWriterTerminateSignal
}
import org.apache.texera.amber.engine.common.AmberLogging
import org.apache.texera.amber.util.VirtualIdentityUtils
import org.apache.texera.amber.engine.common.ambermessage.{ColumnarFrame, LocalColumnarBatch}
import org.apache.texera.amber.util.{ArrowUtils, VirtualIdentityUtils}
import org.apache.arrow.memory.{BufferAllocator, RootAllocator}
import org.apache.arrow.vector.VectorSchemaRoot

import java.net.URI
import scala.collection.mutable
Expand Down Expand Up @@ -112,6 +115,12 @@ class OutputManager(
) extends AmberLogging {

val outputIterator: DPOutputIterator = new DPOutputIterator()

// Set by the worker: whether a receiver worker is co-located (same ActorSystem). A local
// hop never serializes, so the columnar wire is pure overhead there and that edge stays
// on rows. None = not yet known (treated as remote). Wired into each buffer's locality
// check when the partitioner is added.
var isLocalReceiver: ActorVirtualIdentity => Option[Boolean] = _ => None
private val partitioners: mutable.Map[PhysicalLink, Partitioner] =
mutable.HashMap[PhysicalLink, Partitioner]()

Expand All @@ -134,12 +143,18 @@ class OutputManager(
*/
def addPartitionerWithPartitioning(
link: PhysicalLink,
partitioning: Partitioning
partitioning: Partitioning,
// Per-edge columnar-wire decision from the planner: ship this edge's batches as
// Arrow (ColumnarFrame) rather than rows. Defaults off so the row wire is unchanged.
columnar: Boolean = false
): Unit = {
val partitioner = toPartitioner(partitioning, actorId)
partitioners.update(link, partitioner)
partitioner.allReceivers.foreach(receiver => {
val buffer = new NetworkOutputBuffer(receiver, outputGateway, getBatchSize(partitioning))
val buffer =
new NetworkOutputBuffer(receiver, outputGateway, getBatchSize(partitioning), columnar)
// A local (same-JVM) receiver keeps the edge on rows regardless of the planner flag.
buffer.localityCheck = () => isLocalReceiver(receiver)
networkOutputBuffers.update((link, receiver), buffer)
outputGateway.addOutputChannel(ChannelIdentity(actorId, receiver, isControl = false))
})
Expand Down Expand Up @@ -168,6 +183,188 @@ class OutputManager(
}
}

/**
* Provisional (phase 1): turn the columnar output path on or off for every current
* edge. Phase 2 replaces this blanket switch with a per-edge decision made by the
* planner (local vs cross-node, and whether the receiver consumes columns).
*/
def setColumnarOutput(enabled: Boolean): Unit =
networkOutputBuffers.values.foreach(_.columnar = enabled)

// Backs the per-bucket roots the columnar shuffle materializes; each is freed right
// after it is serialized, so this allocator's outstanding bytes return to zero.
private lazy val columnarAllocator: BufferAllocator = new RootAllocator()

/**
* Columnar output path (no round-trip): ship an operator's Arrow result batch straight
* downstream without decoding it to tuples and re-encoding it. Eligible only when the
* output port needs no storage and every edge on it is in columnar mode. A
* non-splitting edge (one-to-one, broadcast) sends the whole batch, serialized once, to
* each receiver. A splitting edge (hash/range/round-robin) assigns each row a bucket
* through the same getBucketIndex the row path uses (reading only the key columns into a
* partial tuple), gathers each bucket's rows in Arrow, and sends one ColumnarFrame per
* non-empty bucket. Bucket assignment is therefore identical to the row path. Does NOT
* close `root` (caller owns it).
*
* @return Some((portId, totalSerializedByteCount)) if it shipped the batch (caller
* records output statistics), or None if the port is not eligible (fall back).
*/
/**
* Local zero-copy fast path: hand the operator's Arrow result root to a co-located
* receiver by reference, with no serialization. Ownership transfers to the receiver (it
* closes the root), so on success the caller must NOT close it. Eligible only for the
* simplest single-consumer shape: the port has one one-to-one edge whose single receiver
* is known-local and columnar-capable, needs no storage, and whose schema matches, so the
* root goes to exactly one consumer. Every other shape returns None (the caller tries the
* remote columnar path, then rows).
*
* @return Some((portId, rootInMemSize)) if the root was handed off, else None.
*/
def tryEmitLocalColumnarRoot(
root: VectorSchemaRoot,
outputPortId: Option[PortIdentity]
): Option[(PortIdentity, Long)] = {
if (outputPortId.isEmpty && ports.size != 1) return None
val portId = outputPortId.getOrElse(ports.head._1)
if (outputPortResultWriterThreads.contains(portId)) return None
val links = partitioners.filter(_._1.fromPortId == portId).toSeq
if (links.size != 1) return None
val (link, partitioner) = links.head
val receivers = partitioner.allReceivers
if (receivers.size != 1) return None
val buffer = networkOutputBuffers((link, receivers.head))
// Receiver must be columnar-capable (planner flag) and known-local.
if (!buffer.columnar || !buffer.localityCheck().contains(true)) return None
val schema = getPort(portId).schema
if (ArrowUtils.toTexeraSchema(root.getSchema) != schema) return None
val batch = LocalColumnarBatch(root, root.getRowCount, schema)
buffer.sendLocalColumnar(batch)
Some((portId, batch.inMemSize))
}

def tryEmitColumnarBatch(
root: VectorSchemaRoot,
outputPortId: Option[PortIdentity]
): Option[(PortIdentity, Long)] = {
if (outputPortId.isEmpty && ports.size != 1) return None
val portId = outputPortId.getOrElse(ports.head._1)
val links = partitioners.filter(_._1.fromPortId == portId).toSeq
// Take the no-decode fast path when at least one receiver is remote-columnar, so those
// receivers get Arrow with no tuple decode; any local receivers on the same edge get
// rows. If no receiver is columnar (all local, or the wire is off), fall back so the
// caller decodes the batch once and routes it through the row path.
val anyColumnar = links.exists {
case (link, partitioner) =>
partitioner.allReceivers.exists(r =>
networkOutputBuffers.get((link, r)).exists(_.effectiveColumnar)
)
}
if (links.isEmpty || outputPortResultWriterThreads.contains(portId) || !anyColumnar) return None
val schema = getPort(portId).schema
// The executor's Arrow schema must match the declared output port schema; if not,
// fall back so the row path's enforceSchema can adjust it.
if (ArrowUtils.toTexeraSchema(root.getSchema) != schema) return None
Some((portId, links.map { case (link, p) => emitColumnarToLink(root, schema, link, p) }.sum))
}

// Ship `root` to one link's receivers, splitting per bucket if the partitioner splits.
// Each receiver gets Arrow if its buffer is columnar (remote) or rows if it is local, so
// a mixed-locality edge keeps the remote buckets columnar while sending rows locally.
// Returns the total bytes accounted for this link (serialized Arrow + local row sizes).
private def emitColumnarToLink(
root: VectorSchemaRoot,
schema: Schema,
link: PhysicalLink,
partitioner: Partitioner
): Long = {
val receivers = partitioner.allReceivers
if (!partitioner.columnarSplits) {
// One-to-one / broadcast: the whole batch goes to each receiver. Serialize to Arrow
// once for the remote receivers, decode to rows once for the local ones.
val (remote, local) =
receivers.partition(r => networkOutputBuffers((link, r)).effectiveColumnar)
var bytes = 0L
if (remote.nonEmpty) {
val arrow = ArrowUtils.serializeRoot(root)
val frame = ColumnarFrame(arrow, root.getRowCount, schema)
remote.foreach(r => networkOutputBuffers((link, r)).sendColumnarFrame(frame))
bytes += arrow.length.toLong
}
if (local.nonEmpty) {
val tuples = ArrowUtils.tuplesOfRoot(root)
local.foreach(r => networkOutputBuffers((link, r)).sendRows(tuples))
bytes += tuples.foldLeft(0L)(_ + _.inMemSize)
}
bytes
} else {
// Splitting shuffle: assign each row a bucket, then send each bucket to its receiver
// as Arrow (remote) or rows (local).
val n = root.getRowCount
val rowsPerBucket = Array.fill(receivers.length)(new mutable.ArrayBuffer[Int]())
// Build only the columns getBucketIndex needs into a partial tuple per row.
val keyCols =
if (partitioner.requiresFullTuple) schema.getAttributes.map(_.getName)
else partitioner.bucketKeyColumns
val keySchema = Schema(keyCols.map(schema.getAttribute).toList)
val keyIdx =
if (keyCols.isEmpty) Array.empty[Int] else ArrowUtils.projectionIndices(root, keyCols)
val keyTypes = keyCols.map(schema.getAttribute(_).getType).toArray
val vectors = root.getFieldVectors
var i = 0
while (i < n) {
val keyValues = new Array[Any](keyIdx.length)
var c = 0
while (c < keyIdx.length) {
keyValues(c) = ArrowUtils.getCell(vectors.get(keyIdx(c)), i, keyTypes(c)); c += 1
}
partitioner.getBucketIndex(Tuple(keySchema, keyValues)).foreach(b => rowsPerBucket(b) += i)
i += 1
}
var bytes = 0L
var b = 0
while (b < receivers.length) {
val rows = rowsPerBucket(b)
if (rows.nonEmpty) {
bytes += sendBucket(
root,
schema,
networkOutputBuffers((link, receivers(b))),
rows.toArray
)
}
b += 1
}
bytes
}
}

// Send one bucket's rows to a receiver: Arrow (gather + ColumnarFrame) if the buffer is
// columnar (remote), otherwise decode just those rows and send them as a DataFrame
// (local). Returns the bytes accounted for the send.
private def sendBucket(
root: VectorSchemaRoot,
schema: Schema,
buffer: NetworkOutputBuffer,
rows: Array[Int]
): Long = {
if (buffer.effectiveColumnar) {
val bucketRoot = ArrowUtils.gather(root, rows, columnarAllocator)
try {
val frameBytes = ArrowUtils.serializeRoot(bucketRoot)
buffer.sendColumnarFrame(ColumnarFrame(frameBytes, rows.length, schema))
frameBytes.length.toLong
} finally bucketRoot.close()
} else {
val tuples = new Array[Tuple](rows.length)
var k = 0
while (k < rows.length) {
tuples(k) = ArrowUtils.getTexeraTuple(rows(k), root, schema); k += 1
}
buffer.sendRows(tuples)
tuples.foldLeft(0L)(_ + _.inMemSize)
}
}

/**
* Flushes the network output buffers based on the specified set of physical links.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,14 @@ class PythonProxyClient(portNumberPromise: Promise[Int], val actorId: ActorVirtu
dataPayload match {
case DataFrame(frame) =>
writeArrowStream(mutable.Queue(ArraySeq.unsafeWrapArray(frame): _*), from, "Data")
case ColumnarFrame(arrowIpcBytes, _, _) =>
// A columnar-wire batch headed to a Python worker: decode it back to tuples
// and hand it over as a Data batch, same as a DataFrame.
writeArrowStream(
mutable.Queue(ArraySeq.unsafeWrapArray(ArrowUtils.deserializeTuples(arrowIpcBytes)): _*),
from,
"Data"
)
case StateFrame(state, loopCounter, loopStartId) =>
// The Arrow wire format for states IS the State row (content +
// loop_counter + loop_start_id), so the envelope rides its own
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,5 +94,8 @@ case object LinkConfig {

case class LinkConfig(
channelConfigs: List[ChannelConfig],
partitioning: Partitioning
partitioning: Partitioning,
// Per-edge columnar-wire decision: ship this link's batches as Arrow when the
// downstream operator consumes columns natively and the columnar wire is enabled.
columnar: Boolean = false
)
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.texera.amber.engine.architecture.scheduling.resourcePolicies

import org.apache.texera.common.config.ApplicationConfig
import org.apache.texera.amber.core.virtualidentity.PhysicalOpIdentity
import org.apache.texera.amber.core.workflow._
import org.apache.texera.amber.engine.architecture.scheduling.Region
Expand Down Expand Up @@ -89,7 +90,14 @@ class DefaultResourceAllocator(
operatorConfigs(physicalLink.toOpId).workerConfigs.map(_.workerId),
linkPartitionInfos(physicalLink),
workflowSettings.dataTransferBatchSize
)
),
// Per-edge columnar-wire decision: enabled only when the wire is on and the
// downstream operator consumes columns natively (otherwise shipping Arrow
// would force a costly per-tuple decode on the receiver). Locality (same-JVM
// links gain nothing) is a later refinement.
columnar =
ApplicationConfig.enableColumnarWire &&
physicalPlan.getOperator(physicalLink.toOpId).isColumnarCapable
)
}.toMap

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,7 @@ case class BroadcastPartitioner(partitioning: BroadcastPartitioning) extends Par
}

override def allReceivers: Seq[ActorVirtualIdentity] = receivers

// The whole batch goes to every receiver; no per-row split.
override def columnarSplits: Boolean = false
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,8 @@ case class HashBasedShufflePartitioner(partitioning: HashBasedShufflePartitionin
}

override def allReceivers: Seq[ActorVirtualIdentity] = receivers

// Hash reads the key columns (or the whole tuple when no keys are named).
override def bucketKeyColumns: Seq[String] = partitioning.hashAttributeNames
override def requiresFullTuple: Boolean = partitioning.hashAttributeNames.isEmpty
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,7 @@ case class OneToOnePartitioner(partitioning: OneToOnePartitioning, actorId: Acto

override def allReceivers: Seq[ActorVirtualIdentity] =
Seq(partitioning.channels.filter(_.fromWorkerId == actorId).head.toWorkerId)

// The whole batch goes to the single receiver; no per-row split.
override def columnarSplits: Boolean = false
}
Loading
Loading