diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala index 37556d2fd9e..8cd66400703 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala @@ -72,12 +72,16 @@ object FriesReconfigurationAlgorithm { var backwardVertices = Set.empty[PhysicalOpIdentity] val topologicalOps = region.topologicalIterator().toList + // Parents and children come from the region's own links. An operator's own link lists can + // also name a link from another operator of the region that carries no data in this + // execution, when the operator reads that port's saved result instead. + val regionLinks = region.getLinks topologicalOps.foreach { opId => val op = region.getOperator(opId) val fromParent = op.inputPorts.valuesIterator .flatMap(_._2) - .exists(link => forwardVertices.contains(link.fromOpId)) + .exists(link => regionLinks.contains(link) && forwardVertices.contains(link.fromOpId)) if (M.contains(opId) || fromParent) { forwardVertices += opId } @@ -87,7 +91,7 @@ object FriesReconfigurationAlgorithm { val op = region.getOperator(opId) val fromChild = op.outputPorts.valuesIterator .flatMap(_._2) - .exists(link => backwardVertices.contains(link.toOpId)) + .exists(link => regionLinks.contains(link) && backwardVertices.contains(link.toOpId)) if (M.contains(opId) || fromChild) { backwardVertices += opId } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithmSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithmSpec.scala index fbee2c30409..433968e67cc 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithmSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithmSpec.scala @@ -70,23 +70,32 @@ class FriesReconfigurationAlgorithmSpec extends AnyFlatSpec { ) /** - * Builds a region whose operators have their per-port link lists wired to match - * the given links; the closure computation traverses those per-op links. + * Builds a region with the given links, whose operators also list those links on their + * ports, as a compiled plan's operators do. `operatorOnlyLinks` are listed by the operators + * of the region at their ends but are not links of the region, like a link from outside the + * region or a link whose reader reads the port's saved result instead. */ private def buildRegion( regionId: Long, ops: Set[PhysicalOp], - links: Set[PhysicalLink] + links: Set[PhysicalLink], + operatorOnlyLinks: Set[PhysicalLink] = Set.empty ): Region = { - val wiredOps = links.foldLeft(ops.map(physicalOp => physicalOp.id -> physicalOp).toMap) { - (currentOps, physicalLink) => - currentOps - .updated( - physicalLink.fromOpId, - currentOps(physicalLink.fromOpId).addOutputLink(physicalLink) + val wiredOps = + (links ++ operatorOnlyLinks).foldLeft( + ops.map(physicalOp => physicalOp.id -> physicalOp).toMap + ) { (currentOps, physicalLink) => + val withOutput = currentOps + .get(physicalLink.fromOpId) + .fold(currentOps)(from => + currentOps.updated(physicalLink.fromOpId, from.addOutputLink(physicalLink)) ) - .updated(physicalLink.toOpId, currentOps(physicalLink.toOpId).addInputLink(physicalLink)) - } + withOutput + .get(physicalLink.toOpId) + .fold(withOutput)(to => + withOutput.updated(physicalLink.toOpId, to.addInputLink(physicalLink)) + ) + } Region(RegionIdentity(regionId), wiredOps.values.toSet, links) } @@ -373,4 +382,69 @@ class FriesReconfigurationAlgorithmSpec extends AnyFlatSpec { assert(componentOfB1.reconfigurations.map(_.targetOpId) == Set(physicalOpId("b1"))) assert(componentOfB2.reconfigurations.map(_.targetOpId) == Set(physicalOpId("b2"))) } + + it should "count only the region's links, not a link that only its operators list" in { + // Split is fed from outside the region. U1 reads split's port 0 from its saved result, so + // the link from split's port 0 to U1 carries no data: split and U1 list it, the region does + // not. U1 is on no path of the region's links from split to U2, so reconfiguring split and + // U2 involves only the two of them. + val split = PhysicalOp( + physicalOpId("split"), + WorkflowIdentity(0), + ExecutionIdentity(0), + OpExecInitInfo.Empty + ).withInputPorts(List(InputPort(PortIdentity()))) + .withOutputPorts(List(OutputPort(PortIdentity()), OutputPort(PortIdentity(1)))) + val region = buildRegion( + 1, + Set(split, op("r"), op("u1"), op("u2")), + Set(link("r", "u1"), link("u1", "u2"), link("split", "u2", fromPort = 1)), + operatorOnlyLinks = Set(link("src", "split"), link("split", "u1", fromPort = 0)) + ) + assert(region.getOperator(physicalOpId("u1")).getInputLinks().size == 2) + val request = reconfigureRequest("split", "u2") + + val components = + FriesReconfigurationAlgorithm.getReconfigurations(managerWithRegions(region), request) + + val component = singleComponent(components) + assert(component.scope == Set(physicalOpId("split"), physicalOpId("u2"))) + assert(component.sources == Set(physicalOpId("split"))) + assert(component.reconfigurations == request.reconfiguration.toSet) + } + + it should "not reach a reconfigured operator over a link that only its operators list" in { + // p feeds x and q, and q feeds y. x runs for r, and y reads x's port 0 from its saved + // result, so the link from x's port 0 to y carries no data: x and y list it, the region + // does not. Reconfiguring p and y takes in q, which lies between them, but not x: no link + // of the region leads from x to y. + val x = PhysicalOp( + physicalOpId("x"), + WorkflowIdentity(0), + ExecutionIdentity(0), + OpExecInitInfo.Empty + ).withInputPorts(List(InputPort(PortIdentity()))) + .withOutputPorts(List(OutputPort(PortIdentity()), OutputPort(PortIdentity(1)))) + val region = buildRegion( + 1, + Set(op("p"), x, op("q"), op("y"), op("r")), + Set( + link("p", "x"), + link("p", "q"), + link("q", "y"), + link("x", "r", fromPort = 1), + link("y", "r") + ), + operatorOnlyLinks = Set(link("x", "y", fromPort = 0)) + ) + assert(region.getOperator(physicalOpId("x")).getOutputLinks(PortIdentity(0)).size == 1) + val request = reconfigureRequest("p", "y") + + val components = + FriesReconfigurationAlgorithm.getReconfigurations(managerWithRegions(region), request) + + val component = singleComponent(components) + assert(component.scope == Set(physicalOpId("p"), physicalOpId("q"), physicalOpId("y"))) + assert(component.sources == Set(physicalOpId("p"))) + } }