Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import org.apache.texera.amber.core.virtualidentity.{
OperatorIdentity,
WorkflowIdentity
}
import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext, WorkflowSettings}
import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowSettings}
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{
ConsoleMessage,
ConsoleMessageType
Expand All @@ -49,7 +49,7 @@ import org.apache.texera.auth.SessionUser
import org.apache.texera.dao.SqlServer
import org.apache.texera.dao.jooq.generated.Tables.OPERATOR_EXECUTIONS
import org.apache.texera.web.model.websocket.request.{LogicalPlanPojo, WorkflowExecuteRequest}
import org.apache.texera.workflow.{LogicalLink, WorkflowCompiler}
import org.apache.texera.workflow.LogicalLink
import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource
import org.apache.texera.web.service.{ExecutionResultService, WorkflowService}
import org.apache.texera.web.storage.ExecutionStateStore.updateWorkflowState
Expand Down Expand Up @@ -887,28 +887,6 @@ class SyncExecutionResource extends LazyLogging {
)
}

// Returns operator-id -> error message; empty map means compilation succeeded.
private def validateWorkflow(
workflowId: Long,
logicalPlan: LogicalPlanPojo
): Map[String, String] = {
try {
val tempContext = new WorkflowContext(WorkflowIdentity(workflowId))
val compiler = new WorkflowCompiler(tempContext)
compiler.compile(logicalPlan)
Map.empty
} catch {
case e: Exception =>
val errorMsg = Option(e.getMessage).getOrElse("Compilation failed")
val operatorIdPattern = """operator[- ]?(\S+)""".r
val operatorId = operatorIdPattern
.findFirstMatchIn(errorMsg.toLowerCase)
.map(_.group(1))
.getOrElse("workflow")
Map(operatorId -> errorMsg)
}
}

@GET
@Path("/health")
def healthCheck: Map[String, String] = Map("status" -> "ok")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.texera.web.service

import com.typesafe.scalalogging.LazyLogging
import org.apache.texera.amber.core.storage.FileResolver
import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
import org.apache.texera.amber.core.workflow.WorkflowContext
import org.apache.texera.amber.engine.architecture.controller.{ControllerConfig, Workflow}
Expand Down Expand Up @@ -61,6 +62,7 @@ class WorkflowExecutionService(
request: WorkflowExecuteRequest,
val executionStateStore: ExecutionStateStore,
errorHandler: Throwable => Unit,
uid: Integer,
userEmailOpt: Option[String],
sessionUri: URI
) extends SubscriptionManager
Expand Down Expand Up @@ -105,7 +107,7 @@ class WorkflowExecutionService(

def executeWorkflow(): Unit = {
try {
workflow = new WorkflowCompiler(workflowContext)
workflow = new WorkflowCompiler(workflowContext, FileResolver.forUser(uid))
.compile(request.logicalPlan)
} catch {
case err: Throwable =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ class WorkflowService(
req,
executionStateStore,
errorHandler,
uid,
userEmailOpt,
sessionUri
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
package org.apache.texera.workflow

import com.typesafe.scalalogging.LazyLogging
import org.apache.texera.amber.core.storage.FileResolver
import org.apache.texera.amber.core.storage.ScanFileResolver
import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
import org.apache.texera.amber.operator.LogicalOp
import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc
Expand Down Expand Up @@ -90,17 +90,19 @@ case class LogicalPlan(
/**
* Resolve all user-given filename for the scan source operators to URIs, and call op.setFileUri to set the URi
*
* @param fileResolver resolves scan source file names to URIs
* @param errorList if given, put errors during resolving to it
*/
def resolveScanSourceOpFileName(
fileResolver: ScanFileResolver,
errorList: Option[ArrayBuffer[(OperatorIdentity, Throwable)]]
): Unit = {
operators.foreach {
case operator @ (scanOp: ScanSourceOpDesc) =>
Try {
// Resolve file path for ScanSourceOpDesc
val fileName = scanOp.fileName.getOrElse(throw new RuntimeException("no input file name"))
val fileUri = FileResolver.resolve(fileName) // Convert to URI
val fileUri = fileResolver.resolve(fileName) // Convert to URI

// Set the URI in the ScanSourceOpDesc
scanOp.setResolvedFileName(fileUri)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.texera.workflow

import com.typesafe.scalalogging.LazyLogging
import org.apache.texera.amber.core.storage.ScanFileResolver
import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
import org.apache.texera.amber.core.workflow._
import org.apache.texera.amber.engine.architecture.controller.Workflow
Expand All @@ -31,7 +32,8 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala
import scala.util.{Failure, Success, Try}

class WorkflowCompiler(
context: WorkflowContext
context: WorkflowContext,
fileResolver: ScanFileResolver
) extends LazyLogging {

/**
Expand Down Expand Up @@ -145,7 +147,7 @@ class WorkflowCompiler(
val logicalPlan: LogicalPlan = LogicalPlan(logicalPlanPojo)

// 2. resolve the file name in each scan source operator
logicalPlan.resolveScanSourceOpFileName(None)
logicalPlan.resolveScanSourceOpFileName(fileResolver, None)

// 3. expand the logical plan to the physical plan, and get a set of output ports that need storage
val (physicalPlan, outputPortsNeedingStorage) =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAg
PAUSED
}
import org.apache.texera.amber.engine.common.client.AmberClient
import org.apache.texera.amber.operator.LogicalOp
import org.apache.texera.amber.operator.{LogicalOp, TestOperators}
import org.apache.texera.dao.SqlServer
import org.apache.texera.dao.jooq.generated.enums.UserRoleEnum
import org.apache.texera.dao.jooq.generated.tables.daos.{
Expand Down Expand Up @@ -90,9 +90,8 @@ object TestUtils {
links: List[LogicalLink],
context: WorkflowContext
): Workflow = {
val workflowCompiler = new WorkflowCompiler(
context
)
// the scan sources in these tests read fixture files on the local disk
val workflowCompiler = new WorkflowCompiler(context, TestOperators.fixtureFileResolver)
workflowCompiler.compile(
LogicalPlanPojo(operators, links, List(), List())
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec {
val keyword = TestOperators.keywordSearchOpDesc("Region", "Asia")
val ctx = new WorkflowContext()

val workflow = new WorkflowCompiler(ctx).compile(
val workflow = new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver).compile(
pojo(
List(csv, keyword),
List(
Expand All @@ -78,7 +78,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec {
val keyword = TestOperators.keywordSearchOpDesc("Region", "Asia")
val ctx = new WorkflowContext()

val workflow = new WorkflowCompiler(ctx).compile(
val workflow = new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver).compile(
pojo(
List(csv, keyword),
List(
Expand Down Expand Up @@ -118,7 +118,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec {
val keyword = TestOperators.keywordSearchOpDesc("Region", "Asia")
val ctx = new WorkflowContext()

new WorkflowCompiler(ctx).compile(
new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver).compile(
pojo(
List(csv, keyword),
List(
Expand Down Expand Up @@ -148,7 +148,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec {
val keyword = TestOperators.keywordSearchOpDesc("Region", "Asia")
val ctx = new WorkflowContext()

new WorkflowCompiler(ctx).compile(
new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver).compile(
pojo(
List(csv, keyword),
List(
Expand Down Expand Up @@ -179,7 +179,8 @@ class WorkflowCompilerSpec extends AnyFlatSpec {
val csv = TestOperators.smallCsvScanOpDesc()
val ctx = new WorkflowContext()

new WorkflowCompiler(ctx).compile(pojo(List(csv), List.empty))
new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver)
.compile(pojo(List(csv), List.empty))

val storage = ctx.workflowSettings.outputPortsNeedingStorage
assert(
Expand All @@ -205,7 +206,8 @@ class WorkflowCompilerSpec extends AnyFlatSpec {
val ctx = new WorkflowContext()

val ex = intercept[RuntimeException] {
new WorkflowCompiler(ctx).compile(pojo(List(orphanCsv), List.empty))
new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver)
.compile(pojo(List(orphanCsv), List.empty))
}
assert(ex.getMessage == "no input file name")
}
Expand Down
7 changes: 6 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -180,14 +180,19 @@ lazy val FileService = (project in file("file-service"))
lazy val WorkflowOperator = (project in file("common/workflow-operator")).settings(asfLicensingSettingsWithVendored).dependsOn(WorkflowCore)
lazy val WorkflowCompilingService = (project in file("workflow-compiling-service"))
.dependsOn(WorkflowOperator, Auth, Config)
.configs(Test)
.dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests
.settings(asfLicensingSettings)
.settings(
dependencyOverrides ++= Seq(
// override it as io.dropwizard 4 require 2.16.1 or higher
"com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion,
"com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
"org.glassfish.jersey.core" % "jersey-common" % "3.0.12"
) ++ nettyDependencyOverrides
) ++ nettyDependencyOverrides,
// MockTexeraDB swaps a JVM-wide singleton (SqlServer's embedded Postgres),
// so run suites serially to avoid cross-suite races.
Test / parallelExecution := false
)

lazy val WorkflowExecutionService = (project in file("amber"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,58 +22,57 @@ package org.apache.texera.amber.core.storage
import org.apache.commons.vfs2.FileNotFoundException
import org.apache.texera.dao.SqlServer
import org.apache.texera.dao.SqlServer.withTransaction
import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum
import org.apache.texera.dao.jooq.generated.tables.Dataset.DATASET
import org.apache.texera.dao.jooq.generated.tables.DatasetUserAccess.DATASET_USER_ACCESS
import org.apache.texera.dao.jooq.generated.tables.DatasetVersion.DATASET_VERSION
import org.apache.texera.dao.jooq.generated.tables.User.USER
import org.apache.texera.dao.jooq.generated.tables.pojos.{Dataset, DatasetVersion}
import org.jooq.DSLContext

import java.net.{URI, URLEncoder}
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Paths}
import java.nio.file.Paths
import scala.jdk.CollectionConverters.IteratorHasAsScala
import scala.util.{Success, Try}
import scala.util.control.NonFatal

/**
* Unified object for resolving both VFS resources and local/dataset files.
* Resolves dataset logical paths to physical URIs.
*/
object FileResolver {

val DATASET_FILE_URI_SCHEME = "dataset"

/**
* Resolves a given fileName to either a file on the local file system or a dataset file.
*
* @param fileName the name of the file to resolve.
* @throws java.io.FileNotFoundException if the file cannot be resolved.
* @return A URI pointing to the resolved file.
* Returns a [[ScanFileResolver]] scoped to the user `uid`.
*/
def resolve(fileName: String): URI = {
if (isFileResolved(fileName)) {
return new URI(fileName)
}
val resolvers: Seq[String => URI] = Seq(localResolveFunc, datasetResolveFunc)
def forUser(uid: Integer): ScanFileResolver = fileName => resolveForUser(fileName, uid)

// Try each resolver function in sequence
resolvers
.map(resolver => Try(resolver(fileName)))
.collectFirst {
case Success(output) => output
}
.getOrElse(throw new FileNotFoundException(fileName))
}
/**
* Resolves a dataset logical path for the user `uid`.
*
* @throws java.io.FileNotFoundException if the path does not resolve to a readable dataset file
*/
def resolveForUser(fileName: String, uid: Integer): URI =
resolveLogicalPath(fileName, reader = Some(uid))

/**
* Attempts to resolve a local file path.
* @throws java.io.FileNotFoundException if the local file does not exist
* @param fileName the name of the file to check
* Resolves a dataset logical path without a user scope.
*
* @throws java.io.FileNotFoundException if the path does not resolve to a dataset file
*/
private def localResolveFunc(fileName: String): URI = {
val filePath = Paths.get(fileName)
if (!Files.exists(filePath)) {
throw new FileNotFoundException(s"Local file $fileName does not exist")
def resolveVersionedResource(fileName: String): URI =
resolveLogicalPath(fileName, reader = None)

private def resolveLogicalPath(fileName: String, reader: Option[Integer]): URI =
try {
if (isFileResolved(fileName)) {
throw new FileNotFoundException(fileName)
}
datasetResolveFunc(fileName, reader)
} catch {
case NonFatal(_) => throw new FileNotFoundException(fileName)
}
filePath.toUri
}

/**
* Parses a dataset file path and extracts its components.
Expand Down Expand Up @@ -109,10 +108,12 @@ object FileResolver {
* e.g. {DATASET_FILE_URI_SCHEME}:///dataset-15/adeq233td/some/dir/file.txt
*
* @param fileName the name of the file to attempt resolving as a DatasetFileDocument
* @param reader optional user ID to check read access for
* @return Either[String, DatasetFileDocument] - Right(document) if creation succeeds
* @throws java.io.FileNotFoundException if the dataset file does not exist or cannot be created
* @throws java.io.FileNotFoundException if the dataset file does not exist, cannot be created,
* or the reader does not have read access
*/
private def datasetResolveFunc(fileName: String): URI = {
private def datasetResolveFunc(fileName: String, reader: Option[Integer]): URI = {
val (ownerEmail, datasetName, versionName, fileRelativePathSegments) =
parseDatasetFilePath(fileName).getOrElse(
throw new FileNotFoundException(s"Dataset file $fileName not found.")
Expand All @@ -138,14 +139,18 @@ object FileResolver {
.and(DATASET.NAME.eq(datasetName))
.fetchOneInto(classOf[Dataset])

if (dataset == null || !readableBy(ctx, dataset, reader)) {
throw new FileNotFoundException(s"Dataset file $fileName not found.")
}

// fetch the dataset version from DB
val datasetVersion = ctx
.selectFrom(DATASET_VERSION)
.where(DATASET_VERSION.DID.eq(dataset.getDid))
.and(DATASET_VERSION.NAME.eq(versionName))
.fetchOneInto(classOf[DatasetVersion])

if (dataset == null || datasetVersion == null) {
if (datasetVersion == null) {
throw new FileNotFoundException(s"Dataset file $fileName not found.")
}
(dataset, datasetVersion)
Expand Down Expand Up @@ -178,6 +183,18 @@ object FileResolver {
}
}

/** Checks whether `reader` has read access when a user scope is provided. */
private def readableBy(ctx: DSLContext, dataset: Dataset, reader: Option[Integer]): Boolean =
reader.forall(uid =>
dataset.getIsPublic || dataset.getOwnerUid == uid || ctx.fetchExists(
DATASET_USER_ACCESS,
DATASET_USER_ACCESS.DID
.eq(dataset.getDid)
.and(DATASET_USER_ACCESS.UID.eq(uid))
.and(DATASET_USER_ACCESS.PRIVILEGE.in(PrivilegeEnum.READ, PrivilegeEnum.WRITE))
)
)

/**
* Checks if a given file path has a valid scheme.
*
Expand Down
Loading
Loading