diff --git a/amber/src/main/scala/org/apache/texera/web/resource/SyncExecutionResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/SyncExecutionResource.scala index d3047db5802..359b2929c27 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/SyncExecutionResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/SyncExecutionResource.scala @@ -32,7 +32,7 @@ import org.apache.texera.amber.core.virtualidentity.{ OperatorIdentity, WorkflowIdentity } -import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext, WorkflowSettings} +import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowSettings} import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ ConsoleMessage, ConsoleMessageType @@ -49,7 +49,7 @@ import org.apache.texera.auth.SessionUser import org.apache.texera.dao.SqlServer import org.apache.texera.dao.jooq.generated.Tables.OPERATOR_EXECUTIONS import org.apache.texera.web.model.websocket.request.{LogicalPlanPojo, WorkflowExecuteRequest} -import org.apache.texera.workflow.{LogicalLink, WorkflowCompiler} +import org.apache.texera.workflow.LogicalLink import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource import org.apache.texera.web.service.{ExecutionResultService, WorkflowService} import org.apache.texera.web.storage.ExecutionStateStore.updateWorkflowState @@ -887,28 +887,6 @@ class SyncExecutionResource extends LazyLogging { ) } - // Returns operator-id -> error message; empty map means compilation succeeded. - private def validateWorkflow( - workflowId: Long, - logicalPlan: LogicalPlanPojo - ): Map[String, String] = { - try { - val tempContext = new WorkflowContext(WorkflowIdentity(workflowId)) - val compiler = new WorkflowCompiler(tempContext) - compiler.compile(logicalPlan) - Map.empty - } catch { - case e: Exception => - val errorMsg = Option(e.getMessage).getOrElse("Compilation failed") - val operatorIdPattern = """operator[- ]?(\S+)""".r - val operatorId = operatorIdPattern - .findFirstMatchIn(errorMsg.toLowerCase) - .map(_.group(1)) - .getOrElse("workflow") - Map(operatorId -> errorMsg) - } - } - @GET @Path("/health") def healthCheck: Map[String, String] = Map("status" -> "ok") diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala index 741687e02c9..e318b8fad1d 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala @@ -20,6 +20,7 @@ package org.apache.texera.web.service import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.storage.FileResolver import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.WorkflowContext import org.apache.texera.amber.engine.architecture.controller.{ControllerConfig, Workflow} @@ -61,6 +62,7 @@ class WorkflowExecutionService( request: WorkflowExecuteRequest, val executionStateStore: ExecutionStateStore, errorHandler: Throwable => Unit, + uid: Integer, userEmailOpt: Option[String], sessionUri: URI ) extends SubscriptionManager @@ -105,7 +107,7 @@ class WorkflowExecutionService( def executeWorkflow(): Unit = { try { - workflow = new WorkflowCompiler(workflowContext) + workflow = new WorkflowCompiler(workflowContext, FileResolver.forUser(uid)) .compile(request.logicalPlan) } catch { case err: Throwable => diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index b1cc506b5d8..cadd629d50f 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -293,6 +293,7 @@ class WorkflowService( req, executionStateStore, errorHandler, + uid, userEmailOpt, sessionUri ) diff --git a/amber/src/main/scala/org/apache/texera/workflow/LogicalPlan.scala b/amber/src/main/scala/org/apache/texera/workflow/LogicalPlan.scala index 974d17f40a4..d4a79c39462 100644 --- a/amber/src/main/scala/org/apache/texera/workflow/LogicalPlan.scala +++ b/amber/src/main/scala/org/apache/texera/workflow/LogicalPlan.scala @@ -20,7 +20,7 @@ package org.apache.texera.workflow import com.typesafe.scalalogging.LazyLogging -import org.apache.texera.amber.core.storage.FileResolver +import org.apache.texera.amber.core.storage.ScanFileResolver import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc @@ -90,9 +90,11 @@ case class LogicalPlan( /** * Resolve all user-given filename for the scan source operators to URIs, and call op.setFileUri to set the URi * + * @param fileResolver resolves scan source file names to URIs * @param errorList if given, put errors during resolving to it */ def resolveScanSourceOpFileName( + fileResolver: ScanFileResolver, errorList: Option[ArrayBuffer[(OperatorIdentity, Throwable)]] ): Unit = { operators.foreach { @@ -100,7 +102,7 @@ case class LogicalPlan( Try { // Resolve file path for ScanSourceOpDesc val fileName = scanOp.fileName.getOrElse(throw new RuntimeException("no input file name")) - val fileUri = FileResolver.resolve(fileName) // Convert to URI + val fileUri = fileResolver.resolve(fileName) // Convert to URI // Set the URI in the ScanSourceOpDesc scanOp.setResolvedFileName(fileUri) diff --git a/amber/src/main/scala/org/apache/texera/workflow/WorkflowCompiler.scala b/amber/src/main/scala/org/apache/texera/workflow/WorkflowCompiler.scala index b93aa3e4db3..5869a4fcd3b 100644 --- a/amber/src/main/scala/org/apache/texera/workflow/WorkflowCompiler.scala +++ b/amber/src/main/scala/org/apache/texera/workflow/WorkflowCompiler.scala @@ -20,6 +20,7 @@ package org.apache.texera.workflow import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.storage.ScanFileResolver import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.core.workflow._ import org.apache.texera.amber.engine.architecture.controller.Workflow @@ -31,7 +32,8 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala import scala.util.{Failure, Success, Try} class WorkflowCompiler( - context: WorkflowContext + context: WorkflowContext, + fileResolver: ScanFileResolver ) extends LazyLogging { /** @@ -145,7 +147,7 @@ class WorkflowCompiler( val logicalPlan: LogicalPlan = LogicalPlan(logicalPlanPojo) // 2. resolve the file name in each scan source operator - logicalPlan.resolveScanSourceOpFileName(None) + logicalPlan.resolveScanSourceOpFileName(fileResolver, None) // 3. expand the logical plan to the physical plan, and get a set of output ports that need storage val (physicalPlan, outputPortsNeedingStorage) = diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala index 20f62afc3f4..9c5c3940464 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala @@ -48,7 +48,7 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAg PAUSED } import org.apache.texera.amber.engine.common.client.AmberClient -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, TestOperators} import org.apache.texera.dao.SqlServer import org.apache.texera.dao.jooq.generated.enums.UserRoleEnum import org.apache.texera.dao.jooq.generated.tables.daos.{ @@ -90,9 +90,8 @@ object TestUtils { links: List[LogicalLink], context: WorkflowContext ): Workflow = { - val workflowCompiler = new WorkflowCompiler( - context - ) + // the scan sources in these tests read fixture files on the local disk + val workflowCompiler = new WorkflowCompiler(context, TestOperators.fixtureFileResolver) workflowCompiler.compile( LogicalPlanPojo(operators, links, List(), List()) ) diff --git a/amber/src/test/scala/org/apache/texera/workflow/WorkflowCompilerSpec.scala b/amber/src/test/scala/org/apache/texera/workflow/WorkflowCompilerSpec.scala index 94a8ffce552..efda68b4bfe 100644 --- a/amber/src/test/scala/org/apache/texera/workflow/WorkflowCompilerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/workflow/WorkflowCompilerSpec.scala @@ -54,7 +54,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { val keyword = TestOperators.keywordSearchOpDesc("Region", "Asia") val ctx = new WorkflowContext() - val workflow = new WorkflowCompiler(ctx).compile( + val workflow = new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver).compile( pojo( List(csv, keyword), List( @@ -78,7 +78,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { val keyword = TestOperators.keywordSearchOpDesc("Region", "Asia") val ctx = new WorkflowContext() - val workflow = new WorkflowCompiler(ctx).compile( + val workflow = new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver).compile( pojo( List(csv, keyword), List( @@ -118,7 +118,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { val keyword = TestOperators.keywordSearchOpDesc("Region", "Asia") val ctx = new WorkflowContext() - new WorkflowCompiler(ctx).compile( + new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver).compile( pojo( List(csv, keyword), List( @@ -148,7 +148,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { val keyword = TestOperators.keywordSearchOpDesc("Region", "Asia") val ctx = new WorkflowContext() - new WorkflowCompiler(ctx).compile( + new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver).compile( pojo( List(csv, keyword), List( @@ -179,7 +179,8 @@ class WorkflowCompilerSpec extends AnyFlatSpec { val csv = TestOperators.smallCsvScanOpDesc() val ctx = new WorkflowContext() - new WorkflowCompiler(ctx).compile(pojo(List(csv), List.empty)) + new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver) + .compile(pojo(List(csv), List.empty)) val storage = ctx.workflowSettings.outputPortsNeedingStorage assert( @@ -205,7 +206,8 @@ class WorkflowCompilerSpec extends AnyFlatSpec { val ctx = new WorkflowContext() val ex = intercept[RuntimeException] { - new WorkflowCompiler(ctx).compile(pojo(List(orphanCsv), List.empty)) + new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver) + .compile(pojo(List(orphanCsv), List.empty)) } assert(ex.getMessage == "no input file name") } diff --git a/build.sbt b/build.sbt index e75286b868b..d74c3e57296 100644 --- a/build.sbt +++ b/build.sbt @@ -180,6 +180,8 @@ lazy val FileService = (project in file("file-service")) lazy val WorkflowOperator = (project in file("common/workflow-operator")).settings(asfLicensingSettingsWithVendored).dependsOn(WorkflowCore) lazy val WorkflowCompilingService = (project in file("workflow-compiling-service")) .dependsOn(WorkflowOperator, Auth, Config) + .configs(Test) + .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests .settings(asfLicensingSettings) .settings( dependencyOverrides ++= Seq( @@ -187,7 +189,10 @@ lazy val WorkflowCompilingService = (project in file("workflow-compiling-service "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, "org.glassfish.jersey.core" % "jersey-common" % "3.0.12" - ) ++ nettyDependencyOverrides + ) ++ nettyDependencyOverrides, + // MockTexeraDB swaps a JVM-wide singleton (SqlServer's embedded Postgres), + // so run suites serially to avoid cross-suite races. + Test / parallelExecution := false ) lazy val WorkflowExecutionService = (project in file("amber")) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala index c8a407df993..74c2d9c8c7e 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala @@ -22,58 +22,57 @@ package org.apache.texera.amber.core.storage import org.apache.commons.vfs2.FileNotFoundException import org.apache.texera.dao.SqlServer import org.apache.texera.dao.SqlServer.withTransaction +import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum import org.apache.texera.dao.jooq.generated.tables.Dataset.DATASET +import org.apache.texera.dao.jooq.generated.tables.DatasetUserAccess.DATASET_USER_ACCESS import org.apache.texera.dao.jooq.generated.tables.DatasetVersion.DATASET_VERSION import org.apache.texera.dao.jooq.generated.tables.User.USER import org.apache.texera.dao.jooq.generated.tables.pojos.{Dataset, DatasetVersion} +import org.jooq.DSLContext import java.net.{URI, URLEncoder} import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Paths} +import java.nio.file.Paths import scala.jdk.CollectionConverters.IteratorHasAsScala -import scala.util.{Success, Try} +import scala.util.control.NonFatal /** - * Unified object for resolving both VFS resources and local/dataset files. + * Resolves dataset logical paths to physical URIs. */ object FileResolver { val DATASET_FILE_URI_SCHEME = "dataset" /** - * Resolves a given fileName to either a file on the local file system or a dataset file. - * - * @param fileName the name of the file to resolve. - * @throws java.io.FileNotFoundException if the file cannot be resolved. - * @return A URI pointing to the resolved file. + * Returns a [[ScanFileResolver]] scoped to the user `uid`. */ - def resolve(fileName: String): URI = { - if (isFileResolved(fileName)) { - return new URI(fileName) - } - val resolvers: Seq[String => URI] = Seq(localResolveFunc, datasetResolveFunc) + def forUser(uid: Integer): ScanFileResolver = fileName => resolveForUser(fileName, uid) - // Try each resolver function in sequence - resolvers - .map(resolver => Try(resolver(fileName))) - .collectFirst { - case Success(output) => output - } - .getOrElse(throw new FileNotFoundException(fileName)) - } + /** + * Resolves a dataset logical path for the user `uid`. + * + * @throws java.io.FileNotFoundException if the path does not resolve to a readable dataset file + */ + def resolveForUser(fileName: String, uid: Integer): URI = + resolveLogicalPath(fileName, reader = Some(uid)) /** - * Attempts to resolve a local file path. - * @throws java.io.FileNotFoundException if the local file does not exist - * @param fileName the name of the file to check + * Resolves a dataset logical path without a user scope. + * + * @throws java.io.FileNotFoundException if the path does not resolve to a dataset file */ - private def localResolveFunc(fileName: String): URI = { - val filePath = Paths.get(fileName) - if (!Files.exists(filePath)) { - throw new FileNotFoundException(s"Local file $fileName does not exist") + def resolveVersionedResource(fileName: String): URI = + resolveLogicalPath(fileName, reader = None) + + private def resolveLogicalPath(fileName: String, reader: Option[Integer]): URI = + try { + if (isFileResolved(fileName)) { + throw new FileNotFoundException(fileName) + } + datasetResolveFunc(fileName, reader) + } catch { + case NonFatal(_) => throw new FileNotFoundException(fileName) } - filePath.toUri - } /** * Parses a dataset file path and extracts its components. @@ -109,10 +108,12 @@ object FileResolver { * e.g. {DATASET_FILE_URI_SCHEME}:///dataset-15/adeq233td/some/dir/file.txt * * @param fileName the name of the file to attempt resolving as a DatasetFileDocument + * @param reader optional user ID to check read access for * @return Either[String, DatasetFileDocument] - Right(document) if creation succeeds - * @throws java.io.FileNotFoundException if the dataset file does not exist or cannot be created + * @throws java.io.FileNotFoundException if the dataset file does not exist, cannot be created, + * or the reader does not have read access */ - private def datasetResolveFunc(fileName: String): URI = { + private def datasetResolveFunc(fileName: String, reader: Option[Integer]): URI = { val (ownerEmail, datasetName, versionName, fileRelativePathSegments) = parseDatasetFilePath(fileName).getOrElse( throw new FileNotFoundException(s"Dataset file $fileName not found.") @@ -138,6 +139,10 @@ object FileResolver { .and(DATASET.NAME.eq(datasetName)) .fetchOneInto(classOf[Dataset]) + if (dataset == null || !readableBy(ctx, dataset, reader)) { + throw new FileNotFoundException(s"Dataset file $fileName not found.") + } + // fetch the dataset version from DB val datasetVersion = ctx .selectFrom(DATASET_VERSION) @@ -145,7 +150,7 @@ object FileResolver { .and(DATASET_VERSION.NAME.eq(versionName)) .fetchOneInto(classOf[DatasetVersion]) - if (dataset == null || datasetVersion == null) { + if (datasetVersion == null) { throw new FileNotFoundException(s"Dataset file $fileName not found.") } (dataset, datasetVersion) @@ -178,6 +183,18 @@ object FileResolver { } } + /** Checks whether `reader` has read access when a user scope is provided. */ + private def readableBy(ctx: DSLContext, dataset: Dataset, reader: Option[Integer]): Boolean = + reader.forall(uid => + dataset.getIsPublic || dataset.getOwnerUid == uid || ctx.fetchExists( + DATASET_USER_ACCESS, + DATASET_USER_ACCESS.DID + .eq(dataset.getDid) + .and(DATASET_USER_ACCESS.UID.eq(uid)) + .and(DATASET_USER_ACCESS.PRIVILEGE.in(PrivilegeEnum.READ, PrivilegeEnum.WRITE)) + ) + ) + /** * Checks if a given file path has a valid scheme. * diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ScanFileResolver.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ScanFileResolver.scala new file mode 100644 index 00000000000..97c63d6fff3 --- /dev/null +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ScanFileResolver.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.core.storage + +import java.net.URI + +/** + * Resolves the file name configured on a scan source operator to a physical URI. + */ +trait ScanFileResolver { + + /** + * @throws java.io.FileNotFoundException if the name does not resolve to a file + */ + def resolve(fileName: String): URI +} diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala index 593ac8d3471..d1acb424cb2 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala @@ -22,9 +22,19 @@ package org.apache.texera.amber.storage import org.apache.texera.amber.core.storage.FileResolver import org.apache.commons.vfs2.FileNotFoundException import org.apache.texera.dao.MockTexeraDB -import org.apache.texera.dao.jooq.generated.enums.UserRoleEnum -import org.apache.texera.dao.jooq.generated.tables.daos.{DatasetDao, DatasetVersionDao, UserDao} -import org.apache.texera.dao.jooq.generated.tables.pojos.{Dataset, DatasetVersion, User} +import org.apache.texera.dao.jooq.generated.enums.{PrivilegeEnum, UserRoleEnum} +import org.apache.texera.dao.jooq.generated.tables.daos.{ + DatasetDao, + DatasetUserAccessDao, + DatasetVersionDao, + UserDao +} +import org.apache.texera.dao.jooq.generated.tables.pojos.{ + Dataset, + DatasetUserAccess, + DatasetVersion, + User +} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} @@ -77,6 +87,47 @@ class FileResolverSpec datasetVersion } + private def userNamed(uid: Int, name: String): User = { + val user = new User + user.setUid(Integer.valueOf(uid)) + user.setName(name) + user.setRole(UserRoleEnum.REGULAR) + user.setPassword("123") + user.setEmail(s"$name@test.com") + user + } + + private val grantedUser: User = userNamed(2, "granted_user") + private val otherUser: User = userNamed(3, "other_user") + + private val privateDataset: Dataset = { + val dataset = new Dataset + dataset.setDid(Integer.valueOf(2)) + dataset.setName("private_dataset") + dataset.setRepositoryName("private_dataset_repo") + dataset.setDescription("private dataset for test") + dataset.setIsPublic(false) + dataset.setOwnerUid(testUser.getUid) + dataset + } + + private val privateDatasetVersion: DatasetVersion = { + val datasetVersion = new DatasetVersion + datasetVersion.setDid(privateDataset.getDid) + datasetVersion.setName("v1") + datasetVersion.setDvid(Integer.valueOf(3)) + datasetVersion.setCreatorUid(testUser.getUid) + datasetVersion.setVersionHash("5f0c7a1be2d94c3fa6b87d10e4c9a2b35d6e7f80") + datasetVersion + } + + private val privateDatasetFilePath = "/test_user@test.com/private_dataset/v1/data.csv" + private val privateDatasetFileUri = + s"${FileResolver.DATASET_FILE_URI_SCHEME}:///${privateDataset.getRepositoryName}/${privateDatasetVersion.getVersionHash}/data.csv" + + // names nothing: the owner exists, the dataset does not + private val missingDatasetFilePath = "/test_user@test.com/no_such_dataset/v1/data.csv" + private val localCsvFilePath = "common/workflow-core/src/test/resources/country_sales_small.csv" private val datasetACsvFilePath = "/test_user@test.com/test_dataset/v2/directory/a.csv" @@ -98,17 +149,23 @@ class FileResolverSpec val datasetVersionDao = new DatasetVersionDao(getDSLContext.configuration()) datasetVersionDao.insert(testDatasetVersion1) datasetVersionDao.insert(testDatasetVersion2) - } - "FileResolver" should "resolve local file correctly" in { - val localUri = FileResolver.resolve(localCsvFilePath) + // add the private dataset, and the two users who do not own it + userDao.insert(grantedUser) + userDao.insert(otherUser) + datasetDao.insert(privateDataset) + datasetVersionDao.insert(privateDatasetVersion) - assert(localUri == Paths.get(localCsvFilePath).toUri) + val datasetGrant = new DatasetUserAccess + datasetGrant.setDid(privateDataset.getDid) + datasetGrant.setUid(grantedUser.getUid) + datasetGrant.setPrivilege(PrivilegeEnum.READ) + new DatasetUserAccessDao(getDSLContext.configuration()).insert(datasetGrant) } - "FileResolver" should "resolve dataset file correctly" in { - val datasetACsvUri = FileResolver.resolve(datasetACsvFilePath) - val dataset1TxtUri = FileResolver.resolve(dataset1TxtFilePath) + "resolveVersionedResource" should "resolve dataset file correctly" in { + val datasetACsvUri = FileResolver.resolveVersionedResource(datasetACsvFilePath) + val dataset1TxtUri = FileResolver.resolveVersionedResource(dataset1TxtFilePath) assert( datasetACsvUri.toString == f"${FileResolver.DATASET_FILE_URI_SCHEME}:///${testDataset.getRepositoryName}/${testDatasetVersion2.getVersionHash}/directory/a.csv" @@ -118,9 +175,108 @@ class FileResolverSpec ) } - "FileResolver" should "throw not found exception" in { + it should "resolve a private dataset without a user scope" in { + assert( + FileResolver + .resolveVersionedResource(privateDatasetFilePath) + .toString == privateDatasetFileUri + ) + } + + it should "throw not found exception" in { + assertThrows[FileNotFoundException] { + FileResolver.resolveVersionedResource("some/random/path") + } + } + + it should "not resolve a file on the local disk, even one that exists" in { + assert(Paths.get(localCsvFilePath).toFile.exists()) + assertThrows[FileNotFoundException] { + FileResolver.resolveVersionedResource(localCsvFilePath) + } + assertThrows[FileNotFoundException] { + FileResolver.resolveVersionedResource(Paths.get(localCsvFilePath).toAbsolutePath.toString) + } + } + + it should "not accept a name that is already a URI" in { + assertThrows[FileNotFoundException] { + FileResolver.resolveVersionedResource(privateDatasetFileUri) + } + assertThrows[FileNotFoundException] { + FileResolver.resolveVersionedResource(Paths.get(localCsvFilePath).toUri.toString) + } + } + + "resolveForUser" should "resolve a private dataset file for its owner" in { + assert( + FileResolver.resolveForUser(privateDatasetFilePath, testUser.getUid).toString + == privateDatasetFileUri + ) + } + + it should "resolve a private dataset file for a user who was granted read access" in { + assert( + FileResolver.resolveForUser(privateDatasetFilePath, grantedUser.getUid).toString + == privateDatasetFileUri + ) + } + + it should "not resolve a private dataset file for any other user" in { + assertThrows[FileNotFoundException] { + FileResolver.resolveForUser(privateDatasetFilePath, otherUser.getUid) + } + } + + it should "resolve a public dataset file for a user with no grant on it" in { + assert( + FileResolver.resolveForUser(dataset1TxtFilePath, otherUser.getUid).toString == + f"${FileResolver.DATASET_FILE_URI_SCHEME}:///${testDataset.getRepositoryName}/${testDatasetVersion1.getVersionHash}/1.txt" + ) + } + + it should "throw FileNotFoundException with the input path when the dataset is not readable or missing" in { + val unreadable = intercept[FileNotFoundException] { + FileResolver.resolveForUser(privateDatasetFilePath, otherUser.getUid) + } + val missing = intercept[FileNotFoundException] { + FileResolver.resolveForUser(missingDatasetFilePath, otherUser.getUid) + } + assert( + unreadable.getMessage.replace(privateDatasetFilePath, "") + == missing.getMessage.replace(missingDatasetFilePath, "") + ) + assert(unreadable.getCause == null && missing.getCause == null) + } + + it should "not resolve a file on the local disk" in { + assertThrows[FileNotFoundException] { + FileResolver.resolveForUser(localCsvFilePath, testUser.getUid) + } + assertThrows[FileNotFoundException] { + FileResolver.resolveForUser( + Paths.get(localCsvFilePath).toAbsolutePath.toString, + testUser.getUid + ) + } + } + + it should "not accept a name that is already a URI" in { + assertThrows[FileNotFoundException] { + FileResolver.resolveForUser(privateDatasetFileUri, testUser.getUid) + } + assertThrows[FileNotFoundException] { + FileResolver.resolveForUser(Paths.get(localCsvFilePath).toUri.toString, testUser.getUid) + } + } + + "forUser" should "resolve names on behalf of the user it was built for" in { + assert( + FileResolver.forUser(grantedUser.getUid).resolve(privateDatasetFilePath).toString + == privateDatasetFileUri + ) assertThrows[FileNotFoundException] { - FileResolver.resolve("some/random/path") + FileResolver.forUser(otherUser.getUid).resolve(privateDatasetFilePath) } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala index b3c38735957..83a89211a68 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala @@ -19,7 +19,8 @@ package org.apache.texera.amber.operator -import org.apache.texera.amber.core.storage.FileResolver +import org.apache.commons.vfs2.FileNotFoundException +import org.apache.texera.amber.core.storage.{FileResolver, ScanFileResolver} import org.apache.texera.amber.core.tuple.{Attribute, AttributeType} import org.apache.texera.amber.operator.aggregate.{ AggregateOpDesc, @@ -34,10 +35,32 @@ import org.apache.texera.amber.operator.source.sql.asterixdb.AsterixDBSourceOpDe import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 import org.apache.texera.amber.operator.udf.python.source.PythonUDFSourceOpDescV2 -import java.nio.file.Path +import java.net.URI +import java.nio.file.{Files, Path, Paths} object TestOperators { + /** + * The URI of a fixture file on the local disk. + * + * @throws FileNotFoundException if there is no such file + */ + def localFileUri(path: String): URI = { + val filePath = Paths.get(path) + if (!Files.exists(filePath)) { + throw new FileNotFoundException(s"Local file $path does not exist") + } + filePath.toUri + } + + /** + * Resolves the fixture files the scan operators below read: a path on the local disk, or the + * `file:` URI one of them was already given. Tests hand this to the compiler in place of a + * user's resolver, which knows nothing about local files. + */ + val fixtureFileResolver: ScanFileResolver = fileName => + if (FileResolver.isFileResolved(fileName)) new URI(fileName) else localFileUri(fileName) + val parentDir = Path .of(sys.env.getOrElse("TEXERA_HOME", ".")) .resolve("common") @@ -93,7 +116,7 @@ object TestOperators { csvHeaderlessOp.fileName = Some(fileName) csvHeaderlessOp.customDelimiter = Some(",") csvHeaderlessOp.hasHeader = header - csvHeaderlessOp.setResolvedFileName(FileResolver.resolve(fileName)) + csvHeaderlessOp.setResolvedFileName(localFileUri(fileName)) csvHeaderlessOp } @@ -102,7 +125,7 @@ object TestOperators { val jsonlOp = new JSONLScanSourceOpDesc jsonlOp.fileName = Some(fileName) jsonlOp.flatten = flatten - jsonlOp.setResolvedFileName(FileResolver.resolve(fileName)) + jsonlOp.setResolvedFileName(localFileUri(fileName)) jsonlOp } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpExec.scala index 4cc60bf2bd9..ed9553c642e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpExec.scala @@ -24,6 +24,9 @@ import org.apache.texera.amber.core.storage.FileResolver import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} import org.apache.texera.amber.util.JSONUtils.objectMapper +import java.net.URI +import java.nio.file.{Files, Paths} + class FileScanOpExec private[scan] ( descString: String ) extends OperatorExecutor { @@ -32,7 +35,7 @@ class FileScanOpExec private[scan] ( override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = { val originalFileName = tuple.getFields.collectFirst { case s: String => s }.get - val fileName = FileResolver.resolve(originalFileName).toASCIIString + val fileName = locate(originalFileName).toASCIIString FileScanUtils.createTuplesFromFile( fileName = fileName, displayFileName = originalFileName, @@ -45,4 +48,14 @@ class FileScanOpExec private[scan] ( ) } + /** Resolves the file name carried by an input tuple to a URI. */ + private def locate(fileName: String): URI = + if (FileResolver.isFileResolved(fileName)) { + new URI(fileName) + } else { + val localPath = Paths.get(fileName) + if (Files.exists(localPath)) localPath.toUri + else FileResolver.resolveVersionedResource(fileName) + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index 87b86fea6dd..8726dc28628 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -19,7 +19,6 @@ package org.apache.texera.amber.operator.source.scan.csv -import org.apache.texera.amber.core.storage.FileResolver import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.WorkflowContext.{ DEFAULT_EXECUTION_ID, @@ -60,7 +59,7 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { parallelCsvScanSourceOpDesc.customDelimiter = Some(",") parallelCsvScanSourceOpDesc.hasHeader = true parallelCsvScanSourceOpDesc.setResolvedFileName( - FileResolver.resolve(parallelCsvScanSourceOpDesc.fileName.get) + TestOperators.localFileUri(parallelCsvScanSourceOpDesc.fileName.get) ) val inferredSchema: Schema = parallelCsvScanSourceOpDesc.sourceSchema() @@ -76,7 +75,7 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { parallelCsvScanSourceOpDesc.customDelimiter = Some(",") parallelCsvScanSourceOpDesc.hasHeader = false parallelCsvScanSourceOpDesc.setResolvedFileName( - FileResolver.resolve(parallelCsvScanSourceOpDesc.fileName.get) + TestOperators.localFileUri(parallelCsvScanSourceOpDesc.fileName.get) ) val inferredSchema: Schema = parallelCsvScanSourceOpDesc.sourceSchema() @@ -91,7 +90,9 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) csvScanSourceOpDesc.customDelimiter = Some(",") csvScanSourceOpDesc.hasHeader = true - csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(csvScanSourceOpDesc.fileName.get)) + csvScanSourceOpDesc.setResolvedFileName( + TestOperators.localFileUri(csvScanSourceOpDesc.fileName.get) + ) val inferredSchema: Schema = csvScanSourceOpDesc.sourceSchema() @@ -105,7 +106,9 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesHeaderlessSmallCsvPath) csvScanSourceOpDesc.customDelimiter = Some(",") csvScanSourceOpDesc.hasHeader = false - csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(csvScanSourceOpDesc.fileName.get)) + csvScanSourceOpDesc.setResolvedFileName( + TestOperators.localFileUri(csvScanSourceOpDesc.fileName.get) + ) val inferredSchema: Schema = csvScanSourceOpDesc.sourceSchema() @@ -120,7 +123,9 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { Some(TestOperators.CountrySalesSmallMultiLineCustomDelimiterCsvPath) csvScanSourceOpDesc.customDelimiter = Some(";") csvScanSourceOpDesc.hasHeader = false - csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(csvScanSourceOpDesc.fileName.get)) + csvScanSourceOpDesc.setResolvedFileName( + TestOperators.localFileUri(csvScanSourceOpDesc.fileName.get) + ) val inferredSchema: Schema = csvScanSourceOpDesc.sourceSchema() @@ -135,7 +140,9 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { Some(TestOperators.CountrySalesSmallMultiLineCustomDelimiterCsvPath) csvScanSourceOpDesc.customDelimiter = Some(";") csvScanSourceOpDesc.hasHeader = false - csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(csvScanSourceOpDesc.fileName.get)) + csvScanSourceOpDesc.setResolvedFileName( + TestOperators.localFileUri(csvScanSourceOpDesc.fileName.get) + ) assert( !csvScanSourceOpDesc @@ -181,7 +188,7 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { csvScanSourceOpDesc.fileName = Some(path) csvScanSourceOpDesc.customDelimiter = Some(",") csvScanSourceOpDesc.hasHeader = true - csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + csvScanSourceOpDesc.setResolvedFileName(TestOperators.localFileUri(path)) val names = csvScanSourceOpDesc.sourceSchema().getAttributes.map(_.getName).toList assert(names == List("id", "name", "column-3", "age")) @@ -192,7 +199,7 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { parallelCsvScanSourceOpDesc.fileName = Some(path) parallelCsvScanSourceOpDesc.customDelimiter = Some(",") parallelCsvScanSourceOpDesc.hasHeader = true - parallelCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + parallelCsvScanSourceOpDesc.setResolvedFileName(TestOperators.localFileUri(path)) val names = parallelCsvScanSourceOpDesc.sourceSchema().getAttributes.map(_.getName).toList assert(names == List("id", "name", "column-3", "age")) @@ -204,7 +211,7 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { oldCsvScanSourceOpDesc.fileName = Some(path) oldCsvScanSourceOpDesc.customDelimiter = Some(",") oldCsvScanSourceOpDesc.hasHeader = true - oldCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + oldCsvScanSourceOpDesc.setResolvedFileName(TestOperators.localFileUri(path)) val names = oldCsvScanSourceOpDesc.sourceSchema().getAttributes.map(_.getName).toList assert(names == List("id", "name", "column-3", "age")) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala index dee4cfc0374..b94ec5341cb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala @@ -19,7 +19,6 @@ package org.apache.texera.amber.operator.source.scan.file -import org.apache.texera.amber.core.storage.FileResolver import org.apache.texera.amber.core.tuple.{AttributeType, Schema, SchemaEnforceable, Tuple} import org.apache.texera.amber.operator.TestOperators import org.apache.texera.amber.operator.source.scan.{FileAttributeType, FileDecodingMethod} @@ -33,7 +32,9 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { before { fileScanSourceOpDesc = new FileScanSourceOpDesc() - fileScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(TestOperators.TestTextFilePath)) + fileScanSourceOpDesc.setResolvedFileName( + TestOperators.localFileUri(TestOperators.TestTextFilePath) + ) fileScanSourceOpDesc.fileEncoding = FileDecodingMethod.UTF_8 } @@ -114,7 +115,7 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { it should "read first 5 lines of the input text file with CRLF separators into corresponding output tuples" in { fileScanSourceOpDesc.setResolvedFileName( - FileResolver.resolve(TestOperators.TestCRLFTextFilePath) + TestOperators.localFileUri(TestOperators.TestCRLFTextFilePath) ) fileScanSourceOpDesc.attributeType = FileAttributeType.STRING fileScanSourceOpDesc.fileScanLimit = Option(5) @@ -159,7 +160,7 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { it should "read first 5 lines of the input text into corresponding output INTEGER tuples" in { fileScanSourceOpDesc.setResolvedFileName( - FileResolver.resolve(TestOperators.TestNumbersFilePath) + TestOperators.localFileUri(TestOperators.TestNumbersFilePath) ) fileScanSourceOpDesc.attributeType = FileAttributeType.INTEGER fileScanSourceOpDesc.fileScanLimit = Option(5) @@ -183,7 +184,7 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { it should "read first 5 lines of the input text file with US_ASCII encoding" in { fileScanSourceOpDesc.setResolvedFileName( - FileResolver.resolve(TestOperators.TestCRLFTextFilePath) + TestOperators.localFileUri(TestOperators.TestCRLFTextFilePath) ) fileScanSourceOpDesc.fileEncoding = FileDecodingMethod.ASCII fileScanSourceOpDesc.attributeType = FileAttributeType.STRING diff --git a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala index 98c37cac13b..476b35bf6f0 100644 --- a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala +++ b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala @@ -1524,7 +1524,7 @@ class DatasetResource extends LazyLogging { case (None, None) => // Case 3: Neither repositoryName nor commitHash are provided, resolve normally val response = withTransaction(context) { ctx => - val fileUri = FileResolver.resolve(decodedPathStr) + val fileUri = FileResolver.resolveVersionedResource(decodedPathStr) val document = DocumentFactory.openReadonlyDocument(fileUri).asInstanceOf[OnDataset] val datasetDao = new DatasetDao(ctx.configuration()) val datasets = @@ -2163,7 +2163,7 @@ class DatasetResource extends LazyLogging { val owner = getOwner(ctx, did) val document = DocumentFactory .openReadonlyDocument( - FileResolver.resolve(s"${owner.getEmail}/${dataset.getName}/$normalized") + FileResolver.resolveVersionedResource(s"${owner.getEmail}/${dataset.getName}/$normalized") ) .asInstanceOf[OnDataset] @@ -2220,7 +2220,7 @@ class DatasetResource extends LazyLogging { val fullPath = s"${owner.getEmail}/${dataset.getName}/$coverImage" val document = DocumentFactory - .openReadonlyDocument(FileResolver.resolve(fullPath)) + .openReadonlyDocument(FileResolver.resolveVersionedResource(fullPath)) .asInstanceOf[OnDataset] val presignedUrl = withLakeFSErrorHandling( @@ -2269,7 +2269,7 @@ class DatasetResource extends LazyLogging { val fullPath = s"${owner.getEmail}/${dataset.getName}/$coverImage" val document = DocumentFactory - .openReadonlyDocument(FileResolver.resolve(fullPath)) + .openReadonlyDocument(FileResolver.resolveVersionedResource(fullPath)) .asInstanceOf[OnDataset] val presignedUrl = withLakeFSErrorHandling( diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/compiler/WorkflowCompiler.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/compiler/WorkflowCompiler.scala index 25166e7ac52..6287cf877ac 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/compiler/WorkflowCompiler.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/compiler/WorkflowCompiler.scala @@ -26,6 +26,7 @@ import org.apache.texera.amber.compiler.WorkflowCompiler.{ convertErrorListToWorkflowFatalErrorMap } import org.apache.texera.amber.compiler.model.{LogicalPlan, LogicalPlanPojo} +import org.apache.texera.amber.core.storage.ScanFileResolver import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.core.workflow.{ @@ -122,7 +123,8 @@ case class WorkflowCompilationResult( ) class WorkflowCompiler( - context: WorkflowContext + context: WorkflowContext, + fileResolver: ScanFileResolver ) extends LazyLogging { // function to expand logical plan to physical plan @@ -208,7 +210,7 @@ class WorkflowCompiler( val logicalPlan: LogicalPlan = LogicalPlan(logicalPlanPojo) // 2. resolve the file name in each scan source operator - logicalPlan.resolveScanSourceOpFileName(Some(errorList)) + logicalPlan.resolveScanSourceOpFileName(fileResolver, Some(errorList)) // 3. expand the logical plan to the physical plan val physicalPlan = expandLogicalPlan(logicalPlan, Some(errorList)) diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/compiler/model/LogicalPlan.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/compiler/model/LogicalPlan.scala index eecb435cc87..b1e61973461 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/compiler/model/LogicalPlan.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/compiler/model/LogicalPlan.scala @@ -20,7 +20,7 @@ package org.apache.texera.amber.compiler.model import com.typesafe.scalalogging.LazyLogging -import org.apache.texera.amber.core.storage.FileResolver +import org.apache.texera.amber.core.storage.ScanFileResolver import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.LogicalOp @@ -108,9 +108,11 @@ case class LogicalPlan( /** * Resolve all user-given filename for the scan source operators to URIs, and call op.setFileUri to set the URi + * @param fileResolver resolves scan source file names to URIs * @param errorList if given, put errors during resolving to it */ def resolveScanSourceOpFileName( + fileResolver: ScanFileResolver, errorList: Option[ArrayBuffer[(OperatorIdentity, Throwable)]] ): Unit = { operators.foreach { @@ -118,7 +120,7 @@ case class LogicalPlan( Try { // Resolve file path for ScanSourceOpDesc val fileName = scanOp.fileName.getOrElse(throw new RuntimeException("no input file name")) - val fileUri = FileResolver.resolve(fileName) // Convert to URI + val fileUri = fileResolver.resolve(fileName) // Convert to URI // Set the URI in the ScanSourceOpDesc scanOp.setResolvedFileName(fileUri) diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowCompilationResource.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowCompilationResource.scala index 501498b1d5c..03e23fc4132 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowCompilationResource.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowCompilationResource.scala @@ -21,16 +21,19 @@ package org.apache.texera.service.resource import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} import com.typesafe.scalalogging.LazyLogging +import io.dropwizard.auth.Auth import jakarta.annotation.security.RolesAllowed import jakarta.ws.rs.core.MediaType import jakarta.ws.rs.{Consumes, POST, Path, Produces} import org.apache.texera.amber.compiler.WorkflowCompiler import org.apache.texera.amber.compiler.model.LogicalPlanPojo +import org.apache.texera.amber.core.storage.FileResolver import org.apache.texera.amber.core.tuple.Attribute import org.apache.texera.amber.core.virtualidentity.WorkflowIdentity import org.apache.texera.amber.core.workflow.{PhysicalPlan, WorkflowContext} import org.apache.texera.amber.core.workflowruntimestate.WorkflowFatalError import org.apache.texera.amber.util.serde.PortIdentityKeySerializer +import org.apache.texera.auth.SessionUser @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -63,13 +66,15 @@ class WorkflowCompilationResource extends LazyLogging { @POST @Path("") def compileWorkflow( - logicalPlanPojo: LogicalPlanPojo + logicalPlanPojo: LogicalPlanPojo, + @Auth user: SessionUser ): WorkflowCompilationResponse = { // a placeholder workflow context, as compiling a workflow doesn't require a wid from the frontend val context = new WorkflowContext(workflowId = WorkflowIdentity(0)) // Compile the pojo using WorkflowCompiler - val compilationResult = new WorkflowCompiler(context).compile(logicalPlanPojo) + val compilationResult = + new WorkflowCompiler(context, FileResolver.forUser(user.getUid)).compile(logicalPlanPojo) val operatorOutputSchemas = compilationResult.operatorIdToOutputSchemas.map { case (operatorIdentity, schemas) => diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/compiler/WorkflowCompilerSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/compiler/WorkflowCompilerSpec.scala index ee221728d31..582f155445b 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/compiler/WorkflowCompilerSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/compiler/WorkflowCompilerSpec.scala @@ -24,6 +24,7 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType} import org.apache.texera.amber.core.virtualidentity.WorkflowIdentity import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} import org.apache.texera.amber.core.workflowruntimestate.FatalErrorType.COMPILATION_ERROR +import org.apache.texera.amber.operator.TestOperators import org.apache.texera.amber.operator.filter.{ ComparisonType, FilterPredicate, @@ -94,7 +95,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { "WorkflowCompiler" should "produce a populated physicalPlan and no errors for a well-formed plan" in { val csv = csvOp(realCsvPath) val proj = projectOp(List("Region", "Total Profit")) - val result = new WorkflowCompiler(newContext()).compile( + val result = new WorkflowCompiler(newContext(), TestOperators.fixtureFileResolver).compile( LogicalPlanPojo( operators = List(csv, proj), links = List( @@ -133,7 +134,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { val filter2 = filterOp(new FilterPredicate("Region", ComparisonType.NOT_EQUAL_TO, "JPN")) val limit2 = limitOp(5) - val result = new WorkflowCompiler(newContext()).compile( + val result = new WorkflowCompiler(newContext(), TestOperators.fixtureFileResolver).compile( LogicalPlanPojo( operators = List(csv, proj, limit1, filter1, filter2, limit2), links = List( @@ -197,7 +198,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { "WorkflowCompiler" should "accumulate, not throw, when a scan source has no fileName" in { val orphan = csvOpNoFile() - val result = new WorkflowCompiler(newContext()).compile( + val result = new WorkflowCompiler(newContext(), TestOperators.fixtureFileResolver).compile( LogicalPlanPojo( operators = List(orphan), links = List.empty, @@ -217,7 +218,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { it should "accumulate when a scan source's fileName points to a non-existent path" in { val broken = csvOp("/does/not/exist/missing.csv") - val result = new WorkflowCompiler(newContext()).compile( + val result = new WorkflowCompiler(newContext(), TestOperators.fixtureFileResolver).compile( LogicalPlanPojo( operators = List(broken), links = List.empty, @@ -228,11 +229,9 @@ class WorkflowCompilerSpec extends AnyFlatSpec { assert(result.physicalPlan.isEmpty) assert(result.operatorIdToError.contains(broken.operatorIdentifier)) - // FileResolver.resolve falls through both resolvers and rethrows - // org.apache.commons.vfs2.FileNotFoundException(fileName); its message bundle - // renders as `Could not read from "" because it is not a file.`, so the - // only stable substring across that wording and any java.io.FileNotFoundException - // fallback is the bad path itself. + // The resolver raises org.apache.commons.vfs2.FileNotFoundException; its message + // bundle wraps whatever text it is given, so the only stable substring across that + // wording and any java.io.FileNotFoundException fallback is the bad path itself. assert( result.operatorIdToError(broken.operatorIdentifier).message.contains("missing.csv"), s"unexpected message: ${result.operatorIdToError(broken.operatorIdentifier).message}" @@ -243,7 +242,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { val csv = csvOp(realCsvPath) val badProjection = projectOp(List("DoesNotExist")) - val result = new WorkflowCompiler(newContext()).compile( + val result = new WorkflowCompiler(newContext(), TestOperators.fixtureFileResolver).compile( LogicalPlanPojo( operators = List(csv, badProjection), links = List( @@ -273,7 +272,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { } it should "not throw when given an empty plan" in { - val result = new WorkflowCompiler(newContext()).compile( + val result = new WorkflowCompiler(newContext(), TestOperators.fixtureFileResolver).compile( LogicalPlanPojo( operators = List.empty, links = List.empty, @@ -295,7 +294,7 @@ class WorkflowCompilerSpec extends AnyFlatSpec { val orphan1 = csvOpNoFile() val orphan2 = csvOpNoFile() - val result = new WorkflowCompiler(newContext()).compile( + val result = new WorkflowCompiler(newContext(), TestOperators.fixtureFileResolver).compile( LogicalPlanPojo( operators = List(orphan1, orphan2), links = List.empty, diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/LoggedInResourceFixture.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/LoggedInResourceFixture.scala new file mode 100644 index 00000000000..b82e40843ba --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/LoggedInResourceFixture.scala @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.service.resource + +import io.dropwizard.auth.AuthValueFactoryProvider +import io.dropwizard.testing.junit5.ResourceExtension +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.auth.{JwtAuth, JwtAuthFilter, SessionUser, UnauthorizedExceptionMapper} +import org.apache.texera.dao.MockTexeraDB +import org.apache.texera.dao.jooq.generated.enums.UserRoleEnum +import org.apache.texera.dao.jooq.generated.tables.daos.{DatasetDao, DatasetVersionDao, UserDao} +import org.apache.texera.dao.jooq.generated.tables.pojos.{Dataset, DatasetVersion, User} +import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature +import org.scalatest.TestSuite + +/** + * What a resource spec of this service needs to make requests as a logged-in user: the + * providers production registers through `AuthFeatures`, and two users around one private + * dataset, owned by `owner` and shared with nobody. + */ +trait LoggedInResourceFixture extends MockTexeraDB { this: TestSuite => + + private def userNamed(uid: Int, name: String): User = { + val user = new User + user.setUid(Integer.valueOf(uid)) + user.setName(name) + user.setEmail(s"$name@test.com") + user.setPassword("123") + user.setRole(UserRoleEnum.REGULAR) + user + } + + protected val owner: User = userNamed(1, "owner") + protected val otherUser: User = userNamed(2, "other_user") + + protected val privateDatasetRepositoryName = "private-dataset-repository" + protected val privateDatasetVersionHash = "5f0c7a1be2d94c3fa6b87d10e4c9a2b35d6e7f80" + + /** A file of the private dataset, named the way a user names it in an operator. */ + protected val privateDatasetFilePath = "/owner@test.com/private_dataset/v1/notes.txt" + + protected def resourcesFor(resource: AnyRef): ResourceExtension = + ResourceExtension + .builder() + .setMapper(objectMapper) + .addProvider(classOf[JwtAuthFilter]) + .addProvider(classOf[UnauthorizedExceptionMapper]) + .addProvider(classOf[RolesAllowedDynamicFeature]) + .addProvider(new AuthValueFactoryProvider.Binder(classOf[SessionUser])) + .addResource(resource) + .build() + + protected def bearerTokenOf(user: User): String = + s"Bearer ${JwtAuth.jwtToken(JwtAuth.jwtClaims(user, 1))}" + + protected def insertUsersAndPrivateDataset(): Unit = { + val configuration = getDSLContext.configuration() + val userDao = new UserDao(configuration) + userDao.insert(owner) + userDao.insert(otherUser) + + val dataset = new Dataset + dataset.setDid(Integer.valueOf(1)) + dataset.setName("private_dataset") + dataset.setRepositoryName(privateDatasetRepositoryName) + dataset.setDescription("private dataset for test") + dataset.setIsPublic(false) + dataset.setOwnerUid(owner.getUid) + new DatasetDao(configuration).insert(dataset) + + val version = new DatasetVersion + version.setDid(dataset.getDid) + version.setDvid(Integer.valueOf(1)) + version.setName("v1") + version.setCreatorUid(owner.getUid) + version.setVersionHash(privateDatasetVersionHash) + new DatasetVersionDao(configuration).insert(version) + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowCompilationResourceSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowCompilationResourceSpec.scala index fd99c3d27dc..ba6f702c48d 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowCompilationResourceSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowCompilationResourceSpec.scala @@ -25,9 +25,13 @@ import jakarta.ws.rs.client.Entity import jakarta.ws.rs.core.{MediaType, Response} import org.apache.texera.amber.compiler.model.{LogicalLink, LogicalPlanPojo} import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.projection.{AttributeUnit, ProjectionOpDesc} +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.limit.LimitOpDesc import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpDesc +import org.apache.texera.amber.operator.source.scan.file.FileScanSourceOpDesc +import org.apache.texera.amber.operator.source.scan.text.TextInputSourceOpDesc import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.dao.jooq.generated.tables.pojos.User import org.assertj.core.api.Assertions.assertThat import org.scalatest.BeforeAndAfterAll import org.scalatest.flatspec.AnyFlatSpec @@ -43,19 +47,23 @@ import org.scalatest.flatspec.AnyFlatSpec * `org.apache.texera.amber.compiler.WorkflowCompilerSpec` so a regression * lands on the right spec. */ -class WorkflowCompilationResourceSpec extends AnyFlatSpec with BeforeAndAfterAll { +class WorkflowCompilationResourceSpec + extends AnyFlatSpec + with BeforeAndAfterAll + with LoggedInResourceFixture { - private val resources: ResourceExtension = ResourceExtension - .builder() - .addResource(new WorkflowCompilationResource()) - .setMapper(objectMapper) - .build() + private val resources: ResourceExtension = resourcesFor(new WorkflowCompilationResource()) - override protected def beforeAll(): Unit = resources.before() - override protected def afterAll(): Unit = resources.after() + override protected def beforeAll(): Unit = { + initializeDBAndReplaceDSLContext() + insertUsersAndPrivateDataset() + resources.before() + } - private val realCsvPath = - "workflow-compiling-service/src/test/resources/country_sales_small.csv" + override protected def afterAll(): Unit = { + resources.after() + shutdownDB() + } private def csvOp(fileName: String): CSVScanSourceOpDesc = { val op = new CSVScanSourceOpDesc() @@ -65,13 +73,34 @@ class WorkflowCompilationResourceSpec extends AnyFlatSpec with BeforeAndAfterAll op } - private def projectOp(columns: List[String]): ProjectionOpDesc = { - val op = new ProjectionOpDesc() - op.attributes = columns.map(name => new AttributeUnit(name, "")) - op.isDrop = false + // A source that needs no file, for the tests that are about the envelope only. + private def textOp(text: String): TextInputSourceOpDesc = { + val op = new TextInputSourceOpDesc() + op.textInput = text + op + } + + private def limitOp(rows: Int): LimitOpDesc = { + val op = new LimitOpDesc() + op.limit = rows + op + } + + // Scans a whole file as one value, so compiling it resolves the name without reading the file. + private def wholeFileScanOp(fileName: String): FileScanSourceOpDesc = { + val op = new FileScanSourceOpDesc() + op.fileName = Some(fileName) op } + private def planOf(operator: LogicalOp): LogicalPlanPojo = + LogicalPlanPojo( + operators = List(operator), + links = List.empty, + opsToViewResult = List.empty, + opsToReuseResult = List.empty + ) + // The frontend serializes LogicalLink with `fromOpId` / `toOpId` as flat // strings, but the Scala case class stores them as nested `OperatorIdentity` // records. This helper mirrors the wire shape so the test exercises the @@ -88,23 +117,24 @@ class WorkflowCompilationResourceSpec extends AnyFlatSpec with BeforeAndAfterAll objectMapper.writeValueAsString(jsonNode) } - private def postCompile(pojo: LogicalPlanPojo): Response = + private def postCompile(pojo: LogicalPlanPojo, as: User = owner): Response = resources .target("/compile") .request(MediaType.APPLICATION_JSON) + .header("Authorization", bearerTokenOf(as)) .post(Entity.json(encodePojoAsFrontendJson(pojo))) "POST /compile" should "return HTTP 200 for a well-formed plan" in { - val csv = csvOp(realCsvPath) - val proj = projectOp(List("Region", "Total Profit")) + val text = textOp("north\nsouth") + val limit = limitOp(1) val response = postCompile( LogicalPlanPojo( - operators = List(csv, proj), + operators = List(text, limit), links = List( LogicalLink( - csv.operatorIdentifier, + text.operatorIdentifier, PortIdentity(0), - proj.operatorIdentifier, + limit.operatorIdentifier, PortIdentity(0) ) ), @@ -120,16 +150,16 @@ class WorkflowCompilationResourceSpec extends AnyFlatSpec with BeforeAndAfterAll // Both polymorphic deserialization (round-tripping through the trait) and // a raw-JSON `type == "success"` check need to hold so the Angular client // can branch without depending on Scala class names. - val csv = csvOp(realCsvPath) - val proj = projectOp(List("Region", "Total Profit")) + val text = textOp("north\nsouth") + val limit = limitOp(1) val response = postCompile( LogicalPlanPojo( - operators = List(csv, proj), + operators = List(text, limit), links = List( LogicalLink( - csv.operatorIdentifier, + text.operatorIdentifier, PortIdentity(0), - proj.operatorIdentifier, + limit.operatorIdentifier, PortIdentity(0) ) ), @@ -172,4 +202,59 @@ class WorkflowCompilationResourceSpec extends AnyFlatSpec with BeforeAndAfterAll assertThat(rootNode.get("type").asText()).isEqualTo("failure") assertThat(rootNode.has("operatorErrors")).isTrue } + + it should "answer 401 to a request that carries no logged-in user" in { + val response = resources + .target("/compile") + .request(MediaType.APPLICATION_JSON) + .post(Entity.json(encodePojoAsFrontendJson(planOf(wholeFileScanOp(privateDatasetFilePath))))) + assertThat(response.getStatus).isEqualTo(401) + } + + it should "compile a scan source whose file the logged-in user can read" in { + val scan = wholeFileScanOp(privateDatasetFilePath) + val response = postCompile(planOf(scan), as = owner) + + assertThat(response.getStatus).isEqualTo(200) + val rootNode = objectMapper.readTree(response.readEntity(classOf[String])) + assert(rootNode.get("type").asText() == "success", s"expected success, got $rootNode") + assert(rootNode.get("operatorOutputSchemas").has(scan.operatorIdentifier.id)) + } + + it should "return WorkflowCompilationFailure when the scan source file is not readable by the user" in { + val scan = csvOp("/owner@test.com/private_dataset/v1/table.csv") + val response = postCompile(planOf(scan), as = otherUser) + + assertThat(response.getStatus).isEqualTo(200) + val body = response.readEntity(classOf[String]) + val rootNode = objectMapper.readTree(body) + assert(rootNode.get("type").asText() == "failure", s"expected failure, got $body") + assert(rootNode.get("operatorErrors").has(scan.operatorIdentifier.id)) + val schemas = rootNode.get("operatorOutputSchemas").get(scan.operatorIdentifier.id) + assert( + schemas == null || !schemas.elements().hasNext || + schemas.elements().next().isNull, + s"expected no schema for the operator, got $schemas" + ) + assert(!body.contains(privateDatasetRepositoryName)) + assert(!body.contains(privateDatasetVersionHash)) + } + + it should "return the same operator error for an unreadable file as for a missing one" in { + val unreadable = wholeFileScanOp(privateDatasetFilePath) + val missingPath = "/owner@test.com/no_such_dataset/v1/notes.txt" + val missing = wholeFileScanOp(missingPath) + + def errorMessageOf(scan: FileScanSourceOpDesc): String = { + val rootNode = objectMapper.readTree( + postCompile(planOf(scan), as = otherUser).readEntity(classOf[String]) + ) + rootNode.get("operatorErrors").get(scan.operatorIdentifier.id).get("message").asText() + } + + assert( + errorMessageOf(unreadable).replace(privateDatasetFilePath, "") + == errorMessageOf(missing).replace(missingPath, "") + ) + } }