diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst index 5953c501ed06e..266aad2d0c408 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst @@ -22,9 +22,10 @@ TypeScript SDK |experimental| -The TypeScript SDK lets you implement Airflow task logic in TypeScript (or plain JavaScript), running on -Node.js. The Dag and its scheduling remain in Python; individual tasks delegate to a Node.js subprocess that -is spawned by :class:`~airflow.sdk.coordinators.node.NodeCoordinator` for each task instance. +The TypeScript SDK lets you group task handlers in a ``Dag`` and implement their logic in TypeScript (or +plain JavaScript), running on Node.js. A matching Python stub Dag still declares the scheduling shape and +dependencies; individual tasks delegate to a Node.js subprocess that is spawned by +:class:`~airflow.sdk.coordinators.node.NodeCoordinator` for each task instance. The SDK is an ESM-only package that ships from the ``ts-sdk/`` directory of the Airflow repository. It is currently in **beta** and its API may change. @@ -36,7 +37,8 @@ The SDK is an ESM-only package that ships from the ``ts-sdk/`` directory of the .. seealso:: - For the full TypeScript API reference (task handlers, ``TaskClient``, and the coordinator runtime), + For the full TypeScript API reference (``Dag``, ``DagRegistry``, ``serveDags``, task handlers, + ``TaskClient``, supporting types, and exceptions), see the `TypeScript SDK API reference `__. .. contents:: Contents diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 1a034f862c9b3..283d36636f913 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -595,8 +595,8 @@ GitHub Actions to pass the list of parameters to a command to execute | run-mypy | Whether mypy check is supposed to run in this build | true | | | run-system-tests | Whether system tests should be run ("true"/"false") | true | | | run-task-sdk-tests | Whether Task SDK tests should be run ("true"/"false") | true | | -| run-ts-sdk-docs | Whether the TypeScript SDK API reference should be built — on `ts-sdk/docs/` or `ts-sdk/src/` changes, including Markdown ("true"/"false") | true | | -| run-ts-sdk-e2e-tests | Whether TypeScript SDK e2e tests should be run — on `ts-sdk/`, TS e2e test, or Node coordinator changes ("true"/"false") | true | | +| run-ts-sdk-docs | Whether the TypeScript SDK API reference should be built — on `ts-sdk/api-docs/`, `ts-sdk/docs/`, or `ts-sdk/src/` changes, including Markdown ("true"/"false") | true | | +| run-ts-sdk-e2e-tests | Whether TypeScript SDK e2e tests should be run — on runtime-affecting `ts-sdk/`, TS e2e test, or Node coordinator changes ("true"/"false") | true | | | run-ui-tests | Whether UI tests should be run ("true"/"false") | true | | | run-unit-tests | Whether unit tests should be run ("true"/"false") | true | | | run-www-tests | Whether Legacy WWW tests should be run ("true"/"false") | true | | diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index ccc80c7fb7d20..33dbaffc35c21 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -276,7 +276,8 @@ def __hash__(self): r"^airflow-e2e-tests/docker/openlineage-compat\.Dockerfile$", ], FileGroupForCi.TS_SDK_E2E_FILES: [ - r"^ts-sdk/(?!.*\.md$).*", + # API documentation entry points and Markdown do not affect runtime e2e tests. + r"^ts-sdk/(?!api-docs/)(?!.*\.md$).*", r"^airflow-e2e-tests/tests/airflow_e2e_tests/ts_sdk_tests/.*", r"^airflow-e2e-tests/docker/ts\.yml$", r"^task-sdk/src/airflow/sdk/coordinators/_subprocess\.py$", @@ -486,20 +487,23 @@ def __hash__(self): r"^java-sdk/(?!.*\.md$).*", ], FileGroupForCi.TS_SDK_DOCS_FILES: [ - # TypeDoc renders the reference from the SDK sources, and the landing page is - # authored in ts-sdk/docs — unlike TS_SDK_FILES, `.md` counts here. tsconfig.json - # and package.json are included too: docs/tsconfig.json `extends` the former, and - # the latter pins the `@msgpack/msgpack` version the checked program depends on. + # TypeDoc renders the reference from the SDK sources and category entry points, + # and the landing page is authored in ts-sdk/docs — unlike TS_SDK_FILES, `.md` + # counts here. tsconfig.json and package.json are included too: docs/tsconfig.json + # `extends` the former, and the latter pins the `@msgpack/msgpack` version the + # checked program depends on. + r"^ts-sdk/api-docs/.*", r"^ts-sdk/docs/.*", r"^ts-sdk/src/.*", r"^ts-sdk/tsconfig\.json$", r"^ts-sdk/package\.json$", ], FileGroupForCi.TS_SDK_FILES: [ - # `.md` excluded — doc-only edits do not affect the generated supervisor schema. - # `ts-sdk/docs/package.json` and its lock file excluded too — they pin the docs - # toolchain's own dependencies and do not affect the SDK build. - r"^ts-sdk/(?!.*\.md$)(?!docs/package(-lock)?\.json$).*", + # Documentation entry points and `.md` files do not affect the generated + # supervisor schema. `ts-sdk/docs/package.json` and its lock file are excluded + # too — they pin the docs toolchain's own dependencies and do not affect the SDK + # build. + r"^ts-sdk/(?!api-docs/)(?!.*\.md$)(?!docs/package(-lock)?\.json$).*", ], FileGroupForCi.ASSET_FILES: [ r"^airflow-core/src/airflow/assets/", diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 8f4e9382d29f8..a91804ff4e832 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1509,6 +1509,15 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): }, id="Build ts-sdk docs for a docs-only Markdown change that skips ts-sdk tests", ), + pytest.param( + ("ts-sdk/api-docs/dag-authoring-api.ts",), + { + "run-ts-sdk-docs": "true", + "run-ts-sdk-e2e-tests": "false", + "prod-image-build": "false", + }, + id="Build only ts-sdk docs when a TypeDoc category entry point changes", + ), pytest.param( ("ts-sdk/tsconfig.json",), { @@ -1873,6 +1882,11 @@ def test_ktlint_hook_only_runs_for_java_sdk_changes(files: tuple[str, ...], ktli True, id="skipped when only the docs toolchain's lock file changes", ), + pytest.param( + ("ts-sdk/api-docs/dag-authoring-api.ts",), + True, + id="skipped when only a TypeDoc category entry point changes", + ), ], ) def test_check_ts_sdk_supervisor_schema_hook_only_runs_for_relevant_changes( diff --git a/ts-sdk/api-docs/dag-authoring-api.ts b/ts-sdk/api-docs/dag-authoring-api.ts new file mode 100644 index 0000000000000..f91f824eabd99 --- /dev/null +++ b/ts-sdk/api-docs/dag-authoring-api.ts @@ -0,0 +1,33 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** @module Authoring */ + +export { Dag, DagRegistry, serveDags } from "../src/index.js"; +export type { + DagSpec, + TaskClient, + TaskContext, + TaskHandler, + TaskHandlerArgs, + TaskInputs, + TaskOptions, + TaskRef, + TaskSpec, +} from "../src/index.js"; diff --git a/ts-sdk/api-docs/exceptions.ts b/ts-sdk/api-docs/exceptions.ts new file mode 100644 index 0000000000000..5e0399299c9bc --- /dev/null +++ b/ts-sdk/api-docs/exceptions.ts @@ -0,0 +1,22 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** @module Exceptions */ + +export { ConnectionNotFoundError, VariableNotFoundError } from "../src/index.js"; diff --git a/ts-sdk/api-docs/internal-runtime-details.ts b/ts-sdk/api-docs/internal-runtime-details.ts new file mode 100644 index 0000000000000..8be97ac6a13f0 --- /dev/null +++ b/ts-sdk/api-docs/internal-runtime-details.ts @@ -0,0 +1,22 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** @module Internals */ + +export { SUPERVISOR_API_VERSION } from "../src/index.js"; diff --git a/ts-sdk/api-docs/supporting-types.ts b/ts-sdk/api-docs/supporting-types.ts new file mode 100644 index 0000000000000..f928752d91d1b --- /dev/null +++ b/ts-sdk/api-docs/supporting-types.ts @@ -0,0 +1,22 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** @module Types */ + +export type { ConnectionResult, GetXComOpts, JsonValue, SetXComOpts } from "../src/index.js"; diff --git a/ts-sdk/docs/tsconfig.json b/ts-sdk/docs/tsconfig.json index e971de7cb96d8..caaf9009d3106 100644 --- a/ts-sdk/docs/tsconfig.json +++ b/ts-sdk/docs/tsconfig.json @@ -1,13 +1,13 @@ { "//": [ "Docs-only tsconfig for the TypeDoc analysis pass. It inherits the SDK's compiler", - "settings verbatim and roots the program at the public entry point only: TypeScript", - "pulls in whatever that entry point transitively imports, but a glob like", + "settings verbatim and roots the program at the docs-only API category entry points: TypeScript", + "pulls in whatever those entry points transitively import, but a glob like", "`../src/**/*.ts` would also root unreachable internal modules (e.g. the `cli/` bin,", "which needs the optional `esbuild` peer dependency) that the docs toolchain has no", "reason to install. `paths` below redirects `@msgpack/msgpack`, a real transitive", "dependency of the checked program (coordinator/frames.ts, reachable from the public", - "startCoordinator export): Node module resolution walks up node_modules from the", + "serveDags export): Node module resolution walks up node_modules from the", "*importing file* (ts-sdk/src/coordinator/), never sideways into this package's own", "node_modules, so the bare specifier would otherwise never resolve here." ], @@ -17,5 +17,5 @@ "@msgpack/msgpack": ["./node_modules/@msgpack/msgpack/dist.esm/index.d.ts"] } }, - "include": ["../src/index.ts"] + "include": ["../api-docs/*.ts"] } diff --git a/ts-sdk/docs/typedoc.json b/ts-sdk/docs/typedoc.json index 6efc7c4e7d9e9..feed7d21ac0ba 100644 --- a/ts-sdk/docs/typedoc.json +++ b/ts-sdk/docs/typedoc.json @@ -2,7 +2,7 @@ "$schema": "https://typedoc.org/schema.json", "plugin": ["@clean-jsdoc-theme/typedoc"], "outputs": [{ "name": "clean-jsdoc-theme", "path": "_build/html" }], - "entryPoints": ["../src/index.ts"], + "entryPoints": ["../api-docs/*.ts"], "tsconfig": "./tsconfig.json", "name": "Apache Airflow TypeScript SDK", "readme": ".typedoc/readme.md", diff --git a/ts-sdk/eslint.config.js b/ts-sdk/eslint.config.js index 4d1f851d186b5..f4ae01d7d9deb 100644 --- a/ts-sdk/eslint.config.js +++ b/ts-sdk/eslint.config.js @@ -27,7 +27,7 @@ export default tseslint.config( js.configs.recommended, ...tseslint.configs.recommended, { - files: ["src/**/*.ts", "tests/**/*.ts"], + files: ["api-docs/**/*.ts", "src/**/*.ts", "tests/**/*.ts"], rules: { "@typescript-eslint/no-explicit-any": "error", "@typescript-eslint/no-unused-vars": [ diff --git a/ts-sdk/package.json b/ts-sdk/package.json index 8bc9560f17be0..1d182c2321564 100644 --- a/ts-sdk/package.json +++ b/ts-sdk/package.json @@ -40,10 +40,10 @@ ], "scripts": { "clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"", - "lint": "eslint src tests", - "lint:fix": "eslint src tests --fix", - "format:check": "prettier --check src tests", - "format": "prettier --write src tests", + "lint": "eslint api-docs src tests", + "lint:fix": "eslint api-docs src tests --fix", + "format:check": "prettier --check api-docs src tests", + "format": "prettier --write api-docs src tests", "typecheck": "tsc --noEmit", "test": "vitest run", "test:watch": "vitest", diff --git a/ts-sdk/src/sdk/client.ts b/ts-sdk/src/sdk/client.ts index a0a9a48bd8d59..8d63eee49c38a 100644 --- a/ts-sdk/src/sdk/client.ts +++ b/ts-sdk/src/sdk/client.ts @@ -43,7 +43,7 @@ export interface TaskClient { * This matches Python `Variable.get` behavior when no default value is * supplied. * - * @throws {@link VariableNotFoundError} when the key is missing. + * @throws {@link Exceptions!VariableNotFoundError | VariableNotFoundError} when the key is missing. */ getVariableOrThrow(key: string): Promise; @@ -86,7 +86,7 @@ export interface TaskClient { * * This matches Python `BaseHook.get_connection` behavior. * - * @throws {@link ConnectionNotFoundError} when the connection does not exist. + * @throws {@link Exceptions!ConnectionNotFoundError | ConnectionNotFoundError} when the connection does not exist. */ getConnectionOrThrow(connId: string): Promise; }