diff --git a/.changeset/azure-template-upload-headers-python.md b/.changeset/azure-template-upload-headers-python.md new file mode 100644 index 0000000000..fbbefca86e --- /dev/null +++ b/.changeset/azure-template-upload-headers-python.md @@ -0,0 +1,5 @@ +--- +"@e2b/python-sdk": patch +--- + +Apply the request headers the API returns with a template layer-file upload link. Azure Blob Storage requires `x-ms-blob-type` on the upload request, which its signed URL cannot carry, so `COPY` instructions failed on Azure-backed clusters. GCS- and S3-backed clusters return no headers and are unaffected. diff --git a/.changeset/azure-template-upload-headers.md b/.changeset/azure-template-upload-headers.md new file mode 100644 index 0000000000..ddf3f9417a --- /dev/null +++ b/.changeset/azure-template-upload-headers.md @@ -0,0 +1,5 @@ +--- +"e2b": patch +--- + +Apply the request headers the API returns with a template layer-file upload link. Azure Blob Storage requires `x-ms-blob-type` on the upload request, which its signed URL cannot carry, so `COPY` instructions failed on Azure-backed clusters. GCS- and S3-backed clusters return no headers and are unaffected. diff --git a/packages/js-sdk/src/api/schema.gen.ts b/packages/js-sdk/src/api/schema.gen.ts index 8943b3385e..fa424a8a4a 100644 --- a/packages/js-sdk/src/api/schema.gen.ts +++ b/packages/js-sdk/src/api/schema.gen.ts @@ -1162,6 +1162,7 @@ export interface paths { }; 400: components["responses"]["400"]; 401: components["responses"]["401"]; + 409: components["responses"]["409"]; 500: components["responses"]["500"]; }; }; @@ -1243,6 +1244,7 @@ export interface paths { }; }; 401: components["responses"]["401"]; + 409: components["responses"]["409"]; 500: components["responses"]["500"]; }; }; @@ -1854,6 +1856,7 @@ export interface paths { }; 400: components["responses"]["400"]; 401: components["responses"]["401"]; + 409: components["responses"]["409"]; 500: components["responses"]["500"]; }; }; @@ -1995,6 +1998,7 @@ export interface paths { 400: components["responses"]["400"]; 401: components["responses"]["401"]; 403: components["responses"]["403"]; + 409: components["responses"]["409"]; 500: components["responses"]["500"]; }; }; @@ -2856,6 +2860,10 @@ export interface components { updatedAt: string; }; TemplateBuildFileUpload: { + /** @description Request headers that must be sent with the upload request */ + headers?: { + [key: string]: string; + }; /** @description Whether the file is already present in the cache */ present: boolean; /** @description Url where the file should be uploaded to */ diff --git a/packages/js-sdk/src/template/buildApi.ts b/packages/js-sdk/src/template/buildApi.ts index 489445c4ee..ef1860a322 100644 --- a/packages/js-sdk/src/template/buildApi.ts +++ b/packages/js-sdk/src/template/buildApi.ts @@ -113,6 +113,7 @@ export async function uploadFile( fileName: string fileContextPath: string url: string + headers?: Record ignorePatterns: string[] resolveSymlinks: boolean gzip: boolean @@ -128,6 +129,7 @@ export async function uploadFile( const { fileName, url, + headers, fileContextPath, ignorePatterns, resolveSymlinks, @@ -154,7 +156,7 @@ export async function uploadFile( abortOpts?.signal ) - const res = await putFileStream(url, tar.path, tar.size, signal) + const res = await putFileStream(url, tar.path, tar.size, signal, headers) if (!res.ok) { throw new FileUploadError( @@ -176,7 +178,8 @@ async function putFileStream( url: string, filePath: string, size: number, - signal: AbortSignal | undefined + signal: AbortSignal | undefined, + headers?: Record ): Promise<{ ok: boolean; statusText: string }> { // Prefer undici's fetch: it honors the explicit Content-Length on stream // bodies on every runtime, while Deno's native fetch ignores the header and @@ -192,7 +195,10 @@ async function putFileStream( body: stream.Readable.toWeb( fs.createReadStream(filePath) ) as ReadableStream, + // Headers the API asked for, applied as given (Azure's Put Blob requires + // x-ms-blob-type, which its SAS cannot carry). Content-Length stays ours. headers: { + ...headers, 'Content-Length': size.toString(), }, // Streaming request bodies require half-duplex mode. diff --git a/packages/js-sdk/src/template/index.ts b/packages/js-sdk/src/template/index.ts index 1cc6401b28..12ba6caeb6 100644 --- a/packages/js-sdk/src/template/index.ts +++ b/packages/js-sdk/src/template/index.ts @@ -1112,7 +1112,7 @@ export class TemplateBase stackTrace = this.stackTraces[index + 1] } - const { present, url } = await getFileUploadLink( + const { present, url, headers } = await getFileUploadLink( client, { templateID, @@ -1131,6 +1131,7 @@ export class TemplateBase fileName: src, fileContextPath: this.fileContextPath.toString(), url, + headers, ignorePatterns: [ ...this.fileIgnorePatterns, ...readDockerignore(this.fileContextPath.toString()), diff --git a/packages/js-sdk/tests/template/uploadFile.test.ts b/packages/js-sdk/tests/template/uploadFile.test.ts index e9f7492c9c..b953297e77 100644 --- a/packages/js-sdk/tests/template/uploadFile.test.ts +++ b/packages/js-sdk/tests/template/uploadFile.test.ts @@ -73,5 +73,45 @@ describe('uploadFile transfer encoding', () => { // Content-Type (e.g. inferred from the archive's file extension) makes // the storage backend reject the upload with 403 Forbidden. expect(capturedHeaders['content-type']).toBeUndefined() + + // S3 and GCS presigned PUTs sign the header set, so the upload must add + // nothing the API did not ask for. + expect(capturedHeaders['x-ms-blob-type']).toBeUndefined() + }) + + test('sends the headers the API returned with the upload link', async () => { + await uploadFile( + { + fileName: '*.txt', + fileContextPath: testDir, + url: baseUrl, + headers: { 'x-ms-blob-type': 'BlockBlob' }, + ignorePatterns: [], + resolveSymlinks: false, + gzip: true, + }, + undefined + ) + + // Azure's Put Blob rejects the request without it, and its SAS cannot + // carry a required request header, so the API hands it back instead. + expect(capturedHeaders['x-ms-blob-type']).toBe('BlockBlob') + }) + + test('keeps its own Content-Length when the API returns one', async () => { + await uploadFile( + { + fileName: '*.txt', + fileContextPath: testDir, + url: baseUrl, + headers: { 'Content-Length': '1' }, + ignorePatterns: [], + resolveSymlinks: false, + gzip: true, + }, + undefined + ) + + expect(Number(capturedHeaders['content-length'])).toBe(capturedBodyLength) }) }) diff --git a/packages/python-sdk/e2b/api/client/api/templates/post_templates.py b/packages/python-sdk/e2b/api/client/api/templates/post_templates.py index e561b2adb2..61f4f7b73e 100644 --- a/packages/python-sdk/e2b/api/client/api/templates/post_templates.py +++ b/packages/python-sdk/e2b/api/client/api/templates/post_templates.py @@ -45,6 +45,10 @@ def _parse_response( response_401 = Error.from_dict(response.json()) return response_401 + if response.status_code == 409: + response_409 = Error.from_dict(response.json()) + + return response_409 if response.status_code == 500: response_500 = Error.from_dict(response.json()) diff --git a/packages/python-sdk/e2b/api/client/api/templates/post_templates_template_id.py b/packages/python-sdk/e2b/api/client/api/templates/post_templates_template_id.py index 067d1ebc6f..35e68d3ddb 100644 --- a/packages/python-sdk/e2b/api/client/api/templates/post_templates_template_id.py +++ b/packages/python-sdk/e2b/api/client/api/templates/post_templates_template_id.py @@ -42,6 +42,10 @@ def _parse_response( response_401 = Error.from_dict(response.json()) return response_401 + if response.status_code == 409: + response_409 = Error.from_dict(response.json()) + + return response_409 if response.status_code == 500: response_500 = Error.from_dict(response.json()) diff --git a/packages/python-sdk/e2b/api/client/api/templates/post_v2_templates.py b/packages/python-sdk/e2b/api/client/api/templates/post_v2_templates.py index 0c3f568d10..0ed4b6d706 100644 --- a/packages/python-sdk/e2b/api/client/api/templates/post_v2_templates.py +++ b/packages/python-sdk/e2b/api/client/api/templates/post_v2_templates.py @@ -45,6 +45,10 @@ def _parse_response( response_401 = Error.from_dict(response.json()) return response_401 + if response.status_code == 409: + response_409 = Error.from_dict(response.json()) + + return response_409 if response.status_code == 500: response_500 = Error.from_dict(response.json()) diff --git a/packages/python-sdk/e2b/api/client/api/templates/post_v3_templates.py b/packages/python-sdk/e2b/api/client/api/templates/post_v3_templates.py index fd390477d7..3ae4afbd29 100644 --- a/packages/python-sdk/e2b/api/client/api/templates/post_v3_templates.py +++ b/packages/python-sdk/e2b/api/client/api/templates/post_v3_templates.py @@ -49,6 +49,10 @@ def _parse_response( response_403 = Error.from_dict(response.json()) return response_403 + if response.status_code == 409: + response_409 = Error.from_dict(response.json()) + + return response_409 if response.status_code == 500: response_500 = Error.from_dict(response.json()) diff --git a/packages/python-sdk/e2b/api/client/models/__init__.py b/packages/python-sdk/e2b/api/client/models/__init__.py index d2acfa930d..8390964fc1 100644 --- a/packages/python-sdk/e2b/api/client/models/__init__.py +++ b/packages/python-sdk/e2b/api/client/models/__init__.py @@ -63,6 +63,7 @@ from .template_alias_response import TemplateAliasResponse from .template_build import TemplateBuild from .template_build_file_upload import TemplateBuildFileUpload +from .template_build_file_upload_headers import TemplateBuildFileUploadHeaders from .template_build_info import TemplateBuildInfo from .template_build_logs_response import TemplateBuildLogsResponse from .template_build_request import TemplateBuildRequest @@ -144,6 +145,7 @@ "TemplateAliasResponse", "TemplateBuild", "TemplateBuildFileUpload", + "TemplateBuildFileUploadHeaders", "TemplateBuildInfo", "TemplateBuildLogsResponse", "TemplateBuildRequest", diff --git a/packages/python-sdk/e2b/api/client/models/template_build_file_upload.py b/packages/python-sdk/e2b/api/client/models/template_build_file_upload.py index a7d4e44a04..aacdf46be7 100644 --- a/packages/python-sdk/e2b/api/client/models/template_build_file_upload.py +++ b/packages/python-sdk/e2b/api/client/models/template_build_file_upload.py @@ -1,11 +1,17 @@ from collections.abc import Mapping -from typing import Any, TypeVar, Union +from typing import TYPE_CHECKING, Any, TypeVar, Union from attrs import define as _attrs_define from attrs import field as _attrs_field from ..types import UNSET, Unset +if TYPE_CHECKING: + from ..models.template_build_file_upload_headers import ( + TemplateBuildFileUploadHeaders, + ) + + T = TypeVar("T", bound="TemplateBuildFileUpload") @@ -15,10 +21,13 @@ class TemplateBuildFileUpload: Attributes: present (bool): Whether the file is already present in the cache url (Union[Unset, str]): Url where the file should be uploaded to + headers (Union[Unset, TemplateBuildFileUploadHeaders]): Request headers that must be sent with the upload + request """ present: bool url: Union[Unset, str] = UNSET + headers: Union[Unset, "TemplateBuildFileUploadHeaders"] = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -26,6 +35,10 @@ def to_dict(self) -> dict[str, Any]: url = self.url + headers: Union[Unset, dict[str, Any]] = UNSET + if not isinstance(self.headers, Unset): + headers = self.headers.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -35,19 +48,33 @@ def to_dict(self) -> dict[str, Any]: ) if url is not UNSET: field_dict["url"] = url + if headers is not UNSET: + field_dict["headers"] = headers return field_dict @classmethod def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.template_build_file_upload_headers import ( + TemplateBuildFileUploadHeaders, + ) + d = dict(src_dict) present = d.pop("present") url = d.pop("url", UNSET) + _headers = d.pop("headers", UNSET) + headers: Union[Unset, TemplateBuildFileUploadHeaders] + if isinstance(_headers, Unset): + headers = UNSET + else: + headers = TemplateBuildFileUploadHeaders.from_dict(_headers) + template_build_file_upload = cls( present=present, url=url, + headers=headers, ) template_build_file_upload.additional_properties = d diff --git a/packages/python-sdk/e2b/api/client/models/template_build_file_upload_headers.py b/packages/python-sdk/e2b/api/client/models/template_build_file_upload_headers.py new file mode 100644 index 0000000000..93bcb32717 --- /dev/null +++ b/packages/python-sdk/e2b/api/client/models/template_build_file_upload_headers.py @@ -0,0 +1,44 @@ +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="TemplateBuildFileUploadHeaders") + + +@_attrs_define +class TemplateBuildFileUploadHeaders: + """Request headers that must be sent with the upload request""" + + additional_properties: dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + template_build_file_upload_headers = cls() + + template_build_file_upload_headers.additional_properties = d + return template_build_file_upload_headers + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/packages/python-sdk/e2b/template_async/build_api.py b/packages/python-sdk/e2b/template_async/build_api.py index 328a002f9a..a8a324321d 100644 --- a/packages/python-sdk/e2b/template_async/build_api.py +++ b/packages/python-sdk/e2b/template_async/build_api.py @@ -1,7 +1,7 @@ import asyncio import os from types import TracebackType -from typing import Callable, Optional, List, Union +from typing import Callable, Dict, Optional, List, Union import httpx from pyqwest import HTTPTransport @@ -115,6 +115,7 @@ async def upload_file( resolve_symlinks: bool, gzip: bool, stack_trace: Optional[TracebackType], + headers: Optional[Dict[str, str]] = None, request_timeout: Optional[float] = None, ): # Uploading a large build-context archive can take far longer than the 60s @@ -156,10 +157,13 @@ async def upload_file( # explicit Content-Length suppresses chunked transfer # encoding, which S3 presigned URLs reject; reqwest keeps the # Content-Length framing for the streamed body. + # Headers the API asked for, applied as given (Azure's Put + # Blob requires x-ms-blob-type, which its SAS cannot carry). + # Content-Length stays ours. response = await client.put( url, content=aiter_io_chunks(tar_file), - headers={"Content-Length": str(size)}, + headers={**(headers or {}), "Content-Length": str(size)}, ) response.raise_for_status() finally: diff --git a/packages/python-sdk/e2b/template_async/main.py b/packages/python-sdk/e2b/template_async/main.py index 2247f8b0ae..0a9c0e0374 100644 --- a/packages/python-sdk/e2b/template_async/main.py +++ b/packages/python-sdk/e2b/template_async/main.py @@ -137,6 +137,7 @@ async def _build( resolve_symlinks, gzip, stack_trace, + headers=file_info.headers.to_dict() if file_info.headers else None, request_timeout=request_timeout, ) if on_build_logs: diff --git a/packages/python-sdk/e2b/template_sync/build_api.py b/packages/python-sdk/e2b/template_sync/build_api.py index 735dcf7970..0b956ff725 100644 --- a/packages/python-sdk/e2b/template_sync/build_api.py +++ b/packages/python-sdk/e2b/template_sync/build_api.py @@ -1,6 +1,6 @@ import time from types import TracebackType -from typing import Callable, Optional, List, Union +from typing import Callable, Dict, Optional, List, Union import httpx from pyqwest import SyncHTTPTransport @@ -113,6 +113,7 @@ def upload_file( resolve_symlinks: bool, gzip: bool, stack_trace: Optional[TracebackType], + headers: Optional[Dict[str, str]] = None, request_timeout: Optional[float] = None, ): # Uploading a large build-context archive can take far longer than the 60s @@ -152,7 +153,9 @@ def upload_file( # Content-Length from the file size—S3 presigned URLs reject # chunked transfer encoding, and reqwest keeps the # Content-Length framing for the streamed body. - response = client.put(url, content=tar_file) + # Headers the API asked for, applied as given (Azure's Put + # Blob requires x-ms-blob-type, which its SAS cannot carry). + response = client.put(url, content=tar_file, headers=headers) response.raise_for_status() finally: # Closing the spooled temp file is best-effort: a failure here diff --git a/packages/python-sdk/e2b/template_sync/main.py b/packages/python-sdk/e2b/template_sync/main.py index 1481966e04..55598cd749 100644 --- a/packages/python-sdk/e2b/template_sync/main.py +++ b/packages/python-sdk/e2b/template_sync/main.py @@ -137,6 +137,7 @@ def _build( resolve_symlinks, gzip, stack_trace, + headers=file_info.headers.to_dict() if file_info.headers else None, request_timeout=request_timeout, ) if on_build_logs: diff --git a/packages/python-sdk/tests/async/template_async/test_upload_file.py b/packages/python-sdk/tests/async/template_async/test_upload_file.py index a8d2993427..2274fc7fb9 100644 --- a/packages/python-sdk/tests/async/template_async/test_upload_file.py +++ b/packages/python-sdk/tests/async/template_async/test_upload_file.py @@ -240,3 +240,61 @@ def failing_close_stream(*args, **kwargs): thread.join(timeout=5) assert state["headers"] is not None + + +async def test_upload_file_sends_the_headers_the_api_returned(tmp_path): + # Azure's Put Blob rejects the request without x-ms-blob-type, and its SAS + # cannot carry a required request header, so the API hands it back with the + # upload link for the client to apply. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + await upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + headers={"x-ms-blob-type": "BlockBlob"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"]["x-ms-blob-type"] == "BlockBlob" + + +async def test_upload_file_adds_no_headers_when_the_api_returns_none(tmp_path): + # S3 and GCS presigned PUTs sign the header set, so the upload must add + # nothing the API did not ask for. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + await upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert "x-ms-blob-type" not in state["headers"] diff --git a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py index a08ae9906a..617ce8a304 100644 --- a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py +++ b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py @@ -236,3 +236,61 @@ def failing_close_stream(*args, **kwargs): thread.join(timeout=5) assert state["headers"] is not None + + +def test_upload_file_sends_the_headers_the_api_returned(tmp_path): + # Azure's Put Blob rejects the request without x-ms-blob-type, and its SAS + # cannot carry a required request header, so the API hands it back with the + # upload link for the client to apply. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + headers={"x-ms-blob-type": "BlockBlob"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"]["x-ms-blob-type"] == "BlockBlob" + + +def test_upload_file_adds_no_headers_when_the_api_returns_none(tmp_path): + # S3 and GCS presigned PUTs sign the header set, so the upload must add + # nothing the API did not ask for. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert "x-ms-blob-type" not in state["headers"] diff --git a/spec/openapi.yml b/spec/openapi.yml index e7787b451f..6df979f231 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -1651,6 +1651,11 @@ components: url: description: Url where the file should be uploaded to type: string + headers: + description: Request headers that must be sent with the upload request + type: object + additionalProperties: + type: string LogLevel: type: string @@ -1948,6 +1953,15 @@ components: type: integer format: uint32 description: Number of sandboxes running on the node + maxSandboxes: + type: integer + format: int64 + description: Node-scoped configured sandbox admission limit. Nonpositive values reject creation. Omitted when unknown or not an orchestrator. + outstandingWork: + type: integer + format: uint64 + minimum: 0 + description: Observed work holds on the node. Omitted when unknown; zero does not authorize deletion. metrics: $ref: "#/components/schemas/NodeMetrics" createSuccesses: @@ -2005,6 +2019,15 @@ components: type: integer format: uint32 description: Number of sandboxes running on the node + maxSandboxes: + type: integer + format: int64 + description: Node-scoped configured sandbox admission limit. Nonpositive values reject creation. Omitted when unknown or not an orchestrator. + outstandingWork: + type: integer + format: uint64 + minimum: 0 + description: Observed work holds on the node. Omitted when unknown; zero does not authorize deletion. metrics: $ref: "#/components/schemas/NodeMetrics" createSuccesses: @@ -3233,6 +3256,8 @@ paths: $ref: "#/components/responses/401" "403": $ref: "#/components/responses/403" + "409": + $ref: "#/components/responses/409" "500": $ref: "#/components/responses/500" @@ -3309,6 +3334,8 @@ paths: $ref: "#/components/responses/400" "401": $ref: "#/components/responses/401" + "409": + $ref: "#/components/responses/409" "500": $ref: "#/components/responses/500" @@ -3410,6 +3437,8 @@ paths: $ref: "#/components/responses/400" "401": $ref: "#/components/responses/401" + "409": + $ref: "#/components/responses/409" "500": $ref: "#/components/responses/500" @@ -3470,6 +3499,8 @@ paths: $ref: "#/components/schemas/TemplateLegacy" "401": $ref: "#/components/responses/401" + "409": + $ref: "#/components/responses/409" "500": $ref: "#/components/responses/500" delete: diff --git a/spec/runtime-ref b/spec/runtime-ref index ce24be88db..3809a41f0c 100644 --- a/spec/runtime-ref +++ b/spec/runtime-ref @@ -1 +1 @@ -debf6bec7a59db73e3407d93ef94da5c82429089 +8cd25be70e2d9e6f9bb457c15fdad51b55bd4bdf