diff --git a/.github/workflows/java-integration-tests.yml b/.github/workflows/java-integration-tests.yml index 51160d9..3257ab1 100644 --- a/.github/workflows/java-integration-tests.yml +++ b/.github/workflows/java-integration-tests.yml @@ -47,7 +47,7 @@ jobs: # Offline unit tests (mock HTTP backend): prove the retry/error/signature logic without the API. - name: Unit tests - run: mvn -B test -Dtest='UnitHttpTest,ReviewFixesTest,ClientMetaTest,SignatureTest,ConfigTest' -DfailIfNoTests=true + run: mvn -B test -Dtest='UnitHttpTest,ReviewFixesTest,ClientMetaTest,SignatureTest,ConfigTest,CompressionTest' -DfailIfNoTests=true # Fail fast if the integration-test secret is missing or empty. Without this guard the # integration tests silently "pass" (JUnit assumptions skip them when APIFY_TOKEN is unset), diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fca00c..df5035b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,35 @@ All notable changes to the Apify Java client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.3] - 2026-07-10 + +### Added + +- Request-body compression now prefers brotli (`Content-Encoding: br`), matching the reference JS + client, via the `brotli4j` native codec. Gzip (`Content-Encoding: gzip`) remains the fallback when + no brotli native binary is available for the running platform. + +### Changed + +- Extended the `User-Agent` OS-token mapping test to assert the emitted token exactly matches the + reference JS client's `os.platform()` value for every platform the JVM can run on (`sunos`, + `freebsd`, `openbsd`, `aix`, in addition to `linux`, `darwin`, `win32`, `android`). + +## [0.1.2] - 2026-07-09 + +### Changed + +- Verified the client against OpenAPI specification version `v2-2026-07-08T143931Z` and bumped + `Version.API_SPEC_VERSION` accordingly. +- Aligned the `User-Agent` OS token with the reference JS client's `os.platform()` token: it now + uses the short, lowercase platform identifier (`linux`, `darwin`, `win32`, `android`, …) instead + of the human-readable `os.name` value. + +### Added + +- Request bodies of 1024 bytes or more are now gzip-compressed and sent with + `Content-Encoding: gzip`. + ## [0.1.1] - 2026-07-07 ### Changed diff --git a/README.md b/README.md index 95fdd5d..ef9d203 100644 --- a/README.md +++ b/README.md @@ -21,18 +21,49 @@ Maven: com.apify apify-client - 0.1.1 + 0.1.3 ``` Gradle: ```groovy -implementation 'com.apify:apify-client:0.1.1' +implementation 'com.apify:apify-client:0.1.3' ``` ## Quick start +A complete, copy-pasteable first program (save as `HelloApify.java`). Populate a `lib/` directory +with the client and its runtime dependencies (from a directory whose `pom.xml` declares the +dependency shown in [Installation](#installation) above), then compile and run against the JVM's +`lib/*` classpath wildcard — quote it so the shell does not expand it: + +```bash +# 1. Collect apify-client and its runtime dependencies (Jackson, brotli4j codecs, …) into lib/. +mvn dependency:copy-dependencies -DoutputDirectory=lib -DincludeScope=runtime + +# 2. Compile and run. '.' is for the compiled HelloApify.class; lib/* is the JVM classpath wildcard. +javac -cp 'lib/*' HelloApify.java +java -cp '.:lib/*' HelloApify # Windows: java -cp ".;lib/*" HelloApify +``` + +```java +import com.apify.client.ApifyClient; +import com.apify.client.ActorRun; +import com.apify.client.ActorStartOptions; + +class HelloApify { + public static void main(String[] args) { + ApifyClient client = ApifyClient.create("my-api-token"); + ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L); + System.out.println("Run " + run.getId() + " finished with status " + run.getStatus()); + } +} +``` + +The remaining snippets below are fragments that assume a configured `client` (see the imports note +after the next block): + ```java ApifyClient client = ApifyClient.create("my-api-token"); @@ -43,7 +74,7 @@ ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOption // Read items from the run's default dataset. PaginationList items = client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()); -System.out.println("Item count: " + items.getCount()); +System.out.println("Items in this page: " + items.getCount()); ``` All public client types live in the `com.apify.client` package (e.g. `import com.apify.client.*;`). @@ -122,9 +153,9 @@ try { ## Versioning -- `Version.CLIENT_VERSION` — the semantic version of this client (`0.1.1`). +- `Version.CLIENT_VERSION` — the semantic version of this client (`0.1.3`). - `Version.API_SPEC_VERSION` — the Apify OpenAPI specification version this client was verified - against (`v2-2026-07-07T132551Z`). + against (`v2-2026-07-08T143931Z`). Changes to the public interface other than additive ones are considered breaking changes and follow [Semantic Versioning](https://semver.org/). diff --git a/docs/README.md b/docs/README.md index d390635..0f6cfbe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,8 +5,12 @@ > in production and report issues on the repository. This directory documents the public API of the Apify Java client, organized by resource. Each page -lists the available methods with their parameters and short, runnable snippets. For an overview, -configuration, error handling and the full resource table, see the [top-level README](../README.md). +lists the available methods with their parameters and short snippets. The snippets are code +fragments that assume a configured `client` and the imports listed below, not standalone `main` +programs; for complete, runnable programs see [examples.md](examples.md) and +[`src/test/java/com/apify/client/examples/`](../src/test/java/com/apify/client/examples). For an +overview, configuration, error handling and the full resource table, see the +[top-level README](../README.md). All snippets assume a configured client: diff --git a/docs/examples.md b/docs/examples.md index 23ab709..0266997 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,9 +1,11 @@ # Runnable examples -Each example below is a self-contained snippet assuming a configured `client`. The same programs -live under [`src/test/java/com/apify/client/examples/`](../src/test/java/com/apify/client/examples) -and are executed end-to-end against the live API by the `Test examples` CI step (see -`ExamplesTest`), so they are guaranteed to stay runnable. +Each example below is a code fragment that assumes a configured `client` and the imports listed in +the [documentation index](README.md#imports-and-dependencies); it is not a standalone `main`. The +complete, runnable programs live under +[`src/test/java/com/apify/client/examples/`](../src/test/java/com/apify/client/examples) and are +executed end-to-end against the live API by the `Test examples` CI step (see `ExamplesTest`), so +they are guaranteed to stay runnable. ## Run a store Actor and read its default dataset @@ -11,7 +13,7 @@ and are executed end-to-end against the live API by the `Test examples` CI step ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L); PaginationList items = client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()); -System.out.println("Item count: " + items.getCount()); +System.out.println("Items in this page: " + items.getCount()); ``` ## Each storage: create, push, read diff --git a/pom.xml b/pom.xml index a1d1372..69c1d5a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.apify apify-client - 0.1.1 + 0.1.3 jar Apify Java Client @@ -41,6 +41,7 @@ UTF-8 2.17.2 5.10.2 + 1.23.0 @@ -55,6 +56,46 @@ ${jackson.version} + + + com.aayushatharva.brotli4j + brotli4j + ${brotli4j.version} + + + com.aayushatharva.brotli4j + native-linux-x86_64 + ${brotli4j.version} + runtime + + + com.aayushatharva.brotli4j + native-linux-aarch64 + ${brotli4j.version} + runtime + + + com.aayushatharva.brotli4j + native-osx-x86_64 + ${brotli4j.version} + runtime + + + com.aayushatharva.brotli4j + native-osx-aarch64 + ${brotli4j.version} + runtime + + + com.aayushatharva.brotli4j + native-windows-x86_64 + ${brotli4j.version} + runtime + + org.junit.jupiter junit-jupiter diff --git a/src/main/java/com/apify/client/ApifyClientBuilder.java b/src/main/java/com/apify/client/ApifyClientBuilder.java index a073f1c..ef4a377 100644 --- a/src/main/java/com/apify/client/ApifyClientBuilder.java +++ b/src/main/java/com/apify/client/ApifyClientBuilder.java @@ -133,7 +133,7 @@ static boolean defaultIsAtHome() { * ApifyClient/{version} ({os}; Java/{javaVersion}); isAtHome/{true|false}}. */ static String buildUserAgent(String suffix, BooleanSupplier isAtHomeFn) { - String os = System.getProperty("os.name", "unknown").toLowerCase(java.util.Locale.ROOT); + String os = platformToken(); String javaVersion = System.getProperty("java.version", "unknown"); String atHome = isAtHomeFn.getAsBoolean() ? "true" : "false"; String ua = @@ -150,4 +150,55 @@ static String buildUserAgent(String suffix, BooleanSupplier isAtHomeFn) { } return ua; } + + /** + * Returns the short, lowercase OS token used in the {@code User-Agent}. It matches the + * identifiers emitted by the reference JS client's {@code os.platform()} (Node) — {@code linux}, + * {@code darwin}, {@code win32}, {@code android}, etc. — so all Apify clients report the platform + * uniformly. Java's {@code os.name} system property returns human-readable names ({@code Linux}, + * {@code Mac OS X}, {@code Windows 10}), so it is mapped to the aligned token here. + */ + static String platformToken() { + return platformToken(System.getProperty("os.name", ""), System.getProperty("java.vm.name", "")); + } + + /** + * Maps the given {@code os.name} / {@code java.vm.name} property values to the aligned platform + * token. Split out from {@link #platformToken()} as a pure function so the mapping can be tested + * for every platform without depending on the host the tests run on. + */ + static String platformToken(String osName, String vmName) { + // Android runs on a Linux kernel (os.name == "Linux"); detect it via its VM name ("Dalvik"). + String vm = vmName.toLowerCase(java.util.Locale.ROOT); + if (vm.contains("dalvik")) { + return "android"; + } + String os = osName.toLowerCase(java.util.Locale.ROOT); + // macOS/Darwin is checked before Windows because the literal "darwin" contains "win"; the + // reverse order would misclassify a "Darwin" os.name as "win32". + if (os.contains("mac") || os.contains("darwin")) { + return "darwin"; + } + if (os.contains("win")) { + return "win32"; + } + if (os.contains("nux") || os.contains("nix")) { + return "linux"; + } + if (os.contains("aix")) { + return "aix"; + } + if (os.contains("sunos") || os.contains("solaris")) { + return "sunos"; + } + if (os.contains("freebsd")) { + return "freebsd"; + } + if (os.contains("openbsd")) { + return "openbsd"; + } + // Fallback: the first whitespace-delimited word, lowercased, to keep the token short and + // stable. + return os.isEmpty() ? "unknown" : os.split("\\s+")[0]; + } } diff --git a/src/main/java/com/apify/client/HttpClientCore.java b/src/main/java/com/apify/client/HttpClientCore.java index 54ce7f5..24d04b7 100644 --- a/src/main/java/com/apify/client/HttpClientCore.java +++ b/src/main/java/com/apify/client/HttpClientCore.java @@ -1,14 +1,19 @@ package com.apify.client; +import com.aayushatharva.brotli4j.Brotli4jLoader; +import com.aayushatharva.brotli4j.encoder.Encoder; import com.fasterxml.jackson.databind.JsonNode; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; +import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.ThreadLocalRandom; +import java.util.zip.GZIPOutputStream; /** * The orchestrating HTTP client shared by every resource client. It owns the backend, the optional @@ -29,6 +34,47 @@ final class HttpClientCore { /** Exponential-backoff multiplier applied to the inter-retry delay after each attempt. */ private static final int BACKOFF_FACTOR = 2; + /** + * Request bodies at or above this size (in bytes) are compressed before being sent. Small bodies + * are left uncompressed because the CPU cost outweighs the transfer saving. Matches the reference + * JS client's 1024-byte threshold. + */ + private static final int MIN_COMPRESS_BYTES = 1024; + + /** Header announcing the request-body content coding to the server. */ + private static final String CONTENT_ENCODING_HEADER = "Content-Encoding"; + + /** {@code Content-Encoding} value for brotli-compressed bodies (matches the reference client). */ + private static final String ENCODING_BROTLI = "br"; + + /** {@code Content-Encoding} value for gzip-compressed bodies (the fallback coding). */ + private static final String ENCODING_GZIP = "gzip"; + + /** + * Whether a brotli native codec loaded for the running platform. Resolved once at class load: + * brotli4j needs a platform-specific native library, so on platforms without one (or if loading + * fails) the client falls back to gzip. Mirrors the reference JS client, which prefers brotli and + * falls back to gzip. + */ + private static final boolean BROTLI_AVAILABLE = detectBrotli(); + + private static boolean detectBrotli() { + try { + Brotli4jLoader.ensureAvailability(); + return true; + } catch (Throwable t) { + // Native codec unavailable (no bundled binary for this OS/arch, or a link error). Catch + // Throwable because native loading can raise UnsatisfiedLinkError/NoClassDefFoundError, not + // just Exception. The client stays fully functional using gzip. + return false; + } + } + + /** Reports whether the brotli path is active on this platform (package-private for tests). */ + static boolean brotliAvailable() { + return BROTLI_AVAILABLE; + } + private final HttpBackend backend; private final String token; private final String userAgent; @@ -105,6 +151,16 @@ ApiResponse callWithHeaders( Duration baseTimeout, boolean doNotRetryTimeouts) { + // Compress the body once, up front, so every retry reuses the same encoded payload. + byte[] requestBody = body; + Map headers = extraHeaders; + if (shouldCompress(requestBody, extraHeaders)) { + Compressed compressed = compress(requestBody, BROTLI_AVAILABLE); + requestBody = compressed.body; + headers = new LinkedHashMap<>(extraHeaders == null ? Map.of() : extraHeaders); + headers.put(CONTENT_ENCODING_HEADER, compressed.encoding); + } + Duration delay = retry.minDelayBetweenRetries; int maxAttempts = retry.maxRetries + 1; String path = extractPath(url); @@ -115,7 +171,12 @@ ApiResponse callWithHeaders( try { ApiResponse resp = doAttempt( - method, url, body, contentType, extraHeaders, attemptTimeout(baseTimeout, attempt)); + method, + url, + requestBody, + contentType, + headers, + attemptTimeout(baseTimeout, attempt)); if (resp.statusCode < MAX_SUCCESS_STATUS) { return resp; } @@ -204,6 +265,78 @@ private Duration attemptTimeout(Duration base, int attempt) { return minDuration(scaled, retry.timeout); } + /** + * Reports whether a request body should be compressed: it must be present, at least {@link + * #MIN_COMPRESS_BYTES} bytes, and the caller must not have already set a {@code Content-Encoding} + * header (which would mean the body is pre-encoded). + */ + private static boolean shouldCompress(byte[] body, Map extraHeaders) { + if (body == null || body.length < MIN_COMPRESS_BYTES) { + return false; + } + if (extraHeaders != null) { + for (String key : extraHeaders.keySet()) { + if (CONTENT_ENCODING_HEADER.equalsIgnoreCase(key)) { + return false; + } + } + } + return true; + } + + /** + * A compressed request body together with the {@code Content-Encoding} token that describes it. + */ + static final class Compressed { + final byte[] body; + final String encoding; + + Compressed(byte[] body, String encoding) { + this.body = body; + this.encoding = encoding; + } + } + + /** + * Compresses a request body, preferring brotli and falling back to gzip, matching the reference + * JS client. When {@code preferBrotli} is {@code true} the body is brotli-encoded ({@code + * Content-Encoding: br}); otherwise it is gzip-encoded ({@code Content-Encoding: gzip}). Callers + * pass {@link #BROTLI_AVAILABLE}; making the coding an explicit parameter keeps this a pure + * function of its inputs rather than of hidden static state. Package-private. + */ + static Compressed compress(byte[] data, boolean preferBrotli) { + return preferBrotli + ? new Compressed(brotli(data), ENCODING_BROTLI) + : new Compressed(gzip(data), ENCODING_GZIP); + } + + /** Brotli-compresses a request body using the loaded native codec. */ + private static byte[] brotli(byte[] data) { + try { + return Encoder.compress(data); + } catch (IOException e) { + // Encoding an in-memory byte[] performs no real I/O, so this is unreachable in practice. + throw new TransportException(e); + } + } + + /** + * Gzip-compresses a request body. Used as the fallback coding when no brotli native codec is + * available for the running platform; the JDK always ships a gzip codec ({@link + * GZIPOutputStream}). + */ + private static byte[] gzip(byte[] data) { + ByteArrayOutputStream out = new ByteArrayOutputStream(data.length); + try (GZIPOutputStream gz = new GZIPOutputStream(out)) { + gz.write(data); + } catch (IOException e) { + // Compressing an in-memory byte[] cannot perform real I/O, so this is unreachable in + // practice. + throw new TransportException(e); + } + return out.toByteArray(); + } + private static boolean isStatusRetryable(int status) { return status == RATE_LIMIT_EXCEEDED || status >= MIN_SERVER_ERROR; } diff --git a/src/main/java/com/apify/client/Version.java b/src/main/java/com/apify/client/Version.java index c15e29d..290ebee 100644 --- a/src/main/java/com/apify/client/Version.java +++ b/src/main/java/com/apify/client/Version.java @@ -13,13 +13,13 @@ public final class Version { * The semantic version of this client library (see SemVer). * Changes to the public interface other than additive ones are considered breaking changes. */ - public static final String CLIENT_VERSION = "0.1.1"; + public static final String CLIENT_VERSION = "0.1.3"; /** * The version of the Apify OpenAPI specification this client was generated and verified against. * Corresponds to the {@code info.version} field of the Apify OpenAPI document. */ - public static final String API_SPEC_VERSION = "v2-2026-07-07T132551Z"; + public static final String API_SPEC_VERSION = "v2-2026-07-08T143931Z"; private Version() {} } diff --git a/src/test/java/com/apify/client/ClientMetaTest.java b/src/test/java/com/apify/client/ClientMetaTest.java index b1159a4..d225e21 100644 --- a/src/test/java/com/apify/client/ClientMetaTest.java +++ b/src/test/java/com/apify/client/ClientMetaTest.java @@ -1,6 +1,7 @@ package com.apify.client; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; @@ -25,6 +26,45 @@ void userAgentFormat() { assertTrue(Character.isDigit(after.charAt(0)), "Java version must be a real version: " + ua); } + @Test + void userAgentOsTokenIsShortAndLowercase() { + // The OS token must be a short, lowercase platform identifier (linux/darwin/win32/…), matching + // the reference JS client's os.platform() output — never the human-readable os.name. + String token = ApifyClientBuilder.platformToken(); + assertFalse(token.isEmpty(), "platform token must not be empty"); + assertEquals( + token.toLowerCase(java.util.Locale.ROOT), token, "token must be lowercase: " + token); + assertFalse(token.contains(" "), "token must not contain spaces: " + token); + + String ua = ApifyClient.create("token").getUserAgent(); + assertTrue(ua.contains("(" + token + "; Java/"), ua); + } + + @Test + void platformTokenMapsEachOsToAlignedIdentifier() { + // Every token must exactly match Node's os.platform() output, so all Apify clients agree. + assertEquals("linux", ApifyClientBuilder.platformToken("Linux", "OpenJDK 64-Bit Server VM")); + // "Mac OS X" and a bare "Darwin" must both map to darwin. "Darwin" contains "win", so this + // guards the ordering fix that keeps it from being misread as win32. + assertEquals( + "darwin", ApifyClientBuilder.platformToken("Mac OS X", "OpenJDK 64-Bit Server VM")); + assertEquals("darwin", ApifyClientBuilder.platformToken("Darwin", "OpenJDK 64-Bit Server VM")); + assertEquals("win32", ApifyClientBuilder.platformToken("Windows 10", "Java HotSpot(TM) VM")); + assertEquals( + "win32", ApifyClientBuilder.platformToken("Windows Server 2022", "Java HotSpot(TM) VM")); + // Android reports os.name == "Linux" but runs on the Dalvik VM. + assertEquals("android", ApifyClientBuilder.platformToken("Linux", "Dalvik")); + // The Unix platforms Node's os.platform() can emit must map to the identical token. + assertEquals("sunos", ApifyClientBuilder.platformToken("SunOS", "OpenJDK 64-Bit Server VM")); + assertEquals("sunos", ApifyClientBuilder.platformToken("Solaris", "OpenJDK 64-Bit Server VM")); + assertEquals( + "freebsd", ApifyClientBuilder.platformToken("FreeBSD", "OpenJDK 64-Bit Server VM")); + assertEquals( + "openbsd", ApifyClientBuilder.platformToken("OpenBSD", "OpenJDK 64-Bit Server VM")); + assertEquals("aix", ApifyClientBuilder.platformToken("AIX", "OpenJDK 64-Bit Server VM")); + assertEquals("unknown", ApifyClientBuilder.platformToken("", "")); + } + @Test void userAgentIsAtHomeFlag() { ApifyClient off = ApifyClient.builder().token("t").isAtHomeFn(() -> false).build(); diff --git a/src/test/java/com/apify/client/CompressionTest.java b/src/test/java/com/apify/client/CompressionTest.java new file mode 100644 index 0000000..f7c6c27 --- /dev/null +++ b/src/test/java/com/apify/client/CompressionTest.java @@ -0,0 +1,160 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import com.aayushatharva.brotli4j.Brotli4jLoader; +import com.aayushatharva.brotli4j.decoder.Decoder; +import com.aayushatharva.brotli4j.decoder.DirectDecompress; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Locale; +import java.util.zip.GZIPInputStream; +import org.junit.jupiter.api.Test; + +/** + * Offline tests for request-body compression. Mirrors the reference JS client: bodies of at least + * 1024 bytes are compressed and announced via {@code Content-Encoding}, smaller bodies are sent + * verbatim. The client prefers brotli ({@code br}) and falls back to gzip; both codings are + * exercised here — the brotli/gzip decision (a pure function) directly, and the live client path + * via a real request. + */ +class CompressionTest { + + private static final String STORE_ID = "store-id"; + private static final String RECORD_KEY = "record"; + private static final String CONTENT_TYPE = "application/octet-stream"; + + private static ApifyClient client(MockBackend backend) { + return ApifyClient.builder().token("t").httpBackend(backend).maxRetries(0).build(); + } + + private static byte[] gunzip(byte[] data) throws IOException { + try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(data))) { + return in.readAllBytes(); + } + } + + private static byte[] unbrotli(byte[] data) throws IOException { + Brotli4jLoader.ensureAvailability(); + DirectDecompress result = Decoder.decompress(data); + return result.getDecompressedData(); + } + + private static byte[] payload(int size, byte fill) { + byte[] p = new byte[size]; + Arrays.fill(p, fill); + return p; + } + + /** + * Whether a brotli native codec is expected to load here. brotli4j bundles glibc x86_64/aarch64 + * binaries (what CI's ubuntu-latest and typical server runtimes use); on musl (Alpine), 32-bit, + * or other CPU arches none ships and the client falls back to gzip. Tests that force the brotli + * codec are gated on this so they run where brotli is expected and skip (not error) where gzip + * applies. + */ + private static boolean nativeBrotliExpected() { + String os = System.getProperty("os.name", "").toLowerCase(Locale.ROOT); + String arch = System.getProperty("os.arch", "").toLowerCase(Locale.ROOT); + boolean linux = os.contains("nux"); + boolean commonArch = + arch.equals("amd64") + || arch.equals("x86_64") + || arch.equals("aarch64") + || arch.equals("arm64"); + boolean musl = + Files.exists(Path.of("/lib/ld-musl-x86_64.so.1")) + || Files.exists(Path.of("/lib/ld-musl-aarch64.so.1")); + return linux && commonArch && !musl; + } + + // --- Pure encoding decision: both paths are directly reachable and verified round-trip. --- + + @Test + void brotliPathEncodesAsBrAndRoundTrips() throws IOException { + assumeTrue(HttpClientCore.brotliAvailable(), "no brotli native codec on this platform"); + byte[] payload = payload(4096, (byte) 'a'); + HttpClientCore.Compressed c = HttpClientCore.compress(payload, true); + + assertEquals("br", c.encoding, "brotli path must announce Content-Encoding: br"); + assertTrue(c.body.length < payload.length, "brotli-compressed body should be smaller"); + assertArrayEquals( + payload, unbrotli(c.body), "server must recover the original body from brotli"); + } + + @Test + void gzipPathEncodesAsGzipAndRoundTrips() throws IOException { + byte[] payload = payload(4096, (byte) 'a'); + HttpClientCore.Compressed c = HttpClientCore.compress(payload, false); + + assertEquals("gzip", c.encoding, "gzip fallback must announce Content-Encoding: gzip"); + assertTrue(c.body.length < payload.length, "gzip-compressed body should be smaller"); + assertArrayEquals(payload, gunzip(c.body), "server must recover the original body from gzip"); + } + + // --- Brotli native codec must load where a native is bundled (CI/runtime), so brotli runs. --- + + @Test + void brotliNativeCodecLoadsWhereBundled() { + assumeTrue(nativeBrotliExpected(), "no brotli native bundled for this platform; gzip fallback"); + assertTrue( + HttpClientCore.brotliAvailable(), + "brotli native codec must load on glibc x86_64/aarch64 so the preferred brotli path runs"); + } + + // --- Live client path: uses the preferred coding and round-trips through the backend. --- + + @Test + void largeBodyIsCompressedWithPreferredEncoding() throws IOException { + MockBackend backend = MockBackend.ofConstant(201, ""); + byte[] payload = payload(4096, (byte) 'a'); + + client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE); + + boolean brotli = HttpClientCore.brotliAvailable(); + String expected = brotli ? "br" : "gzip"; + assertEquals( + expected, + backend.lastHeaders.firstValue("Content-Encoding").orElse(null), + "large body must be sent with the platform's preferred Content-Encoding"); + assertTrue(backend.lastBodyBytes.length < payload.length, "compressed body should be smaller"); + byte[] recovered = brotli ? unbrotli(backend.lastBodyBytes) : gunzip(backend.lastBodyBytes); + assertArrayEquals(payload, recovered, "server must be able to recover the original body"); + } + + @Test + void smallBodyIsNotCompressed() { + MockBackend backend = MockBackend.ofConstant(201, ""); + byte[] payload = payload(16, (byte) 'b'); + + client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE); + + assertFalse( + backend.lastHeaders.firstValue("Content-Encoding").isPresent(), + "sub-threshold body must not carry a Content-Encoding header"); + assertArrayEquals(payload, backend.lastBodyBytes, "small body must be sent verbatim"); + } + + @Test + void bodyExactlyAtThresholdIsCompressed() throws IOException { + MockBackend backend = MockBackend.ofConstant(201, ""); + byte[] payload = payload(1024, (byte) 'c'); + + client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE); + + boolean brotli = HttpClientCore.brotliAvailable(); + assertEquals( + brotli ? "br" : "gzip", + backend.lastHeaders.firstValue("Content-Encoding").orElse(null), + "a body at exactly the 1024-byte threshold must be compressed"); + byte[] recovered = brotli ? unbrotli(backend.lastBodyBytes) : gunzip(backend.lastBodyBytes); + assertArrayEquals(payload, recovered); + } +} diff --git a/src/test/java/com/apify/client/MockBackend.java b/src/test/java/com/apify/client/MockBackend.java index fb54c0d..16dc0d9 100644 --- a/src/test/java/com/apify/client/MockBackend.java +++ b/src/test/java/com/apify/client/MockBackend.java @@ -43,6 +43,7 @@ static final class Scripted { HttpHeaders lastHeaders; String lastUrl; String lastBody; + byte[] lastBodyBytes; final List bodies = new ArrayList<>(); MockBackend(List responses) { @@ -71,7 +72,8 @@ public synchronized HttpResponse send(HttpRequest request) throws IOExce int idx = calls++; lastHeaders = request.headers(); lastUrl = request.uri().toString(); - lastBody = readBody(request); + lastBodyBytes = readBody(request); + lastBody = lastBodyBytes == null ? null : new String(lastBodyBytes, StandardCharsets.UTF_8); if (lastBody != null) { bodies.add(lastBody); } @@ -105,7 +107,7 @@ public synchronized HttpResponse sendStreaming(HttpRequest request) request.uri(), r.status, new java.io.ByteArrayInputStream(r.body)); } - private static String readBody(HttpRequest request) { + private static byte[] readBody(HttpRequest request) { Optional publisher = request.bodyPublisher(); if (publisher.isEmpty() || publisher.get().contentLength() == 0) { return null; @@ -143,7 +145,7 @@ public void onComplete() { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - return out.toString(StandardCharsets.UTF_8); + return out.toByteArray(); } /** Minimal {@link HttpResponse} implementation over a byte[] body. */ diff --git a/src/test/java/com/apify/client/examples/RunAndLastRunStorages.java b/src/test/java/com/apify/client/examples/RunAndLastRunStorages.java index 6f49757..bed3213 100644 --- a/src/test/java/com/apify/client/examples/RunAndLastRunStorages.java +++ b/src/test/java/com/apify/client/examples/RunAndLastRunStorages.java @@ -26,7 +26,7 @@ public static void main(String[] args) { System.out.println("Last run: " + run.getId()); var items = client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()); - System.out.println("Last run dataset item count: " + items.getCount()); + System.out.println("Items in this page of the last run's dataset: " + items.getCount()); client.keyValueStore(run.getDefaultKeyValueStoreId()).getRecord("OUTPUT"); } } diff --git a/src/test/java/com/apify/client/examples/RunStoreActor.java b/src/test/java/com/apify/client/examples/RunStoreActor.java index 480785f..561e255 100644 --- a/src/test/java/com/apify/client/examples/RunStoreActor.java +++ b/src/test/java/com/apify/client/examples/RunStoreActor.java @@ -19,6 +19,6 @@ public static void main(String[] args) { System.out.println("Run " + run.getId() + " finished with status " + run.getStatus()); var items = client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()); - System.out.println("Default dataset item count: " + items.getCount()); + System.out.println("Items in this page of the default dataset: " + items.getCount()); } }