Optimize API clients and Jetty runtime for Virtual Threads and high throughput

api/src/main/java/com/google/appengine/api/datastore/AsyncDatastoreServiceInternal.java

@@ -26,4 +26,7 @@ interface AsyncDatastoreServiceInternal extends AsyncDatastoreService {
 
   /** See {@link DatastoreService#allocateIdRange(KeyRange)}. */
   Future<DatastoreService.KeyRangeState> allocateIdRange(final KeyRange range);
+
+  /** Returns the default transaction stack provider. */
+  TransactionStack getDefaultTxnProvider();
 }

api/src/main/java/com/google/appengine/api/datastore/BaseAsyncDatastoreServiceImpl.java

@@ -393,6 +393,11 @@ public Future<Transaction> beginTransaction(TransactionOptions options) {
     return new FutureHelper.FakeFuture<Transaction>(txn);
   }
 
+  @Override
+  public TransactionStack getDefaultTxnProvider() {
+    return defaultTxnProvider;
+  }
+
   private Transaction createTransaction(TransactionOptions options, boolean isExplicit) {
     return new TransactionImpl(
         datastoreServiceConfig.getAppIdNamespace().getAppId(),

api/src/main/java/com/google/appengine/api/datastore/DatastoreServiceImpl.java

@@ -18,6 +18,7 @@
 
 import static com.google.appengine.api.datastore.FutureHelper.quietGet;
 
+import com.google.apphosting.api.ApiProxy;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -30,6 +31,8 @@
 final class DatastoreServiceImpl implements DatastoreService {
 
   private final AsyncDatastoreServiceInternal async;
+  static final long BEGIN_TXN_RETRY_DELAY_MS = 100;
+  private static final int MAX_RETRIES = Integer.getInteger("appengine.datastore.retries", 1);
 
   public DatastoreServiceImpl(AsyncDatastoreServiceInternal async) {
     this.async = async;
@@ -137,12 +140,41 @@ public KeyRangeState allocateIdRange(KeyRange range) {
 
   @Override
   public Transaction beginTransaction() {
-    return quietGet(async.beginTransaction());
+    return beginTransaction(TransactionOptions.Builder.withDefaults());
   }
 
   @Override
   public Transaction beginTransaction(TransactionOptions options) {
-    return quietGet(async.beginTransaction(options));
+    int retries = 0;
+    long delay = BEGIN_TXN_RETRY_DELAY_MS;
+    while (true) {
+      Transaction tx = null;
+      try {
+        tx = quietGet(async.beginTransaction(options));
+        tx.getId(); // Force handle resolution
+        return tx;
+      } catch (DatastoreFailureException
+          | DatastoreTimeoutException
+          | ApiProxy.RPCFailedException e) {
+        if (tx != null) {
+          try {
+            tx.rollbackAsync();
+          } catch (Exception ignored) {
+          }
+          async.getDefaultTxnProvider().remove(tx);
+        }
+        if (++retries > MAX_RETRIES) {
+          throw e;
+        }
+        try {
+          Thread.sleep(delay);
+        } catch (InterruptedException ie) {
+          Thread.currentThread().interrupt();
+          throw e;
+        }
+        delay *= 2;
+      }
+    }
   }
 
   @Override

runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java

@@ -339,8 +339,8 @@ protected void connectContainer() throws Exception {
     configuration.setSendDateHeader(false);
     configuration.setSendServerVersion(false);
     configuration.setSendXPoweredBy(false);
-    // Try to enable virtual threads if requested on java21:
-    if (Boolean.getBoolean("appengine.use.virtualthreads")) {
+    // Try to enable virtual threads if requested on Java 21+:
+    if (Boolean.getBoolean("appengine.use.virtualthreads") && Runtime.version().feature() >= 21) {
       QueuedThreadPool threadPool = new QueuedThreadPool();
       threadPool.setVirtualThreadsExecutor(VirtualThreads.getDefaultVirtualThreadsExecutor());
       server = new Server(threadPool);

runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/JettyContainerService.java

@@ -344,8 +344,8 @@ protected void connectContainer() throws Exception {
     configuration.setSendDateHeader(false);
     configuration.setSendServerVersion(false);
     configuration.setSendXPoweredBy(false);
-    // Try to enable virtual threads if requested on java21:
-    if (Boolean.getBoolean("appengine.use.virtualthreads")) {
+    // Try to enable virtual threads if requested on Java 21+:
+    if (Boolean.getBoolean("appengine.use.virtualthreads") && Runtime.version().feature() >= 21) {
       QueuedThreadPool threadPool = new QueuedThreadPool();
       threadPool.setVirtualThreadsExecutor(VirtualThreads.getDefaultVirtualThreadsExecutor());
       server = new Server(threadPool);

runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java

@@ -48,8 +48,6 @@
 import java.io.InputStreamReader;
 import java.util.Objects;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ForkJoinPool;
 import org.eclipse.jetty.http.CookieCompliance;
 import org.eclipse.jetty.http.HttpCompliance;
 import org.eclipse.jetty.http.MultiPartCompliance;
@@ -103,17 +101,11 @@ public void start(String serverInfo, ServletEngineAdapter.Config runtimeOptions)
     boolean isHttpConnectorMode = Boolean.getBoolean(HTTP_CONNECTOR_MODE);
     QueuedThreadPool threadPool =
         new QueuedThreadPool(MAX_THREAD_POOL_THREADS, MIN_THREAD_POOL_THREADS);
-    // Try to enable virtual threads if requested and on java21:
+    // Try to enable virtual threads if requested and on Java 21+:
     if (Boolean.getBoolean("appengine.use.virtualthreads")
-        && ("java21".equals(GAE_RUNTIME) || "java25".equals(GAE_RUNTIME))) {
-      int maxParallelism = getMaxSafeCarrierParallelism();
-      Executor virtualThreadsExecutor =
-          new ForkJoinPool(
-              maxParallelism, ForkJoinPool.defaultForkJoinWorkerThreadFactory, null, true);
-      threadPool.setVirtualThreadsExecutor(virtualThreadsExecutor);
-      logger.atInfo().log(
-          "Configuring Appengine web server virtual threads with capped carrier parallelism: %d",
-          maxParallelism);
+        && Runtime.version().feature() >= 21) {
+      threadPool.setVirtualThreadsExecutor(VirtualThreads.getDefaultVirtualThreadsExecutor());
+      logger.atInfo().log("Configuring Appengine web server virtual threads.");
     }
 
     server =
@@ -278,24 +270,4 @@ public void serviceRequest(UPRequest upRequest, MutableUpResponse upResponse) th
       upResponse.setErrorMessage("Unexpected Error: " + error);
     }
   }
-
-  /**
-   * Calculates a safe maximum carrier thread count based on GAE sandbox memory boundaries to
-   * prevent OS scheduling thrashing on fractional/low-core instances.
-   */
-  static int getMaxSafeCarrierParallelism() {
-    return getMaxSafeCarrierParallelism(System.getenv("GAE_MEMORY_MB"));
-  }
-
-  static int getMaxSafeCarrierParallelism(String memoryMbStr) {
-    if (memoryMbStr == null || memoryMbStr.isEmpty()) {
-      return 4; // Conservative default cap for standard runtimes
-    }
-    try {
-      int memoryMb = Integer.parseInt(memoryMbStr);
-      return memoryMb <= 512 ? 1 : memoryMb <= 1024 ? 2 : 4;
-    } catch (NumberFormatException e) {
-      return 4; // Safety Fallback
-    }
-  }
 }

runtime/runtime_impl_jetty121/API_CLIENTS.md

@@ -0,0 +1,156 @@
+<!--
+ Copyright 2021 Google LLC
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+# App Engine API Client Configuration
+
+The App Engine Java runtime communicates with Google Cloud APIs (such as
+Datastore, Task Queue, and Memcache) using an HTTP-based RPC mechanism. The
+runtime includes two HTTP client implementations for this purpose: a default
+client based on Jetty and an alternative client using the JDK's built-in HTTP
+facilities.
+
+This document describes both clients and how to configure them using environment
+variables and Java system properties.
+
+## Jetty HTTP Client (Default)
+
+The Jetty HTTP client is the default client used by the runtime. It is based on
+the [Eclipse Jetty](https://eclipse.dev/jetty/) HTTP client and is optimized for
+high performance and efficient connection management.
+
+By default, the client is configured to allow a maximum of 100 concurrent
+threads and 100 concurrent connections for API calls. These limits help
+prevent memory exhaustion on smaller App Engine instance types (like F1 or F2)
+and avoid overwhelming backend services during sudden traffic spikes or high
+rates of failing requests with aggressive retry logic. If the thread limit is
+reached, subsequent requests are queued until a thread becomes available.
+
+### Configuration
+
+You can configure the Jetty client using the following environment variables
+and system properties:
+
+*   **`APPENGINE_API_MAX_CONNECTIONS`** (Environment Variable): Sets the
+    maximum number of concurrent connections in the HTTP client pool.
+    *   Default: `100`
+*   **`APPENGINE_API_MAX_THREADS`** (Environment Variable): Sets the
+    maximum number of concurrent threads for executing API calls. If unset,
+    this also defaults to 100. This is the most direct way to control API
+    call throughput and prevent backend overload. If set to lower values (e.g. 5 or 8),
+    `minThreads` automatically scales down (`min(10, maxThreads)`) to prevent startup failures.
+    *   Default: `100`
+*   **`APPENGINE_API_CALLS_IDLE_TIMEOUT_MS`** (Environment Variable): Sets
+    the idle timeout in milliseconds for connections in the connection pool.
+    Connections that are idle for longer than this duration may be closed.
+    *   Default: `58000` (58 seconds)
+*   **`appengine.api.use.virtualthreads`** (Java System Property): If set to `true` on
+    Java 21+, the client will use Java Virtual Threads to execute API requests, avoiding
+    platform thread stack memory overhead while honoring pool and connection limits.
+    *   Default: `false`
+
+## JDK HTTP Client
+
+The JDK HTTP client uses Java's built-in `HttpURLConnection` for API calls. It
+is provided as an alternative to the Jetty client and can be useful for
+troubleshooting network- or connection-related issues that might be specific to
+one client implementation. In general, it may be less performant than the
+default Jetty client.
+
+To use the JDK client instead of the Jetty client, set the
+`APPENGINE_API_CALLS_USING_JDK_CLIENT` environment variable to any non-null value
+(e.g., `true`).
+
+### Configuration
+
+When using the JDK client, you can configure its behavior with the following
+settings:
+
+*   **`appengine.api.use.virtualthreads`** (Java System Property): If set to `true`,
+    the JDK client will use Java Virtual Threads (when available on the JVM) to
+    handle API requests. This avoids platform thread stack size overhead while
+    keeping concurrent in-flight calls safely throttled via a semaphore to prevent
+    backend overload and retry storms. Note that this is different than
+    `appengine.use.virtualthreads`.
+    *   Default: `false`
+
+The JDK client controls concurrency using the following environment variables (both when using virtual threads and when using the platform thread pool):
+
+*   **`APPENGINE_API_MAX_CONNECTIONS`** (Environment Variable): Sets the
+    maximum number of concurrent connections in the HTTP client pool.
+    *   Default: `100`
+*   **`APPENGINE_API_MAX_THREADS`** (Environment Variable): Sets the
+    maximum number of concurrent API calls (via thread pool size or virtual thread concurrency semaphore).
+    This limits the number of concurrent in-flight API calls to prevent overwhelming
+    the backend.
+    *   Default: `100`
+
+## Datastore-Specific Configuration
+
+### `beginTransaction` Retries
+
+In addition to the HTTP client settings, the Datastore client library includes
+specific retry logic for `beginTransaction()` calls. These calls can fail with
+transient errors such as `DatastoreFailureException`, `DatastoreTimeoutException`,
+or `ApiProxy.RPCFailedException`, especially under high contention when
+multiple transactions attempt to access the same entity group simultaneously.
+
+To handle this, `DatastoreService.beginTransaction()` automatically retries
+failed attempts with exponential backoff, starting at 100ms. You can
+configure the number of retry attempts using a system property:
+
+*   **`appengine.datastore.retries`** (Java System Property): The maximum
+    number of times to retry a `beginTransaction` call if it fails with
+    `DatastoreFailureException`, `DatastoreTimeoutException`, or
+    `ApiProxy.RPCFailedException`. This retry logic applies only to
+    `beginTransaction` calls; other Datastore operations are not retried
+    by this mechanism.
+    *   Default: `1`
+
+
+```xml
+<system-properties>
+  <property name="appengine.datastore.retries" value="3" />
+</system-properties>
+```
+
+## Recommended value for Jetty 12.1 / Java 25
+The Java 25 runtime environment is highly performant, and in rare cases of very high throughput,
+this can lead to backend services (like Datastore) being temporarily overloaded,
+which may result in exceptions like `DatastoreFailureException: Internal Datastore Error`.
+If you encounter such issues, you can throttle the rate of API calls by reducing
+the maximum number of concurrent threads used by the API client.
+We recommend starting with a lower value and adjusting as needed:
+`APPENGINE_API_MAX_THREADS=50`
+
+## Configuring via `appengine-web.xml`
+
+You can set these options by adding `<env-variables>` and
+`<system-properties>` sections to your `appengine-web.xml` file.
+
+For example, to switch to the JDK client, enable virtual threads, increase
+the thread limit to 200, and set Datastore `beginTransaction` retries to 3,
+you would add:
+
+```xml
+<env-variables>
+  <env-var name="APPENGINE_API_CALLS_USING_JDK_CLIENT" value="true" />
+  <env-var name="APPENGINE_API_MAX_CONNECTIONS" value="200" />
+  <env-var name="APPENGINE_API_MAX_THREADS" value="200" />
+</env-variables>
+<system-properties>
+  <property name="appengine.api.use.virtualthreads" value="true" />
+  <property name="appengine.datastore.retries" value="3" />
+</system-properties>
+```

runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/http/HttpApiHostClient.java

@@ -76,6 +76,12 @@ abstract class HttpApiHostClient implements APIHostClientInterface {
   abstract static class Config {
     abstract double extraTimeoutSeconds();
 
+    /**
+     * The maximum number of concurrent connections to the API host.
+     *
+     * <p>This value is used to configure both the Jetty/JDK HTTP client's connection pool limit
+     * and, when applicable, the maximum number of threads in the client's thread pool.
+     */
     abstract OptionalInt maxConnectionsPerDestination();
 
     /** For testing that we handle missing Content-Length correctly. */
@@ -123,6 +129,33 @@ Config config() {
     return config;
   }
 
+  static int getMaxThreads(Config config) {
+    int maxThreads = config.maxConnectionsPerDestination().orElse(100);
+    if (maxThreads <= 0) {
+      maxThreads = 100;
+    }
+    String maxThreadsEnv = System.getenv("APPENGINE_API_MAX_THREADS");
+    if (maxThreadsEnv != null) {
+      try {
+        int envMaxThreads = Integer.parseInt(maxThreadsEnv);
+        if (envMaxThreads > 0) {
+          logger.atInfo().log(
+              "Overriding API max threads to %d from environment variable.", envMaxThreads);
+          return envMaxThreads;
+        } else {
+          logger.atWarning().log(
+              "APPENGINE_API_MAX_THREADS must be positive: %d, using default %d",
+              envMaxThreads, maxThreads);
+        }
+      } catch (NumberFormatException e) {
+        logger.atWarning().withCause(e).log(
+            "Invalid value for APPENGINE_API_MAX_THREADS: %s, using default %d",
+            maxThreadsEnv, maxThreads);
+      }
+    }
+    return maxThreads;
+  }
+
   static HttpApiHostClient create(String url, Config config) {
     if (System.getenv("APPENGINE_API_CALLS_USING_JDK_CLIENT") != null) {
       logger.atInfo().log("Using JDK HTTP client for API calls");