Fix unkown length response

.builder/action/local-server-setup.py

@@ -49,3 +49,4 @@ def run(self, env):
         def close_local_server():
             p_server.terminate()
             p_non_tls_server.terminate()
+            p_h11_server.terminate()

include/aws/http/private/h1_decoder.h

@@ -73,6 +73,14 @@ AWS_HTTP_API int aws_h1_decode(struct aws_h1_decoder *decoder, struct aws_byte_c
 
 AWS_HTTP_API void aws_h1_decoder_set_logging_id(struct aws_h1_decoder *decoder, const void *id);
 AWS_HTTP_API void aws_h1_decoder_set_body_headers_ignored(struct aws_h1_decoder *decoder, bool body_headers_ignored);
+AWS_HTTP_API void aws_h1_decoder_set_body_headers_ignored_on_2xx(
+    struct aws_h1_decoder *decoder,
+    bool body_headers_ignored_on_2xx);
+
+/**
+ * Signal that the connection has closed
+ */
+AWS_HTTP_API int aws_h1_decoder_on_connection_closed(struct aws_h1_decoder *decoder);
 
 /* RFC-7230 section 4.2 Message Format */
 #define AWS_HTTP_TRANSFER_ENCODING_CHUNKED (1 << 0)

source/h1_connection.c

@@ -142,6 +142,11 @@ static void s_stop(
     AWS_ASSERT(stop_reading || stop_writing || schedule_shutdown); /* You are required to stop at least 1 thing */
 
     if (stop_reading) {
+        /* signal the decoder that the connection has closed. */
+        if (aws_h1_decoder_on_connection_closed(connection->thread_data.incoming_stream_decoder)) {
+            error_code = aws_last_error();
+        }
+
         if (connection->thread_data.read_state == AWS_CONNECTION_READ_OPEN) {
             connection->thread_data.read_state = AWS_CONNECTION_READ_SHUT_DOWN_COMPLETE;
         } else if (connection->thread_data.read_state == AWS_CONNECTION_READ_SHUTTING_DOWN) {
@@ -2037,6 +2042,16 @@ static int s_try_process_next_stream_read_message(struct aws_h1_connection *conn
 
     bool body_headers_ignored = incoming_stream->base.request_method == AWS_HTTP_METHOD_HEAD;
     aws_h1_decoder_set_body_headers_ignored(connection->thread_data.incoming_stream_decoder, body_headers_ignored);
+    /**
+     * RFC-7230 3.3.3
+     * 2. Any 2xx (Successful) response to a CONNECT request implies that
+     * the connection will become a tunnel immediately after the empty
+     * line that concludes the header fields.  A client MUST ignore any
+     * Content-Length or Transfer-Encoding header fields received in
+     * such a message. . */
+    aws_h1_decoder_set_body_headers_ignored_on_2xx(
+        connection->thread_data.incoming_stream_decoder,
+        incoming_stream->base.request_method == AWS_HTTP_METHOD_CONNECT);
 
     if (incoming_stream->base.metrics.receive_start_timestamp_ns == -1) {
         /* That's the first time for the stream receives any message */

source/h1_decoder.c

@@ -37,9 +37,11 @@ struct aws_h1_decoder {
     uint64_t chunk_size;
     bool doing_trailers;
     bool is_done;
+    bool body_headers_ignored_on_2xx;
     bool body_headers_ignored;
     bool body_headers_forbidden;
     bool content_length_received;
+    bool response_body_indeterminate_length;
 
     enum aws_http_header_block header_block;
     const void *logging_id;
@@ -215,12 +217,28 @@ static void s_reset_state(struct aws_h1_decoder *decoder) {
     decoder->doing_trailers = false;
     decoder->is_done = false;
     decoder->body_headers_ignored = false;
+    decoder->body_headers_ignored_on_2xx = false;
     decoder->body_headers_forbidden = false;
     decoder->content_length_received = false;
+    decoder->response_body_indeterminate_length = false;
     /* set to normal by default */
     decoder->header_block = AWS_HTTP_HEADER_BLOCK_MAIN;
 }
 
+/* State for body with indeterminate length - consumes all data until connection closes */
+static int s_state_indeterminate_length_body(struct aws_h1_decoder *decoder, struct aws_byte_cursor *input) {
+    /* indeterminate length only valid for response. */
+    AWS_ASSERT(!decoder->is_decoding_requests);
+    if (input->len > 0) {
+        struct aws_byte_cursor body = aws_byte_cursor_advance(input, input->len);
+        int err = decoder->vtable.on_body(&body, false, decoder->user_data);
+        if (err) {
+            return AWS_OP_ERR;
+        }
+    }
+    return AWS_OP_SUCCESS;
+}
+
 static int s_state_unchunked_body(struct aws_h1_decoder *decoder, struct aws_byte_cursor *input) {
 
     size_t processed_bytes = 0;
@@ -361,6 +379,22 @@ static int s_linestate_header(struct aws_h1_decoder *decoder, struct aws_byte_cu
                 s_set_line_state(decoder, s_linestate_chunk_size);
             } else if (decoder->content_length > 0) {
                 s_set_state(decoder, s_state_unchunked_body);
+            } else if (
+                !decoder->is_decoding_requests && !decoder->content_length_received &&
+                !decoder->body_headers_forbidden && !decoder->body_headers_ignored_on_2xx) {
+                /* RFC-7230 3.3.3: If a message is received without Transfer-Encoding and without Content-Length and NOT
+                 * determined by the response line to have no body, then the message body length is determined by
+                 * reading the stream until connection closure. Note: This only applies to responses. A request without
+                 * Content-Length or Transfer-Encoding has no message body (per RFC 7230 section 3.3.3). */
+                /* The failure response for CONNECT with body is undefined, and since the client will handle the CONNECT
+                 * request, don't read the body to keep it simple. */
+                decoder->response_body_indeterminate_length = true;
+                s_set_state(decoder, s_state_indeterminate_length_body);
+                AWS_LOGF_DEBUG(
+                    AWS_LS_HTTP_STREAM,
+                    "id=%p: Response has no Content-Length or Transfer-Encoding, body length will be determined by "
+                    "connection closure.",
+                    decoder->logging_id);
             } else {
                 err = s_mark_done(decoder);
                 if (err) {
@@ -536,13 +570,6 @@ static int s_linestate_header(struct aws_h1_decoder *decoder, struct aws_byte_cu
                     return aws_raise_error(AWS_ERROR_HTTP_PROTOCOL_ERROR);
                 }
             }
-
-            /* TODO: deal with body of indeterminate length, marking it as successful when connection is closed:
-             *
-             * A response that has neither chunked transfer coding nor Content-Length is terminated by closure of
-             * the connection and, thus, is considered complete regardless of the number of message body octets
-             * received, provided that the header section was received intact.
-             * RFC-7230 3.4 */
         } break;
 
         default:
@@ -688,6 +715,9 @@ static int s_linestate_response(struct aws_h1_decoder *decoder, struct aws_byte_
 
     /* RFC-7230 section 3.3 Message Body */
     decoder->body_headers_ignored |= code_val == AWS_HTTP_STATUS_CODE_304_NOT_MODIFIED;
+    if (decoder->body_headers_ignored_on_2xx) {
+        decoder->body_headers_ignored |= code_val / 100 == 2;
+    }
     decoder->body_headers_forbidden = code_val == AWS_HTTP_STATUS_CODE_204_NO_CONTENT || code_val / 100 == 1;
 
     if (s_check_info_response_status_code(code_val)) {
@@ -777,3 +807,38 @@ void aws_h1_decoder_set_logging_id(struct aws_h1_decoder *decoder, const void *i
 void aws_h1_decoder_set_body_headers_ignored(struct aws_h1_decoder *decoder, bool body_headers_ignored) {
     decoder->body_headers_ignored = body_headers_ignored;
 }
+
+void aws_h1_decoder_set_body_headers_ignored_on_2xx(struct aws_h1_decoder *decoder, bool body_headers_ignored_on_2xx) {
+    decoder->body_headers_ignored_on_2xx = body_headers_ignored_on_2xx;
+}
+
+int aws_h1_decoder_on_connection_closed(struct aws_h1_decoder *decoder) {
+    if (!decoder) {
+        return AWS_OP_SUCCESS;
+    }
+
+    if (!decoder->response_body_indeterminate_length || decoder->is_done) {
+        /* nothing to do */
+        return AWS_OP_SUCCESS;
+    }
+    /* If the decoder is processing the indeterminate length response, the connection close marks the response ends. */
+    /* Signal final body callback with finished=true */
+    struct aws_byte_cursor empty_cursor = {.ptr = NULL, .len = 0};
+    int err = decoder->vtable.on_body(&empty_cursor, true, decoder->user_data);
+    if (err) {
+        return AWS_OP_ERR;
+    }
+
+    /* Mark the message as complete */
+    err = s_mark_done(decoder);
+    if (err) {
+        return AWS_OP_ERR;
+    }
+
+    AWS_LOGF_DEBUG(
+        AWS_LS_HTTP_STREAM,
+        "id=%p: Response with indeterminate body length completed via connection closure.",
+        decoder->logging_id);
+
+    return AWS_OP_SUCCESS;
+}

tests/CMakeLists.txt

@@ -157,6 +157,9 @@ add_test_case(h1_client_connection_close_before_request_finishes_with_buffer)
 add_test_case(h1_client_connection_close_before_request_finishes_with_buffer_incomplete_response)
 add_test_case(h1_client_connection_close_before_request_finishes_with_buffer_force_shutdown)
 add_test_case(h1_client_connection_close_before_request_finishes_with_buffer_stream_cancel)
+add_test_case(h1_client_response_indeterminate_length_body)
+add_test_case(h1_client_response_indeterminate_length_empty_body)
+add_test_case(h1_client_response_204_no_indeterminate_length)
 add_test_case(h1_client_write_data_single_chunk)
 add_test_case(h1_client_write_data_multiple_chunks)
 add_test_case(h1_client_write_data_not_enabled)
@@ -305,6 +308,9 @@ if(ENABLE_LOCALHOST_INTEGRATION_TESTS)
     add_net_test_case(localhost_integ_hpack_compression_stress)
     add_net_test_case(localhost_integ_h2_upload_stress)
     add_net_test_case(localhost_integ_h2_download_stress)
+    # Test for H1 client handling response without Content-Length header
+    # Requires h11mock_server.py to be running on localhost:8081
+    add_net_test_case(localhost_integ_h1_no_content_length_response)
 endif()
 
 add_test_case(h2_header_empty_payload)

tests/mock_server/README.md

@@ -23,6 +23,7 @@ Echoes back request headers and body as JSON.
 ```bash
 curl -k -v -H "foo:bar" https://localhost:3443/echo
 ```
+
 #### Special headers
 
 ##### `/echo` with `x-repeat-data` header - Download test
@@ -79,7 +80,6 @@ Returns 404 Not Found.
 - The code is based the non-tls [example](http://python-hyper.org/projects/h2/en/stable/basic-usage.html) from hyper h2 server.
 - Run python. `python3 ./non_tls_server.py`.
 - To test the server runs correctly, you can do `curl -v --http2-prior-knowledge http://localhost:3280` and check the result.
-
 # HTTP1.1 Local server
 
 ## Requirements
@@ -118,8 +118,9 @@ HTTP_PORT=8080 HTTPS_PORT=8443 python3 mock_server.py
 
 ## Endpoints
 
-- **Any path**: Echoes request body as JSON
-- **/response-headers?HeaderName=value**: Adds custom headers to the response based on query parameters
+- **Any path**: Echoes request body as JSON with all request headers echoed back with "Echo-" prefix
+- **/indeterminate-length**: Returns a response without Content-Length or Transfer-Encoding headers, using `Connection: close` to indicate message boundary
+- **/404**: Returns a 404 Not Found status
 
 ## Stopping the Server
 

tests/mock_server/h11mock_server.py

@@ -42,7 +42,7 @@ async def _read_from_peer(self):
                 request_line_end = data.index(b'\r\n')
                 request_line = data[:request_line_end]
                 rest = data[request_line_end:]
-                
+
                 try:
                     # Try to decode and check if there are non-ASCII chars
                     request_line_str = request_line.decode('ascii')
@@ -66,7 +66,7 @@ async def _read_from_peer(self):
                                 else:
                                     encoded_query_parts.append(param)
                             target_str = path + '?' + '&'.join(encoded_query_parts)
-                        
+
                         request_line = b' '.join([method, target_str.encode('ascii'), version])
                         data = request_line + rest
         except (ConnectionError, trio.BrokenResourceError, trio.ClosedResourceError):
@@ -87,7 +87,7 @@ async def shutdown_and_clean_up(self):
                 await self.stream.send_eof()
         except (trio.BrokenResourceError, AttributeError):
             pass
-        
+
         with trio.move_on_after(TIMEOUT):
             try:
                 while True:
@@ -96,14 +96,14 @@ async def shutdown_and_clean_up(self):
                         break
             except (trio.BrokenResourceError, trio.ClosedResourceError):
                 pass
-        
+
         try:
             await self.stream.aclose()
         except (trio.BrokenResourceError, trio.ClosedResourceError):
             pass
 
     def basic_headers(self):
-        return [("Server", "echo-server")]
+        return [("Server", "crt-local-server")]
 
     def info(self, *args):
         print(f"{self._obj_id}:", *args)
@@ -121,7 +121,7 @@ async def http_serve(stream):
                 event = await wrapper.next_event()
                 wrapper.info("Server main loop got event:", event)
                 if type(event) is h11.Request:
-                    await send_echo_response(wrapper, event)
+                    await send_response(wrapper, event)
         except Exception as exc:
             wrapper.info(f"Error during response handler: {exc!r}")
             await maybe_send_error_response(wrapper, exc)
@@ -175,38 +175,73 @@ async def maybe_send_error_response(wrapper, exc):
         wrapper.info("error while sending error response:", exc)
 
 
-async def send_echo_response(wrapper, request):
-    wrapper.info("Preparing echo response")
-    
+async def send_response(wrapper, request):
+    wrapper.info("Preparing response")
+
     body_data = b""
     while True:
         event = await wrapper.next_event()
         if type(event) is h11.EndOfMessage:
             break
         assert type(event) is h11.Data
         body_data += event.data
-    
+
     target = request.target if isinstance(request.target, bytes) else request.target.encode()
     target_str = target.decode("utf-8", errors="replace")
-
+
+    # Check if this is the /indeterminate-length endpoint
+    if target_str.startswith("/indeterminate-length"):
+        wrapper.info("Sending raw response without Content-Length or Transfer-Encoding")
+        response_body = b"Response body without Content-Length header"
+
+        # Send raw HTTP response to avoid h11 adding Transfer-Encoding: chunked
+        # Build the raw HTTP response
+        response_lines = [
+            b"HTTP/1.1 200 OK",
+            b"Server: crt-local-server",
+            b"Content-Type: text/plain; charset=utf-8",
+            b"Connection: close",
+            b"",  # Empty line separates headers from body
+        ]
+        raw_response = b"\r\n".join(response_lines) + b"\r\n" + response_body
+
+        # Send raw response directly to the stream, bypassing h11
+        await wrapper.stream.send_all(raw_response)
+
+        # Update h11's state to indicate connection must close
+        # We simulate the response flow through h11's state machine without actually sending bytes
+        wrapper.info("Updating h11 state to MUST_CLOSE after raw response with Connection: close")
+        headers = [
+            ("Server", "crt-local-server"),
+            ("Content-Type", "text/plain; charset=utf-8"),
+            ("Connection", "close"),
+        ]
+        res = h11.Response(status_code=200, headers=headers)
+        # Call send() to update state machine, but don't send the bytes (we already did)
+        wrapper.conn.send(res)
+        wrapper.conn.send(h11.Data(data=response_body))
+        wrapper.conn.send(h11.EndOfMessage())
+        # Now h11 knows the connection must close
+        return
+
     # Check if this is the /404 endpoint
     if target_str.startswith("/404"):
         status_code = 404
     else:
         status_code = 200
-    
+
     response_json = {"data": body_data.decode("utf-8")}
     response_body = json.dumps(response_json, indent=4).encode("utf-8")
-    
+
     headers = wrapper.basic_headers()
     headers.append(("Content-Type", "application/json; charset=utf-8"))
     headers.append(("Content-Length", str(len(response_body))))
-    
+
     for header_name, header_value in request.headers:
         echo_name = b"Echo-" + header_name if isinstance(header_name, bytes) else f"Echo-{header_name}".encode()
         echo_value = header_value if isinstance(header_value, bytes) else str(header_value).encode()
         headers.append((echo_name, echo_value))
-    
+
     res = h11.Response(status_code=status_code, headers=headers)
     await wrapper.send(res)
     await wrapper.send(h11.Data(data=response_body))
@@ -225,20 +260,20 @@ async def serve_ssl(port, cert_file=os.path.join(os.path.dirname(__file__),
                           "../resources/unittests.crt"), key_file=os.path.join(os.path.dirname(__file__),
                           "../resources/unittests.key")):
     import ssl
-    
+
     script_dir = os.path.dirname(os.path.abspath(__file__))
     cert_path = os.path.join(script_dir, cert_file)
     key_path = os.path.join(script_dir, key_file)
-    
+
     if not os.path.exists(cert_path) or not os.path.exists(key_path):
         print(f"Warning: SSL certificates not found at {cert_path} and {key_path}")
         print(f"Skipping HTTPS server on port {port}")
         print(f"To enable HTTPS, run: openssl req -x509 -newkey rsa:2048 -keyout {key_file} -out {cert_file} -days 365 -nodes -subj '/C=US/ST=WA/L=Seattle/O=Test/CN=localhost'")
         return
-    
+
     ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
     ssl_context.load_cert_chain(cert_path, key_path)
-    
+
     print(f"listening on https://localhost:{port}")
     try:
         await trio.serve_ssl_over_tcp(http_serve, port, ssl_context)
@@ -249,7 +284,7 @@ async def serve_ssl(port, cert_file=os.path.join(os.path.dirname(__file__),
 async def main():
     http_port = os.environ.get('HTTP_PORT')
     https_port = os.environ.get('HTTPS_PORT')
-    
+
     async with trio.open_nursery() as nursery:
         nursery.start_soon(serve, 8081 if not http_port else int(http_port))
         nursery.start_soon(serve_ssl, 8082 if not https_port else int(https_port))