How to add proper binary I/O (stdin, stdout, stderr) to boa?

[guest271314]

This is what I currently have that fails

use boa_engine::{Context, Source, JsResult, JsValue, JsObject, NativeFunction, JsString};
use boa_engine::object::builtins::{JsArrayBuffer, JsUint8Array};
use boa_engine::property::PropertyDescriptor;
use std::io::{self, Read, Write};

/// High-performance synchronous block parsing from WASI stdin (fd: 0)
fn read_exactly(bytes_to_read: usize) -> Option<Vec<u8>> {
    let mut buffer = vec![0u8; bytes_to_read];
    let mut total_read = 0;
    let mut stdin = io::stdin().lock();

    while total_read < bytes_to_read {
        match stdin.read(&mut buffer[total_read..bytes_to_read]) {
            Ok(0) => return None,
            Ok(n) => total_read += n,
            Err(_) => return None,
        }
    }
    Some(buffer)
}

/// Dispatches binary blocks back across WASI stdout (fd: 1)
fn write_stdout(bytes: &[u8]) -> bool {
    let mut stdout = io::stdout().lock();
    if stdout.write_all(bytes).is_err() {
        return false;
    }
    stdout.flush().is_ok()
}

/// Forwards system panic states safely across WASI stderr (fd: 2)
fn write_stderr(message: String) {
    let mut stderr = io::stderr().lock();
    let _ = stderr.write_all(message.as_bytes());
    let _ = stderr.flush();
}

fn main() -> JsResult<()> {
    // 1. Initialize the official execution context
    let mut context = Context::default();

    // 2. Build the nested Deno namespace structures using public constructors
    let deno_obj = JsObject::with_object_proto(context.intrinsics());
    let core_obj = JsObject::with_object_proto(context.intrinsics());

    // Inject Deno.core.read
    let read_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let rid = args.get(0).and_then(|v| v.as_number()).unwrap_or(-1.0) as i32;
        let len = args.get(1).and_then(|v| v.as_number()).unwrap_or(0.0) as usize;
        
        if rid == 0 {
            if let Some(bytes) = read_exactly(len) {
                // FIX: Use verified from_byte_block constructor
                let ab = JsArrayBuffer::from_byte_block(bytes, ctx)?;
                let ua = JsUint8Array::from_array_buffer(ab, ctx)?;
                return Ok(JsValue::from(ua));
            }
        }
        Ok(JsValue::null())
    });
    // FIX: Use from_proto_and_data alongside constructors namespace
    let read_js_fn = JsObject::from_proto_and_data(
        context.intrinsics().constructors().function().prototype(),
        read_fn,
    );
    let read_desc = PropertyDescriptor::builder()
        .value(JsValue::from(read_js_fn))
        .writable(true)
        .enumerable(true)
        .configurable(true)
        .build();
    // FIX: Map using define_property_or_throw
    core_obj.define_property_or_throw("read", read_desc, &mut context)?;

    // Inject Deno.core.write
    let write_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let rid = args.get(0).and_then(|v| v.as_number()).unwrap_or(-1.0) as i32;
        let js_buffer = args.get(1).and_then(|v| v.as_object().cloned());
        
        if let Some(obj) = js_buffer {
            if let Ok(typed_array) = JsUint8Array::from_object(obj) {
                // FIX: Read byte data views out via the verified .view() array slices
                let view = typed_array.view(ctx)?;
                let bytes = view.as_slice();
                if rid == 1 {
                    return Ok(JsValue::from(write_stdout(bytes)));
                } else if rid == 2 {
                    write_stderr(String::from_utf8_lossy(bytes).into_owned());
                    return Ok(JsValue::from(true));
                }
            }
        }
        Ok(JsValue::from(false))
    });
    let write_js_fn = JsObject::from_proto_and_data(
        context.intrinsics().constructors().function().prototype(),
        write_fn,
    );
    let write_desc = PropertyDescriptor::builder()
        .value(JsValue::from(write_js_fn))
        .writable(true)
        .enumerable(true)
        .configurable(true)
        .build();
    core_obj.define_property_or_throw("write", write_desc, &mut context)?;

    // Inject Deno.core.encode
    let encode_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let js_str = args.get(0).and_then(|v| v.as_string()).cloned().unwrap_or_default();
        let bytes = js_str.to_std_string_escaped().into_bytes();
        
        let ab = JsArrayBuffer::from_byte_block(bytes, ctx)?;
        let ua = JsUint8Array::from_array_buffer(ab, ctx)?;
        Ok(JsValue::from(ua))
    });
    let encode_js_fn = JsObject::from_proto_and_data(
        context.intrinsics().constructors().function().prototype(),
        encode_fn,
    );
    let encode_desc = PropertyDescriptor::builder()
        .value(JsValue::from(encode_js_fn))
        .writable(true)
        .enumerable(true)
        .configurable(true)
        .build();
    core_obj.define_property_or_throw("encode", encode_desc, &mut context)?;

    // Inject Deno.core.decode
    let decode_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let js_buffer = args.get(0).and_then(|v| v.as_object().cloned());
        if let Some(obj) = js_buffer {
            if let Ok(typed_array) = JsUint8Array::from_object(obj) {
                let view = typed_array.view(ctx)?;
                let bytes = view.as_slice();
                let s = String::from_utf8_lossy(bytes).into_owned();
                return Ok(JsValue::from(JsString::from(s)));
            }
        }
        Ok(JsValue::from(JsString::from("")))
    });
    let decode_js_fn = JsObject::from_proto_and_data(
        context.intrinsics().constructors().function().prototype(),
        decode_fn,
    );
    let decode_desc = PropertyDescriptor::builder()
        .value(JsValue::from(decode_js_fn))
        .writable(true)
        .enumerable(true)
        .configurable(true)
        .build();
    core_obj.define_property_or_throw("decode", decode_desc, &mut context)?;

    // Assemble nested architecture modules
    let core_desc = PropertyDescriptor::builder()
        .value(JsValue::from(core_obj))
        .writable(true)
        .enumerable(true)
        .configurable(true)
        .build();
    deno_obj.define_property_or_throw("core", core_desc, &mut context)?;

    let deno_desc = PropertyDescriptor::builder()
        .value(JsValue::from(deno_obj))
        .writable(true)
        .enumerable(true)
        .configurable(true)
        .build();
    context.global_object().define_property_or_throw("Deno", deno_desc, &mut context)?;

    // 3. Exact JavaScript protocol polling loop
    let js_code = r#"
        const MAX_CHUNK_ELEMENTS = 209715;

        async function readExactly(bytesToRead) {
          const buffer = new Uint8Array(bytesToRead);
          let totalRead = 0;
          while (totalRead < bytesToRead) {
            const chunkBytes = Deno.core.read(0, bytesToRead - totalRead);
            if (!chunkBytes) return null; 
            
            buffer.set(chunkBytes, totalRead);
            totalRead += chunkBytes.length;
          }
          return buffer;
        }

        async function sendNativeMessage(payloadData) {
          if (Array.isArray(payloadData) && payloadData.length > MAX_CHUNK_ELEMENTS) {
            for (let i = 0; i < payloadData.length; i += MAX_CHUNK_ELEMENTS) {
              await transmitFrame(payloadData.slice(i, i + MAX_CHUNK_ELEMENTS));
            }
          } else {
            await transmitFrame(payloadData);
          }
        }

        async function transmitFrame(obj) {
          try {
            const encodedPayload = Deno.core.encode(JSON.stringify(obj));
            const packet = new Uint8Array(4 + encodedPayload.length);
            const view = new DataView(packet.buffer);
            view.setUint32(0, encodedPayload.length, true);
            packet.set(encodedPayload, 4);
            Deno.core.write(1, packet);
          } catch (_) {}
        }

        async function runHostLoop() {
          while (true) {
            const headerBytes = await readExactly(4);
            if (!headerBytes) return;

            const view = new DataView(headerBytes.buffer);
            const messageLength = view.getUint32(0, true);

            const bodyBytes = await readExactly(messageLength);
            if (!bodyBytes) return;

            try {
              const parsedMessage = JSON.parse(Deno.core.decode(bodyBytes));
              await sendNativeMessage(parsedMessage);
            } catch (err) {
              try {
                Deno.core.write(2, Deno.core.encode(`Parse error: ${err.message}\n`));
              } catch (_) {}
            }
          }
        }

        runHostLoop();
    "#;

    // 4. Fire execution parsing using your verified syntax format
    context.eval(Source::from_bytes(js_code.as_bytes()))?;

    Ok(())
}

[identity-wael]

There are several independent issues in this sample, before the WASI transport itself:

1. Giving an object Function.prototype does not make it a callable JavaScript function. Build each NativeFunction with FunctionObjectBuilder, as in Boa's typed-array example:

let read_js_fn = FunctionObjectBuilder::new(context.realm(), read_fn)
    .build();

Import it from boa_engine::object::FunctionObjectBuilder. Apply the same pattern to write/encode/decode. Check against your Boa version, since the constructor/API details are version-sensitive.

2. runHostLoop() is async, but the host exits after context.eval(...). Its continuations need the job queue driven; the event-loop example calls context.run_jobs()?. Alternatively, since these host reads/writes are already synchronous, remove the JS async/await layer for a first blocking echo test.

3. Binary stderr currently goes through from_utf8_lossy(...).into_owned() and back to bytes. That changes invalid UTF-8. For a binary write(2, Uint8Array), write the bytes directly just as you do for stdout. Likewise, to_std_string_escaped() is an escaped display representation, not a general UTF-8 encoder.

Also stop swallowing every transmit exception while debugging: it can turn a binding error into an apparently silent I/O failure. Send diagnostics to stderr, leaving stdout exclusively for framed messages. A useful first test is a synchronous byte echo containing NUL and non-UTF-8 bytes, then add JSON framing and promise handling separately.

I haven't compiled your complete example; please share the pinned Boa version and first compiler/runtime error so the remaining typed-array API calls can be checked precisely.

[guest271314]

This is what I wound up with, so far, that works. Suggesttions for improvement welcome

use boa_engine::{Context, Source, JsResult, JsValue, JsObject, NativeFunction, js_string};
use boa_engine::object::builtins::{JsArrayBuffer, JsUint8Array};
use boa_engine::object::FunctionObjectBuilder;
use std::io::{self, Read, Write};

/// High-performance synchronous block parsing from stdin (Works on native and WASI)
fn read_bytes(bytes_to_read: usize) -> Option<Vec<u8>> {
    if bytes_to_read == 0 {
        return Some(vec![]);
    }
    
    let mut buffer = vec![0u8; bytes_to_read];
    let mut stdin = io::stdin().lock();

    // 💡 FIX: Read whatever bytes are available up to capacity (do not use read_exact)
    match stdin.read(&mut buffer) {
        Ok(0) => None, // Clean EOF / Stream closed
        Ok(n) => {
            buffer.truncate(n);
            Some(buffer)
        }
        Err(_) => None,
    }
}

/// Dispatches binary blocks cleanly back across stdout and returns amount written
fn write_stdout(bytes: &[u8]) -> Option<usize> {
    let mut stdout = io::stdout().lock();
    if stdout.write_all(bytes).is_err() {
        return None;
    }
    if stdout.flush().is_err() {
        return None;
    }
    Some(bytes.len())
}

/// Forwards standard error diagnostics safely without corrupting the stdout pipeline
fn write_stderr(message: &str) {
    let mut stderr = io::stderr().lock();
    let _ = stderr.write_all(message.as_bytes());
    let _ = stderr.flush();
}

fn main() -> JsResult<()> {
    // DIRECTLY REFERENCE THE JS FILE IN MAIN.RS
    let js_code = include_str!("__index.js");

    // Initialize execution context
    let mut context = Context::default();

    // Build the top-level 'std' namespace object template
    let std_obj = JsObject::with_object_proto(context.intrinsics());

    // Inject std.read(typedArray) -> returns bytes read or null
    let read_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let js_buffer = args.get(0).and_then(|v| v.as_object().cloned());
        
        if let Some(obj) = js_buffer {
            if let Ok(typed_array) = JsUint8Array::from_object(obj) {
                let len = typed_array.length(ctx).unwrap_or(0);
                let byte_offset = typed_array.byte_offset(ctx).unwrap_or(0);

                if let Some(bytes) = read_bytes(len) {
                    let bytes_read = bytes.len();
                    
                    let ab_val = typed_array.buffer(ctx)?;
                    if let Some(ab_obj) = ab_val.as_object().cloned() {
                        let ab = JsArrayBuffer::from_object(ab_obj)?;
                        
                        let write_result = if let Some(mut gc_ref_mut) = ab.data_mut() {
                            let slice = &mut *gc_ref_mut;
                            slice[byte_offset..(byte_offset + bytes_read)].copy_from_slice(&bytes[..bytes_read]);
                            Some(JsValue::from(bytes_read))
                        } else {
                            None
                        };
                        
                        if let Some(val) = write_result {
                            return Ok(val);
                        }
                    }
                }
            }
        }
        Ok(JsValue::null())
    });
    let read_js_fn = FunctionObjectBuilder::new(context.realm(), read_fn).build();
    std_obj.set(js_string!("read"), JsValue::from(read_js_fn), true, &mut context)?;

    // Inject std.write(typedArray) -> returns bytes written or 0
    let write_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let js_buffer = args.get(0).and_then(|v| v.as_object().cloned());
        if let Some(obj) = js_buffer {
            if let Ok(typed_array) = JsUint8Array::from_object(obj) {
                let byte_offset = typed_array.byte_offset(ctx).unwrap_or(0);
                let length = typed_array.length(ctx).unwrap_or(0);

                let array_buffer_val = typed_array.buffer(ctx)?;
                if let Some(array_buffer_obj) = array_buffer_val.as_object().cloned() {
                    let array_buffer = JsArrayBuffer::from_object(array_buffer_obj)?;
                    
                    let write_result = if let Some(gc_ref) = array_buffer.data() {
                        let bytes: &[u8] = &*gc_ref;
                        let view_window_slice = &bytes[byte_offset..(byte_offset + length)];
                        write_stdout(view_window_slice).map(JsValue::from)
                    } else {
                        None
                    };

                    if let Some(val) = write_result {
                        return Ok(val);
                    }
                }
            }
        }
        Ok(JsValue::from(0))
    });
    let write_js_fn = FunctionObjectBuilder::new(context.realm(), write_fn).build();
    std_obj.set(js_string!("write"), JsValue::from(write_js_fn), true, &mut context)?;

    // Inject std.err(string)
    let err_fn = NativeFunction::from_copy_closure(|_this, args, _ctx| {
        let js_str = args.get(0).and_then(|v| v.as_string()).cloned().unwrap_or_default();
        write_stderr(&js_str.to_std_string_escaped());
        Ok(JsValue::undefined())
    });
    let err_js_fn = FunctionObjectBuilder::new(context.realm(), err_fn).build();
    std_obj.set(js_string!("err"), JsValue::from(err_js_fn), true, &mut context)?;

    // Register 'std' globally
    context.global_object().set(js_string!("std"), JsValue::from(std_obj), true, &mut context)?;

    // Parse execution with explicit error tracking to stderr if JavaScript crashes
    match context.eval(Source::from_bytes(js_code.as_bytes())) {
        Ok(_) => {
            context.run_jobs();
        }
        Err(js_error) => {
            eprintln!("JavaScript Exception: {}", js_error);
        }
    }

    Ok(())
}

[identity-wael]

The explicit byte_offset/length handling is useful here: a Uint8Array.subarray() should only write its view, not the entire backing buffer.

The next change I'd make is to distinguish EOF, an interrupted read, and a real I/O error. Currently Err(_) => None turns all three into an apparently clean end of input. Rust's Read contract says ErrorKind::Interrupted should normally be retried. A helper can preserve those distinctions:

fn read_bytes(capacity: usize) -> io::Result<Option<Vec<u8>>> {
    if capacity == 0 {
        return Ok(Some(Vec::new()));
    }
    let mut buffer = vec![0; capacity];
    let mut stdin = io::stdin().lock();
    loop {
        match stdin.read(&mut buffer) {
            Ok(0) => return Ok(None),
            Ok(n) => {
                buffer.truncate(n);
                return Ok(Some(buffer));
            }
            Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
            Err(e) => return Err(e),
        }
    }
}

The native binding would then map Ok(None) to JS null, an empty successful read to 0, and an error to a JS exception rather than silently treating it as EOF. Likewise, a stdout error after a partial write_all must terminate/report a broken framed stream, not invite retrying the whole frame and duplicating its prefix.

For your framing tests, include a header split across several reads, EOF halfway through a payload, a nonzero-offset subarray, and a zero-length view. Also cap the frame length before allocating the buffer. Those will catch protocol errors that a single successful echo won't expose. This helper is a suggested change; I haven't compiled your complete Boa/WASI wrapper.

[guest271314]

@identity-wael

error[E0308]: mismatched types
   --> src/main.rs:204:16
    |
204 |         if let Some(bytes) = read_bytes(len) {
    |                ^^^^^^^^^^^   --------------- this expression has type `Result<Option<Vec<u8>>, std::io::Error>`
    |                |
    |                expected `Result<Option<Vec<u8>>, Error>`, found `Option<_>`
    |
    = note: expected enum `Result<Option<Vec<u8>>, std::io::Error>`
               found enum `Option<_>`
help: try wrapping the pattern in `Ok`
    |
204 |         if let Ok(Some(bytes)) = read_bytes(len) {
    |                +++           +