Skip to content

Fix Windows/MSVC handling of large WAT input files and increase stack reserve #2748

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aydgelee wants to merge 1 commit into
base: main
Choose a base branch
from
+104 −2
Open

Fix Windows/MSVC handling of large WAT input files and increase stack reserve #2748

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,10 @@ function(wabt_executable)
add_dependencies(everything ${EXE_NAME})
target_link_libraries(${EXE_NAME} ${EXE_LIBS})

if (MSVC)
target_link_options(${EXE_NAME} PRIVATE "/STACK:8388608")
endif ()

if (EMSCRIPTEN)
set(EXTRA_LINK_FLAGS
"${EXTRA_LINK_FLAGS} -sNODERAWFS -Oz -sALLOW_MEMORY_GROWTH"
Expand Down
37 changes: 35 additions & 2 deletions src/common.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@
#include <cstdio>
#include <cstring>

#if COMPILER_IS_MSVC
#include <limits>
#endif

#include <sys/stat.h>
#include <sys/types.h>

Expand All @@ -31,8 +35,13 @@
#include <io.h>
#include <stdlib.h>
#define PATH_MAX _MAX_PATH
#define stat _stat
#define stat _stat64
#define fseek _fseeki64
#define ftell _ftelli64
#define S_IFREG _S_IFREG
#define WABT_FILE_OFFSET __int64
#else
#define WABT_FILE_OFFSET long
#endif

namespace wabt {
Expand Down Expand Up @@ -129,7 +138,7 @@ Result ReadFile(std::string_view filename, std::vector<uint8_t>* out_data) {
return res;
}

long size = ftell(infile);
WABT_FILE_OFFSET size = ftell(infile);
if (size < 0) {
perror("ftell failed");
fclose(infile);
Expand All @@ -142,12 +151,36 @@ Result ReadFile(std::string_view filename, std::vector<uint8_t>* out_data) {
return Result::Error;
}

#if COMPILER_IS_MSVC
if (static_cast<uint64_t>(size) >
static_cast<uint64_t>(std::numeric_limits<size_t>::max())) {
fprintf(stderr, "%s: file too large to read into memory\n", filename_cstr);
fclose(infile);
return Result::Error;
}

static constexpr size_t kReadChunkSize = 64 * 1024 * 1024;
size_t data_size = static_cast<size_t>(size);
out_data->resize(data_size);
size_t offset = 0;
while (offset < data_size) {
size_t remaining = data_size - offset;
size_t chunk_size = remaining < kReadChunkSize ? remaining : kReadChunkSize;
if (fread(out_data->data() + offset, chunk_size, 1, infile) != 1) {
fprintf(stderr, "%s: fread failed: %s\n", filename_cstr, strerror(errno));
fclose(infile);
return Result::Error;
}
offset += chunk_size;
}
#else
out_data->resize(size);
if (size != 0 && fread(out_data->data(), size, 1, infile) != 1) {
fprintf(stderr, "%s: fread failed: %s\n", filename_cstr, strerror(errno));
fclose(infile);
return Result::Error;
}
#endif

fclose(infile);
return Result::Ok;
Expand Down
65 changes: 65 additions & 0 deletions src/stream.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@
#include <cctype>
#include <cerrno>

#if COMPILER_IS_MSVC
#include <cstdint>
#include <limits>
#endif

#define DUMP_OCTETS_PER_LINE 16
#define DUMP_OCTETS_PER_GROUP 2

Expand All @@ -28,6 +33,39 @@

namespace wabt {

#if COMPILER_IS_MSVC
static Result WriteDataChunked(FILE* file,
const void* data,
size_t size,
const char* name) {
static constexpr size_t kWriteChunkSize = 64 * 1024 * 1024;
const uint8_t* src = static_cast<const uint8_t*>(data);
size_t offset = 0;

while (offset < size) {
size_t remaining = size - offset;
size_t chunk_size =
remaining < kWriteChunkSize ? remaining : kWriteChunkSize;
size_t bytes_written = fwrite(src + offset, 1, chunk_size, file);
if (bytes_written != chunk_size) {
ERROR("failed to write %" PRIzd " bytes to %s\n", size, name);
return Result::Error;
}
offset += bytes_written;
}

return Result::Ok;
}

static int SeekFile(FILE* file, size_t offset) {
if (offset > static_cast<size_t>(std::numeric_limits<int64_t>::max())) {

@sbc100 sbc100 May 29, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this SeekFile really needed?

We are never going be seeing outside of int64_t range right? If we did that would fail on all platforms, no just windows.

In fact, fseek on non-windows is limited to long which can be 32-bit on all systems right?

Maybe just add this check for data.size() <= LONG_MAX before the call tofseek` rather than adding a separate function.

errno = EINVAL;
return -1;
}
return _fseeki64(file, static_cast<int64_t>(offset), SEEK_SET);
}
#endif

Stream::Stream(Stream* log_stream)
: offset_(0), result_(Result::Ok), log_stream_(log_stream) {}

Expand Down Expand Up @@ -145,13 +183,21 @@ Result OutputBuffer::WriteToFile(std::string_view filename) const {
return Result::Ok;
}

#if COMPILER_IS_MSVC
if (Failed(WriteDataChunked(file, data.data(), data.size(),
filename_str.c_str()))) {
fclose(file);
return Result::Error;
}
#else
ssize_t bytes = fwrite(data.data(), 1, data.size(), file);
if (bytes < 0 || static_cast<size_t>(bytes) != data.size()) {
ERROR("failed to write %" PRIzd " bytes to %s\n", data.size(),
filename_str.c_str());
fclose(file);
return Result::Error;
}
#endif

fclose(file);
return Result::Ok;
Expand All @@ -161,11 +207,17 @@ Result OutputBuffer::WriteToStdout() const {
if (data.empty()) {
return Result::Ok;
}
#if COMPILER_IS_MSVC
if (Failed(WriteDataChunked(stdout, data.data(), data.size(), "stdout"))) {
return Result::Error;
}
#else
ssize_t bytes = fwrite(data.data(), 1, data.size(), stdout);
if (bytes < 0 || static_cast<size_t>(bytes) != data.size()) {
ERROR("failed to write %" PRIzd " bytes to stdout\n", data.size());
return Result::Error;
}
#endif
return Result::Ok;
}

Expand Down Expand Up @@ -281,16 +333,29 @@ Result FileStream::WriteDataImpl(size_t at, const void* data, size_t size) {
return Result::Ok;
}
if (at != offset_) {
#if COMPILER_IS_MSVC
if (SeekFile(file_, at) != 0) {
ERROR("fseek offset=%" PRIzd " failed, errno=%d\n", at, errno);
return Result::Error;
}
#else
if (fseek(file_, at, SEEK_SET) != 0) {
ERROR("fseek offset=%" PRIzd " failed, errno=%d\n", size, errno);
return Result::Error;
}
#endif
offset_ = at;
}
#if COMPILER_IS_MSVC
if (Failed(WriteDataChunked(file_, data, size, "FileStream"))) {
return Result::Error;
}
#else
if (fwrite(data, size, 1, file_) != 1) {
ERROR("fwrite size=%" PRIzd " failed, errno=%d\n", size, errno);
return Result::Error;
}
#endif
offset_ += size;
return Result::Ok;
}
Expand Down
Loading