Skip to content

Guard meshoptimizer against oversized untrusted meshes #184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
SashaRX wants to merge 1 commit into from
Closed

Guard meshoptimizer against oversized untrusted meshes #184

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 56 additions & 5 deletions Editor/MeshOptimizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@ public struct OptimizeResult
// Some channels may store color data, bone weights, etc.
const int MAX_UV_CHANNELS = 8;

// Keep user-supplied meshes away from allocations large enough to stall the
// editor, and from the unsafe/native packing path if their sizes are invalid.
const int MAX_OPTIMIZE_VERTICES = 2_000_000;
const int MAX_OPTIMIZE_INDICES = 10_000_000;
const int MAX_PACKED_BYTES = 256 * 1024 * 1024;

struct ChannelLayout
{
public bool hasNormal;
Expand Down Expand Up @@ -79,6 +85,23 @@ public static OptimizeResult Optimize(Mesh mesh, float overdrawThreshold = 1.05f

var layout = BuildChannelLayout(mesh);

if (!TryGetPackedByteCount(vertCount, layout.totalStride, out _))
{
result.error = $"Mesh exceeds optimization budget ({vertCount} vertices, stride {layout.totalStride})";
return result;
}

long totalIndexCount = 0;
for (int s = 0; s < subCount; s++)
{
totalIndexCount += mesh.GetIndexCount(s);
if (totalIndexCount > MAX_OPTIMIZE_INDICES)
{
result.error = $"Mesh exceeds optimization budget ({totalIndexCount} indices)";
return result;
}
}

// Build UV dim summary for log
var uvDimStr = new System.Text.StringBuilder();
for (int ch = 0; ch < MAX_UV_CHANNELS; ch++)
Expand Down Expand Up @@ -122,6 +145,7 @@ public static OptimizeResult Optimize(Mesh mesh, float overdrawThreshold = 1.05f

var submeshTriangles = new List<int[]>();
int totalOutVerts = 0;
int totalLocalVerts = 0;

for (int s = 0; s < subCount; s++)
{
Expand All @@ -144,9 +168,17 @@ public static OptimizeResult Optimize(Mesh mesh, float overdrawThreshold = 1.05f
int localVertCount = globalToLocal.Count;
uint localIndexCount = (uint)subTris.Length;

if (!TryGetPackedByteCount(localVertCount, layout.totalStride, out int packedByteCount) ||
totalLocalVerts > MAX_OPTIMIZE_VERTICES - localVertCount)
{
result.error = $"Mesh exceeds optimization budget while processing submesh {s}";
return result;
}
totalLocalVerts += localVertCount;

// Pack vertices into interleaved byte buffer
byte[] vertexBytes = PackVertices(
globalToLocal, layout,
globalToLocal, layout, packedByteCount,
positions, normals, tangents, colors, uvData);

// Build local index buffer
Expand All @@ -155,7 +187,7 @@ public static OptimizeResult Optimize(Mesh mesh, float overdrawThreshold = 1.05f
localIndices[i] = (uint)globalToLocal[subTris[i]];

// Allocate output buffers
byte[] outVertexBytes = new byte[localVertCount * layout.totalStride];
byte[] outVertexBytes = new byte[packedByteCount];
uint[] outIndices = new uint[localIndexCount];
uint outVertCount;

Expand All @@ -173,6 +205,12 @@ public static OptimizeResult Optimize(Mesh mesh, float overdrawThreshold = 1.05f
return result;
}

if (outVertCount > (uint)localVertCount)
{
result.error = $"meshoptOptimize returned an invalid vertex count on submesh {s}";
return result;
}

// Unpack output vertices and append to global lists
UnpackVertices(
outVertexBytes, (int)outVertCount, layout,
Expand Down Expand Up @@ -309,14 +347,27 @@ static ChannelLayout BuildChannelLayout(Mesh mesh)

// ── Packing ──

static bool TryGetPackedByteCount(int vertexCount, int stride, out int byteCount)
{
byteCount = 0;
if (vertexCount < 0 || vertexCount > MAX_OPTIMIZE_VERTICES || stride < 12)
return false;

long requiredBytes = (long)vertexCount * stride;
if (requiredBytes > MAX_PACKED_BYTES || requiredBytes > int.MaxValue)
return false;

byteCount = (int)requiredBytes;
return true;
}

static unsafe byte[] PackVertices(
Dictionary<int, int> globalToLocal,
in ChannelLayout layout,
in ChannelLayout layout, int packedByteCount,
Vector3[] positions, Vector3[] normals, Vector4[] tangents,
Color32[] colors, List<Vector4>[] uvData)
{
int localVertCount = globalToLocal.Count;
byte[] bytes = new byte[localVertCount * layout.totalStride];
byte[] bytes = new byte[packedByteCount];

fixed (byte* pBytes = bytes)
{
Expand Down
Loading