Merge branch 'fix/utf8-string-marshaling-complete'

Author: tinysec

Function/BNCreateMetadataStringListData.cs

@@ -17,12 +17,13 @@ internal static partial class NativeMethods
             EntryPoint = "BNCreateMetadataStringListData"
         )]
 		internal static extern IntPtr BNCreateMetadataStringListData(
-			
-			// const char** data
-		    string[] data  , 
-			
+
+			// const char** data (UTF-8 block built by the caller; string[] cannot be
+			// marshaled as UTF-8 by attribute, so it is passed as a raw pointer)
+		    IntPtr data  ,
+
 			// uint64_t size
-		    ulong size  
+		    ulong size
 		);
 	}
 }

Function/BNCreateMetadataValueStore.cs

@@ -17,15 +17,16 @@ internal static partial class NativeMethods
             EntryPoint = "BNCreateMetadataValueStore"
         )]
 		internal static extern IntPtr BNCreateMetadataValueStore(
-			
-			// const char** keys
-		    string[] keys  , 
-			
+
+			// const char** keys (UTF-8 block built by the caller; string[] cannot be
+			// marshaled as UTF-8 by attribute, so it is passed as a raw pointer)
+		    IntPtr keys  ,
+
 			// BNMetadata** values
-		    IntPtr[] values  , 
-			
+		    IntPtr[] values  ,
+
 			// uint64_t size
-		    ulong size  
+		    ulong size
 		);
 	}
 }

Handle/BNMetadata.cs

@@ -123,10 +123,10 @@ public Metadata(double[] value)
 
 	    }
 
-	    public Metadata(string[] value) 
-		    : this(NativeMethods.BNCreateMetadataStringListData(value , (ulong)value.Length) , true)
+	    public Metadata(string[] value)
+		    : this(Metadata.rawCreateMetadataStringList(value) , true)
 	    {
-	        
+
 	    }
 
 	    public Metadata(Metadata[] value) 
@@ -225,12 +225,21 @@ private static IntPtr rawCreateMetadataDict(IDictionary<string,Metadata> dict)
 		    {
 			    handles.Add(item.DangerousGetHandle());
 		    }
-		    
-		    return NativeMethods.BNCreateMetadataValueStore(
-			    dict.Keys.ToArray() ,
-			    handles.ToArray(),
-				(ulong)handles.Count
-			 );
+
+		    string[] keys = dict.Keys.ToArray();
+
+		    // Keys are a UTF-8 char** block (see rawCreateMetadataStringList); the
+		    // core copies them, so freeing after the call is safe.
+		    using (ScopedAllocator allocator = new ScopedAllocator())
+		    {
+			    IntPtr keyBlock = allocator.AllocUtf8StringArray(keys);
+
+			    return NativeMethods.BNCreateMetadataValueStore(
+				    keyBlock ,
+				    handles.ToArray() ,
+				    (ulong)handles.Count
+			    );
+		    }
 	    }
 
 	    public static Metadata FromBoolArray(bool[] values)
@@ -264,9 +273,27 @@ public static Metadata FromDoubleArray(double[] values)
 	    public static Metadata FromStringArray(string[] values)
 	    {
 		    return Metadata.MustTakeHandle(
-			    NativeMethods.BNCreateMetadataStringListData(values , (ulong)values.Length)
+			    Metadata.rawCreateMetadataStringList(values)
 		    );
 	    }
+
+	    // BN stores string lists as UTF-8 char** blocks. .NET cannot marshal a
+	    // string[] element as UTF-8 via [MarshalAs] (LPUTF8Str is rejected for an
+	    // array subtype), so build the char** block by hand. The core copies the
+	    // strings, so freeing the block after the call (ScopedAllocator dispose)
+	    // is safe.
+	    private static IntPtr rawCreateMetadataStringList(string[] values)
+	    {
+		    using (ScopedAllocator allocator = new ScopedAllocator())
+		    {
+			    IntPtr block = allocator.AllocUtf8StringArray(values);
+
+			    return NativeMethods.BNCreateMetadataStringListData(
+				    block ,
+				    (ulong)values.Length
+			    );
+		    }
+	    }
 
 	    internal static Metadata? NewFromHandle(IntPtr handle)
 	    {
@@ -425,7 +452,7 @@ public string[] ToStringList()
 			    out ulong arrayLength
 		    );
 
-		    string[] values = UnsafeUtils.ReadAnsiStringArray(arrayPointer , arrayLength);
+		    string[] values = UnsafeUtils.ReadUtf8StringArray(arrayPointer , arrayLength);
 
 		    if (arrayPointer != IntPtr.Zero )
 		    {
@@ -730,9 +757,9 @@ public bool SetValue(string key , double[] values)
 	    public bool SetValue(string key , string[] values)
 	    {
 		    return NativeMethods.BNMetadataSetValueForKey(
-			    this.handle, 
-			    key, 
-			    NativeMethods.BNCreateMetadataStringListData(values , (ulong)values.Length)
+			    this.handle,
+			    key,
+			    Metadata.rawCreateMetadataStringList(values)
 		    );
 	    }
 

ScopedAllocator.cs

@@ -38,34 +38,51 @@ internal IntPtr AllocHGlobal(int size)
 			return pointer;
 		}
 
-		internal IntPtr AllocAnsiString(string text)
+		// BN core expects UTF-8 for every string input. Encode into an HGlobal buffer
+		// tracked by this allocator, so disposal keeps using Marshal.FreeHGlobal.
+		internal IntPtr AllocUtf8String(string text)
 		{
-			IntPtr pointer = Marshal.StringToHGlobalAnsi(text);
-			
-			m_pointers.Add(pointer);
+			byte[] utf8 = Encoding.UTF8.GetBytes(text ?? string.Empty);
+
+			IntPtr pointer = this.AllocHGlobal(utf8.Length + 1);
+
+			Marshal.Copy(utf8 , 0 , pointer , utf8.Length);
+			Marshal.WriteByte(pointer , utf8.Length , 0);
 
 			return pointer;
 		}
-		
-		internal IntPtr AllocAnsiStringArray(string[] texts)
+
+		internal IntPtr AllocUtf8StringArray(string[] texts)
 		{
 			if (texts == null || texts.Length == 0)
 			{
 				return IntPtr.Zero;
 			}
 
 			IntPtr arrayPointer = this.AllocHGlobal(IntPtr.Size * texts.Length);
-			
+
 			for (int i = 0; i < texts.Length; i++)
 			{
-				IntPtr textPointer = this.AllocAnsiString(texts[i]);
-				
+				IntPtr textPointer = this.AllocUtf8String(texts[i]);
+
 				Marshal.WriteIntPtr(arrayPointer , i * IntPtr.Size , textPointer);
 			}
 
 			return arrayPointer;
 		}
 
+		// The "Ansi"-named builders are misnomers retained for the generated call
+		// sites; they delegate to the UTF-8 builders so every write path emits UTF-8.
+		internal IntPtr AllocAnsiString(string text)
+		{
+			return this.AllocUtf8String(text);
+		}
+
+		internal IntPtr AllocAnsiStringArray(string[] texts)
+		{
+			return this.AllocUtf8StringArray(texts);
+		}
+
 		internal IntPtr AllocInteger<T>(T value) where T : IConvertible
 		{
 			int elementSize = Marshal.SizeOf<T>();

UnsafeUtils.cs

@@ -136,21 +136,15 @@ internal static T[] ReadNumberArray<T>(IntPtr arrayPointer , ulong arrayLength)
 			return targets.ToArray();
 		}
 
+		// BN core stores and returns every string as UTF-8 (the reference binding
+		// decodes core strings with bytes.decode('utf8'); see the generator's
+		// pyNativeStr helper). This reader and its Take/array variants all funnel
+		// their per-element decode through here, so decoding as UTF-8 here fixes
+		// every core-string reader at once. The "Ansi" name is a misnomer retained
+		// only for the generated call sites that reference it.
 		internal static string ReadAnsiString(IntPtr address)
 		{
-			string text = string.Empty;
-			
-			if (IntPtr.Zero != address)
-			{
-				string? optional = Marshal.PtrToStringAnsi(address);
-
-				if (null != optional)
-				{
-					text = optional;
-				}
-			}
-
-			return text;
+			return UnsafeUtils.ReadUtf8String(address);
 		}
 
 		internal static string ReadUtf8String(IntPtr address)
@@ -595,9 +589,19 @@ internal static TNative[] ConvertToNativeArray<TNative,TManaged>(TManaged[] sour
 		}
 
 
+		// BN core expects UTF-8 for every string input. Encode into an HGlobal buffer
+		// so the existing Marshal.FreeHGlobal ownership at the call sites is unchanged.
+		// The "Ansi" name is a misnomer retained only for the generated call sites.
 		internal static IntPtr AllocAnsiString(string text)
 		{
-			return Marshal.StringToHGlobalAnsi(text);
+			byte[] utf8 = System.Text.Encoding.UTF8.GetBytes(text ?? string.Empty);
+
+			IntPtr pointer = Marshal.AllocHGlobal(utf8.Length + 1);
+
+			Marshal.Copy(utf8 , 0 , pointer , utf8.Length);
+			Marshal.WriteByte(pointer , utf8.Length , 0);
+
+			return pointer;
 		}
 
 		internal static IntPtr AllocUtf8String(string text)