Fix alignment of CSV inlay hints for East Asian wide characters

package.json

@@ -56,6 +56,7 @@
     "typescript": "^5.3.3"
   },
   "dependencies": {
+    "eastasianwidth": "^0.3.0",
     "papaparse": "^5.4.1"
   }
 }

src/csvParser.ts

@@ -8,18 +8,19 @@
  */
 
 import Papa from 'papaparse';
+import * as eaw from 'eastasianwidth';
 
 /**
  * The output type of the ParseCSVWithPapa function.
  * It has the string length of all the data in the csv, and it
  * has the delimiter of the csv
- * @property {number[][]} data - the string length of all the data in the csv.
- * I believe it's matrix notation, so the first index is the row number, and the second
- * index is the column number
- * @property {string} delimiter - the delimiter of the csv.
+ * @property {number[][]} data - the display width of all the cells in the csv.
+ * @property {number[][]} rawLengths - the raw character count of all cells, for position tracking.
+ * @property {number} delimiterLength - the length of the delimiter.
 */
 export interface CsvData {
     data: number[][];
+    rawLengths: number[][];
     delimiterLength: number;
 }
 
@@ -41,12 +42,15 @@ export async function parseCSVWithPapa(csv: string): Promise<CsvData> {
 
     // Step 2: Reconstruct raw text and capture value lengths
     const valueLengths: number[][] = [];
+    const rawLengths: number[][] = [];
     const rows = csv.split(/\r?\n/); // Split raw content by newlines
 
     rows.forEach((row) => {
         let inQuotes = false;
         let valueStart = 0;
+        let displayWidth = 0;
         let rowLengths: number[] = [];
+        let rowRawLengths: number[] = [];
 
         for (let i = 0; i < row.length; i++) {
             const char = row[i];
@@ -56,17 +60,26 @@ export async function parseCSVWithPapa(csv: string): Promise<CsvData> {
             }
 
             if (char === delimiter && !inQuotes) {
-                rowLengths.push(i - valueStart);
+                const rawLen = i - valueStart;
+                rowRawLengths.push(rawLen);
+                rowLengths.push(displayWidth);
+                displayWidth = 0;
                 valueStart = i + 1;
+            } else {
+                displayWidth += eaw.characterLength(char);
             }
         }
 
-        rowLengths.push(row.length - valueStart);
+        const rawLen = row.length - valueStart;
+        rowRawLengths.push(rawLen);
+        rowLengths.push(displayWidth);
+        rawLengths.push(rowRawLengths);
         valueLengths.push(rowLengths);
     });
 
     return {
         data: valueLengths,
+        rawLengths: rawLengths,
         delimiterLength: delimiter.length
     };
 }

src/extension.ts

@@ -78,7 +78,7 @@ class CsvAlignInlayHintsProvider implements vscode.InlayHintsProvider {
             ? headerText
             : `${headerText}\n${bodyText}`;
 
-        const {data: cellLengths, delimiterLength} = await parseCSVWithPapa(textToParse);
+        const {data: cellLengths, rawLengths, delimiterLength} = await parseCSVWithPapa(textToParse);
 
         if (!cellLengths.length) {
             return [];
@@ -88,6 +88,7 @@ class CsvAlignInlayHintsProvider implements vscode.InlayHintsProvider {
         const maxColumnWidths = getMaxColumnWidthsFromCellLengths(cellLengths);
         const hints: VsCodeInlayHintAdapter[] = getHintsFromCellLengths(
             cellLengths,
+            rawLengths,
             maxColumnWidths,
             delimiterLength,
             hintCharacter

src/helpers/getHintAndCurrentPosfromCol.ts

@@ -45,13 +45,14 @@ export interface VsCodeInlayHintAdapter {
 
 export function getHintAndCurrentPosfromCol(
   cellLength: number,
+  rawCellLength: number,
   colMaxWidth: number,
   currentPos: number,
   rowIndex: number,
   delimiterLength: number,
   hintCharacter: string,
 ): [VsCodeInlayHintAdapter, number] {
-    const startPos = currentPos + cellLength;
+    const startPos = currentPos + rawCellLength;
     const spacesNeeded = colMaxWidth - cellLength;
     const position: VsCodePositionAdapter = {rowIndex: rowIndex, startPos: startPos};
 

src/helpers/getHintsFromCellLengths.ts

@@ -4,14 +4,16 @@ import { getHintAndCurrentPosfromCol, VsCodeInlayHintAdapter } from './getHintAn
  * Helper function to generate inlay hints from the cell lengths
  * of a CSV.
  *
- * @param {string[][]} cellLengths - The lengths of the cells in the CSV
- * @param {number[]} columnMaxWidths - The maximum width of each column
+ * @param {number[][]} cellLengths - The display widths of the cells in the CSV
+ * @param {number[][]} rawLengths - The raw character counts of the cells
+ * @param {number[]} columnMaxWidths - The maximum display width of each column
  * @param {number} delimiterLength - The length of the delimiter
  * @param {string} hintCharacter - The character to use for the hint
  * @returns {VsCodeInlayHintAdapter[]} - The inlay hints
  */
 export function getHintsFromCellLengths(
   cellLengths: number[][],
+  rawLengths: number[][],
   columnMaxWidths: number[],
   delimiterLength: number,
   hintCharacter: string
@@ -26,6 +28,7 @@ export function getHintsFromCellLengths(
       rowOfCellLengths.forEach((cellLength, colIndex) => {
           const returns: [VsCodeInlayHintAdapter, number] = getHintAndCurrentPosfromCol(
             cellLength,
+            rawLengths[rowIndex][colIndex],
             columnMaxWidths[colIndex],
             currentPos,
             rowIndex,

src/test/core.test.ts

@@ -24,6 +24,7 @@ suite('Core helpers', () => {
 
 	test('creates an inlay hint at the end of the current cell', () => {
 		const [hint, nextPos] = getHintAndCurrentPosfromCol(
+			5,
 			5,
 			7,
 			0,
@@ -41,6 +42,10 @@ suite('Core helpers', () => {
 
 	test('creates hints for every cell length', () => {
 		const hints = getHintsFromCellLengths(
+			[
+				[5, 6, 7],
+				[7, 2, 6],
+			],
 			[
 				[5, 6, 7],
 				[7, 2, 6],
@@ -102,6 +107,10 @@ suite('CSV parser', () => {
 				[5, 6, 4],
 				[7, 5, 1],
 			],
+			rawLengths: [
+				[5, 6, 4],
+				[7, 5, 1],
+			],
 			delimiterLength: 1,
 		});
 	});
@@ -114,6 +123,10 @@ suite('CSV parser', () => {
 				[15, 1],
 				[7, 1],
 			],
+			rawLengths: [
+				[15, 1],
+				[7, 1],
+			],
 			delimiterLength: 1,
 		});
 	});
@@ -126,6 +139,10 @@ suite('CSV parser', () => {
 				[4, 5],
 				[5, 2],
 			],
+			rawLengths: [
+				[4, 5],
+				[5, 2],
+			],
 			delimiterLength: 1,
 		});
 	});
@@ -138,6 +155,10 @@ suite('CSV parser', () => {
 				[1, 1, 1],
 				[6, 3],
 			],
+			rawLengths: [
+				[1, 1, 1],
+				[6, 3],
+			],
 			delimiterLength: 1,
 		});
 	});