From 0353dbd4576a0aa7d0814def844e5e9378a61708 Mon Sep 17 00:00:00 2001 From: James Faulkner Date: Wed, 1 Jul 2026 11:39:55 +0100 Subject: [PATCH 1/5] feat: populate Glue table description from HMS comment parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TableInput.description was never set during HMS→Glue sync, leaving table documentation invisible in the Glue Data Catalog UI. Now reads the table-level 'comment' TBLPROPERTY and maps it to the Glue description field, consistent with how database descriptions are already handled. Also extends GlueMetadataStringCleaner.cleanTable() to sanitise the description field so the ValidationException retry path is safe if the comment contains non-Unicode characters. Co-Authored-By: Claude Sonnet 4.6 --- .../service/GlueMetadataStringCleaner.java | 4 +- .../service/HiveToGlueTransformer.java | 8 ++- .../GlueMetadataStringCleanerTest.java | 27 ++++++++- .../service/HiveToGlueTransformerTest.java | 56 ++++++++++++++++++- 4 files changed, 90 insertions(+), 5 deletions(-) diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java index 9a4165ca..ec155303 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,6 +34,8 @@ public TableInput cleanTable(TableInput input) { cleanColumns(input.getStorageDescriptor().getColumns()); // Clean Partition Keys cleanColumns(input.getPartitionKeys()); + // Clean Description + input.setDescription(removeNonUnicodeChars(input.getDescription())); return input; } diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformer.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformer.java index e3de4aae..d781dd46 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformer.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformer.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -93,11 +93,15 @@ public TableInput transformTable(final Table table) { .withSortColumns(sortOrders) .withStoredAsSubDirectories(storageDescriptor.isStoredAsSubDirectories()); + Map tableParams = table.getParameters(); + String description = (tableParams != null) ? tableParams.get("comment") : null; + return new TableInput() .withName(table.getTableName()) + .withDescription(description) .withLastAccessTime(date) .withOwner(table.getOwner()) - .withParameters(addClassification(table.getParameters(), storageDescriptor.getInputFormat())) + .withParameters(addClassification(tableParams, storageDescriptor.getInputFormat())) .withPartitionKeys(partitionKeys) .withRetention(table.getRetention()) .withStorageDescriptor(sd) diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java index f6a4d9d3..a348f005 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ */ package com.expediagroup.apiary.extensions.gluesync.listener; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.util.Collections; @@ -64,6 +66,29 @@ public void testTableInput() { assertTrue(partitionColumn.getComment().startsWith("A")); } + @Test + public void testTableInputDescriptionNonUnicodeCharsRemoved() { + TableInput tableInput = new TableInput(); + tableInput.setStorageDescriptor(new StorageDescriptor().withColumns(Collections.emptyList())); + tableInput.setPartitionKeys(Collections.emptyList()); + tableInput.setDescription("\uD999valid description"); + + TableInput result = glueMetadataStringCleaner.cleanTable(tableInput); + + assertEquals("valid description", result.getDescription()); + } + + @Test + public void testTableInputDescriptionNullUnchanged() { + TableInput tableInput = new TableInput(); + tableInput.setStorageDescriptor(new StorageDescriptor().withColumns(Collections.emptyList())); + tableInput.setPartitionKeys(Collections.emptyList()); + + TableInput result = glueMetadataStringCleaner.cleanTable(tableInput); + + assertNull(result.getDescription()); + } + @Test public void testPartitionInput() { PartitionInput partitionInput = new PartitionInput(); diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java index 7211ccc9..9ff3fe45 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import java.util.Arrays; import java.util.Collections; @@ -27,9 +28,11 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; import org.junit.Test; import com.amazonaws.services.glue.model.PartitionInput; +import com.amazonaws.services.glue.model.TableInput; public class HiveToGlueTransformerTest { private final HiveToGlueTransformer transformer = new HiveToGlueTransformer(null); @@ -103,6 +106,57 @@ private Partition createCompleteHivePartition() { return partition; } + @Test + public void testTransformTablePopulatesDescriptionFromCommentParam() { + Table table = createMinimalHiveTable(); + table.getParameters().put("comment", "my table description"); + + TableInput result = transformer.transformTable(table); + + assertEquals("my table description", result.getDescription()); + } + + @Test + public void testTransformTableDescriptionIsNullWhenNoCommentParam() { + Table table = createMinimalHiveTable(); + + TableInput result = transformer.transformTable(table); + + assertNull(result.getDescription()); + } + + private Table createMinimalHiveTable() { + Table table = new Table(); + table.setTableName("test_table"); + table.setDbName("test_db"); + table.setOwner("test_owner"); + table.setTableType("EXTERNAL_TABLE"); + table.setParameters(new HashMap<>()); + + org.apache.hadoop.hive.metastore.api.StorageDescriptor sd = new org.apache.hadoop.hive.metastore.api.StorageDescriptor(); + sd.setCols(Collections.emptyList()); + sd.setBucketCols(Collections.emptyList()); + sd.setCompressed(false); + sd.setInputFormat("org.apache.hadoop.mapred.TextInputFormat"); + sd.setLocation("s3://test-bucket/path"); + sd.setNumBuckets(0); + sd.setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"); + sd.setParameters(Collections.emptyMap()); + + org.apache.hadoop.hive.metastore.api.SerDeInfo serdeInfo = new org.apache.hadoop.hive.metastore.api.SerDeInfo(); + serdeInfo.setName("test-serde"); + serdeInfo.setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"); + serdeInfo.setParameters(Collections.emptyMap()); + sd.setSerdeInfo(serdeInfo); + + sd.setSortCols(Collections.emptyList()); + sd.setStoredAsSubDirectories(false); + table.setSd(sd); + table.setPartitionKeys(Collections.emptyList()); + + return table; + } + /** * Helper method to create test parameters */ From 427bc6d56c55fd06d1548fad0bf27ffd9700dc27 Mon Sep 17 00:00:00 2001 From: James Faulkner Date: Wed, 1 Jul 2026 12:22:47 +0100 Subject: [PATCH 2/5] feat: truncate and sanitise Glue table description in cleaner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend GlueMetadataStringCleaner to also clean the TableInput description field — removing non-Unicode characters and truncating to the Glue API limit of 2048 chars — so the ValidationException retry path is safe if a table comment contains invalid characters. Extracts cleanDescription() as a private method consistent with cleanColumns(). Also adds a test covering tables with parameters but no comment key. Co-Authored-By: Claude Sonnet 4.6 --- .../service/GlueMetadataStringCleaner.java | 15 ++++++++++----- .../listener/GlueMetadataStringCleanerTest.java | 12 ++++++++++++ .../service/HiveToGlueTransformerTest.java | 10 ++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java index ec155303..21b43d1b 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java @@ -35,7 +35,7 @@ public TableInput cleanTable(TableInput input) { // Clean Partition Keys cleanColumns(input.getPartitionKeys()); // Clean Description - input.setDescription(removeNonUnicodeChars(input.getDescription())); + input.setDescription(cleanDescription(input.getDescription())); return input; } @@ -49,6 +49,10 @@ public PartitionInput cleanPartition(PartitionInput input) { return input; } + private String cleanDescription(String description) { + return truncateToMaxAllowedChars(removeNonUnicodeChars(description), 2048); + } + private void cleanColumns(List columns) { for (Column column : columns) { column.setComment(this.truncateToMaxAllowedChars(this.removeNonUnicodeChars(column.getComment()))); @@ -56,13 +60,14 @@ private void cleanColumns(List columns) { } public String truncateToMaxAllowedChars(String input) { + return truncateToMaxAllowedChars(input, 254); + } + + private String truncateToMaxAllowedChars(String input, int maxLength) { if (input == null) { return null; } - if (input.length() > 254) { - return input.substring(0, 254); - } - return input; + return input.length() > maxLength ? input.substring(0, maxLength) : input; } public String removeNonUnicodeChars(String input) { diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java index a348f005..af262be0 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java @@ -78,6 +78,18 @@ public void testTableInputDescriptionNonUnicodeCharsRemoved() { assertEquals("valid description", result.getDescription()); } + @Test + public void testTableInputDescriptionTruncatedTo2048Chars() { + TableInput tableInput = new TableInput(); + tableInput.setStorageDescriptor(new StorageDescriptor().withColumns(Collections.emptyList())); + tableInput.setPartitionKeys(Collections.emptyList()); + tableInput.setDescription("A".repeat(3000)); + + TableInput result = glueMetadataStringCleaner.cleanTable(tableInput); + + assertEquals(2048, result.getDescription().length()); + } + @Test public void testTableInputDescriptionNullUnchanged() { TableInput tableInput = new TableInput(); diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java index 9ff3fe45..4cddc0aa 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java @@ -125,6 +125,16 @@ public void testTransformTableDescriptionIsNullWhenNoCommentParam() { assertNull(result.getDescription()); } + @Test + public void testTransformTableDescriptionIsNullWhenParamsExistButNoComment() { + Table table = createMinimalHiveTable(); + table.getParameters().put("other.param", "some value"); + + TableInput result = transformer.transformTable(table); + + assertNull(result.getDescription()); + } + private Table createMinimalHiveTable() { Table table = new Table(); table.setTableName("test_table"); From 26281bb6476e5604400c2c1c48b2a5cad9bb8b25 Mon Sep 17 00:00:00 2001 From: James Faulkner Date: Wed, 1 Jul 2026 12:24:30 +0100 Subject: [PATCH 3/5] refactor: improve GlueMetadataStringCleaner code clarity Co-Authored-By: Claude Sonnet 4.6 --- .../listener/service/GlueMetadataStringCleaner.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java index 21b43d1b..d70f01d2 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java @@ -34,7 +34,6 @@ public TableInput cleanTable(TableInput input) { cleanColumns(input.getStorageDescriptor().getColumns()); // Clean Partition Keys cleanColumns(input.getPartitionKeys()); - // Clean Description input.setDescription(cleanDescription(input.getDescription())); return input; } @@ -49,6 +48,13 @@ public PartitionInput cleanPartition(PartitionInput input) { return input; } + /** + * Cleans a Glue TableInput description to satisfy API constraints: + *
    + *
  • Unicode characters only
    • + *
  • Maximum 2048 characters
    • + *
+ */ private String cleanDescription(String description) { return truncateToMaxAllowedChars(removeNonUnicodeChars(description), 2048); } From 0d3daf157fe66e085b5b71d95dd9106648b987d8 Mon Sep 17 00:00:00 2001 From: James Faulkner Date: Wed, 1 Jul 2026 12:32:53 +0100 Subject: [PATCH 4/5] refactor: remove truncateToMaxAllowedChars overload, use private constants for limits Co-Authored-By: Claude Sonnet 4.6 --- .../listener/service/GlueMetadataStringCleaner.java | 13 ++++++------- .../service/HiveToGluePartitionComparator.java | 5 ++--- .../service/HiveToGluePartitionComparatorTest.java | 4 ++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java index d70f01d2..4cae1d15 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java @@ -29,6 +29,9 @@ */ public class GlueMetadataStringCleaner { + private static final int MAX_COLUMN_COMMENT_LENGTH = 254; + private static final int MAX_DESCRIPTION_LENGTH = 2048; + public TableInput cleanTable(TableInput input) { // Clean SerDes cleanColumns(input.getStorageDescriptor().getColumns()); @@ -56,20 +59,16 @@ public PartitionInput cleanPartition(PartitionInput input) { * */ private String cleanDescription(String description) { - return truncateToMaxAllowedChars(removeNonUnicodeChars(description), 2048); + return truncateToMaxAllowedChars(removeNonUnicodeChars(description), MAX_DESCRIPTION_LENGTH); } private void cleanColumns(List columns) { for (Column column : columns) { - column.setComment(this.truncateToMaxAllowedChars(this.removeNonUnicodeChars(column.getComment()))); + column.setComment(this.truncateToMaxAllowedChars(this.removeNonUnicodeChars(column.getComment()), MAX_COLUMN_COMMENT_LENGTH)); } } - public String truncateToMaxAllowedChars(String input) { - return truncateToMaxAllowedChars(input, 254); - } - - private String truncateToMaxAllowedChars(String input, int maxLength) { + public String truncateToMaxAllowedChars(String input, int maxLength) { if (input == null) { return null; } diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparator.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparator.java index 40ba8762..93350462 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparator.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparator.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.expediagroup.apiary.extensions.gluesync.listener.service; import java.util.*; @@ -146,7 +145,7 @@ private boolean columnsEqualsCleaned(List Date: Wed, 1 Jul 2026 12:50:27 +0100 Subject: [PATCH 5/5] fix: replace String.repeat() with Java 8 compatible string generation String.repeat() requires Java 11+; GHA runs Java 8. Also fixes the pre-existing bug in generateCharString() which ignored its length param. Co-Authored-By: Claude Sonnet 4.6 --- .../gluesync/listener/GlueMetadataStringCleanerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java index af262be0..0f2d8a6e 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java @@ -83,7 +83,7 @@ public void testTableInputDescriptionTruncatedTo2048Chars() { TableInput tableInput = new TableInput(); tableInput.setStorageDescriptor(new StorageDescriptor().withColumns(Collections.emptyList())); tableInput.setPartitionKeys(Collections.emptyList()); - tableInput.setDescription("A".repeat(3000)); + tableInput.setDescription(generateCharString(3000)); TableInput result = glueMetadataStringCleaner.cleanTable(tableInput); @@ -123,7 +123,7 @@ public void testPartitionInput() { private String generateCharString(int length) { StringBuilder sb = new StringBuilder(length); - for (int i = 0; i < 300; i++) { + for (int i = 0; i < length; i++) { sb.append("A"); } return sb.toString();