diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java index 9a4165ca..4cae1d15 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/GlueMetadataStringCleaner.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,11 +29,15 @@ */ public class GlueMetadataStringCleaner { + private static final int MAX_COLUMN_COMMENT_LENGTH = 254; + private static final int MAX_DESCRIPTION_LENGTH = 2048; + public TableInput cleanTable(TableInput input) { // Clean SerDes cleanColumns(input.getStorageDescriptor().getColumns()); // Clean Partition Keys cleanColumns(input.getPartitionKeys()); + input.setDescription(cleanDescription(input.getDescription())); return input; } @@ -47,20 +51,28 @@ public PartitionInput cleanPartition(PartitionInput input) { return input; } + /** + * Cleans a Glue TableInput description to satisfy API constraints: + * + */ + private String cleanDescription(String description) { + return truncateToMaxAllowedChars(removeNonUnicodeChars(description), MAX_DESCRIPTION_LENGTH); + } + private void cleanColumns(List columns) { for (Column column : columns) { - column.setComment(this.truncateToMaxAllowedChars(this.removeNonUnicodeChars(column.getComment()))); + column.setComment(this.truncateToMaxAllowedChars(this.removeNonUnicodeChars(column.getComment()), MAX_COLUMN_COMMENT_LENGTH)); } } - public String truncateToMaxAllowedChars(String input) { + public String truncateToMaxAllowedChars(String input, int maxLength) { if (input == null) { return null; } - if (input.length() > 254) { - return input.substring(0, 254); - } - return input; + return input.length() > maxLength ? input.substring(0, maxLength) : input; } public String removeNonUnicodeChars(String input) { diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparator.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparator.java index 40ba8762..93350462 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparator.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparator.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.expediagroup.apiary.extensions.gluesync.listener.service; import java.util.*; @@ -146,7 +145,7 @@ private boolean columnsEqualsCleaned(List tableParams = table.getParameters(); + String description = (tableParams != null) ? tableParams.get("comment") : null; + return new TableInput() .withName(table.getTableName()) + .withDescription(description) .withLastAccessTime(date) .withOwner(table.getOwner()) - .withParameters(addClassification(table.getParameters(), storageDescriptor.getInputFormat())) + .withParameters(addClassification(tableParams, storageDescriptor.getInputFormat())) .withPartitionKeys(partitionKeys) .withRetention(table.getRetention()) .withStorageDescriptor(sd) diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java index f6a4d9d3..0f2d8a6e 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/GlueMetadataStringCleanerTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ */ package com.expediagroup.apiary.extensions.gluesync.listener; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.util.Collections; @@ -64,6 +66,41 @@ public void testTableInput() { assertTrue(partitionColumn.getComment().startsWith("A")); } + @Test + public void testTableInputDescriptionNonUnicodeCharsRemoved() { + TableInput tableInput = new TableInput(); + tableInput.setStorageDescriptor(new StorageDescriptor().withColumns(Collections.emptyList())); + tableInput.setPartitionKeys(Collections.emptyList()); + tableInput.setDescription("\uD999valid description"); + + TableInput result = glueMetadataStringCleaner.cleanTable(tableInput); + + assertEquals("valid description", result.getDescription()); + } + + @Test + public void testTableInputDescriptionTruncatedTo2048Chars() { + TableInput tableInput = new TableInput(); + tableInput.setStorageDescriptor(new StorageDescriptor().withColumns(Collections.emptyList())); + tableInput.setPartitionKeys(Collections.emptyList()); + tableInput.setDescription(generateCharString(3000)); + + TableInput result = glueMetadataStringCleaner.cleanTable(tableInput); + + assertEquals(2048, result.getDescription().length()); + } + + @Test + public void testTableInputDescriptionNullUnchanged() { + TableInput tableInput = new TableInput(); + tableInput.setStorageDescriptor(new StorageDescriptor().withColumns(Collections.emptyList())); + tableInput.setPartitionKeys(Collections.emptyList()); + + TableInput result = glueMetadataStringCleaner.cleanTable(tableInput); + + assertNull(result.getDescription()); + } + @Test public void testPartitionInput() { PartitionInput partitionInput = new PartitionInput(); @@ -86,7 +123,7 @@ public void testPartitionInput() { private String generateCharString(int length) { StringBuilder sb = new StringBuilder(length); - for (int i = 0; i < 300; i++) { + for (int i = 0; i < length; i++) { sb.append("A"); } return sb.toString(); diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparatorTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparatorTest.java index afc88033..36139c3a 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparatorTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGluePartitionComparatorTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -135,7 +135,7 @@ private com.amazonaws.services.glue.model.Partition buildGluePartitionFromHive( c.setName(fs.getName()); c.setType(fs.getType()); String cleanedComment = cleaner.removeNonUnicodeChars(fs.getComment()); - cleanedComment = cleaner.truncateToMaxAllowedChars(cleanedComment); + cleanedComment = cleaner.truncateToMaxAllowedChars(cleanedComment, 254); c.setComment(cleanedComment); cols.add(c); } diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java index 7211ccc9..4cddc0aa 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/service/HiveToGlueTransformerTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-2025 Expedia, Inc. + * Copyright (C) 2018-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import java.util.Arrays; import java.util.Collections; @@ -27,9 +28,11 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; import org.junit.Test; import com.amazonaws.services.glue.model.PartitionInput; +import com.amazonaws.services.glue.model.TableInput; public class HiveToGlueTransformerTest { private final HiveToGlueTransformer transformer = new HiveToGlueTransformer(null); @@ -103,6 +106,67 @@ private Partition createCompleteHivePartition() { return partition; } + @Test + public void testTransformTablePopulatesDescriptionFromCommentParam() { + Table table = createMinimalHiveTable(); + table.getParameters().put("comment", "my table description"); + + TableInput result = transformer.transformTable(table); + + assertEquals("my table description", result.getDescription()); + } + + @Test + public void testTransformTableDescriptionIsNullWhenNoCommentParam() { + Table table = createMinimalHiveTable(); + + TableInput result = transformer.transformTable(table); + + assertNull(result.getDescription()); + } + + @Test + public void testTransformTableDescriptionIsNullWhenParamsExistButNoComment() { + Table table = createMinimalHiveTable(); + table.getParameters().put("other.param", "some value"); + + TableInput result = transformer.transformTable(table); + + assertNull(result.getDescription()); + } + + private Table createMinimalHiveTable() { + Table table = new Table(); + table.setTableName("test_table"); + table.setDbName("test_db"); + table.setOwner("test_owner"); + table.setTableType("EXTERNAL_TABLE"); + table.setParameters(new HashMap<>()); + + org.apache.hadoop.hive.metastore.api.StorageDescriptor sd = new org.apache.hadoop.hive.metastore.api.StorageDescriptor(); + sd.setCols(Collections.emptyList()); + sd.setBucketCols(Collections.emptyList()); + sd.setCompressed(false); + sd.setInputFormat("org.apache.hadoop.mapred.TextInputFormat"); + sd.setLocation("s3://test-bucket/path"); + sd.setNumBuckets(0); + sd.setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"); + sd.setParameters(Collections.emptyMap()); + + org.apache.hadoop.hive.metastore.api.SerDeInfo serdeInfo = new org.apache.hadoop.hive.metastore.api.SerDeInfo(); + serdeInfo.setName("test-serde"); + serdeInfo.setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"); + serdeInfo.setParameters(Collections.emptyMap()); + sd.setSerdeInfo(serdeInfo); + + sd.setSortCols(Collections.emptyList()); + sd.setStoredAsSubDirectories(false); + table.setSd(sd); + table.setPartitionKeys(Collections.emptyList()); + + return table; + } + /** * Helper method to create test parameters */