From e1d8da30522d6be07aa9cf2e4d41673d7184ce9b Mon Sep 17 00:00:00 2001 From: James Faulkner Date: Fri, 3 Jul 2026 13:28:58 +0100 Subject: [PATCH 1/2] fix: Iceberg renames now delete the old Glue entry via doRenameOperation (#140) Removes the !isIcebergPredicate guard in onAlterTable so Iceberg renames go through doRenameOperation (create new + copy partitions + deleteIfUnchanged old) the same as Hive renames. Previously, Iceberg renames fell through to updateOrCreateTable which only wrote the new entry, leaving the old _iceberg table as an orphan in Glue. IsIcebergTablePredicate is no longer used in ApiaryGlueSync and has been removed. Fixes Bug 2 from #140. Co-Authored-By: Claude Sonnet 4.6 --- .../gluesync/listener/ApiaryGlueSync.java | 8 +------- .../gluesync/listener/ApiaryGlueSyncTest.java | 16 +++++++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSync.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSync.java index 7681b395..3ca4dd66 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSync.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSync.java @@ -48,7 +48,6 @@ import com.expediagroup.apiary.extensions.gluesync.listener.service.GlueDatabaseService; import com.expediagroup.apiary.extensions.gluesync.listener.service.GluePartitionService; import com.expediagroup.apiary.extensions.gluesync.listener.service.GlueTableService; -import com.expediagroup.apiary.extensions.gluesync.listener.service.IsIcebergTablePredicate; public class ApiaryGlueSync extends MetaStoreEventListener { @@ -68,7 +67,6 @@ public class ApiaryGlueSync extends MetaStoreEventListener { private final GlueDatabaseService glueDatabaseService; private final GlueTableService glueTableService; private final GluePartitionService gluePartitionService; - private final IsIcebergTablePredicate isIcebergPredicate; private final MetricService metricService; private final boolean throwExceptions; @@ -83,7 +81,6 @@ public ApiaryGlueSync(Configuration config, boolean throwExceptions) { this.glueDatabaseService = new GlueDatabaseService(glueClient, gluePrefix); this.gluePartitionService = new GluePartitionService(glueClient, gluePrefix); this.glueTableService = new GlueTableService(glueClient, gluePartitionService, gluePrefix); - this.isIcebergPredicate = new IsIcebergTablePredicate(); this.metricService = new MetricService(); this.throwExceptions = throwExceptions; log.debug("ApiaryGlueSync created"); @@ -109,7 +106,6 @@ public ApiaryGlueSync(Configuration config, AWSGlue glueClient, String gluePrefi this.glueDatabaseService = new GlueDatabaseService(glueClient, gluePrefix); this.gluePartitionService = new GluePartitionService(glueClient, gluePrefix, defaultSkipArchive); this.glueTableService = new GlueTableService(glueClient, gluePartitionService, gluePrefix); - this.isIcebergPredicate = new IsIcebergTablePredicate(); this.metricService = metricService; this.throwExceptions = throwExceptions; log.debug("ApiaryGlueSync created"); @@ -214,9 +210,7 @@ public void onAlterTable(AlterTableEvent event) throws MetaException { Table oldTable = event.getOldTable(); Table newTable = event.getNewTable(); try { - // Only Iceberg rename is supported by Glue, for Hive tables we need to delete - // table and create again - if (isTableRename(oldTable, newTable) && !isIcebergPredicate.test(oldTable.getParameters())) { + if (isTableRename(oldTable, newTable)) { doRenameOperation(oldTable, newTable); return; } diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSyncTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSyncTest.java index 458ca2c5..7e84d91e 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSyncTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSyncTest.java @@ -870,23 +870,25 @@ public void onCreateTable_exceptionRethrownWhenThrowExceptionsEnabled() throws M } @Test - public void onAlterIcebergTable_RenameTableSkipsRenameOperation() throws MetaException { + public void onAlterIcebergTable_RenameTable() throws MetaException { AlterTableEvent event = mock(AlterTableEvent.class); when(event.getStatus()).thenReturn(true); Table oldTable = simpleIcebergTable(dbName, tableName, simpleIcebergSchema(), simpleIcebergPartitionSpec(), null); Table newTable = simpleIcebergTable(dbName, "table_renamed", simpleIcebergSchema(), simpleIcebergPartitionSpec(), null); when(event.getOldTable()).thenReturn(oldTable); when(event.getNewTable()).thenReturn(newTable); + when(glueClient.getPartitions(any())).thenReturn(new GetPartitionsResult().withPartitions()); + when(glueClient.getTable(any())).thenReturn(new GetTableResult().withTable(new com.amazonaws.services.glue.model.Table())); glueSync.onAlterTable(event); - verify(glueClient).updateTable(updateTableRequestCaptor.capture()); + verify(glueClient).createTable(createTableRequestCaptor.capture()); + assertThat(createTableRequestCaptor.getValue().getTableInput().getName(), is("table_renamed")); + verify(glueClient).deleteTable(deleteTableRequestCaptor.capture()); + assertThat(deleteTableRequestCaptor.getValue().getName(), is(tableName)); + verify(glueClient, never()).updateTable(any()); verify(metricService).incrementCounter(MetricConstants.LISTENER_TABLE_SUCCESS); - verify(metricService).recordEvent(MetricConstants.ALTER_TABLE, MetricConstants.RESULT_SUCCESS, MetricConstants.OUTCOME_UPDATED); - assertThat(updateTableRequestCaptor.getValue().getTableInput().getName(), is("table_renamed")); - // rename operation (copy+delete) must not be triggered for Iceberg tables - verify(glueClient, times(0)).deleteTable(any()); - verify(glueClient, times(0)).batchCreatePartition(any()); + verify(metricService).recordEvent(MetricConstants.ALTER_TABLE, MetricConstants.RESULT_SUCCESS, "renamed"); } @Test From e8d9885d472656ac90479d53bb2b117d70e2725b Mon Sep 17 00:00:00 2001 From: James Faulkner Date: Mon, 6 Jul 2026 12:32:28 +0100 Subject: [PATCH 2/2] fix: use createOrUpdate in doRenameOperation to handle duplicate Kafka events Bare create() throws AlreadyExistsException if a rename event is replayed (Kafka at-least-once delivery, or a retry after a partial failure). Switching to createOrUpdateTable() makes the rename idempotent: if the new Glue entry already exists it falls through to update, then proceeds to deleteIfUnchanged. Co-Authored-By: Claude Sonnet 4.6 --- .../gluesync/listener/ApiaryGlueSync.java | 2 +- .../gluesync/listener/ApiaryGlueSyncTest.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSync.java b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSync.java index 3ca4dd66..0b1c243b 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSync.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSync.java @@ -285,7 +285,7 @@ private String updateOrCreatePartition(Table table, Partition partition) { private void doRenameOperation(Table oldTable, Table newTable) { log.info("{} glue table rename detected to {}", oldTable.getTableName(), newTable.getTableName()); long startTime = System.currentTimeMillis(); - glueTableService.create(newTable); + createOrUpdateTable(newTable); gluePartitionService.copyPartitions(newTable, gluePartitionService.getPartitions(oldTable)); glueTableService.deleteIfUnchanged(oldTable); metricService.incrementCounter(MetricConstants.LISTENER_TABLE_SUCCESS); diff --git a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSyncTest.java b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSyncTest.java index 7e84d91e..778c6a24 100644 --- a/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSyncTest.java +++ b/hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSyncTest.java @@ -891,6 +891,28 @@ public void onAlterIcebergTable_RenameTable() throws MetaException { verify(metricService).recordEvent(MetricConstants.ALTER_TABLE, MetricConstants.RESULT_SUCCESS, "renamed"); } + @Test + public void onAlterIcebergTable_RenameTable_idempotentWhenNewTableAlreadyExists() throws MetaException { + AlterTableEvent event = mock(AlterTableEvent.class); + when(event.getStatus()).thenReturn(true); + Table oldTable = simpleIcebergTable(dbName, tableName, simpleIcebergSchema(), simpleIcebergPartitionSpec(), null); + Table newTable = simpleIcebergTable(dbName, "table_renamed", simpleIcebergSchema(), simpleIcebergPartitionSpec(), null); + when(event.getOldTable()).thenReturn(oldTable); + when(event.getNewTable()).thenReturn(newTable); + when(glueClient.getPartitions(any())).thenReturn(new GetPartitionsResult().withPartitions()); + when(glueClient.getTable(any())).thenReturn(new GetTableResult().withTable(new com.amazonaws.services.glue.model.Table())); + when(glueClient.createTable(any())).thenThrow(new AlreadyExistsException("already exists")); + + glueSync.onAlterTable(event); + + verify(glueClient).updateTable(updateTableRequestCaptor.capture()); + assertThat(updateTableRequestCaptor.getValue().getTableInput().getName(), is("table_renamed")); + verify(glueClient).deleteTable(deleteTableRequestCaptor.capture()); + assertThat(deleteTableRequestCaptor.getValue().getName(), is(tableName)); + verify(metricService).incrementCounter(MetricConstants.LISTENER_TABLE_SUCCESS); + verify(metricService).recordEvent(MetricConstants.ALTER_TABLE, MetricConstants.RESULT_SUCCESS, "renamed"); + } + @Test public void onDropPartition_partitionNotFoundInGlue() throws MetaException { DropPartitionEvent event = mock(DropPartitionEvent.class);