From 6f0fed9e23ca2f0041cf60872ecc26a567ef17ee Mon Sep 17 00:00:00 2001 From: Tigran Manasyan Date: Mon, 15 Dec 2025 18:01:56 +0400 Subject: [PATCH 1/2] [ADH-7260] Support Ozone file access events collection on SmartServer --- .../java/org/smartdata/conf/SmartConf.java | 7 + .../metrics/GeneralFileInfoSource.java | 23 +- .../org/smartdata/server/SmartEngine.java | 7 +- .../server/engine/StatesManager.java | 6 +- .../server/engine/rule/RuleExecutor.java | 19 +- .../server/engine/rule/RuleInfoRepo.java | 1 + .../server/engine/rule/TestRuleExecutor.java | 5 +- .../org/smartdata/metastore/MetaStore.java | 17 +- .../accesscount/DbAccessEventAggregator.java | 12 +- .../accesscount/HdfsFileAccessManager.java | 104 +++++++ .../smartdata/metastore/dao/DaoProvider.java | 3 + .../metastore/dao/DaoProviderFactory.java | 5 +- .../metastore/dao/FileAccessDao.java | 1 - .../metastore/dao/FileAccessPartitionDao.java | 2 + .../smartdata/metastore/dao/FileInfoDao.java | 8 +- .../metastore/dao/impl/BaseFileInfoDao.java | 70 +++++ .../dao/impl/DefaultDaoProvider.java | 28 +- .../dao/impl/DefaultFileAccessDao.java | 16 +- .../impl/DefaultFileAccessPartitionDao.java | 21 +- .../dao/impl/DefaultFileInfoDao.java | 20 +- .../dao/impl/DefaultOzoneFileInfoDao.java | 3 +- .../dao/postgres/PostgresDaoProvider.java | 5 +- .../metastore/utils/MetaStoreUtils.java | 3 +- ...angelog-13.add-ozone-file-access-table.xml | 57 ++++ .../resources/db/changelog/changelog-root.xml | 1 + ...te_file_access_partition_by_table_name.sql | 30 ++ .../org/smartdata/metastore/TestDaoBase.java | 2 +- .../org/smartdata/metastore/TestDruid.java | 2 +- .../smart-ozone-fs-client/pom.xml | 2 +- .../hadoop/fs/ozone/SmartOzoneFileSystem.java | 92 ++++++ .../fs/ozone/SmartRootedOzoneFileSystem.java | 92 ++++++ .../ozone/client/FileAccessReportSupport.java | 67 +++++ .../ozone/client/SmartOzoneClientAdapter.java | 269 +----------------- .../ozone/client/SmartOzoneFileSystem.java | 37 --- .../ozone}/SmartOzoneClientAdapterTest.java | 36 +-- .../org/smartdata/ozone/OzoneFileInfoDao.java | 3 +- .../tools/docker/ozone/conf/core-site.xml | 19 ++ .../tools/docker/ozone/docker-compose.yaml | 25 +- .../tools/docker/ozone/fs-client/common.sh | 66 +++++ .../docker/ozone/fs-client/entrypoint.sh | 10 + 40 files changed, 767 insertions(+), 429 deletions(-) rename smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartRootedOzoneFileSystem.java => smart-common/src/main/java/org/smartdata/metrics/GeneralFileInfoSource.java (56%) create mode 100644 smart-metastore/src/main/java/org/smartdata/metastore/accesscount/HdfsFileAccessManager.java create mode 100644 smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/BaseFileInfoDao.java create mode 100644 smart-metastore/src/main/resources/db/changelog/changelog-13.add-ozone-file-access-table.xml create mode 100644 smart-metastore/src/main/resources/db/changelog/sql/create_file_access_partition_by_table_name.sql create mode 100644 smart-ozone-support/smart-ozone-fs-client/src/main/java/org/apache/hadoop/fs/ozone/SmartOzoneFileSystem.java create mode 100644 smart-ozone-support/smart-ozone-fs-client/src/main/java/org/apache/hadoop/fs/ozone/SmartRootedOzoneFileSystem.java create mode 100644 smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/FileAccessReportSupport.java delete mode 100644 smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartOzoneFileSystem.java rename smart-ozone-support/smart-ozone-fs-client/src/test/java/org/{smartdata/ozone/client => apache/hadoop/fs/ozone}/SmartOzoneClientAdapterTest.java (67%) create mode 100644 supports/tools/docker/ozone/conf/core-site.xml create mode 100755 supports/tools/docker/ozone/fs-client/common.sh create mode 100755 supports/tools/docker/ozone/fs-client/entrypoint.sh diff --git a/smart-common/src/main/java/org/smartdata/conf/SmartConf.java b/smart-common/src/main/java/org/smartdata/conf/SmartConf.java index f75845a15e..6da73a3427 100644 --- a/smart-common/src/main/java/org/smartdata/conf/SmartConf.java +++ b/smart-common/src/main/java/org/smartdata/conf/SmartConf.java @@ -35,6 +35,9 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static org.smartdata.conf.SmartConfKeys.SMART_FS_TYPE; +import static org.smartdata.conf.SmartConfKeys.SMART_FS_TYPE_DEFAULT; + /** * SSM related configurations as well as HDFS configurations. */ @@ -130,6 +133,10 @@ private void parseHostsFiles() { } } + public SmartFsType getFsType() { + return getEnum(SMART_FS_TYPE, SMART_FS_TYPE_DEFAULT); + } + private Set parseHostsFile( SsmHostsFileReader hostFileReader, String fileName) throws IOException { String configDir = get( diff --git a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartRootedOzoneFileSystem.java b/smart-common/src/main/java/org/smartdata/metrics/GeneralFileInfoSource.java similarity index 56% rename from smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartRootedOzoneFileSystem.java rename to smart-common/src/main/java/org/smartdata/metrics/GeneralFileInfoSource.java index f444ffd067..c4bfede746 100644 --- a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartRootedOzoneFileSystem.java +++ b/smart-common/src/main/java/org/smartdata/metrics/GeneralFileInfoSource.java @@ -15,23 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.ozone.client; +package org.smartdata.metrics; -import org.apache.hadoop.fs.ozone.OzoneClientAdapter; -import org.apache.hadoop.fs.ozone.RootedOzoneFileSystem; -import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import java.sql.SQLException; +import java.util.Collection; +import java.util.Map; -import java.io.IOException; - -public class SmartRootedOzoneFileSystem extends RootedOzoneFileSystem { - - @Override - protected OzoneClientAdapter createAdapter( - ConfigurationSource conf, String omHost, int omPort) throws IOException { - return SmartOzoneClientAdapter.wrap( - super.createAdapter(conf, omHost, omPort), - OzoneConfiguration.of(conf) - ); - } +public interface GeneralFileInfoSource { + Map getPathsToIdsMapping(Collection paths) throws SQLException; } diff --git a/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java b/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java index d12097cfd5..2a8c6c8911 100644 --- a/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java +++ b/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java @@ -39,8 +39,6 @@ import java.util.ArrayList; import java.util.List; -import static org.smartdata.conf.SmartConfKeys.SMART_FS_TYPE; -import static org.smartdata.conf.SmartConfKeys.SMART_FS_TYPE_DEFAULT; import static org.smartdata.conf.SmartConfKeys.SMART_HMS_EVENT_FETCH_DEFAULT; import static org.smartdata.conf.SmartConfKeys.SMART_HMS_EVENT_FETCH_ENABLED; @@ -147,10 +145,7 @@ private void maybeEnableHiveEventsFetcher() { } private void maybeEnableOzoneFetcher() { - SmartFsType smartFsType = serverContext.getConf().getEnum( - SMART_FS_TYPE, - SMART_FS_TYPE_DEFAULT); - if (smartFsType != SmartFsType.OZONE) { + if (serverContext.getConf().getFsType() != SmartFsType.OZONE) { return; } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java b/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java index 47a1a5e764..ddf570b1bb 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java @@ -47,6 +47,7 @@ import static org.smartdata.conf.SmartConfKeys.ACCESS_EVENT_SOURCE_DEFAULT; import static org.smartdata.conf.SmartConfKeys.ACCESS_EVENT_SOURCE_KEY; +import static org.smartdata.utils.PathUtil.addPathSeparator; import static org.springframework.transaction.annotation.Isolation.SERIALIZABLE; /** @@ -95,7 +96,7 @@ public void init() throws IOException { AccessCountFailoverFactory accessCountFailoverFactory = new AccessCountFailoverFactory(serverContext.getConf()); DbAccessEventAggregator accessEventAggregator = new DbAccessEventAggregator( - serverContext.getMetaStore().fileInfoDao(), + serverContext.getMetaStore().generalFileInfoSource(), fileAccessManager, accessCountFailoverFactory.create()); this.accessEventFetcher = new AccessEventFetcher( @@ -199,8 +200,7 @@ public void stop() throws IOException { } public void reportFileAccessEvent(FileAccessEvent event) { - String path = event.getPath(); - path = path + (path.endsWith("/") ? "" : "/"); + String path = addPathSeparator(event.getPath()); if (pathChecker.isIgnored(path)) { LOG.debug("Path {} is in the ignore list. Skip report file access event.", path); diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleExecutor.java b/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleExecutor.java index f96675b8c1..c322cbfe0f 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleExecutor.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleExecutor.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.smartdata.conf.SmartConf; import org.smartdata.exception.NotFoundException; import org.smartdata.exception.QueueFullException; import org.smartdata.metastore.MetaStore; @@ -41,6 +42,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.smartdata.conf.SmartConfKeys.SMART_FS_TYPE_DEFAULT; +import static org.smartdata.metastore.dao.FileAccessPartitionDao.HDFS_FILE_ACCESS_TABLE; +import static org.smartdata.metastore.dao.FileAccessPartitionDao.OZONE_FILE_ACCESS_TABLE; + /** * Execute rule queries and return result. */ @@ -61,16 +66,19 @@ public class RuleExecutor implements Runnable { private final MetaStore metastore; private final Stack dynamicCleanups; private final List executorPlugins; + private final SmartConf conf; private volatile boolean exited; private long exitTime; public RuleExecutor( + SmartConf conf, RuleManager ruleManager, ExecutionContext executionCtx, RuleTranslationResult translationResult, MetaStore metastore, List executorPlugins) { + this.conf = conf; this.ruleManager = ruleManager; this.executionCtx = executionCtx; this.metastore = metastore; @@ -199,12 +207,14 @@ public String genVirtualAccessCountTable(List parameters) { long interval = paraList.isEmpty() ? 0L : (long) paraList.get(0); String countFilter = ""; long currentTimeMillis = System.currentTimeMillis(); - return generateSQL(newTable, countFilter, metastore, currentTimeMillis - interval, + return generateSQL(conf, newTable, countFilter, metastore, currentTimeMillis - interval, currentTimeMillis); } + // todo refactor dynamic calls to access count tables @VisibleForTesting static String generateSQL( + SmartConf conf, String newTable, String countFilter, MetaStore adapter, @@ -217,11 +227,16 @@ static String generateSQL( } catch (MetaStoreException e) { LOG.error("Cannot create table " + newTable, e); } + + String fileAccessTable = conf.getFsType() == SMART_FS_TYPE_DEFAULT + ? HDFS_FILE_ACCESS_TABLE + : OZONE_FILE_ACCESS_TABLE; + String sqlCountFilter = (countFilter == null || countFilter.isEmpty()) ? "" : " HAVING count(*) " + countFilter; - sqlFinal = "INSERT INTO " + newTable + " SELECT fid, count(*) AS count FROM file_access\n" + sqlFinal = "INSERT INTO " + newTable + " SELECT fid, count(*) AS count FROM " + fileAccessTable + "\n" + "WHERE access_time >= " + startTime + " AND access_time <= " + endTime + " GROUP BY fid" + sqlCountFilter + " ;"; return sqlFinal; diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleInfoRepo.java b/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleInfoRepo.java index 46ab37e1d3..1e58b6177e 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleInfoRepo.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleInfoRepo.java @@ -152,6 +152,7 @@ private RuleExecutor doLaunchExecutor(RuleManager ruleManager) : new SmartRuleStringParser(ruleInfo.getRuleText(), translationCtx, conf).translate(); ruleExecutor = new RuleExecutor( + conf, ruleManager, executionCtx, translationResult, diff --git a/smart-engine/src/test/java/org/smartdata/server/engine/rule/TestRuleExecutor.java b/smart-engine/src/test/java/org/smartdata/server/engine/rule/TestRuleExecutor.java index f88e7211aa..fcfb8bfe47 100644 --- a/smart-engine/src/test/java/org/smartdata/server/engine/rule/TestRuleExecutor.java +++ b/smart-engine/src/test/java/org/smartdata/server/engine/rule/TestRuleExecutor.java @@ -19,6 +19,7 @@ import org.junit.Before; import org.junit.Test; +import org.smartdata.conf.SmartConf; import org.smartdata.metastore.TestDaoBase; import org.smartdata.metastore.dao.MetaStoreHelper; @@ -39,7 +40,7 @@ public void generateSQL() { String sql; long interval = 60000; long currentTimeMillis = System.currentTimeMillis(); - sql = RuleExecutor.generateSQL(newTable, countFilter, metaStore, currentTimeMillis - interval, + sql = RuleExecutor.generateSQL(new SmartConf(), newTable, countFilter, metaStore, currentTimeMillis - interval, currentTimeMillis); try { metaStoreHelper.execute(sql); @@ -50,7 +51,7 @@ public void generateSQL() { } // Test with count filter countFilter = "> 10"; - sql = RuleExecutor.generateSQL(newTable, countFilter, metaStore, currentTimeMillis - interval, + sql = RuleExecutor.generateSQL(new SmartConf(), newTable, countFilter, metaStore, currentTimeMillis - interval, currentTimeMillis); try { metaStoreHelper.execute(sql); diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java index c8fefbcb91..62680d986f 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java @@ -54,6 +54,7 @@ import org.smartdata.metastore.model.AggregatedAccessCounts; import org.smartdata.metastore.transaction.TransactionRunner; import org.smartdata.metastore.utils.MetaStoreUtils; +import org.smartdata.metrics.GeneralFileInfoSource; import org.smartdata.model.ActionInfo; import org.smartdata.model.BackUpInfo; import org.smartdata.model.CachedFileStatus; @@ -131,6 +132,7 @@ public class MetaStore implements CopyMetaService, private final WhitelistDao whitelistDao; private final OzoneFileInfoDao ozoneFileInfoDao; private final UserActivityDao userActivityDao; + private final GeneralFileInfoSource generalFileInfoSource; private final DBPool dbPool; public MetaStore(DBPool pool, @@ -168,10 +170,7 @@ public MetaStore(DBPool pool, hmsIgnoredEventDao = daoProvider.hmsIgnoredEventDao(); hmsSyncProgressDao = daoProvider.hmsSyncProgressDao(); ozoneFileInfoDao = daoProvider.ozoneFileInfoDao(); - } - - public DbMetadataProvider dbMetadataProvider() { - return dbMetadataProvider; + generalFileInfoSource = daoProvider.generalFileInfoSource(); } public UserActivityDao userActivityDao() { @@ -222,6 +221,10 @@ public OzoneFileInfoDao ozoneFileInfoDao() { return ozoneFileInfoDao; } + public GeneralFileInfoSource generalFileInfoSource() { + return generalFileInfoSource; + } + public PlatformTransactionManager transactionManager() { return defaultTransactionRunner.getTransactionManager(); } @@ -365,7 +368,7 @@ public List getFilesByPaths(Collection paths) public Map getFileIDs(Collection paths) throws MetaStoreException { try { - return fileInfoDao.getPathFids(paths); + return fileInfoDao.getPathsToIdsMapping(paths); } catch (EmptyResultDataAccessException e) { return new HashMap<>(); } catch (Exception e) { @@ -1475,7 +1478,7 @@ public List getLastFetchedDirs() throws MetaStoreException { String[] oldList = fetchedList.split(","); lastFetchedDirs.addAll(Arrays.asList(oldList)); } - LOG.info("Last fetch dirs are " + lastFetchedDirs.toString()); + LOG.info("Last fetch dirs are {}", lastFetchedDirs); return lastFetchedDirs; } catch (Exception e) { throw new MetaStoreException(e); @@ -1488,7 +1491,7 @@ public List getLastFetchedDirs() throws MetaStoreException { public void updateWhitelistTable(String newWhitelist) throws MetaStoreException { try { whitelistDao.updateTable(newWhitelist); - LOG.info("Success to update whitelist table with " + newWhitelist); + LOG.info("Success to update whitelist table with {}", newWhitelist); } catch (Exception e) { throw new MetaStoreException(e); } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/accesscount/DbAccessEventAggregator.java b/smart-metastore/src/main/java/org/smartdata/metastore/accesscount/DbAccessEventAggregator.java index 15903da4a9..9f6951ae4c 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/accesscount/DbAccessEventAggregator.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/accesscount/DbAccessEventAggregator.java @@ -20,9 +20,9 @@ import lombok.extern.slf4j.Slf4j; import org.smartdata.metastore.accesscount.failover.AccessCountContext; import org.smartdata.metastore.accesscount.failover.Failover; -import org.smartdata.metastore.dao.FileInfoDao; import org.smartdata.metastore.model.AggregatedAccessCounts; import org.smartdata.metrics.FileAccessEvent; +import org.smartdata.metrics.GeneralFileInfoSource; import java.util.Collections; import java.util.List; @@ -33,13 +33,13 @@ @Slf4j public class DbAccessEventAggregator implements AccessEventAggregator { - private final FileInfoDao fileInfoDao; + private final GeneralFileInfoSource fileInfoDao; private final FileAccessManager dbTableManager; private final Failover accessCountFailover; - public DbAccessEventAggregator(FileInfoDao fileInfoDao, - FileAccessManager dbTableManager, - Failover failover) { + public DbAccessEventAggregator(GeneralFileInfoSource fileInfoDao, + FileAccessManager dbTableManager, + Failover failover) { this.fileInfoDao = fileInfoDao; this.dbTableManager = dbTableManager; this.accessCountFailover = failover; @@ -75,7 +75,7 @@ private List getAggregatedAccessCounts(List getFileIdMap(List paths) { try { - return fileInfoDao.getPathFids(paths); + return fileInfoDao.getPathsToIdsMapping(paths); } catch (Exception e) { log.error("Error fetching file ids for paths {}", paths, e); return Collections.emptyMap(); diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/accesscount/HdfsFileAccessManager.java b/smart-metastore/src/main/java/org/smartdata/metastore/accesscount/HdfsFileAccessManager.java new file mode 100644 index 0000000000..e12212bbb2 --- /dev/null +++ b/smart-metastore/src/main/java/org/smartdata/metastore/accesscount/HdfsFileAccessManager.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.metastore.accesscount; + +import lombok.extern.slf4j.Slf4j; +import org.smartdata.metastore.MetaStoreException; +import org.smartdata.metastore.dao.CacheFileDao; +import org.smartdata.metastore.dao.FileAccessDao; +import org.smartdata.metastore.dao.SearchableService; +import org.smartdata.metastore.model.AggregatedAccessCounts; +import org.smartdata.metastore.queries.sort.FileAccessInfoSortField; +import org.smartdata.metastore.transaction.TransactionRunner; +import org.smartdata.model.FileAccessInfo; +import org.smartdata.model.request.FileAccessInfoSearchRequest; + +import java.util.Collection; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public class HdfsFileAccessManager extends + SearchableService { + + private final TransactionRunner transactionRunner; + private final FileAccessDao fileAccessDao; + private final CacheFileDao cacheFileDao; + + public HdfsFileAccessManager( + TransactionRunner transactionRunner, + FileAccessDao fileAccessDao, + CacheFileDao cacheFileDao) { + super(fileAccessDao, "file accesses"); + this.fileAccessDao = fileAccessDao; + this.cacheFileDao = cacheFileDao; + this.transactionRunner = transactionRunner; + } + + public void save(Collection accessCounts) { + if (accessCounts.isEmpty()) { + return; + } + try { + transactionRunner.inTransaction(() -> { + insertFileAccesses(accessCounts); + updateCachedFilesInMetastore(getAggregatedAccessCounts(accessCounts)); + }); + } catch (MetaStoreException e) { + log.error("Failed to save access counts", e); + throw new RuntimeException(e); + } + } + + private void insertFileAccesses( + Collection accessCounts) throws MetaStoreException { + try { + fileAccessDao.insert(accessCounts); + log.debug("Inserted values {} to file access table", accessCounts); + } catch (Exception e) { + log.error("Error inserting file accesses {}", accessCounts, e); + throw new MetaStoreException(e); + } + } + + private void updateCachedFilesInMetastore(Collection accessCounts) + throws MetaStoreException { + try { + cacheFileDao.update(accessCounts); + } catch (Exception e) { + log.error("Error updating cached files {}", accessCounts, e); + throw new MetaStoreException(e); + } + } + + private Collection getAggregatedAccessCounts( + Collection accessCounts) { + Map aggregatedAccessCounts = + accessCounts.stream() + .collect(Collectors.toMap( + AggregatedAccessCounts::getFileId, + Function.identity(), + AggregatedAccessCounts::merge + )); + return aggregatedAccessCounts.values().stream() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } +} diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/DaoProvider.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/DaoProvider.java index 89a7ba67df..f4fb7f657e 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/DaoProvider.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/DaoProvider.java @@ -19,6 +19,7 @@ import org.smartdata.hive.HmsEventDao; import org.smartdata.hive.rule.HmsSyncProgressDao; +import org.smartdata.metrics.GeneralFileInfoSource; import org.smartdata.ozone.OzoneFileInfoDao; public interface DaoProvider { @@ -73,4 +74,6 @@ public interface DaoProvider { HmsSyncProgressDao hmsSyncProgressDao(); OzoneFileInfoDao ozoneFileInfoDao(); + + GeneralFileInfoSource generalFileInfoSource(); } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/DaoProviderFactory.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/DaoProviderFactory.java index 63156300b2..f05e16ae4c 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/DaoProviderFactory.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/DaoProviderFactory.java @@ -17,6 +17,7 @@ */ package org.smartdata.metastore.dao; +import org.smartdata.conf.SmartConf; import org.smartdata.metastore.DBPool; import org.smartdata.metastore.DBType; import org.smartdata.metastore.dao.postgres.PostgresDaoProvider; @@ -24,11 +25,11 @@ public class DaoProviderFactory { public DaoProvider createDaoProvider( - DBPool dbPool, PlatformTransactionManager transactionManager, DBType dbType) { + SmartConf conf, DBPool dbPool, PlatformTransactionManager transactionManager, DBType dbType) { switch (dbType) { case POSTGRES: default: - return new PostgresDaoProvider(dbPool, transactionManager); + return new PostgresDaoProvider(conf, dbPool, transactionManager); } } } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileAccessDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileAccessDao.java index 0a3ae75670..3c902e4ebf 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileAccessDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileAccessDao.java @@ -28,7 +28,6 @@ public interface FileAccessDao extends Searchable { - String TABLE_NAME = "file_access"; String FILE_ID_FIELD = "fid"; String ACCESS_COUNT_FIELD = "count"; String ACCESS_TIME_FIELD = "access_time"; diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileAccessPartitionDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileAccessPartitionDao.java index 0c392f5629..776bb22e4f 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileAccessPartitionDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileAccessPartitionDao.java @@ -24,6 +24,8 @@ import java.util.List; public interface FileAccessPartitionDao { + String HDFS_FILE_ACCESS_TABLE = "file_access"; + String OZONE_FILE_ACCESS_TABLE = "ofile_access"; void create(LocalDateTime date) throws MetaStoreException; diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java index d72f34b06e..9575309b2f 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java @@ -17,15 +17,14 @@ */ package org.smartdata.metastore.dao; +import org.smartdata.metrics.GeneralFileInfoSource; import org.smartdata.model.FileInfo; import org.smartdata.model.FileInfoDiff; -import java.sql.SQLException; import java.util.Collection; import java.util.List; -import java.util.Map; -public interface FileInfoDao { +public interface FileInfoDao extends GeneralFileInfoSource { String FILE_PATH_FIELD = "path"; List getAll(); @@ -40,9 +39,6 @@ public interface FileInfoDao { FileInfo getByPath(String path); - Map getPathFids(Collection paths) - throws SQLException; - void insert(FileInfo fileInfo, boolean generateId); void insert(FileInfo[] fileInfos, boolean generateIds); diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/BaseFileInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/BaseFileInfoDao.java new file mode 100644 index 0000000000..bae24548c5 --- /dev/null +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/BaseFileInfoDao.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.metastore.dao.impl; + +import lombok.Data; +import org.smartdata.metastore.dao.AbstractDao; +import org.smartdata.metrics.GeneralFileInfoSource; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import javax.sql.DataSource; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public abstract class BaseFileInfoDao extends AbstractDao implements GeneralFileInfoSource { + + protected final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + public BaseFileInfoDao(DataSource dataSource, String tableName) { + super(dataSource, tableName); + this.namedParameterJdbcTemplate = new NamedParameterJdbcTemplate(dataSource); + } + + @Override + public Map getPathsToIdsMapping(Collection paths) { + List files = namedParameterJdbcTemplate.query( + "SELECT * FROM " + + tableName + + " WHERE path IN (:paths)", + new MapSqlParameterSource("paths", paths), + this::mapRow); + return files.stream() + .collect(Collectors.toMap( + GeneralFileInfo::getPath, + GeneralFileInfo::getId)); + } + + private GeneralFileInfo mapRow(ResultSet resultSet, int i) throws SQLException { + return new GeneralFileInfo( + resultSet.getLong("fid"), + resultSet.getString("path") + ); + } + + @Data + private static class GeneralFileInfo { + private final long id; + private final String path; + } +} diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultDaoProvider.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultDaoProvider.java index 16dc266791..624801a10c 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultDaoProvider.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultDaoProvider.java @@ -17,6 +17,8 @@ */ package org.smartdata.metastore.dao.impl; +import org.smartdata.conf.SmartConf; +import org.smartdata.conf.SmartFsType; import org.smartdata.metastore.DBPool; import org.smartdata.metastore.dao.ActionDao; import org.smartdata.metastore.dao.BackUpInfoDao; @@ -41,17 +43,24 @@ import org.smartdata.metastore.dao.SystemInfoDao; import org.smartdata.metastore.dao.UserActivityDao; import org.smartdata.metastore.dao.WhitelistDao; +import org.smartdata.metrics.GeneralFileInfoSource; import org.smartdata.ozone.OzoneFileInfoDao; import org.springframework.transaction.PlatformTransactionManager; import javax.sql.DataSource; +import static org.smartdata.metastore.dao.FileAccessPartitionDao.HDFS_FILE_ACCESS_TABLE; +import static org.smartdata.metastore.dao.FileAccessPartitionDao.OZONE_FILE_ACCESS_TABLE; + public abstract class DefaultDaoProvider implements DaoProvider { + protected final DataSource dataSource; protected final PlatformTransactionManager transactionManager; + protected final SmartConf smartConf; - public DefaultDaoProvider(DBPool dbPool, PlatformTransactionManager transactionManager) { + public DefaultDaoProvider(SmartConf smartConf, DBPool dbPool, PlatformTransactionManager transactionManager) { this.dataSource = dbPool.getDataSource(); + this.smartConf = smartConf; this.transactionManager = transactionManager; } @@ -92,7 +101,7 @@ public FileDiffDao fileDiffDao() { @Override public FileAccessDao fileAccessDao() { - return new DefaultFileAccessDao(dataSource, transactionManager); + return new DefaultFileAccessDao(dataSource, transactionManager, fileAccessTableName()); } @Override @@ -162,11 +171,24 @@ public UserActivityDao userActivityDao() { @Override public FileAccessPartitionDao fileAccessPartitionDao() { - return new DefaultFileAccessPartitionDao(dataSource); + return new DefaultFileAccessPartitionDao(dataSource, fileAccessTableName()); } @Override public OzoneFileInfoDao ozoneFileInfoDao() { return new DefaultOzoneFileInfoDao(dataSource); } + + @Override + public GeneralFileInfoSource generalFileInfoSource() { + return smartConf.getFsType() == SmartFsType.HDFS + ? fileInfoDao() + : ozoneFileInfoDao(); + } + + private String fileAccessTableName() { + return smartConf.getFsType() == SmartFsType.HDFS + ? HDFS_FILE_ACCESS_TABLE + : OZONE_FILE_ACCESS_TABLE; + } } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileAccessDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileAccessDao.java index 516bc4b3c7..ec773d564c 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileAccessDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileAccessDao.java @@ -26,7 +26,6 @@ import org.smartdata.metastore.queries.sort.FileAccessInfoSortField; import org.smartdata.model.FileAccessInfo; import org.smartdata.model.request.FileAccessInfoSearchRequest; -import org.springframework.jdbc.core.simple.SimpleJdbcInsert; import org.springframework.transaction.PlatformTransactionManager; import javax.sql.DataSource; @@ -48,14 +47,13 @@ public class DefaultFileAccessDao implements FileAccessDao { public DefaultFileAccessDao( - DataSource dataSource, PlatformTransactionManager transactionManager) { - super(dataSource, transactionManager, TABLE_NAME); + DataSource dataSource, PlatformTransactionManager transactionManager, String tableName) { + super(dataSource, transactionManager, tableName); } @Override public void insert(Collection aggregatedAccessCounts) { - insert(new SimpleJdbcInsert(dataSource).withTableName(TABLE_NAME), aggregatedAccessCounts, - this::toMap); + insert(aggregatedAccessCounts, this::toMap); } protected Map toMap(AggregatedAccessCounts accessCounts) { @@ -68,7 +66,7 @@ protected Map toMap(AggregatedAccessCounts accessCounts) { @Override public void updateFileIds(long srcFileId, long destFileId) throws MetaStoreException { - String statement = "UPDATE " + TABLE_NAME + String statement = "UPDATE " + tableName + " SET fid = " + destFileId + " WHERE fid = " + srcFileId; try { @@ -85,9 +83,9 @@ protected MetastoreQuery searchQuery(FileAccessInfoSearchRequest searchRequest) "access_time", "path") .fromSubQuery("SELECT file.fid, count(*) AS count,\n" - + "MAX(file_access.access_time) as access_time, file.path as path\n" - + "FROM file_access\n" - + " JOIN file ON file_access.fid = file.fid\n" + + "MAX(fa.access_time) as access_time, file.path as path\n" + + "FROM " + tableName + " as fa\n" + + " JOIN file ON fa.fid = file.fid\n" + "GROUP BY file.fid, file.path", "f") .where( in("fid", searchRequest.getIds()), diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileAccessPartitionDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileAccessPartitionDao.java index b7bc37ea11..458f441c6c 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileAccessPartitionDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileAccessPartitionDao.java @@ -33,36 +33,33 @@ @Slf4j public class DefaultFileAccessPartitionDao extends AbstractDao implements FileAccessPartitionDao { - private static final String CREATE_NEW_PARTITION_ERR_MSG = - "Failed to create new partition for file_access table"; - - public DefaultFileAccessPartitionDao(DataSource dataSource) { - super(dataSource, ""); + public DefaultFileAccessPartitionDao(DataSource dataSource, String accessTableName) { + super(dataSource, accessTableName); } @Override public void create(LocalDateTime date) throws MetaStoreException { try { Integer result = - jdbcTemplate.queryForObject("select create_file_access_partition(?);", Integer.class, - date); + jdbcTemplate.queryForObject("select create_file_access_partition(?, ?);", Integer.class, + tableName, date); if (result == null) { - throw new MetaStoreException(CREATE_NEW_PARTITION_ERR_MSG); + throw new MetaStoreException("Failed to create new partition for table " + tableName); } if (result == 1) { - log.info("Created partition for file_access table for date {}", date); + log.info("Created partition for {} table for date {}", tableName, date); } } catch (Exception e) { - throw new MetaStoreException(CREATE_NEW_PARTITION_ERR_MSG, e); + throw new MetaStoreException("Failed to create new partition for table " + tableName, e); } } @Override public List getAll() { String query = "SELECT inhrelid AS id, inhrelid::regclass AS name, " - + "cast(REPLACE(REPLACE(inhrelid::regclass::text, 'file_access_', ''),'_','-') as date) " + + "cast(REPLACE(REPLACE(inhrelid::regclass::text, '" + tableName + "_', ''),'_','-') as date) " + "as partition_date FROM pg_catalog.pg_inherits " - + "WHERE inhparent = 'file_access'::regclass " + + "WHERE inhparent = '" + tableName + "'::regclass " + "ORDER BY partition_date ASC;"; try { return jdbcTemplate.query(query, diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileInfoDao.java index ece73f34e7..be4bff191a 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileInfoDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultFileInfoDao.java @@ -17,7 +17,6 @@ */ package org.smartdata.metastore.dao.impl; -import org.smartdata.metastore.dao.AbstractDao; import org.smartdata.metastore.dao.FileInfoDao; import org.smartdata.model.FileInfo; import org.smartdata.model.FileInfoDiff; @@ -36,7 +35,7 @@ import java.util.List; import java.util.Map; -public class DefaultFileInfoDao extends AbstractDao implements FileInfoDao { +public class DefaultFileInfoDao extends BaseFileInfoDao implements FileInfoDao { private static final String TABLE_NAME = "file"; @@ -85,23 +84,6 @@ public FileInfo getByPath(String path) { new Object[]{path}, new DefaultFileInfoDao.FileInfoRowMapper()); } - @Override - public Map getPathFids(Collection paths) - throws SQLException { - NamedParameterJdbcTemplate namedParameterJdbcTemplate = - new NamedParameterJdbcTemplate(dataSource); - Map pathToId = new HashMap<>(); - String sql = "SELECT * FROM file WHERE path IN (:paths)"; - MapSqlParameterSource parameterSource = new MapSqlParameterSource(); - parameterSource.addValue("paths", paths); - List files = namedParameterJdbcTemplate.query(sql, - parameterSource, new FileInfoRowMapper()); - for (FileInfo file : files) { - pathToId.put(file.getPath(), file.getFileId()); - } - return pathToId; - } - @Override public void insert(FileInfo fileInfo, boolean generateId) { SimpleJdbcInsert simpleJdbcInsert = simpleJdbcInsert(); diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultOzoneFileInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultOzoneFileInfoDao.java index c46987f94e..a7c1f975a4 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultOzoneFileInfoDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultOzoneFileInfoDao.java @@ -17,7 +17,6 @@ */ package org.smartdata.metastore.dao.impl; -import org.smartdata.metastore.dao.AbstractDao; import org.smartdata.ozone.OzoneFileInfoDao; import org.smartdata.ozone.model.OzoneFileInfo; import org.springframework.jdbc.core.simple.SimpleJdbcInsert; @@ -27,7 +26,7 @@ import java.util.HashMap; import java.util.Map; -public class DefaultOzoneFileInfoDao extends AbstractDao implements OzoneFileInfoDao { +public class DefaultOzoneFileInfoDao extends BaseFileInfoDao implements OzoneFileInfoDao { private static final String TABLE_NAME = "ofile"; private static final String FILE_ID_FIELD = "fid"; diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/postgres/PostgresDaoProvider.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/postgres/PostgresDaoProvider.java index 31d95456f3..477e92d846 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/postgres/PostgresDaoProvider.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/postgres/PostgresDaoProvider.java @@ -17,6 +17,7 @@ */ package org.smartdata.metastore.dao.postgres; +import org.smartdata.conf.SmartConf; import org.smartdata.hive.HmsEventDao; import org.smartdata.hive.rule.HmsSyncProgressDao; import org.smartdata.metastore.DBPool; @@ -30,8 +31,8 @@ import org.springframework.transaction.PlatformTransactionManager; public class PostgresDaoProvider extends DefaultDaoProvider { - public PostgresDaoProvider(DBPool dbPool, PlatformTransactionManager transactionManager) { - super(dbPool, transactionManager); + public PostgresDaoProvider(SmartConf smartConf, DBPool dbPool, PlatformTransactionManager transactionManager) { + super(smartConf, dbPool, transactionManager); } @Override diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java b/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java index d57f29c964..dd80164c84 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java @@ -58,6 +58,7 @@ public class MetaStoreUtils { "ec_policy", "file", "file_access", + "ofile_access", "storage", "storage_hist", "storage_policy", @@ -119,7 +120,7 @@ public static MetaStore getDBAdapter( new JdbcTransactionManager(druidPool.getDataSource()); DaoProvider daoProvider = daoProviderFactory - .createDaoProvider(druidPool, transactionManager, dbType); + .createDaoProvider(conf, druidPool, transactionManager, dbType); DbSchemaManager dbSchemaManager = dbHandlersFactory.createDbManager(druidPool, conf); DbMetadataProvider dbMetadataProvider = dbHandlersFactory .createDbMetadataProvider(druidPool, dbType); diff --git a/smart-metastore/src/main/resources/db/changelog/changelog-13.add-ozone-file-access-table.xml b/smart-metastore/src/main/resources/db/changelog/changelog-13.add-ozone-file-access-table.xml new file mode 100644 index 0000000000..b87d844c20 --- /dev/null +++ b/smart-metastore/src/main/resources/db/changelog/changelog-13.add-ozone-file-access-table.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + select count(*) from pg_proc where proname = 'create_file_access_partition' + + + drop function create_file_access_partition(timestamp); + + + + + + select count(*) from pg_proc where proname = 'create_file_access_partition' + + + + select create_file_access_partition('ofile_access', cast(now() as date)); + + + + + + + + + + + + + + \ No newline at end of file diff --git a/smart-metastore/src/main/resources/db/changelog/changelog-root.xml b/smart-metastore/src/main/resources/db/changelog/changelog-root.xml index 6103512be4..fc1f2632f0 100644 --- a/smart-metastore/src/main/resources/db/changelog/changelog-root.xml +++ b/smart-metastore/src/main/resources/db/changelog/changelog-root.xml @@ -20,4 +20,5 @@ + \ No newline at end of file diff --git a/smart-metastore/src/main/resources/db/changelog/sql/create_file_access_partition_by_table_name.sql b/smart-metastore/src/main/resources/db/changelog/sql/create_file_access_partition_by_table_name.sql new file mode 100644 index 0000000000..c0ab6a3903 --- /dev/null +++ b/smart-metastore/src/main/resources/db/changelog/sql/create_file_access_partition_by_table_name.sql @@ -0,0 +1,30 @@ +CREATE OR REPLACE FUNCTION create_file_access_partition(source_table_name text, input_date timestamp) + RETURNS INTEGER AS ' + DECLARE + current_date_part DATE; + current_date_part_text TEXT; + partition_table_name TEXT; + first_day_of_month DATE; + last_day_of_month DATE; + result INTEGER; + create_query TEXT; + BEGIN + result := 0; + current_date_part := CAST(DATE_TRUNC(''month'', input_date::date) AS DATE); + current_date_part_text := REGEXP_REPLACE(current_date_part::TEXT, ''-'',''_'',''g''); + partition_table_name := FORMAT(''%s_%s'', source_table_name, current_date_part_text::TEXT); + IF (TO_REGCLASS(partition_table_name::TEXT) ISNULL) THEN + first_day_of_month := current_date_part; + last_day_of_month := current_date_part + ''1 month''::INTERVAL; + result := 1; + RAISE NOTICE ''table: %'', partition_table_name; + create_query := FORMAT( + ''CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (extract(epoch from %L::DATE) * 1000) TO (extract(epoch from %L::DATE) * 1000);'', + partition_table_name, source_table_name, first_day_of_month, last_day_of_month); + RAISE NOTICE ''query: %'', create_query; + EXECUTE create_query; + EXECUTE FORMAT(''CREATE INDEX %1$s__access_time ON %1$I (access_time);'', partition_table_name); + END IF; + RETURN result; + END; +' LANGUAGE plpgsql; diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/TestDaoBase.java b/smart-metastore/src/test/java/org/smartdata/metastore/TestDaoBase.java index 6599825bd6..fe1ce62de4 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/TestDaoBase.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/TestDaoBase.java @@ -84,7 +84,7 @@ public static void initDao(DBType dbType, String driverClassName, String dbUrl) PlatformTransactionManager transactionManager = new JdbcTransactionManager(druidPool.getDataSource()); daoProvider = new DaoProviderFactory() - .createDaoProvider(druidPool, transactionManager, dbType); + .createDaoProvider(new SmartConf(), druidPool, transactionManager, dbType); dbMetadataProvider = dbHandlersFactory.createDbMetadataProvider(druidPool, dbType); metaStore = new MetaStore( druidPool, dbSchemaManager, daoProvider, dbMetadataProvider, transactionManager); diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/TestDruid.java b/smart-metastore/src/test/java/org/smartdata/metastore/TestDruid.java index 29e60062cf..8d5ca52678 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/TestDruid.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/TestDruid.java @@ -54,7 +54,7 @@ public void test() throws Exception { .createDbManager(druidPool, new Configuration()); PlatformTransactionManager transactionManager = new JdbcTransactionManager(druidPool.getDataSource()); - DaoProvider daoProvider = new PostgresDaoProvider(druidPool, transactionManager); + DaoProvider daoProvider = new PostgresDaoProvider(new SmartConf(), druidPool, transactionManager); DbMetadataProvider dbMetadataProvider = dbHandlersFactory.createDbMetadataProvider(druidPool, DBType.POSTGRES); MetaStore adapter = new MetaStore( diff --git a/smart-ozone-support/smart-ozone-fs-client/pom.xml b/smart-ozone-support/smart-ozone-fs-client/pom.xml index f29b5ee720..136e8ef103 100644 --- a/smart-ozone-support/smart-ozone-fs-client/pom.xml +++ b/smart-ozone-support/smart-ozone-fs-client/pom.xml @@ -123,7 +123,7 @@ ${shaded.dependencies.prefix}.org.apache.commons - com.google + com.google.guava ${shaded.dependencies.prefix}.com.google diff --git a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/apache/hadoop/fs/ozone/SmartOzoneFileSystem.java b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/apache/hadoop/fs/ozone/SmartOzoneFileSystem.java new file mode 100644 index 0000000000..73d78f2323 --- /dev/null +++ b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/apache/hadoop/fs/ozone/SmartOzoneFileSystem.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.ozone; + +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.util.Preconditions; +import org.smartdata.client.SmartClient; +import org.smartdata.ozone.client.FileAccessReportSupport; +import org.smartdata.ozone.client.SmartOzoneClientAdapter; +import org.smartdata.protocol.SmartClientProtocol; + +import java.io.IOException; +import java.io.InputStream; + +public class SmartOzoneFileSystem extends OzoneFileSystem { + + @Override + protected OzoneClientAdapter createAdapter(ConfigurationSource conf, String bucketStr, String volumeStr, + String omHost, int omPort) throws IOException { + return new SmartClientAdapter( + omHost, omPort, conf, volumeStr, bucketStr, + (OzoneFSStorageStatistics) getOzoneFSOpsCountStatistics()); + } + + @Slf4j + static class SmartClientAdapter extends OzoneClientAdapterImpl implements SmartOzoneClientAdapter { + private final FileAccessReportSupport accessReportSupport; + + SmartClientAdapter( + String omHost, int omPort, ConfigurationSource hadoopConf, + String volumeStr, String bucketStr, OzoneFSStorageStatistics storageStatistics) throws IOException { + this(omHost, omPort, hadoopConf, volumeStr, bucketStr, storageStatistics, + new SmartClient(Preconditions.checkNotNull(OzoneConfiguration.of(hadoopConf)))); + } + + SmartClientAdapter( + String omHost, + int omPort, + ConfigurationSource hadoopConf, + String volumeStr, + String bucketStr, + OzoneFSStorageStatistics storageStatistics, + SmartClientProtocol ssmClient) throws IOException { + super(omHost, omPort, hadoopConf, volumeStr, bucketStr, storageStatistics); + this.accessReportSupport = new FileAccessReportSupport(ssmClient, volumeStr, bucketStr); + } + + @Override + public void close() { + try { + super.close(); + } catch (IOException e) { + log.error("Error closing OzoneClient", e); + } + + try { + accessReportSupport.close(); + } catch (IOException e) { + log.error("Error closing SmartClient", e); + } + } + + @Override + public InputStream readFile(String key) throws IOException { + InputStream inputStream = super.readFile(key); + accessReportSupport.reportFileAccess(key); + return inputStream; + } + + @Override + public String getBasePath() { + return accessReportSupport.getBasePath(); + } + } +} diff --git a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/apache/hadoop/fs/ozone/SmartRootedOzoneFileSystem.java b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/apache/hadoop/fs/ozone/SmartRootedOzoneFileSystem.java new file mode 100644 index 0000000000..d9c851979c --- /dev/null +++ b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/apache/hadoop/fs/ozone/SmartRootedOzoneFileSystem.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.ozone; + +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.util.Preconditions; +import org.smartdata.client.SmartClient; +import org.smartdata.ozone.client.FileAccessReportSupport; +import org.smartdata.ozone.client.SmartOzoneClientAdapter; +import org.smartdata.protocol.SmartClientProtocol; + +import java.io.IOException; +import java.io.InputStream; + +public class SmartRootedOzoneFileSystem extends RootedOzoneFileSystem { + + @Override + protected OzoneClientAdapter createAdapter( + ConfigurationSource conf, String omHost, int omPort) throws IOException { + return new SmartClientAdapter( + omHost, omPort, conf, (OzoneFSStorageStatistics) getOzoneFSOpsCountStatistics() + ); + } + + @Slf4j + static class SmartClientAdapter extends RootedOzoneClientAdapterImpl implements SmartOzoneClientAdapter { + private final FileAccessReportSupport accessReportSupport; + + SmartClientAdapter( + String omHost, + int omPort, + ConfigurationSource hadoopConf, + OzoneFSStorageStatistics storageStatistics) throws IOException { + this(omHost, omPort, hadoopConf, storageStatistics, + new SmartClient(Preconditions.checkNotNull(OzoneConfiguration.of(hadoopConf)))); + } + + SmartClientAdapter( + String omHost, + int omPort, + ConfigurationSource hadoopConf, + OzoneFSStorageStatistics storageStatistics, + SmartClientProtocol ssmClient) throws IOException { + super(omHost, omPort, hadoopConf, storageStatistics); + this.accessReportSupport = new FileAccessReportSupport(ssmClient); + } + + @Override + public void close() { + try { + super.close(); + } catch (IOException e) { + log.error("Error closing OzoneClient", e); + } + + try { + accessReportSupport.close(); + } catch (IOException e) { + log.error("Error closing SmartClient", e); + } + } + + @Override + public InputStream readFile(String key) throws IOException { + InputStream inputStream = super.readFile(key); + accessReportSupport.reportFileAccess(key); + return inputStream; + } + + @Override + public String getBasePath() { + return accessReportSupport.getBasePath(); + } + } +} diff --git a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/FileAccessReportSupport.java b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/FileAccessReportSupport.java new file mode 100644 index 0000000000..37726f4fef --- /dev/null +++ b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/FileAccessReportSupport.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.ozone.client; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.fs.Path; +import org.smartdata.metrics.FileAccessEvent; +import org.smartdata.protocol.SmartClientProtocol; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Arrays; +import java.util.stream.Collectors; + +import static org.smartdata.utils.SecurityUtil.getCurrentUsername; + +@Slf4j +public class FileAccessReportSupport implements Closeable { + private final SmartClientProtocol ssmClient; + + @Getter + private final String basePath; + + public FileAccessReportSupport(SmartClientProtocol ssmClient, String... basePathSegments) { + this.ssmClient = ssmClient; + this.basePath = Arrays.stream(basePathSegments) + .collect(Collectors.joining("/", "/", "")); + } + + public void reportFileAccess(String path) { + String pathWithoutAuthority = new Path(basePath, path) + .toUri() + .getPath(); + + FileAccessEvent accessEvent = new FileAccessEvent( + pathWithoutAuthority, + getCurrentUsername().orElse(null)); + + try { + ssmClient.reportFileAccessEvent(accessEvent); + } catch (IOException exception) { + // todo add retry mechanism for both HDFS and Ozone clients + log.error("Error reporting file access event", exception); + } + } + + @Override + public void close() throws IOException { + ssmClient.close(); + } +} diff --git a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartOzoneClientAdapter.java b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartOzoneClientAdapter.java index 70042b5f3a..cb5b9fbbb8 100644 --- a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartOzoneClientAdapter.java +++ b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartOzoneClientAdapter.java @@ -17,271 +17,8 @@ */ package org.smartdata.ozone.client; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.crypto.key.KeyProvider; -import org.apache.hadoop.fs.FileChecksum; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.SafeModeAction; -import org.apache.hadoop.fs.ozone.BasicKeyInfo; -import org.apache.hadoop.fs.ozone.FileStatusAdapter; import org.apache.hadoop.fs.ozone.OzoneClientAdapter; -import org.apache.hadoop.fs.ozone.OzoneFSDataStreamOutput; -import org.apache.hadoop.fs.ozone.OzoneFSOutputStream; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; -import org.apache.hadoop.ozone.OzoneFsServerDefaults; -import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.util.Preconditions; -import org.smartdata.client.SmartClient; -import org.smartdata.metrics.FileAccessEvent; -import org.smartdata.protocol.SmartClientProtocol; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.stream.Collectors; - -import static org.smartdata.utils.SecurityUtil.getCurrentUsername; - -@Slf4j -public class SmartOzoneClientAdapter implements OzoneClientAdapter { - private final OzoneClientAdapter delegate; - - private final SmartClientProtocol ssmClient; - - @Getter - private final String basePath; - - private SmartOzoneClientAdapter( - OzoneClientAdapter delegate, - Configuration configuration, - String... basePathSegments) throws IOException { - this(delegate, - new SmartClient(Preconditions.checkNotNull(configuration)), - basePathSegments); - } - - SmartOzoneClientAdapter( - OzoneClientAdapter delegate, - SmartClientProtocol ssmClient, - String... basePathSegments) { - this.delegate = Preconditions.checkNotNull(delegate); - this.ssmClient = ssmClient; - this.basePath = Arrays.stream(basePathSegments) - .collect(Collectors.joining("/", "/", "")); - } - - @Override - public short getDefaultReplication() { - return delegate.getDefaultReplication(); - } - - @Override - public void close() { - try { - delegate.close(); - } catch (IOException e) { - log.error("Error closing OzoneClientAdapter", e); - } - - try { - ssmClient.close(); - } catch (IOException e) { - log.error("Error closing SmartClient", e); - } - } - - @Override - public InputStream readFile(String key) throws IOException { - InputStream inputStream = delegate.readFile(key); - reportFileAccess(key); - return inputStream; - } - - @Override - public OzoneFSOutputStream createFile(String key, short replication, - boolean overWrite, boolean recursive) throws IOException { - return delegate.createFile(key, replication, overWrite, recursive); - } - - @Override - public OzoneFSDataStreamOutput createStreamFile(String key, short replication, - boolean overWrite, boolean recursive) throws IOException { - return delegate.createStreamFile(key, replication, overWrite, recursive); - } - - @Override - public void renameKey(String key, String newKeyName) throws IOException { - delegate.renameKey(key, newKeyName); - } - - @Override - public void rename(String pathStr, String newPath) throws IOException { - delegate.rename(pathStr, newPath); - } - - @Override - public boolean createDirectory(String keyName) throws IOException { - return delegate.createDirectory(keyName); - } - - @Override - public boolean deleteObject(String keyName) throws IOException { - return delegate.deleteObject(keyName); - } - - @Override - public boolean deleteObject(String keyName, boolean recursive) - throws IOException { - return delegate.deleteObject(keyName, recursive); - } - - @Override - public boolean deleteObjects(List keyNameList) { - return delegate.deleteObjects(keyNameList); - } - - @Override - public FileStatusAdapter getFileStatus(String key, URI uri, - Path qualifiedPath, String userName) - throws IOException { - return delegate.getFileStatus(key, uri, qualifiedPath, userName); - } - - @Override - public Iterator listKeys(String pathKey) throws IOException { - return delegate.listKeys(pathKey); - } - - @Override - public List listStatus(String keyName, boolean recursive, - String startKey, long numEntries, URI uri, - Path workingDir, String username, boolean lite) throws IOException { - return delegate.listStatus(keyName, recursive, startKey, numEntries, uri, workingDir, username, lite); - } - - @Override - public Token getDelegationToken(String renewer) - throws IOException { - return delegate.getDelegationToken(renewer); - } - - @Override - public OzoneFsServerDefaults getServerDefaults() throws IOException { - return delegate.getServerDefaults(); - } - - @Override - public KeyProvider getKeyProvider() throws IOException { - return delegate.getKeyProvider(); - } - - @Override - public URI getKeyProviderUri() throws IOException { - return delegate.getKeyProviderUri(); - } - - @Override - public String getCanonicalServiceName() { - return delegate.getCanonicalServiceName(); - } - - @Override - public boolean isFSOptimizedBucket() { - return delegate.isFSOptimizedBucket(); - } - - @Override - public FileChecksum getFileChecksum(String keyName, long length) - throws IOException { - return delegate.getFileChecksum(keyName, length); - } - - @Override - public String createSnapshot(String pathStr, String snapshotName) - throws IOException { - return delegate.createSnapshot(pathStr, snapshotName); - } - - @Override - public void renameSnapshot(String pathStr, String snapshotOldName, String snapshotNewName) - throws IOException { - delegate.renameSnapshot(pathStr, snapshotOldName, snapshotNewName); - } - - @Override - public void deleteSnapshot(String pathStr, String snapshotName) - throws IOException { - delegate.deleteSnapshot(pathStr, snapshotName); - } - - @Override - public SnapshotDiffReport getSnapshotDiffReport(Path snapshotDir, - String fromSnapshot, String toSnapshot) - throws IOException, InterruptedException { - return delegate.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot); - } - - @Override - public LeaseKeyInfo recoverFilePrepare(final String pathStr, boolean force) throws IOException { - return delegate.recoverFilePrepare(pathStr, force); - } - - @Override - public void recoverFile(OmKeyArgs keyArgs) throws IOException { - delegate.recoverFile(keyArgs); - } - - @Override - public long finalizeBlock(OmKeyLocationInfo block) throws IOException { - return delegate.finalizeBlock(block); - } - - @Override - public void setTimes(String key, long mtime, long atime) throws IOException { - delegate.setTimes(key, mtime, atime); - } - - @Override - public boolean isFileClosed(String pathStr) throws IOException { - return delegate.isFileClosed(pathStr); - } - - @Override - public boolean setSafeMode(SafeModeAction action, boolean isChecked) - throws IOException { - return delegate.setSafeMode(action, isChecked); - } - - private void reportFileAccess(String path) { - String pathWithoutAuthority = new Path(basePath, path) - .toUri() - .getPath(); - - FileAccessEvent accessEvent = new FileAccessEvent( - pathWithoutAuthority, - getCurrentUsername().orElse(null)); - - try { - ssmClient.reportFileAccessEvent(accessEvent); - } catch (IOException exception) { - // todo add retry mechanism for both HDFS and Ozone clients - log.error("Error reporting file access event", exception); - } - } - - public static SmartOzoneClientAdapter wrap( - OzoneClientAdapter delegate, - Configuration configuration, - String... basePathSegments) throws IOException { - return new SmartOzoneClientAdapter(delegate, configuration, basePathSegments); - } -} \ No newline at end of file +public interface SmartOzoneClientAdapter extends OzoneClientAdapter { + String getBasePath(); +} diff --git a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartOzoneFileSystem.java b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartOzoneFileSystem.java deleted file mode 100644 index 5108e00133..0000000000 --- a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/SmartOzoneFileSystem.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.ozone.client; - -import org.apache.hadoop.fs.ozone.OzoneClientAdapter; -import org.apache.hadoop.fs.ozone.OzoneFileSystem; -import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; - -import java.io.IOException; - -public class SmartOzoneFileSystem extends OzoneFileSystem { - - @Override - protected OzoneClientAdapter createAdapter(ConfigurationSource conf, String bucketStr, String volumeStr, - String omHost, int omPort) throws IOException { - return SmartOzoneClientAdapter.wrap( - super.createAdapter(conf, bucketStr, volumeStr, omHost, omPort), - OzoneConfiguration.of(conf), volumeStr, bucketStr - ); - } -} diff --git a/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/smartdata/ozone/client/SmartOzoneClientAdapterTest.java b/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java similarity index 67% rename from smart-ozone-support/smart-ozone-fs-client/src/test/java/org/smartdata/ozone/client/SmartOzoneClientAdapterTest.java rename to smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java index 9db0bca34f..11824a701f 100644 --- a/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/smartdata/ozone/client/SmartOzoneClientAdapterTest.java +++ b/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java @@ -15,14 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.ozone.client; +package org.apache.hadoop.fs.ozone; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.ozone.OzoneClientAdapter; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.smartdata.metrics.FileAccessEvent; import org.smartdata.model.FileState; +import org.smartdata.ozone.client.SmartOzoneClientAdapter; import org.smartdata.protocol.SmartClientProtocol; import java.io.IOException; @@ -33,43 +35,43 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; public class SmartOzoneClientAdapterTest { - private OzoneClientAdapter delegate; private MockSsmClient ssmClient; @Before - public void initMocks() { - this.delegate = mock(OzoneClientAdapter.class); + public void init() { this.ssmClient = new MockSsmClient(); } @Test + @Ignore("Unignore when testing environment for Ozone will be added (ADH-7291)") public void testReportAccessEventOfs() throws IOException { - testReportAccessEventInternal(); + SmartOzoneClientAdapter clientAdapter = new SmartRootedOzoneFileSystem.SmartClientAdapter( + "TODO", -1, new OzoneConfiguration(), null, ssmClient); + testReportAccessEventInternal(clientAdapter); } @Test + @Ignore("Unignore when testing environment for Ozone will be added (ADH-7291)") public void testReportAccessEventO3fs() throws IOException { - testReportAccessEventInternal("someVolume", "someBucket"); + SmartOzoneClientAdapter clientAdapter = new SmartOzoneFileSystem.SmartClientAdapter( + "TODO", -1, new OzoneConfiguration(), "someVolume", "someBucket", null, ssmClient); + testReportAccessEventInternal(clientAdapter); } - public void testReportAccessEventInternal(String... basePathSegments) throws IOException { - SmartOzoneClientAdapter ssmOzoneClient = new SmartOzoneClientAdapter( - delegate, ssmClient, basePathSegments); - - ssmOzoneClient.createFile("key", (short) 1, false, false); + public void testReportAccessEventInternal(SmartOzoneClientAdapter clientAdapter) throws IOException { + clientAdapter.createFile("key", (short) 1, false, false); assertTrue(ssmClient.accessEvents.isEmpty()); - ssmOzoneClient.getFileStatus("key1", null, null, null); + clientAdapter.getFileStatus("key1", null, null, null); assertTrue(ssmClient.accessEvents.isEmpty()); - ssmOzoneClient.readFile("someKey"); + clientAdapter.readFile("someKey"); assertEquals(1, ssmClient.accessEvents.size()); - ssmOzoneClient.readFile("someDir/anotherKey"); + clientAdapter.readFile("someDir/anotherKey"); assertEquals(2, ssmClient.accessEvents.size()); List actualAccessedFiles = ssmClient.accessEvents.stream() @@ -77,7 +79,7 @@ public void testReportAccessEventInternal(String... basePathSegments) throws IOE .collect(Collectors.toList()); List expectedAccessFiles = Stream.of("someKey", "someDir/anotherKey") - .map(path -> new Path(ssmOzoneClient.getBasePath(), path)) + .map(path -> new Path(clientAdapter.getBasePath(), path)) .map(path -> path.toUri().getPath()) .collect(Collectors.toList()); diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFileInfoDao.java b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFileInfoDao.java index 67de5f4944..23c5699fdb 100644 --- a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFileInfoDao.java +++ b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFileInfoDao.java @@ -17,8 +17,9 @@ */ package org.smartdata.ozone; +import org.smartdata.metrics.GeneralFileInfoSource; import org.smartdata.ozone.model.OzoneFileInfo; -public interface OzoneFileInfoDao { +public interface OzoneFileInfoDao extends GeneralFileInfoSource { void insert(OzoneFileInfo fileInfo); } diff --git a/supports/tools/docker/ozone/conf/core-site.xml b/supports/tools/docker/ozone/conf/core-site.xml new file mode 100644 index 0000000000..5aa49b1704 --- /dev/null +++ b/supports/tools/docker/ozone/conf/core-site.xml @@ -0,0 +1,19 @@ + + + + fs.defaultFS + ofs://om/ + + + fs.ofs.impl + org.apache.hadoop.fs.ozone.SmartRootedOzoneFileSystem + + + fs.o3fs.impl + org.apache.hadoop.fs.ozone.SmartOzoneFileSystem + + + smart.server.rpc.address + localhost:8081 + + diff --git a/supports/tools/docker/ozone/docker-compose.yaml b/supports/tools/docker/ozone/docker-compose.yaml index de953d2e4f..d750b51a42 100644 --- a/supports/tools/docker/ozone/docker-compose.yaml +++ b/supports/tools/docker/ozone/docker-compose.yaml @@ -40,9 +40,20 @@ services: <<: *image ports: - 9864 - command: ["ozone","datanode"] + command: [ "ozone","datanode" ] environment: <<: *common-config + + ozone-fs-client: + image: hub.adsw.io/java/hadoop-openjdk8:${HADOOP_VERSION:-3.3.6} + restart: unless-stopped + entrypoint: [ "/entrypoint.sh" ] + volumes: + - ./conf:/etc/conf + - ./fs-client/common.sh:/common.sh + - ./fs-client/entrypoint.sh:/entrypoint.sh + - ../../../../smart-ozone-support/target/smart-ozone-fs-client-shaded-2.2.0-SNAPSHOT.jar:/etc/jars/smart-ozone-client.jar + om: <<: *image ports: @@ -54,7 +65,8 @@ services: CORE-SITE.XML_hadoop.proxyuser.hadoop.groups: "*" ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION WAITFOR: scm:9876 - command: ["ozone","om"] + command: [ "ozone","om" ] + scm: <<: *image ports: @@ -62,21 +74,24 @@ services: environment: <<: *common-config ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["ozone","scm"] + command: [ "ozone","scm" ] + recon: <<: *image ports: - 9888:9888 environment: <<: *common-config - command: ["ozone","recon"] + command: [ "ozone","recon" ] + s3g: <<: *image ports: - 9878:9878 environment: <<: *common-config - command: ["ozone","s3g"] + command: [ "ozone","s3g" ] + httpfs: <<: *image ports: diff --git a/supports/tools/docker/ozone/fs-client/common.sh b/supports/tools/docker/ozone/fs-client/common.sh new file mode 100755 index 0000000000..c967002991 --- /dev/null +++ b/supports/tools/docker/ozone/fs-client/common.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +function wait_for_it() +{ + local serviceport=$1 + local service=${serviceport%%:*} + local port=${serviceport#*:} + local retry_seconds=5 + local max_try=100 + let i=1 + + nc -z $service $port + result=$? + + until [ $result -eq 0 ]; do + echo "[$i/$max_try] check for ${service}:${port}..." + echo "[$i/$max_try] ${service}:${port} is not available yet" + if (( $i == $max_try )); then + echo "[$i/$max_try] ${service}:${port} is still not available; giving up after ${max_try} tries. :/" + exit 1 + fi + + echo "[$i/$max_try] try in ${retry_seconds}s once again ..." + let "i++" + sleep $retry_seconds + + nc -z $service $port + result=$? + done + echo "[$i/$max_try] $service:${port} is available." +} + +function addProperty() { + local path=$1 + local name=$2 + local value=$3 + + local entry="$name${value}" + local escapedEntry=$(echo $entry | sed 's/\//\\\//g') + sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path +} + +function moveHadoopConfFiles() { + local srcDirectory=$1 + local target=$2 + + cp ${srcDirectory}/*.xml ${target} +} + +function configure() { + local path=$1 + local module=$2 + local envPrefix=$3 + + local var + local value + + echo "Configuring $module" + for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do + name=`echo ${c} | perl -pe 's/___/-/g; s/__/@/g; s/_/./g; s/@/_/g;'` + var="${envPrefix}_${c}" + value=${!var} + echo " - Setting $name=$value" + addProperty $path $name "$value" + done +} \ No newline at end of file diff --git a/supports/tools/docker/ozone/fs-client/entrypoint.sh b/supports/tools/docker/ozone/fs-client/entrypoint.sh new file mode 100755 index 0000000000..6a7eaeed43 --- /dev/null +++ b/supports/tools/docker/ozone/fs-client/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +. ./common.sh + +echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc + +cp /etc/jars/*.jar $HADOOP_HOME/share/hadoop/common/lib/ +moveHadoopConfFiles /etc/conf ${HADOOP_CONF_DIR} + +tail -f /dev/null \ No newline at end of file From 099139ac67769fafbb8049ef6f8b64e184a87a56 Mon Sep 17 00:00:00 2001 From: Tigran Manasyan Date: Wed, 17 Dec 2025 18:02:17 +0400 Subject: [PATCH 2/2] [ADH-7260] Fix migration script --- .../db/changelog/changelog-13.add-ozone-file-access-table.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smart-metastore/src/main/resources/db/changelog/changelog-13.add-ozone-file-access-table.xml b/smart-metastore/src/main/resources/db/changelog/changelog-13.add-ozone-file-access-table.xml index b87d844c20..a400936ff2 100644 --- a/smart-metastore/src/main/resources/db/changelog/changelog-13.add-ozone-file-access-table.xml +++ b/smart-metastore/src/main/resources/db/changelog/changelog-13.add-ozone-file-access-table.xml @@ -28,6 +28,8 @@ select count(*) from pg_proc where proname = 'create_file_access_partition' + AND array_length(proargtypes, 1) = 1 + AND proargtypes[0] = 'timestamp'::regtype; drop function create_file_access_partition(timestamp);