diff --git a/tests/functional/countItems.js b/tests/functional/countItems.js index a72079c5..a52fc760 100644 --- a/tests/functional/countItems.js +++ b/tests/functional/countItems.js @@ -20,7 +20,7 @@ const USERSBUCKET = '__usersbucket'; const expectedCountItems = { objects: 90, versions: 60, - buckets: 9, + buckets: 10, dataManaged: { total: { curr: 15000, prev: 12000 }, byLocation: { @@ -100,6 +100,10 @@ const expectedDataMetrics = { objectCount: { current: 10n, deleteMarker: 0n, nonCurrent: 10n }, usedCapacity: { current: 1000n, nonCurrent: 1000n }, }, + [`bucket_test-bucket-empty_${testBucketCreationDate}`]: { + objectCount: { current: 0n, deleteMarker: 0n, nonCurrent: 0n }, + usedCapacity: { current: 0n, nonCurrent: 0n }, + }, 'location_secondary-location-1': { objectCount: { current: 30n, deleteMarker: 0n, nonCurrent: 30n }, usedCapacity: { current: 3000n, nonCurrent: 3000n }, }, @@ -199,6 +203,25 @@ function populateMongo(client, callback) { ], cb), callback); } +const emptyBucket = BucketInfo.fromObj({ ...testBucketMD, _name: 'test-bucket-empty' }); + +function populateEmptyBucket(client, callback) { + return async.series([ + next => client.createBucket(emptyBucket.getName(), emptyBucket, logger, next), + next => client.putObject( + USERSBUCKET, + `${emptyBucket.getOwner()}${constants.splitter}${emptyBucket.getName()}`, + testUserBucketInfo.value, + { + versioning: false, + versionId: null, + }, + logger, + next, + ), + ], callback); +} + jest.setTimeout(120000); describe('CountItems', () => { const oldEnv = process.env; @@ -222,6 +245,7 @@ describe('CountItems', () => { async.series([ next => client.setup(next), next => populateMongo(client, next), + next => populateEmptyBucket(client, next), ], done); setTimeoutSpy = jest.spyOn(global, 'setTimeout'); setTimeoutSpy.mockImplementation((callback, delay) => callback()); diff --git a/tests/unit/utils/S3UtilsMongoClient.js b/tests/unit/utils/S3UtilsMongoClient.js index 1f1929a4..37739150 100644 --- a/tests/unit/utils/S3UtilsMongoClient.js +++ b/tests/unit/utils/S3UtilsMongoClient.js @@ -1059,6 +1059,41 @@ function uploadObjects(client, bucketName, objectList, callback) { }, callback); } +describe('S3UtilsMongoClient::_seedEmptyBucketMetrics', () => { + const bucketInfo = BucketInfo.fromObj({ ...testBucketMD }); + const locationConfig = { 'us-east-1': { objectId: 'us-east-1' } }; + const bucketResource = `test-bucket_${testBucketCreationDate}`; + + it('should seed zero-value bucket, account and location entries and return true', () => { + const collRes = { bucket: {}, location: {}, account: {} }; + const log = { warn: sinon.spy() }; + const seeded = mongoTestClient._seedEmptyBucketMetrics(collRes, bucketResource, bucketInfo, locationConfig, log); + assert.strictEqual(seeded, true); + assert.deepStrictEqual(Object.keys(collRes.bucket), [bucketResource]); + assert.deepStrictEqual(Object.keys(collRes.account), [testAccountCanonicalId]); + assert.deepStrictEqual(Object.keys(collRes.location), ['us-east-1']); + assert.strictEqual(log.warn.called, false); + }); + + it('should warn and seed nothing when no __usersbucket creation date is available', () => { + const collRes = { bucket: {}, location: {}, account: {} }; + const log = { warn: sinon.spy() }; + const seeded = mongoTestClient._seedEmptyBucketMetrics(collRes, undefined, bucketInfo, locationConfig, log); + assert.strictEqual(seeded, false); + assert.deepStrictEqual(collRes, { bucket: {}, location: {}, account: {} }); + assert.strictEqual(log.warn.calledOnce, true); + }); + + it('should seed nothing when the bucket already produced metrics', () => { + const collRes = { bucket: { existing: {} }, location: {}, account: {} }; + const log = { warn: sinon.spy() }; + const seeded = mongoTestClient._seedEmptyBucketMetrics(collRes, bucketResource, bucketInfo, locationConfig, log); + assert.strictEqual(seeded, false); + assert.deepStrictEqual(collRes, { bucket: { existing: {} }, location: {}, account: {} }); + assert.strictEqual(log.warn.called, false); + }); +}); + describe('S3UtilsMongoClient, tests', () => { const hr = 1000 * 60 * 60; let client; @@ -1150,14 +1185,76 @@ describe('S3UtilsMongoClient, tests', () => { dataStore: 'us-east-1', }; + const zeroUsedCapacity = { + current: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }; + const zeroObjectCount = { + current: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + deleteMarker: 0n, + }; + const zeroMetricsEntry = { + usedCapacity: zeroUsedCapacity, + objectCount: zeroObjectCount, + }; const tests = [ [ - 'getObjectMDStats() should return zero-result when no objects in the bucket', + 'getObjectMDStats() should seed zero-value metrics for an empty bucket, account and location', { bucketName: 'test-bucket', isVersioned: false, objectList: [], }, + { + dataManaged: { + locations: { 'us-east-1': { curr: 0, prev: 0 } }, + total: { curr: 0, prev: 0 }, + }, + objects: 0, + stalled: 0, + versions: 0, + dataMetrics: { + bucket: { + [`test-bucket_${testBucketCreationDate}`]: { + ...zeroMetricsEntry, + locations: { 'us-east-1': { ...zeroMetricsEntry } }, + }, + }, + location: { + 'us-east-1': { ...zeroMetricsEntry }, + }, + account: { + [testAccountCanonicalId]: { + ...zeroMetricsEntry, + locations: { 'us-east-1': { ...zeroMetricsEntry } }, + }, + }, + }, + }, + ], + [ + 'getObjectMDStats() should not seed metrics when the bucket has no __usersbucket creation date', + { + bucketName: 'test-bucket-no-usersbucket', + isVersioned: false, + objectList: [], + skipUsersBucket: true, + }, { dataManaged: { locations: {}, @@ -2806,20 +2903,23 @@ describe('S3UtilsMongoClient, tests', () => { isVersioned, objectList, inflights, + skipUsersBucket, } = testCase; return async.waterfall([ next => createBucket(client, bucketName, isVersioned, err => next(err)), - next => client.putObject( - USERSBUCKET, - `${testBucketMD._owner}${constants.splitter}${bucketName}`, - testUserBucketInfo.value, - { - versioning: false, - versionId: null, - }, - logger, - next, - ), + next => (skipUsersBucket + ? next() + : client.putObject( + USERSBUCKET, + `${testBucketMD._owner}${constants.splitter}${bucketName}`, + testUserBucketInfo.value, + { + versioning: false, + versionId: null, + }, + logger, + next, + )), next => uploadObjects(client, bucketName, objectList, err => next(err)), next => client.getBucketAttributes(bucketName, logger, next), (bucketInfo, next) => { diff --git a/utils/S3UtilsMongoClient.js b/utils/S3UtilsMongoClient.js index 2c9d0df8..21fff7e4 100644 --- a/utils/S3UtilsMongoClient.js +++ b/utils/S3UtilsMongoClient.js @@ -164,6 +164,7 @@ class S3UtilsMongoClient extends MongoClientInterface { }; let stalledCount = 0; let bucketKey; + let bucketResource; let inflightsPreScan = 0n; let accountBucket; const cmpDate = new Date(); @@ -184,7 +185,8 @@ class S3UtilsMongoClient extends MongoClientInterface { || bucketStatus.Status === 'Suspended')); if (bucketCreationDate) { - bucketKey = `bucket_${bucketName}_${new Date(bucketCreationDate).getTime()}`; + bucketResource = `${bucketName}_${new Date(bucketCreationDate).getTime()}`; + bucketKey = `bucket_${bucketResource}`; inflightsPreScan = await this.readStorageConsumptionInflights(bucketKey, log); } @@ -400,6 +402,10 @@ class S3UtilsMongoClient extends MongoClientInterface { }, ); + if (this._seedEmptyBucketMetrics(collRes, bucketResource, bucketInfo, locationConfig, log)) { + accountBucket = bucketInfo.getOwner(); + } + const retResult = this._handleResults(collRes, isVer); retResult.stalled = stalledCount; @@ -432,6 +438,39 @@ class S3UtilsMongoClient extends MongoClientInterface { } } + /** + * Seed zero-value metrics for empty buckets. + * @param{object} collRes - per-resource metrics accumulator, mutated in place + * @param{string} [bucketResource] - `${bucketName}_${creationDate}` key, absent when no __usersbucket date + * @param{object} bucketInfo - bucket attributes + * @param{object} locationConfig - locationConfig.json + * @param{object} log - werelogs logger + * @returns{boolean} true when zero metrics were seeded, false otherwise + */ + _seedEmptyBucketMetrics(collRes, bucketResource, bucketInfo, locationConfig, log) { + if (Object.keys(collRes.bucket).length > 0) { + return false; + } + if (!bucketResource) { + log.warn('cannot seed empty-bucket metrics: bucket has no __usersbucket creation date entry', { + method: 'getObjectMDStats', + bucketName: bucketInfo.getName(), + owner: bucketInfo.getOwner(), + }); + return false; + } + // eslint-disable-next-line no-param-reassign + collRes.bucket[bucketResource] = { ...baseMetricsObject, locations: {} }; + // eslint-disable-next-line no-param-reassign + collRes.account[bucketInfo.getOwner()] = { ...baseMetricsObject, locations: {} }; + const bucketLocation = bucketInfo.getLocationConstraint(); + if (bucketLocation && locationConfig && locationConfig[bucketLocation]) { + // eslint-disable-next-line no-param-reassign + collRes.location[locationConfig[bucketLocation].objectId] = { ...baseMetricsObject }; + } + return true; + } + /** * @param{string} bucketName - * @param{object} bucketInfo - bucket attributes