From 970dfe53d6c5c7771aa46f48eb61c426e8c3dfe1 Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:03:09 -0600 Subject: [PATCH 1/7] feat(buckets): track per-bucket used space reported via gateway Add a usedSpaceBytes field to buckets and a gateway endpoint to set it, so products whose content lives outside the network (mail/Stalwart) can report the space they consume against a network bucket. --- lib/core/buckets/Bucket.ts | 1 + lib/core/buckets/MongoDBBucketsRepository.ts | 13 +++++++++++++ lib/core/buckets/Repository.ts | 7 ++++++- lib/core/users/usecase.ts | 2 ++ lib/models/bucket.ts | 2 ++ lib/server/http/gateway/controller.ts | 8 ++++++++ tests/lib/core/users/usecase.test.ts | 2 ++ 7 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/core/buckets/Bucket.ts b/lib/core/buckets/Bucket.ts index a7b59b8b..1851ae09 100644 --- a/lib/core/buckets/Bucket.ts +++ b/lib/core/buckets/Bucket.ts @@ -11,6 +11,7 @@ export interface Bucket { status: string; transfer: number; storage: number; + usedSpaceBytes?: number; created?: Date; maxFrameSize?: number; publicPermissions?: string[]; diff --git a/lib/core/buckets/MongoDBBucketsRepository.ts b/lib/core/buckets/MongoDBBucketsRepository.ts index 8ce69daf..0cb4df8d 100644 --- a/lib/core/buckets/MongoDBBucketsRepository.ts +++ b/lib/core/buckets/MongoDBBucketsRepository.ts @@ -63,6 +63,19 @@ export class MongoDBBucketsRepository implements BucketsRepository { }); } + async setUsedSpaceBytes( + bucketId: Bucket['id'], + userId: Bucket['userId'], + usedSpaceBytes: number + ): Promise { + const result = await this.model.updateOne( + { _id: bucketId, userId }, + { $set: { usedSpaceBytes } } + ); + + return result.matchedCount > 0; + } + async removeByIdAndUser(bucketId: Bucket['id'], userId: Bucket['userId']): Promise { await this.model.deleteOne({ userId, diff --git a/lib/core/buckets/Repository.ts b/lib/core/buckets/Repository.ts index e24da97d..163ac3a8 100644 --- a/lib/core/buckets/Repository.ts +++ b/lib/core/buckets/Repository.ts @@ -7,7 +7,12 @@ export interface BucketsRepository { findByIds(ids: Bucket['id'][]): Promise; find(where: Partial): Promise; findUserBucketsFromDate(userId: Bucket['id'], date?: Date, limit?: number): Promise; + setUsedSpaceBytes( + bucketId: Bucket['id'], + userId: Bucket['userId'], + usedSpaceBytes: number + ): Promise; destroyByUser(userId: Bucket['userId']): Promise; removeAll(where: Partial): Promise; - removeByIdAndUser(bucketId: Bucket['id'], userId: Bucket['userId']): Promise + removeByIdAndUser(bucketId: Bucket['id'], userId: Bucket['userId']): Promise } diff --git a/lib/core/users/usecase.ts b/lib/core/users/usecase.ts index 4ffd06a8..a141e85e 100644 --- a/lib/core/users/usecase.ts +++ b/lib/core/users/usecase.ts @@ -2,6 +2,7 @@ import { createHash, randomBytes } from 'crypto'; import { UsersRepository } from './Repository'; import { BucketsRepository } from '../buckets/Repository'; +import { BucketNotFoundError } from '../buckets/usecase'; import { MailUsecase } from '../mail/usecase'; import { EventBus, EventBusEvents } from '../../server/eventBus'; import { FramesRepository } from '../frames/Repository'; @@ -286,6 +287,7 @@ export class UsersUsecase { status: 'Active', transfer: 0, storage: 0, + usedSpaceBytes: 0, encryptionKey: '', }); diff --git a/lib/models/bucket.ts b/lib/models/bucket.ts index 8db590a2..1a2efa78 100644 --- a/lib/models/bucket.ts +++ b/lib/models/bucket.ts @@ -5,6 +5,7 @@ const errors = require("storj-service-error-types"); interface IBucket extends Document { storage: number; + usedSpaceBytes: number; transfer: number; status: "Active" | "Inactive"; pubkeys: string[]; @@ -20,6 +21,7 @@ interface IBucket extends Document { const BucketSchema = new Schema( { storage: { type: Number, default: 0 }, + usedSpaceBytes: { type: Number, default: 0 }, transfer: { type: Number, default: 0 }, status: { type: String, diff --git a/lib/server/http/gateway/controller.ts b/lib/server/http/gateway/controller.ts index 37b0d088..47b06838 100644 --- a/lib/server/http/gateway/controller.ts +++ b/lib/server/http/gateway/controller.ts @@ -2,6 +2,7 @@ import { Request, Response } from 'express'; import { Logger } from 'winston'; import { EmailIsAlreadyInUseError, InvalidDataFormatError, UserAlreadyExistsError, UserNotFoundError, UsersUsecase } from '../../../core'; import { BucketEntriesUsecase } from '../../../core/bucketEntries/usecase'; +import { BucketNotFoundError } from '../../../core/buckets/usecase'; import { GatewayUsecase } from '../../../core/gateway/Usecase'; import { EventBus, EventBusEvents, UserStorageChangedPayload } from '../../eventBus'; @@ -16,6 +17,13 @@ type DeleteFilesInBulkResponse = { type CreateBucketBody = { name: string }; type CreateBucketResponse = { id: string; name: string }; +type SetBucketUsageBody = { usedSpaceBytes: number }; + +const OBJECT_ID_PATTERN = /^[a-f0-9]{24}$/i; + +const isValidUsedSpaceBytes = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value) && value >= 0; + export class HTTPGatewayController { constructor( private gatewayUsecase: GatewayUsecase, diff --git a/tests/lib/core/users/usecase.test.ts b/tests/lib/core/users/usecase.test.ts index e49d28fb..dbcea0bc 100644 --- a/tests/lib/core/users/usecase.test.ts +++ b/tests/lib/core/users/usecase.test.ts @@ -18,6 +18,7 @@ import { import { MongoDBFramesRepository } from '../../../../lib/core/frames/MongoDBFramesRepository'; import { FramesRepository } from '../../../../lib/core/frames/Repository'; import { BucketsRepository } from '../../../../lib/core/buckets/Repository'; +import { BucketNotFoundError } from '../../../../lib/core/buckets/usecase'; import { Mailer, MailUsecase, SendGridMailUsecase } from '../../../../lib/core/mail/usecase'; import { EventBus, EventBusEvents } from '../../../../lib/server/eventBus'; import { User } from '../../../../lib/core/users/User'; @@ -451,6 +452,7 @@ describe('Users usecases', () => { status: 'Active', transfer: 0, storage: 0, + usedSpaceBytes: 0, encryptionKey: '', }]); expect(result).toStrictEqual({ id: createdBucket.id, name: createdBucket.name }); From 97bef98d4490bfa7076d2c93fe41843ff7b69b92 Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Wed, 10 Jun 2026 08:14:28 -0600 Subject: [PATCH 2/7] feat: enhance bucket usage reporting with user space snapshot Add a new method to sum used space bytes in MongoDBBucketsRepository and update the UsersUsecase to return a UserSpaceSnapshot when setting bucket usage. Update HTTPGatewayController to send the snapshot in the response. Include tests to verify the new functionality. --- lib/core/buckets/MongoDBBucketsRepository.ts | 9 +++++++++ lib/core/buckets/Repository.ts | 1 + lib/core/users/usecase.ts | 5 +++++ lib/server/http/gateway/controller.ts | 2 +- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/core/buckets/MongoDBBucketsRepository.ts b/lib/core/buckets/MongoDBBucketsRepository.ts index 0cb4df8d..157618d4 100644 --- a/lib/core/buckets/MongoDBBucketsRepository.ts +++ b/lib/core/buckets/MongoDBBucketsRepository.ts @@ -57,6 +57,15 @@ export class MongoDBBucketsRepository implements BucketsRepository { return formatFromMongoToBucket(rawModel); } + async sumUsedSpaceBytes(userId: Bucket['userId']): Promise { + const [result] = await this.model.aggregate([ + { $match: { userId } }, + { $group: { _id: null, total: { $sum: { $ifNull: ['$usedSpaceBytes', 0] } } } } + ]); + + return result ? result.total : 0; + } + async destroyByUser(userId: Bucket['userId']): Promise { await this.model.deleteMany({ userId, diff --git a/lib/core/buckets/Repository.ts b/lib/core/buckets/Repository.ts index 163ac3a8..3dc400cc 100644 --- a/lib/core/buckets/Repository.ts +++ b/lib/core/buckets/Repository.ts @@ -12,6 +12,7 @@ export interface BucketsRepository { userId: Bucket['userId'], usedSpaceBytes: number ): Promise; + sumUsedSpaceBytes(userId: Bucket['userId']): Promise; destroyByUser(userId: Bucket['userId']): Promise; removeAll(where: Partial): Promise; removeByIdAndUser(bucketId: Bucket['id'], userId: Bucket['userId']): Promise diff --git a/lib/core/users/usecase.ts b/lib/core/users/usecase.ts index a141e85e..5dcf8ea4 100644 --- a/lib/core/users/usecase.ts +++ b/lib/core/users/usecase.ts @@ -21,6 +21,11 @@ function isEmailValid(email: string) { export const RESET_PASSWORD_TOKEN_BYTES_LENGTH = 256; export const SHA256_HASH_BYTES_LENGTH = 32; +export interface UserSpaceSnapshot { + maxSpaceBytes: number; + totalUsedSpaceBytes: number; +} + export class UserAlreadyExistsError extends Error { constructor() { super('User already exists'); diff --git a/lib/server/http/gateway/controller.ts b/lib/server/http/gateway/controller.ts index 47b06838..bb99eddd 100644 --- a/lib/server/http/gateway/controller.ts +++ b/lib/server/http/gateway/controller.ts @@ -1,6 +1,6 @@ import { Request, Response } from 'express'; import { Logger } from 'winston'; -import { EmailIsAlreadyInUseError, InvalidDataFormatError, UserAlreadyExistsError, UserNotFoundError, UsersUsecase } from '../../../core'; +import { EmailIsAlreadyInUseError, InvalidDataFormatError, UserAlreadyExistsError, UserNotFoundError, UserSpaceSnapshot, UsersUsecase } from '../../../core'; import { BucketEntriesUsecase } from '../../../core/bucketEntries/usecase'; import { BucketNotFoundError } from '../../../core/buckets/usecase'; From 929518b617b2c62693226c2577a1877e6b79599c Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Fri, 12 Jun 2026 15:43:35 -0600 Subject: [PATCH 3/7] feat: implement create and delete entry functionality Add methods to create and delete bucket entries in the MongoDBBucketEntriesRepository and BucketEntriesUsecase. Introduce a new endpoint in the HTTPGatewayController for creating entries, which includes validation for entry keys and sizes. Update tests to cover the new functionality --- .../MongoDBBucketEntriesRepository.ts | 16 ++ lib/core/bucketEntries/Repository.ts | 4 + lib/core/bucketEntries/usecase.ts | 96 ++++++++++ lib/core/buckets/Bucket.ts | 1 - lib/core/buckets/MongoDBBucketsRepository.ts | 22 --- lib/core/buckets/Repository.ts | 6 - lib/core/users/usecase.ts | 2 - lib/models/bucket.ts | 2 - lib/server/http/gateway/controller.ts | 89 ++++++++- lib/server/http/gateway/index.ts | 2 + tests/lib/core/bucketentries/usecase.test.ts | 159 +++++++++++++++- tests/lib/core/users/usecase.test.ts | 2 - tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts | 171 ++++++++++++++++++ 13 files changed, 533 insertions(+), 39 deletions(-) diff --git a/lib/core/bucketEntries/MongoDBBucketEntriesRepository.ts b/lib/core/bucketEntries/MongoDBBucketEntriesRepository.ts index db9f8134..467836a9 100644 --- a/lib/core/bucketEntries/MongoDBBucketEntriesRepository.ts +++ b/lib/core/bucketEntries/MongoDBBucketEntriesRepository.ts @@ -141,6 +141,22 @@ export class MongoDBBucketEntriesRepository implements BucketEntriesRepository { return rawModel.toObject(); } + async findOneOrCreate( + where: Partial, + data: Omit + ): Promise<{ entry: BucketEntry; created: boolean }> { + const result = await this.model.findOneAndUpdate( + where, + { $setOnInsert: { ...data, created: new Date() } }, + { upsert: true, new: true, includeResultMetadata: true } + ); + + return { + entry: formatFromMongoToBucketEntry(result.value), + created: !result.lastErrorObject?.updatedExisting, + }; + } + async deleteByIds(ids: BucketEntry['id'][]): Promise { await this.model.deleteMany({ _id: { $in: ids } }); } diff --git a/lib/core/bucketEntries/Repository.ts b/lib/core/bucketEntries/Repository.ts index 4cc7c926..b677bada 100644 --- a/lib/core/bucketEntries/Repository.ts +++ b/lib/core/bucketEntries/Repository.ts @@ -10,5 +10,9 @@ export interface BucketEntriesRepository { findOneWithFrame(where: Partial): Promise & { frame?: Frame } | null>; findByIdsWithFrames(ids: BucketEntry['id'][]): Promise<(Omit & { frame?: Frame })[]>; create(data: Omit): Promise; + findOneOrCreate( + where: Partial, + data: Omit + ): Promise<{ entry: BucketEntry; created: boolean }>; deleteByIds(ids: BucketEntry['id'][]): Promise; } diff --git a/lib/core/bucketEntries/usecase.ts b/lib/core/bucketEntries/usecase.ts index 733a3263..47a5a0a3 100644 --- a/lib/core/bucketEntries/usecase.ts +++ b/lib/core/bucketEntries/usecase.ts @@ -1,4 +1,5 @@ import lodash from 'lodash'; +import { createHash } from 'crypto'; import { BucketsRepository } from '../buckets/Repository'; import { BucketEntriesRepository } from './Repository'; @@ -11,6 +12,7 @@ import { PointersRepository } from '../pointers/Repository'; import { MirrorsRepository } from '../mirrors/Repository'; import { BucketEntry } from './BucketEntry'; import { UsersRepository } from '../users/Repository'; +import { UserNotFoundError, UserSpaceSnapshot } from '../users'; import { User } from '../users/User'; import { Bucket } from '../buckets/Bucket'; import { FileStateRepository } from '../fileState/Repository'; @@ -181,4 +183,98 @@ export class BucketEntriesUsecase { await this.bucketEntriesRepository.deleteByIds(fileIds); await this.fileStateRepository.deleteByBucketEntryIds(fileIds); } + + private hashEntryKey(key: string): string { + return createHash('sha256').update(key).digest('hex'); + } + + private async findUserAndOwnedBucket( + userUuid: User['uuid'], + bucketId: Bucket['id'] + ): Promise { + const user = await this.usersRepository.findByUuid(userUuid); + + if (!user) { + throw new UserNotFoundError(userUuid); + } + + const bucket = await this.bucketsRepository.findOne({ id: bucketId, userId: userUuid }); + + if (!bucket) { + throw new BucketNotFoundError(); + } + + return user; + } + + async createEntryByKey( + userUuid: User['uuid'], + bucketId: Bucket['id'], + key: string, + size: number + ): Promise<{ id: BucketEntry['id']; snapshot: UserSpaceSnapshot }> { + const user = await this.findUserAndOwnedBucket(userUuid, bucketId); + + const index = this.hashEntryKey(key); + + const { entry, created } = await this.bucketEntriesRepository.findOneOrCreate( + { bucket: bucketId, index }, + { bucket: bucketId, index, name: key, size, version: 2 } + ); + + if (created) { + await this.usersRepository.addTotalUsedSpaceBytes(userUuid, size); + } + + return { + id: entry.id, + snapshot: { + maxSpaceBytes: user.maxSpaceBytes, + totalUsedSpaceBytes: created + ? user.totalUsedSpaceBytes + size + : user.totalUsedSpaceBytes, + }, + }; + } + + async removeEntryByKey( + userUuid: User['uuid'], + bucketId: Bucket['id'], + key: string + ): Promise { + const user = await this.findUserAndOwnedBucket(userUuid, bucketId); + + const index = this.hashEntryKey(key); + const entry = await this.bucketEntriesRepository.findOne({ bucket: bucketId, index }); + + if (!entry) { + return { + maxSpaceBytes: user.maxSpaceBytes, + totalUsedSpaceBytes: user.totalUsedSpaceBytes, + }; + } + + await this.removeFile(entry.id); + + return { + maxSpaceBytes: user.maxSpaceBytes, + totalUsedSpaceBytes: user.totalUsedSpaceBytes - (entry.size || 0), + }; + } + + async removeAllEntriesFromUserBucket( + userUuid: User['uuid'], + bucketId: Bucket['id'] + ): Promise { + await this.findUserAndOwnedBucket(userUuid, bucketId); + + const batchSize = 100; + + let entries = await this.bucketEntriesRepository.findByBucket(bucketId, batchSize, 0); + + while (entries.length > 0) { + await this.removeFiles(entries.map((e) => e.id)); + entries = await this.bucketEntriesRepository.findByBucket(bucketId, batchSize, 0); + } + } } diff --git a/lib/core/buckets/Bucket.ts b/lib/core/buckets/Bucket.ts index 1851ae09..a7b59b8b 100644 --- a/lib/core/buckets/Bucket.ts +++ b/lib/core/buckets/Bucket.ts @@ -11,7 +11,6 @@ export interface Bucket { status: string; transfer: number; storage: number; - usedSpaceBytes?: number; created?: Date; maxFrameSize?: number; publicPermissions?: string[]; diff --git a/lib/core/buckets/MongoDBBucketsRepository.ts b/lib/core/buckets/MongoDBBucketsRepository.ts index 157618d4..8ce69daf 100644 --- a/lib/core/buckets/MongoDBBucketsRepository.ts +++ b/lib/core/buckets/MongoDBBucketsRepository.ts @@ -57,34 +57,12 @@ export class MongoDBBucketsRepository implements BucketsRepository { return formatFromMongoToBucket(rawModel); } - async sumUsedSpaceBytes(userId: Bucket['userId']): Promise { - const [result] = await this.model.aggregate([ - { $match: { userId } }, - { $group: { _id: null, total: { $sum: { $ifNull: ['$usedSpaceBytes', 0] } } } } - ]); - - return result ? result.total : 0; - } - async destroyByUser(userId: Bucket['userId']): Promise { await this.model.deleteMany({ userId, }); } - async setUsedSpaceBytes( - bucketId: Bucket['id'], - userId: Bucket['userId'], - usedSpaceBytes: number - ): Promise { - const result = await this.model.updateOne( - { _id: bucketId, userId }, - { $set: { usedSpaceBytes } } - ); - - return result.matchedCount > 0; - } - async removeByIdAndUser(bucketId: Bucket['id'], userId: Bucket['userId']): Promise { await this.model.deleteOne({ userId, diff --git a/lib/core/buckets/Repository.ts b/lib/core/buckets/Repository.ts index 3dc400cc..4dce295f 100644 --- a/lib/core/buckets/Repository.ts +++ b/lib/core/buckets/Repository.ts @@ -7,12 +7,6 @@ export interface BucketsRepository { findByIds(ids: Bucket['id'][]): Promise; find(where: Partial): Promise; findUserBucketsFromDate(userId: Bucket['id'], date?: Date, limit?: number): Promise; - setUsedSpaceBytes( - bucketId: Bucket['id'], - userId: Bucket['userId'], - usedSpaceBytes: number - ): Promise; - sumUsedSpaceBytes(userId: Bucket['userId']): Promise; destroyByUser(userId: Bucket['userId']): Promise; removeAll(where: Partial): Promise; removeByIdAndUser(bucketId: Bucket['id'], userId: Bucket['userId']): Promise diff --git a/lib/core/users/usecase.ts b/lib/core/users/usecase.ts index 5dcf8ea4..22747574 100644 --- a/lib/core/users/usecase.ts +++ b/lib/core/users/usecase.ts @@ -2,7 +2,6 @@ import { createHash, randomBytes } from 'crypto'; import { UsersRepository } from './Repository'; import { BucketsRepository } from '../buckets/Repository'; -import { BucketNotFoundError } from '../buckets/usecase'; import { MailUsecase } from '../mail/usecase'; import { EventBus, EventBusEvents } from '../../server/eventBus'; import { FramesRepository } from '../frames/Repository'; @@ -292,7 +291,6 @@ export class UsersUsecase { status: 'Active', transfer: 0, storage: 0, - usedSpaceBytes: 0, encryptionKey: '', }); diff --git a/lib/models/bucket.ts b/lib/models/bucket.ts index 1a2efa78..8db590a2 100644 --- a/lib/models/bucket.ts +++ b/lib/models/bucket.ts @@ -5,7 +5,6 @@ const errors = require("storj-service-error-types"); interface IBucket extends Document { storage: number; - usedSpaceBytes: number; transfer: number; status: "Active" | "Inactive"; pubkeys: string[]; @@ -21,7 +20,6 @@ interface IBucket extends Document { const BucketSchema = new Schema( { storage: { type: Number, default: 0 }, - usedSpaceBytes: { type: Number, default: 0 }, transfer: { type: Number, default: 0 }, status: { type: String, diff --git a/lib/server/http/gateway/controller.ts b/lib/server/http/gateway/controller.ts index bb99eddd..5c75e798 100644 --- a/lib/server/http/gateway/controller.ts +++ b/lib/server/http/gateway/controller.ts @@ -17,12 +17,17 @@ type DeleteFilesInBulkResponse = { type CreateBucketBody = { name: string }; type CreateBucketResponse = { id: string; name: string }; -type SetBucketUsageBody = { usedSpaceBytes: number }; +type CreateBucketEntryBody = { key: string; size: number }; +type CreateBucketEntryResponse = UserSpaceSnapshot & { id: string }; const OBJECT_ID_PATTERN = /^[a-f0-9]{24}$/i; +const MAX_ENTRY_KEY_LENGTH = 256; -const isValidUsedSpaceBytes = (value: unknown): value is number => - typeof value === 'number' && Number.isFinite(value) && value >= 0; +const isValidEntryKey = (value: unknown): value is string => + typeof value === 'string' && value.length > 0 && value.length <= MAX_ENTRY_KEY_LENGTH; + +const isValidEntrySize = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; export class HTTPGatewayController { constructor( @@ -172,4 +177,82 @@ export class HTTPGatewayController { return res.status(500).send({ message: 'Internal server error' }); } } + + async createBucketEntry( + req: Request<{ uuid: string; id: string }, {}, Partial, {}>, + res: Response + ) { + const { uuid, id } = req.params; + const { key, size } = req.body; + + if (!uuid || !id || !OBJECT_ID_PATTERN.test(id)) { + return res.status(400).send({ message: 'Invalid params' }); + } + + if (!isValidEntryKey(key)) { + return res.status(400).send({ message: 'key must be a non-empty string' }); + } + + if (!isValidEntrySize(size)) { + return res + .status(400) + .send({ message: 'size must be a non-negative integer' }); + } + + try { + const { id: entryId, snapshot } = await this.bucketEntriesUsecase.createEntryByKey( + uuid, + id, + key, + size + ); + + return res.status(200).send({ id: entryId, ...snapshot }); + } catch (err) { + if (err instanceof UserNotFoundError || err instanceof BucketNotFoundError) { + return res.status(404).send({ message: err.message }); + } + + this.logger.error( + '[GATEWAY/CREATE_ENTRY] Error creating entry on bucket %s of user %s: %s. %s', + id, + uuid, + (err as Error).message, + (err as Error).stack || 'NO STACK' + ); + + return res.status(500).send({ message: 'Internal server error' }); + } + } + + async deleteBucketEntry( + req: Request<{ uuid: string; id: string; key: string }>, + res: Response + ) { + const { uuid, id, key } = req.params; + + if (!uuid || !id || !OBJECT_ID_PATTERN.test(id) || !isValidEntryKey(key)) { + return res.status(400).send({ message: 'Invalid params' }); + } + + try { + const snapshot = await this.bucketEntriesUsecase.removeEntryByKey(uuid, id, key); + + return res.status(200).send(snapshot); + } catch (err) { + if (err instanceof UserNotFoundError || err instanceof BucketNotFoundError) { + return res.status(404).send({ message: err.message }); + } + + this.logger.error( + '[GATEWAY/DELETE_ENTRY] Error deleting entry on bucket %s of user %s: %s. %s', + id, + uuid, + (err as Error).message, + (err as Error).stack || 'NO STACK' + ); + + return res.status(500).send({ message: 'Internal server error' }); + } + } } diff --git a/lib/server/http/gateway/index.ts b/lib/server/http/gateway/index.ts index ad9155e3..a5a27429 100644 --- a/lib/server/http/gateway/index.ts +++ b/lib/server/http/gateway/index.ts @@ -11,6 +11,8 @@ export const createGatewayHTTPRouter = ( router.patch('/users/:uuid', jwtMiddleware, controller.updateUserEmail.bind(controller)); router.put('/storage/users/:uuid', jwtMiddleware, controller.changeStorage.bind(controller)); router.post('/users/:uuid/buckets', jwtMiddleware, controller.createUserBucket.bind(controller)); + router.post('/users/:uuid/buckets/:id/entries', jwtMiddleware, controller.createBucketEntry.bind(controller)); + router.delete('/users/:uuid/buckets/:id/entries/:key', jwtMiddleware, controller.deleteBucketEntry.bind(controller)); router.delete('/storage/files', jwtMiddleware, controller.deleteFilesInBulk.bind(controller)); return router; diff --git a/tests/lib/core/bucketentries/usecase.test.ts b/tests/lib/core/bucketentries/usecase.test.ts index c37f3898..ec8d16be 100644 --- a/tests/lib/core/bucketentries/usecase.test.ts +++ b/tests/lib/core/bucketentries/usecase.test.ts @@ -1,5 +1,6 @@ import { restore, stub } from 'sinon'; import { Model } from 'mongoose'; +import { createHash } from 'node:crypto'; import { BucketEntriesRepository } from '../../../../lib/core/bucketEntries/Repository'; import { FramesRepository } from '../../../../lib/core/frames/Repository'; @@ -19,7 +20,7 @@ import { MongoDBMirrorsRepository } from '../../../../lib/core/mirrors/MongoDBMi import { MongoDBPointersRepository } from '../../../../lib/core/pointers/MongoDBPointersRepository'; import { MongoDBShardsRepository } from '../../../../lib/core/shards/MongoDBShardsRepository'; import { MongoDBBucketEntryShardsRepository } from '../../../../lib/core/bucketEntryShards/MongoDBBucketEntryShardsRepository'; -import { MongoDBUsersRepository } from '../../../../lib/core/users'; +import { MongoDBUsersRepository, UserNotFoundError } from '../../../../lib/core/users'; import { ShardsUsecase } from '../../../../lib/core/shards/usecase'; import fixtures from '../fixtures'; @@ -658,4 +659,160 @@ describe('BucketEntriesUsecase', function () { expect(removeFileStub.calledWith(fileId)).toBeTruthy(); }); }); + + describe('createEntryByKey()', () => { + const entryKey = '12:345'; + const hashedKey = createHash('sha256').update(entryKey).digest('hex'); + + it('When the user does not exist, then it throws UserNotFoundError', async () => { + stub(usersRepository, 'findByUuid').resolves(null); + const findOrCreate = stub(bucketEntriesRepository, 'findOneOrCreate'); + + try { + await bucketEntriesUsecase.createEntryByKey('unknown-uuid', 'bucket-id', entryKey, 100); + expect(true).toBeFalsy(); + } catch (err) { + expect(err).toBeInstanceOf(UserNotFoundError); + } + + expect(findOrCreate.called).toBeFalsy(); + }); + + it('When the bucket does not belong to the user or does not exist, then it throws BucketNotFoundError', async () => { + const user = fixtures.getUser(); + + stub(usersRepository, 'findByUuid').resolves(user); + const findBucket = stub(bucketsRepository, 'findOne').resolves(null); + const findOrCreate = stub(bucketEntriesRepository, 'findOneOrCreate'); + + try { + await bucketEntriesUsecase.createEntryByKey(user.uuid, 'bucket-id', entryKey, 100); + expect(true).toBeFalsy(); + } catch (err) { + expect(err).toBeInstanceOf(BucketNotFoundError); + } + + expect(findBucket.calledOnceWithExactly({ id: 'bucket-id', userId: user.uuid })).toBeTruthy(); + expect(findOrCreate.called).toBeFalsy(); + }); + + it('When the entry is new, then it stores the hashed key and adds the size to the user total', async () => { + const user = fixtures.getUser({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); + const bucket = fixtures.getBucket({ userId: user.uuid }); + const entry = fixtures.getBucketEntry({ bucket: bucket.id, index: hashedKey, size: 500 }); + + stub(usersRepository, 'findByUuid').resolves(user); + stub(bucketsRepository, 'findOne').resolves(bucket); + const findOrCreate = stub(bucketEntriesRepository, 'findOneOrCreate').resolves({ entry, created: true }); + const addUsage = stub(usersRepository, 'addTotalUsedSpaceBytes').resolves(); + + const result = await bucketEntriesUsecase.createEntryByKey(user.uuid, bucket.id, entryKey, 500); + + expect(findOrCreate.calledOnceWithExactly( + { bucket: bucket.id, index: hashedKey }, + { bucket: bucket.id, index: hashedKey, name: entryKey, size: 500, version: 2 } + )).toBeTruthy(); + expect(addUsage.calledOnceWithExactly(user.uuid, 500)).toBeTruthy(); + expect(result).toStrictEqual({ + id: entry.id, + snapshot: { maxSpaceBytes: 10000, totalUsedSpaceBytes: 4500 }, + }); + }); + + it('When the entry already exists, then it does not touch the user total', async () => { + const user = fixtures.getUser({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); + const bucket = fixtures.getBucket({ userId: user.uuid }); + const entry = fixtures.getBucketEntry({ bucket: bucket.id, index: hashedKey, size: 500 }); + + stub(usersRepository, 'findByUuid').resolves(user); + stub(bucketsRepository, 'findOne').resolves(bucket); + stub(bucketEntriesRepository, 'findOneOrCreate').resolves({ entry, created: false }); + const addUsage = stub(usersRepository, 'addTotalUsedSpaceBytes').resolves(); + + const result = await bucketEntriesUsecase.createEntryByKey(user.uuid, bucket.id, entryKey, 500); + + expect(addUsage.called).toBeFalsy(); + expect(result).toStrictEqual({ + id: entry.id, + snapshot: { maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }, + }); + }); + }); + + describe('removeEntryByKey()', () => { + const entryKey = '12:345'; + const hashedKey = createHash('sha256').update(entryKey).digest('hex'); + + it('When the entry does not exist, then it is a no-op and the snapshot is unchanged', async () => { + const user = fixtures.getUser({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); + const bucket = fixtures.getBucket({ userId: user.uuid }); + + stub(usersRepository, 'findByUuid').resolves(user); + stub(bucketsRepository, 'findOne').resolves(bucket); + const findEntry = stub(bucketEntriesRepository, 'findOne').resolves(null); + const removeFile = stub(bucketEntriesUsecase, 'removeFile'); + + const snapshot = await bucketEntriesUsecase.removeEntryByKey(user.uuid, bucket.id, entryKey); + + expect(findEntry.calledOnceWithExactly({ bucket: bucket.id, index: hashedKey })).toBeTruthy(); + expect(removeFile.called).toBeFalsy(); + expect(snapshot).toStrictEqual({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); + }); + + it('When the entry exists, then it removes it and returns the decremented snapshot', async () => { + const user = fixtures.getUser({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); + const bucket = fixtures.getBucket({ userId: user.uuid }); + const entry = fixtures.getBucketEntry({ bucket: bucket.id, index: hashedKey, size: 500 }); + + stub(usersRepository, 'findByUuid').resolves(user); + stub(bucketsRepository, 'findOne').resolves(bucket); + stub(bucketEntriesRepository, 'findOne').resolves(entry); + const removeFile = stub(bucketEntriesUsecase, 'removeFile').resolves(); + + const snapshot = await bucketEntriesUsecase.removeEntryByKey(user.uuid, bucket.id, entryKey); + + expect(removeFile.calledOnceWithExactly(entry.id)).toBeTruthy(); + expect(snapshot).toStrictEqual({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 3500 }); + }); + }); + + describe('removeAllEntriesFromUserBucket()', () => { + it('When the bucket does not belong to the user, then it throws BucketNotFoundError', async () => { + const user = fixtures.getUser(); + + stub(usersRepository, 'findByUuid').resolves(user); + stub(bucketsRepository, 'findOne').resolves(null); + const findByBucket = stub(bucketEntriesRepository, 'findByBucket'); + + try { + await bucketEntriesUsecase.removeAllEntriesFromUserBucket(user.uuid, 'bucket-id'); + expect(true).toBeFalsy(); + } catch (err) { + expect(err).toBeInstanceOf(BucketNotFoundError); + } + + expect(findByBucket.called).toBeFalsy(); + }); + + it('When the bucket has entries, then it removes them in batches until none are left', async () => { + const user = fixtures.getUser(); + const bucket = fixtures.getBucket({ userId: user.uuid }); + const entries = [ + fixtures.getBucketEntry({ bucket: bucket.id }), + fixtures.getBucketEntry({ bucket: bucket.id }), + ]; + + stub(usersRepository, 'findByUuid').resolves(user); + stub(bucketsRepository, 'findOne').resolves(bucket); + const findByBucket = stub(bucketEntriesRepository, 'findByBucket'); + findByBucket.onFirstCall().resolves(entries); + findByBucket.onSecondCall().resolves([]); + const removeFiles = stub(bucketEntriesUsecase, 'removeFiles').resolves(); + + await bucketEntriesUsecase.removeAllEntriesFromUserBucket(user.uuid, bucket.id); + + expect(removeFiles.calledOnceWithExactly(entries.map(e => e.id))).toBeTruthy(); + expect(findByBucket.calledTwice).toBeTruthy(); + }); + }); }); diff --git a/tests/lib/core/users/usecase.test.ts b/tests/lib/core/users/usecase.test.ts index dbcea0bc..e49d28fb 100644 --- a/tests/lib/core/users/usecase.test.ts +++ b/tests/lib/core/users/usecase.test.ts @@ -18,7 +18,6 @@ import { import { MongoDBFramesRepository } from '../../../../lib/core/frames/MongoDBFramesRepository'; import { FramesRepository } from '../../../../lib/core/frames/Repository'; import { BucketsRepository } from '../../../../lib/core/buckets/Repository'; -import { BucketNotFoundError } from '../../../../lib/core/buckets/usecase'; import { Mailer, MailUsecase, SendGridMailUsecase } from '../../../../lib/core/mail/usecase'; import { EventBus, EventBusEvents } from '../../../../lib/server/eventBus'; import { User } from '../../../../lib/core/users/User'; @@ -452,7 +451,6 @@ describe('Users usecases', () => { status: 'Active', transfer: 0, storage: 0, - usedSpaceBytes: 0, encryptionKey: '', }]); expect(result).toStrictEqual({ id: createdBucket.id, name: createdBucket.name }); diff --git a/tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts b/tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts index 6b0c4aa4..fc7e204c 100644 --- a/tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts +++ b/tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts @@ -115,6 +115,177 @@ describe('Gateway V2 e2e tests', () => { }) }) + describe('Bucket entries', () => { + const createBucketForUser = async (userUuid: string, jwt: string) => { + const { body } = await testServer + .post(`/v2/gateway/users/${userUuid}/buckets`) + .set('Authorization', `Bearer ${jwt}`) + .send({ name: `mail-account-${crypto.randomUUID()}` }) + + return body as { id: string; name: string } + } + + const hashKey = (key: string) => crypto.createHash('sha256').update(key).digest('hex') + + it('When creating an entry, then it is persisted with the hashed key and the user total grows by its size', async () => { + const testUser = await createTestUser() + const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) + const bucket = await createBucketForUser(testUser.uuid, jwt) + const key = `1:${Date.now()}` + + const userBefore = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) + + const response = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ key, size: 5000 }) + + expect(response.status).toBe(200) + expect(response.body.id).toBeDefined() + + const entryInDatabase = await databaseConnection.models.BucketEntry.findOne({ _id: response.body.id }) + expect(entryInDatabase).not.toBeNull() + expect(entryInDatabase.index).toBe(hashKey(key)) + expect(entryInDatabase.name).toBe(key) + expect(entryInDatabase.size).toBe(5000) + expect(entryInDatabase.version).toBe(2) + + const userAfter = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) + expect(userAfter.totalUsedSpaceBytes).toBe(userBefore.totalUsedSpaceBytes + 5000) + expect(response.body.maxSpaceBytes).toBe(userAfter.maxSpaceBytes) + expect(response.body.totalUsedSpaceBytes).toBe(userAfter.totalUsedSpaceBytes) + }) + + it('When creating the same entry twice, then it is counted only once', async () => { + const testUser = await createTestUser() + const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) + const bucket = await createBucketForUser(testUser.uuid, jwt) + const key = `1:${Date.now()}` + + const userBefore = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) + + const first = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ key, size: 5000 }) + + const second = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ key, size: 5000 }) + + expect(first.status).toBe(200) + expect(second.status).toBe(200) + expect(second.body.id).toBe(first.body.id) + expect(second.body.totalUsedSpaceBytes).toBe(first.body.totalUsedSpaceBytes) + + const entries = await databaseConnection.models.BucketEntry.find({ bucket: bucket.id }) + expect(entries.length).toBe(1) + + const userAfter = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) + expect(userAfter.totalUsedSpaceBytes).toBe(userBefore.totalUsedSpaceBytes + 5000) + }) + + it('When deleting an entry, then it is removed and the user total shrinks by its size', async () => { + const testUser = await createTestUser() + const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) + const bucket = await createBucketForUser(testUser.uuid, jwt) + const key = `1:${Date.now()}` + + const created = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ key, size: 5000 }) + + const userBefore = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) + + const response = await testServer + .delete(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries/${encodeURIComponent(key)}`) + .set('Authorization', `Bearer ${jwt}`) + + expect(response.status).toBe(200) + + const entryInDatabase = await databaseConnection.models.BucketEntry.findOne({ _id: created.body.id }) + expect(entryInDatabase).toBeNull() + + const userAfter = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) + expect(userAfter.totalUsedSpaceBytes).toBe(userBefore.totalUsedSpaceBytes - 5000) + expect(response.body).toEqual({ + maxSpaceBytes: userAfter.maxSpaceBytes, + totalUsedSpaceBytes: userAfter.totalUsedSpaceBytes, + }) + }) + + it('When deleting an entry that does not exist, then it is a no-op and the user total is unchanged', async () => { + const testUser = await createTestUser() + const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) + const bucket = await createBucketForUser(testUser.uuid, jwt) + + const userBefore = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) + + const response = await testServer + .delete(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries/${encodeURIComponent('1:404')}`) + .set('Authorization', `Bearer ${jwt}`) + + expect(response.status).toBe(200) + + const userAfter = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) + expect(userAfter.totalUsedSpaceBytes).toBe(userBefore.totalUsedSpaceBytes) + }) + + it('When creating an entry on a bucket of another user, then it returns 404 and changes nothing', async () => { + const owner = await createTestUser() + const otherUser = await createTestUser() + const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) + const bucket = await createBucketForUser(owner.uuid, jwt) + + const response = await testServer + .post(`/v2/gateway/users/${otherUser.uuid}/buckets/${bucket.id}/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ key: '1:1', size: 5000 }) + + expect(response.status).toBe(404) + + const entries = await databaseConnection.models.BucketEntry.find({ bucket: bucket.id }) + expect(entries.length).toBe(0) + }) + + it('When the entry key or size is invalid, then it returns 400', async () => { + const testUser = await createTestUser() + const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) + const bucket = await createBucketForUser(testUser.uuid, jwt) + + const missingKey = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ size: 5000 }) + + const negativeSize = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ key: '1:1', size: -1 }) + + const nonIntegerSize = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ key: '1:1', size: 10.5 }) + + expect(missingKey.status).toBe(400) + expect(negativeSize.status).toBe(400) + expect(nonIntegerSize.status).toBe(400) + }) + + it('When no auth token is provided, then it returns 401', async () => { + const testUser = await createTestUser() + + const response = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/${'a'.repeat(24)}/entries`) + .send({ key: '1:1', size: 1000 }) + + expect(response.status).toBe(401) + }) + }) + describe('Deleting user files', () => { let axiosGetStub: sinon.SinonStub const FAKE_UPLOAD_URL = 'http://fake-upload-url' From c8aa165e2b80c97b7150a8dfa2f79c46889a7335 Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Tue, 16 Jun 2026 18:34:15 -0600 Subject: [PATCH 4/7] refactor: remove unused removeAllEntriesFromUserBucket method from BucketEntriesUsecase and its associated tests --- lib/core/bucketEntries/usecase.ts | 16 -------- tests/lib/core/bucketentries/usecase.test.ts | 40 -------------------- 2 files changed, 56 deletions(-) diff --git a/lib/core/bucketEntries/usecase.ts b/lib/core/bucketEntries/usecase.ts index 47a5a0a3..219b18de 100644 --- a/lib/core/bucketEntries/usecase.ts +++ b/lib/core/bucketEntries/usecase.ts @@ -261,20 +261,4 @@ export class BucketEntriesUsecase { totalUsedSpaceBytes: user.totalUsedSpaceBytes - (entry.size || 0), }; } - - async removeAllEntriesFromUserBucket( - userUuid: User['uuid'], - bucketId: Bucket['id'] - ): Promise { - await this.findUserAndOwnedBucket(userUuid, bucketId); - - const batchSize = 100; - - let entries = await this.bucketEntriesRepository.findByBucket(bucketId, batchSize, 0); - - while (entries.length > 0) { - await this.removeFiles(entries.map((e) => e.id)); - entries = await this.bucketEntriesRepository.findByBucket(bucketId, batchSize, 0); - } - } } diff --git a/tests/lib/core/bucketentries/usecase.test.ts b/tests/lib/core/bucketentries/usecase.test.ts index ec8d16be..86e6f9bd 100644 --- a/tests/lib/core/bucketentries/usecase.test.ts +++ b/tests/lib/core/bucketentries/usecase.test.ts @@ -775,44 +775,4 @@ describe('BucketEntriesUsecase', function () { expect(snapshot).toStrictEqual({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 3500 }); }); }); - - describe('removeAllEntriesFromUserBucket()', () => { - it('When the bucket does not belong to the user, then it throws BucketNotFoundError', async () => { - const user = fixtures.getUser(); - - stub(usersRepository, 'findByUuid').resolves(user); - stub(bucketsRepository, 'findOne').resolves(null); - const findByBucket = stub(bucketEntriesRepository, 'findByBucket'); - - try { - await bucketEntriesUsecase.removeAllEntriesFromUserBucket(user.uuid, 'bucket-id'); - expect(true).toBeFalsy(); - } catch (err) { - expect(err).toBeInstanceOf(BucketNotFoundError); - } - - expect(findByBucket.called).toBeFalsy(); - }); - - it('When the bucket has entries, then it removes them in batches until none are left', async () => { - const user = fixtures.getUser(); - const bucket = fixtures.getBucket({ userId: user.uuid }); - const entries = [ - fixtures.getBucketEntry({ bucket: bucket.id }), - fixtures.getBucketEntry({ bucket: bucket.id }), - ]; - - stub(usersRepository, 'findByUuid').resolves(user); - stub(bucketsRepository, 'findOne').resolves(bucket); - const findByBucket = stub(bucketEntriesRepository, 'findByBucket'); - findByBucket.onFirstCall().resolves(entries); - findByBucket.onSecondCall().resolves([]); - const removeFiles = stub(bucketEntriesUsecase, 'removeFiles').resolves(); - - await bucketEntriesUsecase.removeAllEntriesFromUserBucket(user.uuid, bucket.id); - - expect(removeFiles.calledOnceWithExactly(entries.map(e => e.id))).toBeTruthy(); - expect(findByBucket.calledTwice).toBeTruthy(); - }); - }); }); From 6e1bfe582a59db6422e289872343c58bf3219fa6 Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Fri, 19 Jun 2026 07:29:14 -0600 Subject: [PATCH 5/7] refactor: rename findUserAndOwnedBucket to findBucketOwner and update addTotalUsedSpaceBytes to return total used space --- lib/core/bucketEntries/usecase.ts | 16 +++++++--------- lib/core/users/MongoDBUsersRepository.ts | 11 +++++++---- lib/core/users/Repository.ts | 2 +- lib/server/http/gateway/controller.ts | 4 ++-- tests/lib/core/bucketentries/usecase.test.ts | 2 +- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/core/bucketEntries/usecase.ts b/lib/core/bucketEntries/usecase.ts index 219b18de..0652706e 100644 --- a/lib/core/bucketEntries/usecase.ts +++ b/lib/core/bucketEntries/usecase.ts @@ -188,7 +188,7 @@ export class BucketEntriesUsecase { return createHash('sha256').update(key).digest('hex'); } - private async findUserAndOwnedBucket( + private async findBucketOwner( userUuid: User['uuid'], bucketId: Bucket['id'] ): Promise { @@ -213,7 +213,7 @@ export class BucketEntriesUsecase { key: string, size: number ): Promise<{ id: BucketEntry['id']; snapshot: UserSpaceSnapshot }> { - const user = await this.findUserAndOwnedBucket(userUuid, bucketId); + const user = await this.findBucketOwner(userUuid, bucketId); const index = this.hashEntryKey(key); @@ -222,17 +222,15 @@ export class BucketEntriesUsecase { { bucket: bucketId, index, name: key, size, version: 2 } ); - if (created) { - await this.usersRepository.addTotalUsedSpaceBytes(userUuid, size); - } + const totalUsedSpaceBytes = created + ? await this.usersRepository.addTotalUsedSpaceBytes(userUuid, size) + : user.totalUsedSpaceBytes; return { id: entry.id, snapshot: { maxSpaceBytes: user.maxSpaceBytes, - totalUsedSpaceBytes: created - ? user.totalUsedSpaceBytes + size - : user.totalUsedSpaceBytes, + totalUsedSpaceBytes, }, }; } @@ -242,7 +240,7 @@ export class BucketEntriesUsecase { bucketId: Bucket['id'], key: string ): Promise { - const user = await this.findUserAndOwnedBucket(userUuid, bucketId); + const user = await this.findBucketOwner(userUuid, bucketId); const index = this.hashEntryKey(key); const entry = await this.bucketEntriesRepository.findOne({ bucket: bucketId, index }); diff --git a/lib/core/users/MongoDBUsersRepository.ts b/lib/core/users/MongoDBUsersRepository.ts index 7c41e1ae..a2847c87 100644 --- a/lib/core/users/MongoDBUsersRepository.ts +++ b/lib/core/users/MongoDBUsersRepository.ts @@ -100,14 +100,17 @@ export class MongoDBUsersRepository implements UsersRepository { await this.userModel.updateOne({ uuid }, { $set: update }); } - addTotalUsedSpaceBytes( + async addTotalUsedSpaceBytes( uuid: string, totalUsedSpaceBytes: number - ): Promise { - return this.userModel.updateOne( + ): Promise { + const user = await this.userModel.findOneAndUpdate( { uuid }, - { $inc: { totalUsedSpaceBytes } } + { $inc: { totalUsedSpaceBytes } }, + { new: true } ); + + return user?.totalUsedSpaceBytes ?? 0; } removeById(id: User['id']) { diff --git a/lib/core/users/Repository.ts b/lib/core/users/Repository.ts index c36374a2..099fa7c6 100644 --- a/lib/core/users/Repository.ts +++ b/lib/core/users/Repository.ts @@ -7,7 +7,7 @@ export interface UsersRepository { findByEmail(email: User['email']): Promise; findByIds(ids: User['id'][]): Promise; create(data: CreateUserData): Promise; - addTotalUsedSpaceBytes(uuid: User['uuid'], totalUsedSpaceBytes: User['totalUsedSpaceBytes']): Promise; + addTotalUsedSpaceBytes(uuid: User['uuid'], totalUsedSpaceBytes: User['totalUsedSpaceBytes']): Promise; updateById(id: User['id'], update: Partial): Promise; updateByEmail(email: User['email'], update: Partial): Promise; updateByUuid(uuid: User['uuid'], update: Partial): Promise; diff --git a/lib/server/http/gateway/controller.ts b/lib/server/http/gateway/controller.ts index 5c75e798..ebe53b09 100644 --- a/lib/server/http/gateway/controller.ts +++ b/lib/server/http/gateway/controller.ts @@ -27,7 +27,7 @@ const isValidEntryKey = (value: unknown): value is string => typeof value === 'string' && value.length > 0 && value.length <= MAX_ENTRY_KEY_LENGTH; const isValidEntrySize = (value: unknown): value is number => - typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; + typeof value === 'number' && Number.isSafeInteger(value) && value > 0; export class HTTPGatewayController { constructor( @@ -196,7 +196,7 @@ export class HTTPGatewayController { if (!isValidEntrySize(size)) { return res .status(400) - .send({ message: 'size must be a non-negative integer' }); + .send({ message: 'size must be a positive integer' }); } try { diff --git a/tests/lib/core/bucketentries/usecase.test.ts b/tests/lib/core/bucketentries/usecase.test.ts index 86e6f9bd..6fee115f 100644 --- a/tests/lib/core/bucketentries/usecase.test.ts +++ b/tests/lib/core/bucketentries/usecase.test.ts @@ -704,7 +704,7 @@ describe('BucketEntriesUsecase', function () { stub(usersRepository, 'findByUuid').resolves(user); stub(bucketsRepository, 'findOne').resolves(bucket); const findOrCreate = stub(bucketEntriesRepository, 'findOneOrCreate').resolves({ entry, created: true }); - const addUsage = stub(usersRepository, 'addTotalUsedSpaceBytes').resolves(); + const addUsage = stub(usersRepository, 'addTotalUsedSpaceBytes').resolves(4500); const result = await bucketEntriesUsecase.createEntryByKey(user.uuid, bucket.id, entryKey, 500); From a31de0fdb6f58184a818a4abc8e5a8c1a692cb29 Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Tue, 23 Jun 2026 18:15:50 -0600 Subject: [PATCH 6/7] refactor: update BucketEntry interface and simplify entry creation and deletion methods --- lib/core/bucketEntries/BucketEntry.ts | 2 +- lib/core/bucketEntries/usecase.ts | 30 +++------ lib/server/http/gateway/controller.ts | 23 +++---- lib/server/http/gateway/index.ts | 2 +- tests/lib/core/bucketentries/usecase.test.ts | 64 ++++++-------------- 5 files changed, 38 insertions(+), 83 deletions(-) diff --git a/lib/core/bucketEntries/BucketEntry.ts b/lib/core/bucketEntries/BucketEntry.ts index 547c9fbe..cc08bff8 100644 --- a/lib/core/bucketEntries/BucketEntry.ts +++ b/lib/core/bucketEntries/BucketEntry.ts @@ -8,7 +8,7 @@ export interface BucketEntry { name?: string; // filename is deprecated filename?: string; - index: string; + index?: string; bucket: Bucket['id']; created?: Date; mimetype?: string; diff --git a/lib/core/bucketEntries/usecase.ts b/lib/core/bucketEntries/usecase.ts index 0652706e..42171ff6 100644 --- a/lib/core/bucketEntries/usecase.ts +++ b/lib/core/bucketEntries/usecase.ts @@ -1,5 +1,4 @@ import lodash from 'lodash'; -import { createHash } from 'crypto'; import { BucketsRepository } from '../buckets/Repository'; import { BucketEntriesRepository } from './Repository'; @@ -184,10 +183,6 @@ export class BucketEntriesUsecase { await this.fileStateRepository.deleteByBucketEntryIds(fileIds); } - private hashEntryKey(key: string): string { - return createHash('sha256').update(key).digest('hex'); - } - private async findBucketOwner( userUuid: User['uuid'], bucketId: Bucket['id'] @@ -207,24 +202,20 @@ export class BucketEntriesUsecase { return user; } - async createEntryByKey( + async createEntry( userUuid: User['uuid'], bucketId: Bucket['id'], - key: string, size: number ): Promise<{ id: BucketEntry['id']; snapshot: UserSpaceSnapshot }> { const user = await this.findBucketOwner(userUuid, bucketId); - const index = this.hashEntryKey(key); - - const { entry, created } = await this.bucketEntriesRepository.findOneOrCreate( - { bucket: bucketId, index }, - { bucket: bucketId, index, name: key, size, version: 2 } - ); + const entry = await this.bucketEntriesRepository.create({ + bucket: bucketId, + size, + version: 2, + }); - const totalUsedSpaceBytes = created - ? await this.usersRepository.addTotalUsedSpaceBytes(userUuid, size) - : user.totalUsedSpaceBytes; + const totalUsedSpaceBytes = await this.usersRepository.addTotalUsedSpaceBytes(userUuid, size); return { id: entry.id, @@ -235,15 +226,14 @@ export class BucketEntriesUsecase { }; } - async removeEntryByKey( + async removeEntry( userUuid: User['uuid'], bucketId: Bucket['id'], - key: string + entryId: BucketEntry['id'] ): Promise { const user = await this.findBucketOwner(userUuid, bucketId); - const index = this.hashEntryKey(key); - const entry = await this.bucketEntriesRepository.findOne({ bucket: bucketId, index }); + const entry = await this.bucketEntriesRepository.findOne({ id: entryId, bucket: bucketId }); if (!entry) { return { diff --git a/lib/server/http/gateway/controller.ts b/lib/server/http/gateway/controller.ts index ebe53b09..4d7a83c2 100644 --- a/lib/server/http/gateway/controller.ts +++ b/lib/server/http/gateway/controller.ts @@ -17,14 +17,10 @@ type DeleteFilesInBulkResponse = { type CreateBucketBody = { name: string }; type CreateBucketResponse = { id: string; name: string }; -type CreateBucketEntryBody = { key: string; size: number }; +type CreateBucketEntryBody = { size: number }; type CreateBucketEntryResponse = UserSpaceSnapshot & { id: string }; const OBJECT_ID_PATTERN = /^[a-f0-9]{24}$/i; -const MAX_ENTRY_KEY_LENGTH = 256; - -const isValidEntryKey = (value: unknown): value is string => - typeof value === 'string' && value.length > 0 && value.length <= MAX_ENTRY_KEY_LENGTH; const isValidEntrySize = (value: unknown): value is number => typeof value === 'number' && Number.isSafeInteger(value) && value > 0; @@ -183,16 +179,12 @@ export class HTTPGatewayController { res: Response ) { const { uuid, id } = req.params; - const { key, size } = req.body; + const { size } = req.body; if (!uuid || !id || !OBJECT_ID_PATTERN.test(id)) { return res.status(400).send({ message: 'Invalid params' }); } - if (!isValidEntryKey(key)) { - return res.status(400).send({ message: 'key must be a non-empty string' }); - } - if (!isValidEntrySize(size)) { return res .status(400) @@ -200,10 +192,9 @@ export class HTTPGatewayController { } try { - const { id: entryId, snapshot } = await this.bucketEntriesUsecase.createEntryByKey( + const { id: entryId, snapshot } = await this.bucketEntriesUsecase.createEntry( uuid, id, - key, size ); @@ -226,17 +217,17 @@ export class HTTPGatewayController { } async deleteBucketEntry( - req: Request<{ uuid: string; id: string; key: string }>, + req: Request<{ uuid: string; id: string; entryId: string }>, res: Response ) { - const { uuid, id, key } = req.params; + const { uuid, id, entryId } = req.params; - if (!uuid || !id || !OBJECT_ID_PATTERN.test(id) || !isValidEntryKey(key)) { + if (!uuid || !id || !OBJECT_ID_PATTERN.test(id) || !OBJECT_ID_PATTERN.test(entryId)) { return res.status(400).send({ message: 'Invalid params' }); } try { - const snapshot = await this.bucketEntriesUsecase.removeEntryByKey(uuid, id, key); + const snapshot = await this.bucketEntriesUsecase.removeEntry(uuid, id, entryId); return res.status(200).send(snapshot); } catch (err) { diff --git a/lib/server/http/gateway/index.ts b/lib/server/http/gateway/index.ts index a5a27429..cf851cb6 100644 --- a/lib/server/http/gateway/index.ts +++ b/lib/server/http/gateway/index.ts @@ -12,7 +12,7 @@ export const createGatewayHTTPRouter = ( router.put('/storage/users/:uuid', jwtMiddleware, controller.changeStorage.bind(controller)); router.post('/users/:uuid/buckets', jwtMiddleware, controller.createUserBucket.bind(controller)); router.post('/users/:uuid/buckets/:id/entries', jwtMiddleware, controller.createBucketEntry.bind(controller)); - router.delete('/users/:uuid/buckets/:id/entries/:key', jwtMiddleware, controller.deleteBucketEntry.bind(controller)); + router.delete('/users/:uuid/buckets/:id/entries/:entryId', jwtMiddleware, controller.deleteBucketEntry.bind(controller)); router.delete('/storage/files', jwtMiddleware, controller.deleteFilesInBulk.bind(controller)); return router; diff --git a/tests/lib/core/bucketentries/usecase.test.ts b/tests/lib/core/bucketentries/usecase.test.ts index 6fee115f..9dd89929 100644 --- a/tests/lib/core/bucketentries/usecase.test.ts +++ b/tests/lib/core/bucketentries/usecase.test.ts @@ -1,6 +1,5 @@ import { restore, stub } from 'sinon'; import { Model } from 'mongoose'; -import { createHash } from 'node:crypto'; import { BucketEntriesRepository } from '../../../../lib/core/bucketEntries/Repository'; import { FramesRepository } from '../../../../lib/core/frames/Repository'; @@ -660,22 +659,19 @@ describe('BucketEntriesUsecase', function () { }); }); - describe('createEntryByKey()', () => { - const entryKey = '12:345'; - const hashedKey = createHash('sha256').update(entryKey).digest('hex'); - + describe('createEntry()', () => { it('When the user does not exist, then it throws UserNotFoundError', async () => { stub(usersRepository, 'findByUuid').resolves(null); - const findOrCreate = stub(bucketEntriesRepository, 'findOneOrCreate'); + const create = stub(bucketEntriesRepository, 'create'); try { - await bucketEntriesUsecase.createEntryByKey('unknown-uuid', 'bucket-id', entryKey, 100); + await bucketEntriesUsecase.createEntry('unknown-uuid', 'bucket-id', 100); expect(true).toBeFalsy(); } catch (err) { expect(err).toBeInstanceOf(UserNotFoundError); } - expect(findOrCreate.called).toBeFalsy(); + expect(create.called).toBeFalsy(); }); it('When the bucket does not belong to the user or does not exist, then it throws BucketNotFoundError', async () => { @@ -683,34 +679,33 @@ describe('BucketEntriesUsecase', function () { stub(usersRepository, 'findByUuid').resolves(user); const findBucket = stub(bucketsRepository, 'findOne').resolves(null); - const findOrCreate = stub(bucketEntriesRepository, 'findOneOrCreate'); + const create = stub(bucketEntriesRepository, 'create'); try { - await bucketEntriesUsecase.createEntryByKey(user.uuid, 'bucket-id', entryKey, 100); + await bucketEntriesUsecase.createEntry(user.uuid, 'bucket-id', 100); expect(true).toBeFalsy(); } catch (err) { expect(err).toBeInstanceOf(BucketNotFoundError); } expect(findBucket.calledOnceWithExactly({ id: 'bucket-id', userId: user.uuid })).toBeTruthy(); - expect(findOrCreate.called).toBeFalsy(); + expect(create.called).toBeFalsy(); }); - it('When the entry is new, then it stores the hashed key and adds the size to the user total', async () => { + it('When the entry is created, then it adds the size to the user total and returns the entry id', async () => { const user = fixtures.getUser({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); const bucket = fixtures.getBucket({ userId: user.uuid }); - const entry = fixtures.getBucketEntry({ bucket: bucket.id, index: hashedKey, size: 500 }); + const entry = fixtures.getBucketEntry({ bucket: bucket.id, size: 500 }); stub(usersRepository, 'findByUuid').resolves(user); stub(bucketsRepository, 'findOne').resolves(bucket); - const findOrCreate = stub(bucketEntriesRepository, 'findOneOrCreate').resolves({ entry, created: true }); + const create = stub(bucketEntriesRepository, 'create').resolves(entry); const addUsage = stub(usersRepository, 'addTotalUsedSpaceBytes').resolves(4500); - const result = await bucketEntriesUsecase.createEntryByKey(user.uuid, bucket.id, entryKey, 500); + const result = await bucketEntriesUsecase.createEntry(user.uuid, bucket.id, 500); - expect(findOrCreate.calledOnceWithExactly( - { bucket: bucket.id, index: hashedKey }, - { bucket: bucket.id, index: hashedKey, name: entryKey, size: 500, version: 2 } + expect(create.calledOnceWithExactly( + { bucket: bucket.id, size: 500, version: 2 } )).toBeTruthy(); expect(addUsage.calledOnceWithExactly(user.uuid, 500)).toBeTruthy(); expect(result).toStrictEqual({ @@ -718,43 +713,22 @@ describe('BucketEntriesUsecase', function () { snapshot: { maxSpaceBytes: 10000, totalUsedSpaceBytes: 4500 }, }); }); - - it('When the entry already exists, then it does not touch the user total', async () => { - const user = fixtures.getUser({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); - const bucket = fixtures.getBucket({ userId: user.uuid }); - const entry = fixtures.getBucketEntry({ bucket: bucket.id, index: hashedKey, size: 500 }); - - stub(usersRepository, 'findByUuid').resolves(user); - stub(bucketsRepository, 'findOne').resolves(bucket); - stub(bucketEntriesRepository, 'findOneOrCreate').resolves({ entry, created: false }); - const addUsage = stub(usersRepository, 'addTotalUsedSpaceBytes').resolves(); - - const result = await bucketEntriesUsecase.createEntryByKey(user.uuid, bucket.id, entryKey, 500); - - expect(addUsage.called).toBeFalsy(); - expect(result).toStrictEqual({ - id: entry.id, - snapshot: { maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }, - }); - }); }); - describe('removeEntryByKey()', () => { - const entryKey = '12:345'; - const hashedKey = createHash('sha256').update(entryKey).digest('hex'); - + describe('removeEntry()', () => { it('When the entry does not exist, then it is a no-op and the snapshot is unchanged', async () => { const user = fixtures.getUser({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); const bucket = fixtures.getBucket({ userId: user.uuid }); + const entryId = 'aaaaaaaaaaaaaaaaaaaaaaaa'; stub(usersRepository, 'findByUuid').resolves(user); stub(bucketsRepository, 'findOne').resolves(bucket); const findEntry = stub(bucketEntriesRepository, 'findOne').resolves(null); const removeFile = stub(bucketEntriesUsecase, 'removeFile'); - const snapshot = await bucketEntriesUsecase.removeEntryByKey(user.uuid, bucket.id, entryKey); + const snapshot = await bucketEntriesUsecase.removeEntry(user.uuid, bucket.id, entryId); - expect(findEntry.calledOnceWithExactly({ bucket: bucket.id, index: hashedKey })).toBeTruthy(); + expect(findEntry.calledOnceWithExactly({ id: entryId, bucket: bucket.id })).toBeTruthy(); expect(removeFile.called).toBeFalsy(); expect(snapshot).toStrictEqual({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); }); @@ -762,14 +736,14 @@ describe('BucketEntriesUsecase', function () { it('When the entry exists, then it removes it and returns the decremented snapshot', async () => { const user = fixtures.getUser({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); const bucket = fixtures.getBucket({ userId: user.uuid }); - const entry = fixtures.getBucketEntry({ bucket: bucket.id, index: hashedKey, size: 500 }); + const entry = fixtures.getBucketEntry({ bucket: bucket.id, size: 500 }); stub(usersRepository, 'findByUuid').resolves(user); stub(bucketsRepository, 'findOne').resolves(bucket); stub(bucketEntriesRepository, 'findOne').resolves(entry); const removeFile = stub(bucketEntriesUsecase, 'removeFile').resolves(); - const snapshot = await bucketEntriesUsecase.removeEntryByKey(user.uuid, bucket.id, entryKey); + const snapshot = await bucketEntriesUsecase.removeEntry(user.uuid, bucket.id, entry.id); expect(removeFile.calledOnceWithExactly(entry.id)).toBeTruthy(); expect(snapshot).toStrictEqual({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 3500 }); From dd8fb24140140c0447e9a39fab77554cbd2fd6db Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:04:35 -0600 Subject: [PATCH 7/7] refactor: remove findOneOrCreate method from BucketEntriesRepository and update related use case and tests --- .../MongoDBBucketEntriesRepository.ts | 16 ---- lib/core/bucketEntries/Repository.ts | 4 - lib/core/bucketEntries/usecase.ts | 9 ++- tests/lib/core/bucketentries/usecase.test.ts | 12 ++- tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts | 80 +++++++++++++------ 5 files changed, 69 insertions(+), 52 deletions(-) diff --git a/lib/core/bucketEntries/MongoDBBucketEntriesRepository.ts b/lib/core/bucketEntries/MongoDBBucketEntriesRepository.ts index 467836a9..db9f8134 100644 --- a/lib/core/bucketEntries/MongoDBBucketEntriesRepository.ts +++ b/lib/core/bucketEntries/MongoDBBucketEntriesRepository.ts @@ -141,22 +141,6 @@ export class MongoDBBucketEntriesRepository implements BucketEntriesRepository { return rawModel.toObject(); } - async findOneOrCreate( - where: Partial, - data: Omit - ): Promise<{ entry: BucketEntry; created: boolean }> { - const result = await this.model.findOneAndUpdate( - where, - { $setOnInsert: { ...data, created: new Date() } }, - { upsert: true, new: true, includeResultMetadata: true } - ); - - return { - entry: formatFromMongoToBucketEntry(result.value), - created: !result.lastErrorObject?.updatedExisting, - }; - } - async deleteByIds(ids: BucketEntry['id'][]): Promise { await this.model.deleteMany({ _id: { $in: ids } }); } diff --git a/lib/core/bucketEntries/Repository.ts b/lib/core/bucketEntries/Repository.ts index b677bada..4cc7c926 100644 --- a/lib/core/bucketEntries/Repository.ts +++ b/lib/core/bucketEntries/Repository.ts @@ -10,9 +10,5 @@ export interface BucketEntriesRepository { findOneWithFrame(where: Partial): Promise & { frame?: Frame } | null>; findByIdsWithFrames(ids: BucketEntry['id'][]): Promise<(Omit & { frame?: Frame })[]>; create(data: Omit): Promise; - findOneOrCreate( - where: Partial, - data: Omit - ): Promise<{ entry: BucketEntry; created: boolean }>; deleteByIds(ids: BucketEntry['id'][]): Promise; } diff --git a/lib/core/bucketEntries/usecase.ts b/lib/core/bucketEntries/usecase.ts index 42171ff6..f1ec0c93 100644 --- a/lib/core/bucketEntries/usecase.ts +++ b/lib/core/bucketEntries/usecase.ts @@ -242,11 +242,16 @@ export class BucketEntriesUsecase { }; } - await this.removeFile(entry.id); + await this.removeFilesV2([entry]); + + const totalUsedSpaceBytes = await this.usersRepository.addTotalUsedSpaceBytes( + userUuid, + -(entry.size || 0) + ); return { maxSpaceBytes: user.maxSpaceBytes, - totalUsedSpaceBytes: user.totalUsedSpaceBytes - (entry.size || 0), + totalUsedSpaceBytes, }; } } diff --git a/tests/lib/core/bucketentries/usecase.test.ts b/tests/lib/core/bucketentries/usecase.test.ts index 9dd89929..6cac4614 100644 --- a/tests/lib/core/bucketentries/usecase.test.ts +++ b/tests/lib/core/bucketentries/usecase.test.ts @@ -724,12 +724,14 @@ describe('BucketEntriesUsecase', function () { stub(usersRepository, 'findByUuid').resolves(user); stub(bucketsRepository, 'findOne').resolves(bucket); const findEntry = stub(bucketEntriesRepository, 'findOne').resolves(null); - const removeFile = stub(bucketEntriesUsecase, 'removeFile'); + const removeFilesV2 = stub(bucketEntriesUsecase, 'removeFilesV2'); + const addUsage = stub(usersRepository, 'addTotalUsedSpaceBytes'); const snapshot = await bucketEntriesUsecase.removeEntry(user.uuid, bucket.id, entryId); expect(findEntry.calledOnceWithExactly({ id: entryId, bucket: bucket.id })).toBeTruthy(); - expect(removeFile.called).toBeFalsy(); + expect(removeFilesV2.called).toBeFalsy(); + expect(addUsage.called).toBeFalsy(); expect(snapshot).toStrictEqual({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 4000 }); }); @@ -741,11 +743,13 @@ describe('BucketEntriesUsecase', function () { stub(usersRepository, 'findByUuid').resolves(user); stub(bucketsRepository, 'findOne').resolves(bucket); stub(bucketEntriesRepository, 'findOne').resolves(entry); - const removeFile = stub(bucketEntriesUsecase, 'removeFile').resolves(); + const removeFilesV2 = stub(bucketEntriesUsecase, 'removeFilesV2').resolves(); + const addUsage = stub(usersRepository, 'addTotalUsedSpaceBytes').resolves(3500); const snapshot = await bucketEntriesUsecase.removeEntry(user.uuid, bucket.id, entry.id); - expect(removeFile.calledOnceWithExactly(entry.id)).toBeTruthy(); + expect(removeFilesV2.calledOnceWithExactly([entry])).toBeTruthy(); + expect(addUsage.calledOnceWithExactly(user.uuid, -500)).toBeTruthy(); expect(snapshot).toStrictEqual({ maxSpaceBytes: 10000, totalUsedSpaceBytes: 3500 }); }); }); diff --git a/tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts b/tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts index fc7e204c..c53858af 100644 --- a/tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts +++ b/tests/lib/e2e/gateway/gateway-v2.e2e-spec.ts @@ -125,28 +125,26 @@ describe('Gateway V2 e2e tests', () => { return body as { id: string; name: string } } - const hashKey = (key: string) => crypto.createHash('sha256').update(key).digest('hex') + const randomObjectId = () => crypto.randomBytes(12).toString('hex') - it('When creating an entry, then it is persisted with the hashed key and the user total grows by its size', async () => { + it('When creating an entry, then it is persisted as a shard-less v2 entry and the user total grows by its size', async () => { const testUser = await createTestUser() const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) const bucket = await createBucketForUser(testUser.uuid, jwt) - const key = `1:${Date.now()}` const userBefore = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) const response = await testServer .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) .set('Authorization', `Bearer ${jwt}`) - .send({ key, size: 5000 }) + .send({ size: 5000 }) expect(response.status).toBe(200) expect(response.body.id).toBeDefined() const entryInDatabase = await databaseConnection.models.BucketEntry.findOne({ _id: response.body.id }) expect(entryInDatabase).not.toBeNull() - expect(entryInDatabase.index).toBe(hashKey(key)) - expect(entryInDatabase.name).toBe(key) + expect(entryInDatabase.bucket.toString()).toBe(bucket.id) expect(entryInDatabase.size).toBe(5000) expect(entryInDatabase.version).toBe(2) @@ -156,51 +154,48 @@ describe('Gateway V2 e2e tests', () => { expect(response.body.totalUsedSpaceBytes).toBe(userAfter.totalUsedSpaceBytes) }) - it('When creating the same entry twice, then it is counted only once', async () => { + it('When posting the same object twice, then each call mints a distinct entry and the total grows by the sum (dedup is the caller\'s job)', async () => { const testUser = await createTestUser() const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) const bucket = await createBucketForUser(testUser.uuid, jwt) - const key = `1:${Date.now()}` const userBefore = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) const first = await testServer .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) .set('Authorization', `Bearer ${jwt}`) - .send({ key, size: 5000 }) + .send({ size: 5000 }) const second = await testServer .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) .set('Authorization', `Bearer ${jwt}`) - .send({ key, size: 5000 }) + .send({ size: 5000 }) expect(first.status).toBe(200) expect(second.status).toBe(200) - expect(second.body.id).toBe(first.body.id) - expect(second.body.totalUsedSpaceBytes).toBe(first.body.totalUsedSpaceBytes) + expect(second.body.id).not.toBe(first.body.id) const entries = await databaseConnection.models.BucketEntry.find({ bucket: bucket.id }) - expect(entries.length).toBe(1) + expect(entries.length).toBe(2) const userAfter = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) - expect(userAfter.totalUsedSpaceBytes).toBe(userBefore.totalUsedSpaceBytes + 5000) + expect(userAfter.totalUsedSpaceBytes).toBe(userBefore.totalUsedSpaceBytes + 10000) }) it('When deleting an entry, then it is removed and the user total shrinks by its size', async () => { const testUser = await createTestUser() const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) const bucket = await createBucketForUser(testUser.uuid, jwt) - const key = `1:${Date.now()}` const created = await testServer .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) .set('Authorization', `Bearer ${jwt}`) - .send({ key, size: 5000 }) + .send({ size: 5000 }) const userBefore = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) const response = await testServer - .delete(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries/${encodeURIComponent(key)}`) + .delete(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries/${created.body.id}`) .set('Authorization', `Bearer ${jwt}`) expect(response.status).toBe(200) @@ -224,7 +219,7 @@ describe('Gateway V2 e2e tests', () => { const userBefore = await databaseConnection.models.User.findOne({ uuid: testUser.uuid }) const response = await testServer - .delete(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries/${encodeURIComponent('1:404')}`) + .delete(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries/${randomObjectId()}`) .set('Authorization', `Bearer ${jwt}`) expect(response.status).toBe(200) @@ -233,6 +228,18 @@ describe('Gateway V2 e2e tests', () => { expect(userAfter.totalUsedSpaceBytes).toBe(userBefore.totalUsedSpaceBytes) }) + it('When deleting with a malformed entry id, then it returns 400', async () => { + const testUser = await createTestUser() + const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) + const bucket = await createBucketForUser(testUser.uuid, jwt) + + const response = await testServer + .delete(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries/${encodeURIComponent('1:404')}`) + .set('Authorization', `Bearer ${jwt}`) + + expect(response.status).toBe(400) + }) + it('When creating an entry on a bucket of another user, then it returns 404 and changes nothing', async () => { const owner = await createTestUser() const otherUser = await createTestUser() @@ -242,7 +249,7 @@ describe('Gateway V2 e2e tests', () => { const response = await testServer .post(`/v2/gateway/users/${otherUser.uuid}/buckets/${bucket.id}/entries`) .set('Authorization', `Bearer ${jwt}`) - .send({ key: '1:1', size: 5000 }) + .send({ size: 5000 }) expect(response.status).toBe(404) @@ -250,29 +257,50 @@ describe('Gateway V2 e2e tests', () => { expect(entries.length).toBe(0) }) - it('When the entry key or size is invalid, then it returns 400', async () => { + it('When the entry size is invalid, then it returns 400 and creates nothing', async () => { const testUser = await createTestUser() const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) const bucket = await createBucketForUser(testUser.uuid, jwt) - const missingKey = await testServer + const missingSize = await testServer .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) .set('Authorization', `Bearer ${jwt}`) - .send({ size: 5000 }) + .send({}) + + const zeroSize = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ size: 0 }) const negativeSize = await testServer .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) .set('Authorization', `Bearer ${jwt}`) - .send({ key: '1:1', size: -1 }) + .send({ size: -1 }) const nonIntegerSize = await testServer .post(`/v2/gateway/users/${testUser.uuid}/buckets/${bucket.id}/entries`) .set('Authorization', `Bearer ${jwt}`) - .send({ key: '1:1', size: 10.5 }) + .send({ size: 10.5 }) - expect(missingKey.status).toBe(400) + expect(missingSize.status).toBe(400) + expect(zeroSize.status).toBe(400) expect(negativeSize.status).toBe(400) expect(nonIntegerSize.status).toBe(400) + + const entries = await databaseConnection.models.BucketEntry.find({ bucket: bucket.id }) + expect(entries.length).toBe(0) + }) + + it('When the bucket id is malformed, then it returns 400', async () => { + const testUser = await createTestUser() + const jwt = signRS256JWT('5m', engine._config.gateway.SIGN_JWT_SECRET) + + const response = await testServer + .post(`/v2/gateway/users/${testUser.uuid}/buckets/not-an-object-id/entries`) + .set('Authorization', `Bearer ${jwt}`) + .send({ size: 5000 }) + + expect(response.status).toBe(400) }) it('When no auth token is provided, then it returns 401', async () => { @@ -280,7 +308,7 @@ describe('Gateway V2 e2e tests', () => { const response = await testServer .post(`/v2/gateway/users/${testUser.uuid}/buckets/${'a'.repeat(24)}/entries`) - .send({ key: '1:1', size: 1000 }) + .send({ size: 1000 }) expect(response.status).toBe(401) })