diff --git a/.env b/.env index 1893e023ba1..eee7164b5dc 100644 --- a/.env +++ b/.env @@ -87,7 +87,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_USAGE_AGGREGATION_INTERVAL=30 -_APP_STATS_RESOURCES_INTERVAL=3600 +_APP_STATS_RESOURCES_INTERVAL=15 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d35fec3c71..9290c408638 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -163,6 +163,7 @@ jobs: - name: Load and Start Appwrite run: | + export _APP_STATS_RESOURCES_INTERVAL=5 docker load --input /tmp/${{ env.IMAGE }}.tar docker compose up -d sleep 30 diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index d8d496089cf..016aa23ce39 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -534,7 +534,6 @@ function updateAttribute( } $queueForEvents->setParam('databaseId', $database->getId()); - $queueForStatsUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -830,9 +829,6 @@ function updateAttribute( ->setParam('databaseId', $database->getId()) ->setPayload($response->output($database, Response::MODEL_DATABASE)); - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_STORAGE, 1); // Global, deletion forces full recalculation - $response->noContent(); }); @@ -2732,9 +2728,6 @@ function updateAttribute( ->setContext('database', $db) ->setPayload($response->output($attribute, $model)); - $queueForStatsUsage - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$db->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->noContent(); }); @@ -3356,8 +3349,7 @@ function updateAttribute( $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -4141,8 +4133,7 @@ function updateAttribute( $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); $response->addHeader('X-Debug-Operations', 1); @@ -4186,28 +4177,51 @@ function updateAttribute( ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $range, Response $response, Database $dbForProject) { + ->inject('dbForLogs') + ->action(function (string $range, Response $response, Database $dbForProject, Database $dbForLogs) { $periods = Config::getParam('usage', []); $stats = $usage = []; $days = $periods[$range]; - $metrics = [ + $logsDBMetrics = [ METRIC_DATABASES, METRIC_COLLECTIONS, METRIC_DOCUMENTS, METRIC_DATABASES_STORAGE, + ]; + + Authorization::skip(function () use ($dbForLogs, $days, $logsDBMetrics, &$stats) { + foreach ($logsDBMetrics as $metric) { + $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); + $stats[$metric]['total'] = $result->getAttribute('value', 0); + + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForLogs->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + + $metrics = [ METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, ]; Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - + $result = $dbForProject->getDocument('stats', md5('_inf_'. $metric)); $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; $period = $days['period']; $results = $dbForProject->find('stats', [ @@ -4230,7 +4244,7 @@ function updateAttribute( '1d' => 'Y-m-d\T00:00:00.000P', }; - foreach ($metrics as $metric) { + foreach ([...$logsDBMetrics,...$metrics] as $metric) { $usage[$metric]['total'] = $stats[$metric]['total']; $usage[$metric]['data'] = []; $leap = time() - ($days['limit'] * $days['factor']); @@ -4245,18 +4259,18 @@ function updateAttribute( } $response->dynamic(new Document([ 'range' => $range, - 'databasesTotal' => $usage[$metrics[0]]['total'], - 'collectionsTotal' => $usage[$metrics[1]]['total'], - 'documentsTotal' => $usage[$metrics[2]]['total'], - 'storageTotal' => $usage[$metrics[3]]['total'], - 'databasesReadsTotal' => $usage[$metrics[4]]['total'], - 'databasesWritesTotal' => $usage[$metrics[5]]['total'], - 'databases' => $usage[$metrics[0]]['data'], - 'collections' => $usage[$metrics[1]]['data'], - 'documents' => $usage[$metrics[2]]['data'], - 'storage' => $usage[$metrics[3]]['data'], - 'databasesReads' => $usage[$metrics[4]]['data'], - 'databasesWrites' => $usage[$metrics[5]]['data'], + 'databasesTotal' => $usage[$logsDBMetrics[0]]['total'], + 'collectionsTotal' => $usage[$logsDBMetrics[1]]['total'], + 'documentsTotal' => $usage[$logsDBMetrics[2]]['total'], + 'storageTotal' => $usage[$logsDBMetrics[3]]['total'], + 'databasesReadsTotal' => $usage[$metrics[0]]['total'], + 'databasesWritesTotal' => $usage[$metrics[1]]['total'], + 'databases' => $usage[$logsDBMetrics[0]]['data'], + 'collections' => $usage[$logsDBMetrics[1]]['data'], + 'documents' => $usage[$logsDBMetrics[2]]['data'], + 'storage' => $usage[$logsDBMetrics[3]]['data'], + 'databasesReads' => $usage[$metrics[0]]['data'], + 'databasesWrites' => $usage[$metrics[1]]['data'], ]), Response::MODEL_USAGE_DATABASES); }); @@ -4282,7 +4296,8 @@ function updateAttribute( ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) { + ->inject('dbForLogs') + ->action(function (string $databaseId, string $range, Response $response, Database $dbForProject, Database $dbForLogs) { $database = $dbForProject->getDocument('databases', $databaseId); @@ -4293,20 +4308,43 @@ function updateAttribute( $periods = Config::getParam('usage', []); $stats = $usage = []; $days = $periods[$range]; - $metrics = [ + + $logsDBMetrics = [ str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS), str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS), str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_STORAGE), + ]; + + Authorization::skip(function () use ($dbForLogs, $days, $logsDBMetrics, &$stats) { + foreach ($logsDBMetrics as $metric) { + $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); + $stats[$metric]['total'] = $result->getAttribute('value', 0); + + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForLogs->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + + $metrics = [ str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_READS), str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_WRITES) ]; Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; @@ -4331,7 +4369,7 @@ function updateAttribute( '1d' => 'Y-m-d\T00:00:00.000P', }; - foreach ($metrics as $metric) { + foreach ([...$logsDBMetrics,...$metrics] as $metric) { $usage[$metric]['total'] = $stats[$metric]['total']; $usage[$metric]['data'] = []; $leap = time() - ($days['limit'] * $days['factor']); @@ -4347,16 +4385,16 @@ function updateAttribute( $response->dynamic(new Document([ 'range' => $range, - 'collectionsTotal' => $usage[$metrics[0]]['total'], - 'documentsTotal' => $usage[$metrics[1]]['total'], - 'storageTotal' => $usage[$metrics[2]]['total'], - 'databaseReadsTotal' => $usage[$metrics[3]]['total'], - 'databaseWritesTotal' => $usage[$metrics[4]]['total'], - 'collections' => $usage[$metrics[0]]['data'], - 'documents' => $usage[$metrics[1]]['data'], - 'storage' => $usage[$metrics[2]]['data'], - 'databaseReads' => $usage[$metrics[3]]['data'], - 'databaseWrites' => $usage[$metrics[4]]['data'], + 'collectionsTotal' => $usage[$logsDBMetrics[0]]['total'], + 'documentsTotal' => $usage[$logsDBMetrics[1]]['total'], + 'storageTotal' => $usage[$logsDBMetrics[2]]['total'], + 'databaseReadsTotal' => $usage[$metrics[0]]['total'], + 'databaseWritesTotal' => $usage[$metrics[1]]['total'], + 'collections' => $usage[$logsDBMetrics[0]]['data'], + 'documents' => $usage[$logsDBMetrics[1]]['data'], + 'storage' => $usage[$logsDBMetrics[2]]['data'], + 'databaseReads' => $usage[$metrics[0]]['data'], + 'databaseWrites' => $usage[$metrics[1]]['data'], ]), Response::MODEL_USAGE_DATABASE); }); @@ -4384,7 +4422,8 @@ function updateAttribute( ->param('collectionId', '', new UID(), 'Collection ID.') ->inject('response') ->inject('dbForProject') - ->action(function (string $databaseId, string $range, string $collectionId, Response $response, Database $dbForProject) { + ->inject('dbForLogs') + ->action(function (string $databaseId, string $range, string $collectionId, Response $response, Database $dbForProject, Database $dbForLogs) { $database = $dbForProject->getDocument('databases', $databaseId); $collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); @@ -4401,17 +4440,14 @@ function updateAttribute( str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), ]; - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + Authorization::skip(function () use ($dbForLogs, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - + $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForLogs->find('stats', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 583468f6c1a..d2a1c303d3c 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -620,7 +620,8 @@ ->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $functionId, string $range, Response $response, Database $dbForProject) { + ->inject('dbForLogs') + ->action(function (string $functionId, string $range, Response $response, Database $dbForProject, Database $dbForLogs) { $function = $dbForProject->getDocument('functions', $functionId); @@ -631,11 +632,35 @@ $periods = Config::getParam('usage', []); $stats = $usage = []; $days = $periods[$range]; + $logsDBMetrics = [ + str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), + str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), + str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), + str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), + ]; + + Authorization::skip(function () use ($dbForLogs, $days, $logsDBMetrics, &$stats) { + foreach ($logsDBMetrics as $metric) { + $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); + $stats[$metric]['total'] = $result->getAttribute('value', 0); + + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForLogs->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); $metrics = [ - str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS), - str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), - str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), - str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), @@ -645,12 +670,9 @@ Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; $period = $days['period']; $results = $dbForProject->find('stats', [ @@ -673,7 +695,7 @@ '1d' => 'Y-m-d\T00:00:00.000P', }; - foreach ($metrics as $metric) { + foreach ([...$logsDBMetrics,...$metrics] as $metric) { $usage[$metric]['total'] = $stats[$metric]['total']; $usage[$metric]['data'] = []; $leap = time() - ($days['limit'] * $days['factor']); @@ -689,24 +711,24 @@ $response->dynamic(new Document([ 'range' => $range, - 'deploymentsTotal' => $usage[$metrics[0]]['total'], - 'deploymentsStorageTotal' => $usage[$metrics[1]]['total'], - 'buildsTotal' => $usage[$metrics[2]]['total'], - 'buildsStorageTotal' => $usage[$metrics[3]]['total'], - 'buildsTimeTotal' => $usage[$metrics[4]]['total'], - 'executionsTotal' => $usage[$metrics[5]]['total'], - 'executionsTimeTotal' => $usage[$metrics[6]]['total'], - 'deployments' => $usage[$metrics[0]]['data'], - 'deploymentsStorage' => $usage[$metrics[1]]['data'], + 'deploymentsTotal' => $usage[$logsDBMetrics[0]]['total'], + 'deploymentsStorageTotal' => $usage[$logsDBMetrics[1]]['total'], + 'buildsTotal' => $usage[$logsDBMetrics[2]]['total'], + 'buildsStorageTotal' => $usage[$logsDBMetrics[3]]['total'], + 'buildsTimeTotal' => $usage[$metrics[0]]['total'], + 'executionsTotal' => $usage[$metrics[1]]['total'], + 'executionsTimeTotal' => $usage[$metrics[2]]['total'], + 'deployments' => $usage[$logsDBMetrics[0]]['data'], + 'deploymentsStorage' => $usage[$logsDBMetrics[1]]['data'], 'builds' => $usage[$metrics[2]]['data'], 'buildsStorage' => $usage[$metrics[3]]['data'], - 'buildsTime' => $usage[$metrics[4]]['data'], - 'executions' => $usage[$metrics[5]]['data'], - 'executionsTime' => $usage[$metrics[6]]['data'], - 'buildsMbSecondsTotal' => $usage[$metrics[7]]['total'], - 'buildsMbSeconds' => $usage[$metrics[7]]['data'], - 'executionsMbSeconds' => $usage[$metrics[8]]['data'], - 'executionsMbSecondsTotal' => $usage[$metrics[8]]['total'] + 'buildsTime' => $usage[$metrics[0]]['data'], + 'executions' => $usage[$metrics[1]]['data'], + 'executionsTime' => $usage[$metrics[2]]['data'], + 'buildsMbSecondsTotal' => $usage[$metrics[3]]['total'], + 'buildsMbSeconds' => $usage[$metrics[3]]['data'], + 'executionsMbSeconds' => $usage[$metrics[4]]['data'], + 'executionsMbSecondsTotal' => $usage[$metrics[4]]['total'] ]), Response::MODEL_USAGE_FUNCTION); }); @@ -730,17 +752,42 @@ ->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $range, Response $response, Database $dbForProject) { + ->inject('dbForLogs') + ->action(function (string $range, Response $response, Database $dbForProject, Database $dbForLogs) { $periods = Config::getParam('usage', []); $stats = $usage = []; $days = $periods[$range]; - $metrics = [ + $logsDBMetrics = [ METRIC_FUNCTIONS, METRIC_DEPLOYMENTS, METRIC_DEPLOYMENTS_STORAGE, METRIC_BUILDS, METRIC_BUILDS_STORAGE, + ]; + + Authorization::skip(function () use ($dbForLogs, $days, $logsDBMetrics, &$stats) { + foreach ($logsDBMetrics as $metric) { + $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); + $stats[$metric]['total'] = $result['value'] ?? 0; + + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForLogs->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + $metrics = [ METRIC_BUILDS_COMPUTE, METRIC_EXECUTIONS, METRIC_EXECUTIONS_COMPUTE, @@ -750,12 +797,9 @@ Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; $period = $days['period']; $results = $dbForProject->find('stats', [ @@ -778,7 +822,7 @@ '1d' => 'Y-m-d\T00:00:00.000P', }; - foreach ($metrics as $metric) { + foreach ([...$logsDBMetrics,...$metrics] as $metric) { $usage[$metric]['total'] = $stats[$metric]['total']; $usage[$metric]['data'] = []; $leap = time() - ($days['limit'] * $days['factor']); @@ -793,26 +837,26 @@ } $response->dynamic(new Document([ 'range' => $range, - 'functionsTotal' => $usage[$metrics[0]]['total'], - 'deploymentsTotal' => $usage[$metrics[1]]['total'], - 'deploymentsStorageTotal' => $usage[$metrics[2]]['total'], - 'buildsTotal' => $usage[$metrics[3]]['total'], - 'buildsStorageTotal' => $usage[$metrics[4]]['total'], - 'buildsTimeTotal' => $usage[$metrics[5]]['total'], - 'executionsTotal' => $usage[$metrics[6]]['total'], - 'executionsTimeTotal' => $usage[$metrics[7]]['total'], - 'functions' => $usage[$metrics[0]]['data'], - 'deployments' => $usage[$metrics[1]]['data'], - 'deploymentsStorage' => $usage[$metrics[2]]['data'], - 'builds' => $usage[$metrics[3]]['data'], - 'buildsStorage' => $usage[$metrics[4]]['data'], - 'buildsTime' => $usage[$metrics[5]]['data'], - 'executions' => $usage[$metrics[6]]['data'], - 'executionsTime' => $usage[$metrics[7]]['data'], - 'buildsMbSecondsTotal' => $usage[$metrics[8]]['total'], - 'buildsMbSeconds' => $usage[$metrics[8]]['data'], - 'executionsMbSeconds' => $usage[$metrics[9]]['data'], - 'executionsMbSecondsTotal' => $usage[$metrics[9]]['total'], + 'functionsTotal' => $usage[$logsDBMetrics[0]]['total'], + 'deploymentsTotal' => $usage[$logsDBMetrics[1]]['total'], + 'deploymentsStorageTotal' => $usage[$logsDBMetrics[2]]['total'], + 'buildsTotal' => $usage[$logsDBMetrics[3]]['total'], + 'buildsStorageTotal' => $usage[$logsDBMetrics[4]]['total'], + 'buildsTimeTotal' => $usage[$metrics[0]]['total'], + 'executionsTotal' => $usage[$metrics[1]]['total'], + 'executionsTimeTotal' => $usage[$metrics[2]]['total'], + 'functions' => $usage[$logsDBMetrics[0]]['data'], + 'deployments' => $usage[$logsDBMetrics[1]]['data'], + 'deploymentsStorage' => $usage[$logsDBMetrics[2]]['data'], + 'builds' => $usage[$logsDBMetrics[3]]['data'], + 'buildsStorage' => $usage[$logsDBMetrics[4]]['data'], + 'buildsTime' => $usage[$metrics[0]]['data'], + 'executions' => $usage[$metrics[1]]['data'], + 'executionsTime' => $usage[$metrics[2]]['data'], + 'buildsMbSecondsTotal' => $usage[$metrics[3]]['total'], + 'buildsMbSeconds' => $usage[$metrics[3]]['data'], + 'executionsMbSeconds' => $usage[$metrics[4]]['data'], + 'executionsMbSecondsTotal' => $usage[$metrics[4]]['total'], ]), Response::MODEL_USAGE_FUNCTIONS); }); diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index b267e8e51ea..15d75d9ea94 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -42,15 +42,24 @@ ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('getLogsDB') + ->inject('dbForLogs') ->inject('smsRates') - ->action(function (string $startDate, string $endDate, string $period, Response $response, Document $project, Database $dbForProject, callable $getLogsDB, array $smsRates) { + ->action(function (string $startDate, string $endDate, string $period, Response $response, Document $project, Database $dbForProject, Database $dbForLogs, array $smsRates) { $stats = $total = $usage = []; $format = 'Y-m-d 00:00:00'; $firstDay = (new DateTime($startDate))->format($format); $lastDay = (new DateTime($endDate))->format($format); - $dbForLogs = call_user_func($getLogsDB, $project); + $metricsFromLogsDB = [ + METRIC_DOCUMENTS => true, + METRIC_DATABASES => true, + METRIC_BUCKETS => true, + METRIC_FILES_STORAGE => true, + METRIC_DATABASES_STORAGE => true, + METRIC_DEPLOYMENTS_STORAGE => true, + METRIC_BUILDS_STORAGE => true, + METRIC_FILES_IMAGES_TRANSFORMED => true, + ]; $metrics = [ 'total' => [ @@ -99,19 +108,16 @@ '1d' => 'Y-m-d\T00:00:00.000P', }; - Authorization::skip(function () use ($dbForProject, $dbForLogs, $firstDay, $lastDay, $period, $metrics, $limit, &$total, &$stats) { + Authorization::skip(function () use ($dbForProject, $dbForLogs, $firstDay, $lastDay, $period, $metrics, $metricsFromLogsDB, $limit, &$total, &$stats) { foreach ($metrics['total'] as $metric) { - $db = ($metric === METRIC_FILES_IMAGES_TRANSFORMED) ? $dbForLogs : $dbForProject; + $db = array_key_exists($metric, $metricsFromLogsDB) ? $dbForLogs : $dbForProject; - $result = $db->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - $total[$metric] = $result['value'] ?? 0; + $result = $db->getDocument('stats', md5('_inf_' . $metric)); + $total[$metric] = $result->getAttribute('value', 0); } foreach ($metrics['period'] as $metric) { - $db = ($metric === METRIC_FILES_IMAGES_TRANSFORMED) ? $dbForLogs : $dbForProject; + $db = array_key_exists($metric, $metricsFromLogsDB) ? $dbForLogs : $dbForProject; $results = $db->find('stats', [ Query::equal('metric', [$metric]), @@ -149,15 +155,13 @@ $id = $function->getId(); $name = $function->getAttribute('name'); $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS); - $value = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); + + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); return [ 'resourceId' => $id, 'name' => $name, - 'value' => $value['value'] ?? 0, + 'value' => $result->getAttribute('value', 0), ]; }, $dbForProject->find('functions')); @@ -165,15 +169,12 @@ $id = $function->getId(); $name = $function->getAttribute('name'); $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS); - $value = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); return [ 'resourceId' => $id, 'name' => $name, - 'value' => $value['value'] ?? 0, + 'value' => $result->getAttribute('value', 0), ]; }, $dbForProject->find('functions')); @@ -181,15 +182,12 @@ $id = $function->getId(); $name = $function->getAttribute('name'); $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS); - $value = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); return [ 'resourceId' => $id, 'name' => $name, - 'value' => $value['value'] ?? 0, + 'value' => $result->getAttribute('value', 0), ]; }, $dbForProject->find('functions')); @@ -197,15 +195,12 @@ $id = $bucket->getId(); $name = $bucket->getAttribute('name'); $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE); - $value = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); return [ 'resourceId' => $id, 'name' => $name, - 'value' => $value['value'] ?? 0, + 'value' => $result->getAttribute('value', 0), ]; }, $dbForProject->find('buckets')); @@ -214,15 +209,12 @@ $name = $database->getAttribute('name'); $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_STORAGE); - $value = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); return [ 'resourceId' => $id, 'name' => $name, - 'value' => $value['value'] ?? 0, + 'value' => $result->getAttribute('value', 0), ]; }, $dbForProject->find('databases')); @@ -230,16 +222,10 @@ $id = $function->getId(); $name = $function->getAttribute('name'); $deploymentMetric = str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE); - $deploymentValue = $dbForProject->findOne('stats', [ - Query::equal('metric', [$deploymentMetric]), - Query::equal('period', ['inf']) - ]); + $deploymentValue = $dbForProject->getDocument('stats', md5('_inf_' . $deploymentMetric)); $buildMetric = str_replace(['{functionInternalId}'], [$function->getInternalId()], METRIC_FUNCTION_ID_BUILDS_STORAGE); - $buildValue = $dbForProject->findOne('stats', [ - Query::equal('metric', [$buildMetric]), - Query::equal('period', ['inf']) - ]); + $buildValue = $dbForProject->getDocument('stats', md5('_inf_' . $buildMetric)); $value = ($buildValue['value'] ?? 0) + ($deploymentValue['value'] ?? 0); @@ -254,15 +240,11 @@ $id = $function->getId(); $name = $function->getAttribute('name'); $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS); - $value = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); return [ 'resourceId' => $id, 'name' => $name, - 'value' => $value['value'] ?? 0, + 'value' => $result->getAttribute('value', 0), ]; }, $dbForProject->find('functions')); @@ -270,15 +252,12 @@ $id = $function->getId(); $name = $function->getAttribute('name'); $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS); - $value = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); + $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); return [ 'resourceId' => $id, 'name' => $name, - 'value' => $value['value'] ?? 0, + 'value' => $result->getAttribute('value', 0), ]; }, $dbForProject->find('functions')); diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f13c9703c58..9d67a57f80e 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1784,8 +1784,8 @@ )) ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) ->inject('response') - ->inject('dbForProject') - ->action(function (string $range, Response $response, Database $dbForProject) { + ->inject('dbForLogs') + ->action(function (string $range, Response $response, Database $dbForLogs) { $periods = Config::getParam('usage', []); $stats = $usage = []; @@ -1797,17 +1797,14 @@ ]; $total = []; - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { + Authorization::skip(function () use ($dbForLogs, $days, $metrics, &$stats, &$total) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - + $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForLogs->find('stats', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), @@ -1873,10 +1870,8 @@ ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('getLogsDB') - ->action(function (string $bucketId, string $range, Response $response, Document $project, Database $dbForProject, callable $getLogsDB) { - - $dbForLogs = call_user_func($getLogsDB, $project); + ->inject('dbForLogs') + ->action(function (string $bucketId, string $range, Response $response, Document $project, Database $dbForProject, Database $dbForLogs) { $bucket = $dbForProject->getDocument('buckets', $bucketId); if ($bucket->isEmpty()) { @@ -1892,21 +1887,14 @@ str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED), ]; - Authorization::skip(function () use ($dbForProject, $dbForLogs, $bucket, $days, $metrics, &$stats) { + Authorization::skip(function () use ($dbForLogs, $bucket, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $db = ($metric === str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED)) - ? $dbForLogs - : $dbForProject; - - $result = $db->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - + $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; $period = $days['period']; - $results = $db->find('stats', [ + $results = $dbForLogs->find('stats', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 1015400a126..e4d85715f99 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -109,66 +109,6 @@ case $document->getCollection() === 'sessions': // sessions $queueForStatsUsage->addMetric(METRIC_SESSIONS, $value); //per project break; - case $document->getCollection() === 'databases': // databases - $queueForStatsUsage->addMetric(METRIC_DATABASES, $value); // per project - - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); - } - break; - case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections - $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; - $queueForStatsUsage - ->addMetric(METRIC_COLLECTIONS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value); - - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); - } - break; - case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents - $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; - $collectionInternalId = $parts[3] ?? 0; - $queueForStatsUsage - ->addMetric(METRIC_DOCUMENTS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection - break; - case $document->getCollection() === 'buckets': //buckets - $queueForStatsUsage - ->addMetric(METRIC_BUCKETS, $value); // per project - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage - ->addReduce($document); - } - break; - case str_starts_with($document->getCollection(), 'bucket_'): // files - $parts = explode('_', $document->getCollection()); - $bucketInternalId = $parts[1]; - $queueForStatsUsage - ->addMetric(METRIC_FILES, $value) // per project - ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project - ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket - ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket - break; - case $document->getCollection() === 'functions': - $queueForStatsUsage - ->addMetric(METRIC_FUNCTIONS, $value); // per project - - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage - ->addReduce($document); - } - break; - case $document->getCollection() === 'deployments': - $queueForStatsUsage - ->addMetric(METRIC_DEPLOYMENTS, $value) // per project - ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value) // per function - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value); - break; default: break; } diff --git a/app/init.php b/app/init.php index 1311a1429f7..3c399481738 100644 --- a/app/init.php +++ b/app/init.php @@ -1561,6 +1561,10 @@ function (mixed $value) { }; }, ['pools', 'dbForPlatform', 'cache']); +App::setResource('dbForLogs', function (Document $project, callable $getLogsDB) { + return call_user_func($getLogsDB, $project); +}, ['project', 'getLogsDB']); + App::setResource('getLogsDB', function (Group $pools, Cache $cache) { $database = null; return function (?Document $project = null) use ($pools, $cache, $database) { diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 6f26e9a80cf..168ab1da668 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -709,12 +709,8 @@ protected function buildDeployment(Device $deviceForFunctions, Webhook $queueFor } $queueForStatsUsage - ->addMetric(METRIC_BUILDS, 1) // per project - ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) ->setProject($project) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index cf2b8bfc840..36ba70e5a10 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -300,7 +300,6 @@ private function sendExternalMessage( $errorTotal = \count($deliveryErrors); $queueForStatsUsage ->setProject($project) - ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) ->addMetric(METRIC_MESSAGES_FAILED, $errorTotal) ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal)) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 969d43e895c..cfee1be4a9d 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -170,13 +170,13 @@ protected function countForProject(Database $dbForPlatform, callable $getLogsDB, } try { - $this->countForBuckets($dbForProject, $dbForLogs, $region); + $this->countForBuckets($dbForProject, $region); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]); } try { - $this->countImageTransformations($dbForProject, $dbForLogs, $region); + $this->countImageTransformations($dbForProject, $region); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]); } @@ -188,7 +188,7 @@ protected function countForProject(Database $dbForPlatform, callable $getLogsDB, } try { - $this->countForFunctions($dbForProject, $dbForLogs, $region); + $this->countForFunctions($dbForProject, $region); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]); } @@ -201,11 +201,11 @@ protected function countForProject(Database $dbForPlatform, callable $getLogsDB, Console::info('End of count for: ' . $project->getId()); } - protected function countForBuckets(Database $dbForProject, Database $dbForLogs, string $region) + protected function countForBuckets(Database $dbForProject, string $region) { $totalFiles = 0; $totalStorage = 0; - $this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalFiles, &$totalStorage) { + $this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $region, &$totalFiles, &$totalStorage) { $files = $dbForProject->count('bucket_' . $bucket->getInternalId()); $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES); @@ -226,7 +226,7 @@ protected function countForBuckets(Database $dbForProject, Database $dbForLogs, /** * Need separate function to count per period data */ - protected function countImageTransformations(Database $dbForProject, Database $dbForLogs, string $region) + protected function countImageTransformations(Database $dbForProject, string $region) { $totalImageTransformations = 0; $last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00'); @@ -292,7 +292,7 @@ protected function countForCollections(Database $dbForProject, Document $databas return [$databaseDocuments, $databaseStorage]; } - protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region) + protected function countForFunctions(Database $dbForProject, string $region) { $deploymentsStorage = $dbForProject->sum('deployments', 'size'); $buildsStorage = $dbForProject->sum('builds', 'size'); @@ -305,7 +305,7 @@ protected function countForFunctions(Database $dbForProject, Database $dbForLogs $this->createStatsDocuments($region, METRIC_BUILDS, $builds); - $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) { + $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $region) { $functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [ Query::equal('resourceInternalId', [$function->getInternalId()]), Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 119a9e72885..9aa551ab96b 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -4,7 +4,6 @@ use Appwrite\Extend\Exception; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; @@ -14,59 +13,10 @@ class StatsUsageDump extends Action { - public const METRIC_COLLECTION_LEVEL_STORAGE = 4; - public const METRIC_DATABASE_LEVEL_STORAGE = 3; - public const METRIC_PROJECT_LEVEL_STORAGE = 2; protected array $stats = []; protected Registry $register; - /** - * Metrics to skip writing to logsDB - * As these metrics are calculated separately - * by logs DB - * @var array - */ - protected array $skipBaseMetrics = [ - METRIC_DATABASES => true, - METRIC_BUCKETS => true, - METRIC_USERS => true, - METRIC_FUNCTIONS => true, - METRIC_TEAMS => true, - METRIC_MESSAGES => true, - METRIC_MAU => true, - METRIC_WEBHOOKS => true, - METRIC_PLATFORMS => true, - METRIC_PROVIDERS => true, - METRIC_TOPICS => true, - METRIC_KEYS => true, - METRIC_FILES => true, - METRIC_FILES_STORAGE => true, - METRIC_DEPLOYMENTS_STORAGE => true, - METRIC_BUILDS_STORAGE => true, - METRIC_DEPLOYMENTS => true, - METRIC_BUILDS => true, - METRIC_COLLECTIONS => true, - METRIC_DOCUMENTS => true, - METRIC_DATABASES_STORAGE => true, - ]; - - /** - * Skip metrics associated with parent IDs - * these need to be checked individually with `str_ends_with` - */ - protected array $skipParentIdMetrics = [ - '.files', - '.files.storage', - '.collections', - '.documents', - '.deployments', - '.deployments.storage', - '.builds', - '.builds.storage', - '.databases.storage' - ]; - /** * @var callable */ @@ -129,20 +79,13 @@ public function action(Message $message, callable $getProjectDB, callable $getLo try { /** @var \Utopia\Database\Database $dbForProject */ $dbForProject = $getProjectDB($project); + $documents = []; + $documentsClone = []; foreach ($stats['keys'] ?? [] as $key => $value) { if ($value == 0) { continue; } - if (str_contains($key, METRIC_DATABASES_STORAGE)) { - try { - $this->handleDatabaseStorage($key, $dbForProject, $project, $receivedAt); - } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); - } - continue; - } - foreach ($this->periods as $period => $format) { $time = null; @@ -159,191 +102,39 @@ public function action(Message $message, callable $getProjectDB, callable $getLo 'value' => $value, 'region' => System::getEnv('_APP_REGION', 'default'), ]); + $documents[] = $document; $documentClone = new Document($document->getArrayCopy()); - - $dbForProject->createOrUpdateDocumentsWithIncrease( - 'stats', - 'value', - [$document] - ); - - $this->writeToLogsDB($project, $documentClone); + $documentsClone[] = $documentClone; } } - } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); - } - } - } - - private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project, string $receivedAt): void - { - $data = explode('.', $key); - $start = microtime(true); - - $updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) use ($receivedAt) { - $id = \md5("{$time}_{$period}_{$key}"); - - $document = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); - $documentClone = new Document($document->getArrayCopy()); - $dbForProject->createOrUpdateDocumentsWithIncrease( - 'stats', - 'value', - [$document] - ); - $this->writeToLogsDB($project, $documentClone); - }; - - foreach ($this->periods as $period => $format) { - $time = null; - if ($period !== 'inf') { - $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); - } - $id = \md5("{$time}_{$period}_{$key}"); - - $value = 0; - $previousValue = 0; - try { - $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); + $dbForProject->createOrUpdateDocumentsWithIncrease( + 'stats', + 'value', + $documents + ); + $this->writeToLogsDB($project, $documentsClone); } catch (\Exception $e) { - // No previous value - } - - switch (count($data)) { - // Collection Level - case self::METRIC_COLLECTION_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - $collectionInternalId = $data[1]; - - try { - $value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - // Compare with previous value - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Collection - $updateMetric($dbForProject, $project, $diff, $key, $period, $time); - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); - break; - // Database Level - case self::METRIC_DATABASE_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - - $collections = []; - try { - $collections = $dbForProject->find('database_' . $databaseInternalId); - } catch (\Exception $e) { - // Database not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); - break; - // Project Level - case self::METRIC_PROJECT_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); - // Get all project databases - $databases = $dbForProject->find('database'); - - // Recalculate all databases - foreach ($databases as $database) { - $collections = $dbForProject->find('database_' . $database->getInternalId()); - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - } - - $diff = $value - $previousValue; - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); - break; + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); } } - - $end = microtime(true); - - console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); } - protected function writeToLogsDB(Document $project, Document $document): void + /** + * Write to logs DB + * + * @param Document $project + * @param array $documents + * @return void + */ + protected function writeToLogsDB(Document $project, array $documents): void { if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { Console::log('Dual Writing is disabled. Skipping...'); return; } - if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) { - return; - } - foreach ($this->skipParentIdMetrics as $skipMetric) { - if (str_ends_with($document->getAttribute('metric'), $skipMetric)) { - return; - } - } - /** @var \Utopia\Database\Database $dbForLogs*/ $dbForLogs = call_user_func($this->getLogsDB, $project); @@ -351,7 +142,7 @@ protected function writeToLogsDB(Document $project, Document $document): void $dbForLogs->createOrUpdateDocumentsWithIncrease( 'stats', 'value', - [$document] + $documents ); Console::success('Usage logs pushed to Logs DB'); } catch (\Throwable $th) { diff --git a/src/Appwrite/Utopia/Response/Model/File.php b/src/Appwrite/Utopia/Response/Model/File.php index 11a128abcc4..553cc7fe9af 100644 --- a/src/Appwrite/Utopia/Response/Model/File.php +++ b/src/Appwrite/Utopia/Response/Model/File.php @@ -65,6 +65,12 @@ public function __construct() 'default' => 0, 'example' => 17890, ]) + ->addRule('sizeActual', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'File actual size in bytes.', + 'default' => 0, + 'example' => 17890, + ]) ->addRule('chunksTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total number of chunks available', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index e614e2e1853..845547422ba 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -269,7 +269,7 @@ public function testPrepareStorageStats(array $data): array $this->assertEquals(201, $response['headers']['status-code']); $this->assertNotEmpty($response['body']['$id']); - $fileSize = $response['body']['sizeOriginal']; + $fileSize = $response['body']['sizeActual']; $storageTotal += $fileSize; $filesTotal += 1; @@ -295,6 +295,9 @@ public function testPrepareStorageStats(array $data): array } } + // stats-resources runs every 5s in test env + sleep(self::WAIT + 5); + return array_merge($data, [ 'bucketId' => $bucketId, 'bucketsTotal' => $bucketsTotal, @@ -514,6 +517,7 @@ public function testPrepareDatabaseStats(array $data): array } } + sleep(self::WAIT); return array_merge($data, [ 'databaseId' => $databaseId, 'collectionId' => $collectionId, @@ -648,6 +652,8 @@ public function testDatabaseStoragePrepare(): array ] ); + sleep(self::WAIT); + return [ 'databaseId' => $databaseId, 'collectionId' => $collectionId,