-
Notifications
You must be signed in to change notification settings - Fork 0
Fix remove duplicate metrics from stats usage #1
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: 1.9.x
Are you sure you want to change the base?
Changes from all commits
22d2a3b
59f9ed8
706723e
1397bfd
e1a6046
23c6f4b
466cad2
3f2b89e
1f870ab
1384834
dfe1df7
655cb3f
45924d4
85011ce
077aa32
f44dd0f
6dafc67
d4bea35
65ef126
1f27fc2
139b414
6ff6a7e
918b4c3
ef2fe4a
02fa2bd
e570dda
0b1a218
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -87,7 +87,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 | |||||
| _APP_MAINTENANCE_RETENTION_ABUSE=86400 | ||||||
| _APP_MAINTENANCE_RETENTION_AUDIT=1209600 | ||||||
| _APP_USAGE_AGGREGATION_INTERVAL=30 | ||||||
| _APP_STATS_RESOURCES_INTERVAL=3600 | ||||||
| _APP_STATS_RESOURCES_INTERVAL=15 | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Prompt for AI assistanceCopy the prompt below and paste it into ChatGPT, Claude, or any LLM: |
||||||
| _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Prompt for AI assistanceCopy the prompt below and paste it into ChatGPT, Claude, or any LLM: |
||||||
| _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 | ||||||
| _APP_USAGE_STATS=enabled | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -534,7 +534,6 @@ function updateAttribute( | |
| } | ||
|
|
||
| $queueForEvents->setParam('databaseId', $database->getId()); | ||
| $queueForStatsUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database | ||
|
|
||
| $response | ||
| ->setStatusCode(Response::STATUS_CODE_CREATED) | ||
|
|
@@ -830,9 +829,6 @@ function updateAttribute( | |
| ->setParam('databaseId', $database->getId()) | ||
| ->setPayload($response->output($database, Response::MODEL_DATABASE)); | ||
|
|
||
| $queueForStatsUsage | ||
| ->addMetric(METRIC_DATABASES_STORAGE, 1); // Global, deletion forces full recalculation | ||
|
|
||
| $response->noContent(); | ||
| }); | ||
|
|
||
|
|
@@ -2732,9 +2728,6 @@ function updateAttribute( | |
| ->setContext('database', $db) | ||
| ->setPayload($response->output($attribute, $model)); | ||
|
|
||
| $queueForStatsUsage | ||
| ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$db->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection | ||
|
|
||
| $response->noContent(); | ||
| }); | ||
|
|
||
|
|
@@ -3356,8 +3349,7 @@ function updateAttribute( | |
|
|
||
| $queueForStatsUsage | ||
| ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) | ||
| ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) | ||
| ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection | ||
| ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); | ||
|
|
||
| $response->addHeader('X-Debug-Operations', $operations); | ||
|
|
||
|
|
@@ -4141,8 +4133,7 @@ function updateAttribute( | |
|
|
||
| $queueForStatsUsage | ||
| ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) | ||
| ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) | ||
| ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection | ||
| ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); | ||
|
|
||
| $response->addHeader('X-Debug-Operations', 1); | ||
|
|
||
|
|
@@ -4186,28 +4177,51 @@ function updateAttribute( | |
| ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true) | ||
| ->inject('response') | ||
| ->inject('dbForProject') | ||
| ->action(function (string $range, Response $response, Database $dbForProject) { | ||
| ->inject('dbForLogs') | ||
| ->action(function (string $range, Response $response, Database $dbForProject, Database $dbForLogs) { | ||
|
|
||
| $periods = Config::getParam('usage', []); | ||
| $stats = $usage = []; | ||
| $days = $periods[$range]; | ||
| $metrics = [ | ||
| $logsDBMetrics = [ | ||
| METRIC_DATABASES, | ||
| METRIC_COLLECTIONS, | ||
| METRIC_DOCUMENTS, | ||
| METRIC_DATABASES_STORAGE, | ||
| ]; | ||
|
|
||
| Authorization::skip(function () use ($dbForLogs, $days, $logsDBMetrics, &$stats) { | ||
| foreach ($logsDBMetrics as $metric) { | ||
| $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); | ||
| $stats[$metric]['total'] = $result->getAttribute('value', 0); | ||
|
|
||
| $limit = $days['limit']; | ||
| $period = $days['period']; | ||
| $results = $dbForLogs->find('stats', [ | ||
| Query::equal('metric', [$metric]), | ||
| Query::equal('period', [$period]), | ||
| Query::limit($limit), | ||
| Query::orderDesc('time'), | ||
| ]); | ||
| $stats[$metric]['data'] = []; | ||
| foreach ($results as $result) { | ||
| $stats[$metric]['data'][$result->getAttribute('time')] = [ | ||
| 'value' => $result->getAttribute('value'), | ||
| ]; | ||
| } | ||
| } | ||
| }); | ||
|
|
||
| $metrics = [ | ||
| METRIC_DATABASES_OPERATIONS_READS, | ||
| METRIC_DATABASES_OPERATIONS_WRITES, | ||
| ]; | ||
|
|
||
| Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { | ||
| foreach ($metrics as $metric) { | ||
| $result = $dbForProject->findOne('stats', [ | ||
| Query::equal('metric', [$metric]), | ||
| Query::equal('period', ['inf']) | ||
| ]); | ||
|
|
||
| $result = $dbForProject->getDocument('stats', md5('_inf_'. $metric)); | ||
| $stats[$metric]['total'] = $result['value'] ?? 0; | ||
|
|
||
| $limit = $days['limit']; | ||
| $period = $days['period']; | ||
| $results = $dbForProject->find('stats', [ | ||
|
|
@@ -4230,7 +4244,7 @@ function updateAttribute( | |
| '1d' => 'Y-m-d\T00:00:00.000P', | ||
| }; | ||
|
|
||
| foreach ($metrics as $metric) { | ||
| foreach ([...$logsDBMetrics,...$metrics] as $metric) { | ||
| $usage[$metric]['total'] = $stats[$metric]['total']; | ||
| $usage[$metric]['data'] = []; | ||
| $leap = time() - ($days['limit'] * $days['factor']); | ||
|
|
@@ -4245,18 +4259,18 @@ function updateAttribute( | |
| } | ||
| $response->dynamic(new Document([ | ||
| 'range' => $range, | ||
| 'databasesTotal' => $usage[$metrics[0]]['total'], | ||
| 'collectionsTotal' => $usage[$metrics[1]]['total'], | ||
| 'documentsTotal' => $usage[$metrics[2]]['total'], | ||
| 'storageTotal' => $usage[$metrics[3]]['total'], | ||
| 'databasesReadsTotal' => $usage[$metrics[4]]['total'], | ||
| 'databasesWritesTotal' => $usage[$metrics[5]]['total'], | ||
| 'databases' => $usage[$metrics[0]]['data'], | ||
| 'collections' => $usage[$metrics[1]]['data'], | ||
| 'documents' => $usage[$metrics[2]]['data'], | ||
| 'storage' => $usage[$metrics[3]]['data'], | ||
| 'databasesReads' => $usage[$metrics[4]]['data'], | ||
| 'databasesWrites' => $usage[$metrics[5]]['data'], | ||
| 'databasesTotal' => $usage[$logsDBMetrics[0]]['total'], | ||
| 'collectionsTotal' => $usage[$logsDBMetrics[1]]['total'], | ||
| 'documentsTotal' => $usage[$logsDBMetrics[2]]['total'], | ||
| 'storageTotal' => $usage[$logsDBMetrics[3]]['total'], | ||
| 'databasesReadsTotal' => $usage[$metrics[0]]['total'], | ||
| 'databasesWritesTotal' => $usage[$metrics[1]]['total'], | ||
| 'databases' => $usage[$logsDBMetrics[0]]['data'], | ||
| 'collections' => $usage[$logsDBMetrics[1]]['data'], | ||
| 'documents' => $usage[$logsDBMetrics[2]]['data'], | ||
| 'storage' => $usage[$logsDBMetrics[3]]['data'], | ||
| 'databasesReads' => $usage[$metrics[0]]['data'], | ||
| 'databasesWrites' => $usage[$metrics[1]]['data'], | ||
| ]), Response::MODEL_USAGE_DATABASES); | ||
| }); | ||
|
|
||
|
|
@@ -4282,7 +4296,8 @@ function updateAttribute( | |
| ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true) | ||
| ->inject('response') | ||
| ->inject('dbForProject') | ||
| ->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) { | ||
| ->inject('dbForLogs') | ||
| ->action(function (string $databaseId, string $range, Response $response, Database $dbForProject, Database $dbForLogs) { | ||
|
|
||
| $database = $dbForProject->getDocument('databases', $databaseId); | ||
|
|
||
|
|
@@ -4293,20 +4308,43 @@ function updateAttribute( | |
| $periods = Config::getParam('usage', []); | ||
| $stats = $usage = []; | ||
| $days = $periods[$range]; | ||
| $metrics = [ | ||
|
|
||
| $logsDBMetrics = [ | ||
| str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS), | ||
| str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS), | ||
| str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_STORAGE), | ||
| ]; | ||
|
|
||
| Authorization::skip(function () use ($dbForLogs, $days, $logsDBMetrics, &$stats) { | ||
| foreach ($logsDBMetrics as $metric) { | ||
| $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); | ||
| $stats[$metric]['total'] = $result->getAttribute('value', 0); | ||
|
|
||
| $limit = $days['limit']; | ||
| $period = $days['period']; | ||
| $results = $dbForLogs->find('stats', [ | ||
| Query::equal('metric', [$metric]), | ||
| Query::equal('period', [$period]), | ||
| Query::limit($limit), | ||
| Query::orderDesc('time'), | ||
| ]); | ||
| $stats[$metric]['data'] = []; | ||
| foreach ($results as $result) { | ||
| $stats[$metric]['data'][$result->getAttribute('time')] = [ | ||
| 'value' => $result->getAttribute('value'), | ||
| ]; | ||
| } | ||
| } | ||
| }); | ||
|
|
||
| $metrics = [ | ||
| str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_READS), | ||
| str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_WRITES) | ||
|
Comment on lines
+4340
to
4342
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The per-database usage metrics use the global metric names instead of the database-scoped metric constants, so database-specific read and write totals will always be empty; use the METRIC_DATABASE_ID_* templates here. Suggested fix $metrics = [
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS),
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES)
];Prompt for AI assistanceCopy the prompt below and paste it into ChatGPT, Claude, or any LLM: |
||
| ]; | ||
|
|
||
| Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { | ||
| foreach ($metrics as $metric) { | ||
| $result = $dbForProject->findOne('stats', [ | ||
| Query::equal('metric', [$metric]), | ||
| Query::equal('period', ['inf']) | ||
| ]); | ||
| $result = $dbForProject->getDocument('stats', md5('_inf_' . $metric)); | ||
|
|
||
| $stats[$metric]['total'] = $result['value'] ?? 0; | ||
| $limit = $days['limit']; | ||
|
|
@@ -4331,7 +4369,7 @@ function updateAttribute( | |
| '1d' => 'Y-m-d\T00:00:00.000P', | ||
| }; | ||
|
|
||
| foreach ($metrics as $metric) { | ||
| foreach ([...$logsDBMetrics,...$metrics] as $metric) { | ||
| $usage[$metric]['total'] = $stats[$metric]['total']; | ||
| $usage[$metric]['data'] = []; | ||
| $leap = time() - ($days['limit'] * $days['factor']); | ||
|
|
@@ -4347,16 +4385,16 @@ function updateAttribute( | |
|
|
||
| $response->dynamic(new Document([ | ||
| 'range' => $range, | ||
| 'collectionsTotal' => $usage[$metrics[0]]['total'], | ||
| 'documentsTotal' => $usage[$metrics[1]]['total'], | ||
| 'storageTotal' => $usage[$metrics[2]]['total'], | ||
| 'databaseReadsTotal' => $usage[$metrics[3]]['total'], | ||
| 'databaseWritesTotal' => $usage[$metrics[4]]['total'], | ||
| 'collections' => $usage[$metrics[0]]['data'], | ||
| 'documents' => $usage[$metrics[1]]['data'], | ||
| 'storage' => $usage[$metrics[2]]['data'], | ||
| 'databaseReads' => $usage[$metrics[3]]['data'], | ||
| 'databaseWrites' => $usage[$metrics[4]]['data'], | ||
| 'collectionsTotal' => $usage[$logsDBMetrics[0]]['total'], | ||
| 'documentsTotal' => $usage[$logsDBMetrics[1]]['total'], | ||
| 'storageTotal' => $usage[$logsDBMetrics[2]]['total'], | ||
| 'databaseReadsTotal' => $usage[$metrics[0]]['total'], | ||
| 'databaseWritesTotal' => $usage[$metrics[1]]['total'], | ||
| 'collections' => $usage[$logsDBMetrics[0]]['data'], | ||
| 'documents' => $usage[$logsDBMetrics[1]]['data'], | ||
| 'storage' => $usage[$logsDBMetrics[2]]['data'], | ||
| 'databaseReads' => $usage[$metrics[0]]['data'], | ||
| 'databaseWrites' => $usage[$metrics[1]]['data'], | ||
| ]), Response::MODEL_USAGE_DATABASE); | ||
| }); | ||
|
|
||
|
|
@@ -4384,7 +4422,8 @@ function updateAttribute( | |
| ->param('collectionId', '', new UID(), 'Collection ID.') | ||
| ->inject('response') | ||
| ->inject('dbForProject') | ||
| ->action(function (string $databaseId, string $range, string $collectionId, Response $response, Database $dbForProject) { | ||
| ->inject('dbForLogs') | ||
| ->action(function (string $databaseId, string $range, string $collectionId, Response $response, Database $dbForProject, Database $dbForLogs) { | ||
|
|
||
| $database = $dbForProject->getDocument('databases', $databaseId); | ||
|
Comment on lines
+4426
to
4428
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The collection usage endpoint dereferences Suggested fix ->action(function (string $databaseId, string $range, string $collectionId, Response $response, Database $dbForProject, Database $dbForLogs) {
$database = $dbForProject->getDocument('databases', $databaseId);
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
$collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId);Prompt for AI assistanceCopy the prompt below and paste it into ChatGPT, Claude, or any LLM: |
||
| $collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); | ||
|
|
@@ -4401,17 +4440,14 @@ function updateAttribute( | |
| str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), | ||
| ]; | ||
|
|
||
| Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { | ||
| Authorization::skip(function () use ($dbForLogs, $days, $metrics, &$stats) { | ||
| foreach ($metrics as $metric) { | ||
| $result = $dbForProject->findOne('stats', [ | ||
| Query::equal('metric', [$metric]), | ||
| Query::equal('period', ['inf']) | ||
| ]); | ||
|
|
||
| $result = $dbForLogs->getDocument('stats', md5('_inf_' . $metric)); | ||
| $stats[$metric]['total'] = $result['value'] ?? 0; | ||
|
|
||
| $limit = $days['limit']; | ||
| $period = $days['period']; | ||
| $results = $dbForProject->find('stats', [ | ||
| $results = $dbForLogs->find('stats', [ | ||
| Query::equal('metric', [$metric]), | ||
| Query::equal('period', [$period]), | ||
| Query::limit($limit), | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
.env. This file is the shipped default for production deployments, and at 15 s the StatsResources worker will continuously count all documents in every bucket, function, and database — a very expensive set of queries — 240× more often than before. The CI workflow already overrides this viaexport _APP_STATS_RESOURCES_INTERVAL=5, so the.envvalue is not needed for tests to pass.