Skip to content

perf: bulk INSERT secret events to reduce DB round-trips #784

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nimish-ks wants to merge 4 commits into
base: main
Choose a base branch
from
Open

perf: bulk INSERT secret events to reduce DB round-trips #784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3579d33
perf: bulk INSERT secret events to reduce DB round-trips
nimish-ks Feb 24, 2026
af50e9d
Merge branch 'main' into perf/bulk-secret-event-logging
rohan-chaturvedi Mar 9, 2026
daf8cad
Merge branch 'main' into perf/bulk-secret-event-logging
nimish-ks Apr 30, 2026
ff1f7a1
Merge branch 'main' into perf/bulk-secret-event-logging
nimish-ks May 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions backend/api/utils/audit_logging.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,65 @@ def log_secret_event(
)

event.tags.set(secret.tags.all())


def log_secret_events_bulk(
secrets,
event_type,
user=None,
service_token=None,
service_account_token=None,
ip_address=None,
user_agent=None,
):
"""
Bulk version of log_secret_event. Logs events for multiple secrets
using bulk_create to reduce database round-trips.
"""

if not secrets:
return []

service_account = None
if service_account_token is not None:
service_account = service_account_token.service_account

now = timezone.now()

events = [
SecretEvent(
secret=secret,
environment=secret.environment,
folder=secret.folder,
path=secret.path,
user=user,
service_token=service_token,
service_account=service_account,
service_account_token=service_account_token,
key=secret.key,
key_digest=secret.key_digest,
value=secret.value,
version=secret.version,
comment=secret.comment,
event_type=event_type,
timestamp=now,
ip_address=ip_address,
user_agent=user_agent,
)
for secret in secrets
]

created = SecretEvent.objects.bulk_create(events)

# Bulk M2M: collect all tag associations and insert at once
through_model = SecretEvent.tags.through
m2m_rows = []
for event, secret in zip(created, secrets):
for tag in secret.tags.all():
m2m_rows.append(
through_model(secretevent_id=event.pk, secrettag_id=tag.pk)
)
if m2m_rows:
through_model.objects.bulk_create(m2m_rows)

return created
182 changes: 97 additions & 85 deletions backend/api/views/secrets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from api.utils.access.permissions import (
user_has_permission,
)
from api.utils.audit_logging import log_secret_event
from api.utils.audit_logging import log_secret_event, log_secret_events_bulk

from api.utils.crypto import encrypt_asymmetric, validate_encrypted_string
from api.utils.rest import (
Expand Down Expand Up @@ -137,18 +137,17 @@ def get(self, request, *args, **kwargs):
except:
pass

secrets = Secret.objects.filter(**secrets_filter)
secrets = Secret.objects.filter(**secrets_filter).prefetch_related('tags')

for secret in secrets:
log_secret_event(
secret,
SecretEvent.READ,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
log_secret_events_bulk(
list(secrets),
SecretEvent.READ,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

serializer = SecretSerializer(
secrets, many=True, context={"org_member": request.auth["org_member"]}
Expand Down Expand Up @@ -294,6 +293,8 @@ def post(self, request, *args, **kwargs):
):
return JsonResponse({"error": "Duplicate secret found"}, status=409)

created_secrets = []

for secret in request_body["secrets"]:

# Check that all encrypted fields are valid
Expand Down Expand Up @@ -335,16 +336,7 @@ def post(self, request, *args, **kwargs):

secret_obj = Secret.objects.create(**secret_data)
secret_obj.tags.set(tags)

log_secret_event(
secret_obj,
SecretEvent.CREATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
created_secrets.append(secret_obj)

# If the request is authenticated as a user and an override is supplied
if request.auth["org_member"] and "override" in secret:
Expand All @@ -354,6 +346,16 @@ def post(self, request, *args, **kwargs):
value=secret["override"]["value"],
)

log_secret_events_bulk(
created_secrets,
SecretEvent.CREATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

return Response(status=status.HTTP_200_OK)

def put(self, request, *args, **kwargs):
Expand All @@ -372,6 +374,8 @@ def put(self, request, *args, **kwargs):
):
return JsonResponse({"error": "Duplicate secret found"}, status=409)

updated_secrets = []

for secret in request_body["secrets"]:

secret_obj = Secret.objects.get(id=secret["id"])
Expand Down Expand Up @@ -448,16 +452,7 @@ def put(self, request, *args, **kwargs):
secret_obj.updated_at = timezone.now()
secret_obj.tags.set(tags)
secret_obj.save()

log_secret_event(
secret_obj,
SecretEvent.UPDATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
updated_secrets.append(secret_obj)

# If the request is authenticated as a user and an override is supplied
if request.auth["org_member"] and "override" in secret:
Expand All @@ -471,6 +466,16 @@ def put(self, request, *args, **kwargs):
},
)

log_secret_events_bulk(
updated_secrets,
SecretEvent.UPDATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

return Response(status=status.HTTP_200_OK)

def delete(self, request, *args, **kwargs):
Expand All @@ -479,27 +484,31 @@ def delete(self, request, *args, **kwargs):

ip_address, user_agent = get_resolver_request_meta(request)

secrets_to_delete = Secret.objects.filter(id__in=request_body["secrets"])
secrets_to_delete = Secret.objects.filter(
id__in=request_body["secrets"]
).prefetch_related('tags')

if not secrets_to_delete.exists():
return Response(status=status.HTTP_200_OK)

env = secrets_to_delete[0].environment

deleted_secrets = []
for secret in secrets_to_delete:
secret.updated_at = timezone.now()
secret.deleted_at = timezone.now()
secret.save()

log_secret_event(
secret,
SecretEvent.DELETE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
deleted_secrets.append(secret)

log_secret_events_bulk(
deleted_secrets,
SecretEvent.DELETE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

return Response(status=status.HTTP_200_OK)

Expand Down Expand Up @@ -583,18 +592,17 @@ def get(self, request, *args, **kwargs):
# Filter secrets based on these tags
secrets_filter["tags__in"] = tags

secrets = Secret.objects.filter(**secrets_filter)
secrets = Secret.objects.filter(**secrets_filter).prefetch_related('tags')

for secret in secrets:
log_secret_event(
secret,
SecretEvent.READ,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
log_secret_events_bulk(
list(secrets),
SecretEvent.READ,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# Pre-compute crypto context for N+1 optimization
crypto_context = get_environment_crypto_context(env)
Expand Down Expand Up @@ -814,16 +822,6 @@ def post(self, request, *args, **kwargs):
)
secret_obj.tags.set(tags)

log_secret_event(
secret_obj,
SecretEvent.CREATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# If the request is authenticated as a user and an override is supplied
if request.auth["org_member"] and "override" in secret:
PersonalSecret.objects.create(
Expand All @@ -834,6 +832,16 @@ def post(self, request, *args, **kwargs):

created_secrets.append(secret_obj)

log_secret_events_bulk(
created_secrets,
SecretEvent.CREATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# Pre-compute crypto context for N+1 optimization
crypto_context = get_environment_crypto_context(env)
context_cache = {}
Expand Down Expand Up @@ -955,16 +963,6 @@ def put(self, request, *args, **kwargs):

secret_obj.save()

log_secret_event(
secret_obj,
SecretEvent.UPDATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# If the request is authenticated as a user and an override is supplied
if request.auth["org_member"] and "override" in secret:
PersonalSecret.objects.update_or_create(
Expand All @@ -979,6 +977,16 @@ def put(self, request, *args, **kwargs):

updated_secrets.append(secret_obj)

log_secret_events_bulk(
updated_secrets,
SecretEvent.UPDATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# Pre-compute crypto context for N+1 optimization
crypto_context = get_environment_crypto_context(env)
context_cache = {}
Expand Down Expand Up @@ -1008,26 +1016,30 @@ def delete(self, request, *args, **kwargs):

ip_address, user_agent = get_resolver_request_meta(request)

secrets_to_delete = Secret.objects.filter(id__in=request_body["secrets"])
secrets_to_delete = Secret.objects.filter(
id__in=request_body["secrets"]
).prefetch_related('tags')

for secret in secrets_to_delete:
if not Secret.objects.filter(id=secret.id).exists():
return JsonResponse({"error": "Secret does not exist"}, status=404)

deleted_secrets = []
for secret in secrets_to_delete:
secret.updated_at = timezone.now()
secret.deleted_at = timezone.now()
secret.save()

log_secret_event(
secret,
SecretEvent.DELETE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
deleted_secrets.append(secret)

log_secret_events_bulk(
deleted_secrets,
SecretEvent.DELETE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

return Response(
{"message": f"Deleted {len(secrets_to_delete)} secrets"},
Expand Down
Loading
Loading