From b6e651c43e56dbf8d76b070c58f79e60ce744c22 Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Tue, 1 Sep 2026 15:16:47 -0700 Subject: [PATCH 1/3] Add beta gdrive-log connector for Drive events from Workspace audit logs. Restricts the proxy to the Drive Reports API activity endpoint, covering views and comments that the files API misses, with Google's ~6-month retention limit. Co-authored-by: Cursor --- docs/README.md | 1 + docs/SUMMARY.md | 1 + docs/sources/README.md | 1 + docs/sources/gdrive-log | 1 + docs/sources/google-workspace/README.md | 4 +- .../google-workspace/gdrive-log/README.md | 46 +++++ .../original/drive-activities.json | 168 ++++++++++++++++++ .../sanitized/drive-activities.json | 152 ++++++++++++++++ .../gdrive-log/gdrive-log.yaml | 16 ++ .../sources/google-workspace/gdrive/README.md | 2 + .../aws/google-workspace-variables.tf | 2 +- .../gcp/google-workspace-variables.tf | 2 +- .../google-workspace.tf | 21 +++ .../rules/google/PrebuiltSanitizerRules.java | 41 +++++ .../psoxy/rules/google/GDriveLogTests.java | 59 ++++++ 15 files changed, 514 insertions(+), 3 deletions(-) create mode 120000 docs/sources/gdrive-log create mode 100644 docs/sources/google-workspace/gdrive-log/README.md create mode 100644 docs/sources/google-workspace/gdrive-log/example-api-responses/original/drive-activities.json create mode 100644 docs/sources/google-workspace/gdrive-log/example-api-responses/sanitized/drive-activities.json create mode 100644 docs/sources/google-workspace/gdrive-log/gdrive-log.yaml create mode 100644 java/core/src/test/java/co/worklytics/psoxy/rules/google/GDriveLogTests.java diff --git a/docs/README.md b/docs/README.md index 1793b604b6..00fbd1b349 100644 --- a/docs/README.md +++ b/docs/README.md @@ -90,6 +90,7 @@ If you use our provided Terraform modules, specific instructions that you can pa | Google Chat | [data](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/google-chat/example-api-responses) - [rules](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/google-chat/google-chat.yaml) | `admin.reports.audit.readonly` | | Google Directory | [data](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/directory/example-api-responses) - [rules](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/directory/directory.yaml) | `admin.directory.user.readonly admin.directory.domain.readonly admin.directory.group.readonly admin.directory.orgunit.readonly` | | Google Drive | [data](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/gdrive/example-api-responses) - [rules](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/gdrive/gdrive.yaml) | `drive.metadata.readonly` | +| Google Drive Audit Log (**beta**) | [data](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/gdrive-log/example-api-responses) - [rules](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/gdrive-log/gdrive-log.yaml) | `admin.reports.audit.readonly` | | GMail | [data](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/gmail/example-api-responses) - [rules](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/gmail/gmail.yaml) | `gmail.metadata` | | Google Meet | [data](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/meet/example-api-responses) - [rules](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/meet/meet.yaml) | `admin.reports.audit.readonly` | | Gemini Bulk (**deprecated**) | [docs](https://github.com/Worklytics/psoxy/tree/main/docs/sources/google-workspace/gemini-usage-bulk) | n/a; bulk export of Gemini logs | diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index f70dce95ce..d0408f5b61 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -90,6 +90,7 @@ * [Google Chat](sources/google-workspace/google-chat/README.md) * [Google Directory](sources/google-workspace/directory/README.md) * [Google Drive](sources/google-workspace/gdrive/README.md) + * [Google Drive Audit Log](sources/google-workspace/gdrive-log/README.md) * [Google Meet](sources/google-workspace/meet/README.md) * [Gmail](sources/google-workspace/gmail/README.md) * [Gemini in Workspace Apps](sources/google-workspace/gemini-in-workspace-apps/README.md) diff --git a/docs/sources/README.md b/docs/sources/README.md index 0de73ff22e..219fef2a2c 100644 --- a/docs/sources/README.md +++ b/docs/sources/README.md @@ -23,6 +23,7 @@ To add a source, add its Connector ID to the `enabled_connectors` list in your ` | `gcal` | [Google Calendar](google-workspace/calendar/README.md) | API | GA | | `gdirectory` | [Google Directory](google-workspace/directory/README.md) | API | GA | | `gdrive` | [Google Drive](google-workspace/gdrive/README.md) | API | GA | +| `gdrive-log` | [Google Drive Audit Log](google-workspace/gdrive-log/README.md) | API | BETA | | `gemini-in-workspace-apps` | [Google Gemini in Workspace Apps](google-workspace/gemini-in-workspace-apps/README.md) | API | BETA | | `gemini-usage` | [Gemini Usage](google-workspace/gemini-usage-bulk/README.md) | Bulk | DEPRECATED | | `github` | [GitHub Enterprise](github/github/README.md) | API | GA | diff --git a/docs/sources/gdrive-log b/docs/sources/gdrive-log new file mode 120000 index 0000000000..4fe094f308 --- /dev/null +++ b/docs/sources/gdrive-log @@ -0,0 +1 @@ +google-workspace/gdrive-log/README.md \ No newline at end of file diff --git a/docs/sources/google-workspace/README.md b/docs/sources/google-workspace/README.md index 536a71a88c..f95e79e068 100644 --- a/docs/sources/google-workspace/README.md +++ b/docs/sources/google-workspace/README.md @@ -14,6 +14,7 @@ Within those, the `google-workspace.tf` and `google-workspace-variables.tf` file - [calendar](calendar/README.md) (Google Calendar™) - [directory](directory/README.md) (Google Workspace™ Directory) - [gdrive](gdrive/README.md) (Google Drive™) +- [gdrive-log](gdrive-log/README.md) (Google Drive™ Audit Log) - [gemini-in-workspace-apps](gemini-in-workspace-apps/README.md) - [gemini-usage-bulk](gemini-usage-bulk/README.md) - [gmail](gmail/README.md) (Gmail™) @@ -30,6 +31,7 @@ Each connector page includes the full comma-separated OAuth scope string to past | [google-chat](google-chat/README.md) | `google-chat` | `admin.googleapis.com` | `admin.reports.audit.readonly` | | [directory](directory/README.md) | `gdirectory` | `admin.googleapis.com` | `admin.directory.user.readonly` `admin.directory.domain.readonly` `admin.directory.group.readonly` `admin.directory.orgunit.readonly` | | [gdrive](gdrive/README.md) | `gdrive` | `drive.googleapis.com` | `drive.metadata.readonly` | +| [gdrive-log](gdrive-log/README.md) | `gdrive-log` | `admin.googleapis.com` | `admin.reports.audit.readonly` | | [gmail](gmail/README.md) | `gmail` | `gmail.googleapis.com` | `gmail.metadata` | | [meet](meet/README.md) | `google-meet` | `admin.googleapis.com` | `admin.reports.audit.readonly` | | [gemini-in-workspace-apps](gemini-in-workspace-apps/README.md) | `gemini-in-workspace-apps` | `admin.googleapis.com` | `admin.reports.audit.readonly` | @@ -86,7 +88,7 @@ Assign the account a sufficiently privileged role. At minimum, the role must gra | **Groups** → Read | Yes | Directory group membership | | **Organizational Units** → Read | Optional | Org-unit segmentation | | **Domain Management** | Optional | List of internal domains | -| **Reports** | Only if using [Google Chat](google-chat/README.md), [Google Meet](meet/README.md), or other audit-log connectors | Audit / usage reports | +| **Reports** | Only if using [Google Chat](google-chat/README.md), [Google Meet](meet/README.md), [Google Drive Audit Log](gdrive-log/README.md), or other audit-log connectors | Audit / usage reports | All of the above are found under **Admin settings privileges** in the Custom Role editor. Google reorganized administrator privileges in 2025; expand each category and enable only the **Read** sub-action where available. See Google's [privilege definitions](https://knowledge.workspace.google.com/admin/users/administrator-privilege-definitions) for the full list. diff --git a/docs/sources/google-workspace/gdrive-log/README.md b/docs/sources/google-workspace/gdrive-log/README.md new file mode 100644 index 0000000000..89347e4a48 --- /dev/null +++ b/docs/sources/google-workspace/gdrive-log/README.md @@ -0,0 +1,46 @@ +# Google Drive Audit Log + +**Connector ID:** `gdrive-log` + +**Availability:** Beta + +Please review the [Google Workspace™ README](../README.md) for general information applicable to +all Google Workspace connectors. + +This connector pulls Drive events from the Google Workspace audit log (Reports API), rather than the Drive files API used by the [`gdrive`](../gdrive/README.md) connector. + +## Caveats + +Relative to [`gdrive`](../gdrive/README.md), this source includes additional activity types that the Drive files API does not expose as an event stream — for example views, comments, downloads, and previews, in addition to creates, edits, and sharing changes. + +The tradeoff is lookback. Google retains Drive audit-log events for about **6 months** (180 days). Older activity is not available from the Reports API, even if the files themselves still exist. Historical collaboration that predates that window is better covered by the files API connector. + +See Google's [Drive activity report](https://developers.google.com/workspace/admin/reports/v1/guides/manage-audit-drive) and [data retention](https://support.google.com/a/answer/7061566) documentation. + +## Required OAuth Scopes + +- `admin.reports.audit.readonly` + +For Domain-wide Delegation in the Google Workspace Admin console, paste the following comma-separated list into the **Scopes** field: + +``` +https://www.googleapis.com/auth/admin.reports.audit.readonly +``` + +## Required GCP APIs + +Enable the following API in the GCP project where you provision the OAuth client: + +- `admin.googleapis.com` (Admin SDK API) + +## Examples + +- [Example Rules](gdrive-log.yaml) +- Example Data: + - [original/drive-activities.json](example-api-responses/original/drive-activities.json) | + [sanitized/drive-activities.json](example-api-responses/sanitized/drive-activities.json) + + +--- +Google Workspace™ and related marks are trademarks of Google LLC. +Worklytics™ is a trademark of Worklytics, Corp. diff --git a/docs/sources/google-workspace/gdrive-log/example-api-responses/original/drive-activities.json b/docs/sources/google-workspace/gdrive-log/example-api-responses/original/drive-activities.json new file mode 100644 index 0000000000..a6c1fb8895 --- /dev/null +++ b/docs/sources/google-workspace/gdrive-log/example-api-responses/original/drive-activities.json @@ -0,0 +1,168 @@ +{ + "kind": "admin#reports#activities", + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogExampleEtag\"", + "nextPageToken": "A:1634053603289000:5470579845645479028:913471609310:C03v4gz0f", + "items": [ + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T15:46:45.286Z", + "uniqueQualifier": "6127585580824196360", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem1\"", + "actor": { + "callerType": "USER", + "email": "alice@worklytics.co", + "profileId": "117927411761819390794" + }, + "ipAddress": "203.0.113.42", + "ownerDomain": "worklytics.co", + "events": [ + { + "type": "access", + "name": "edit", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "owner_is_shared_drive", + "boolValue": false + }, + { + "name": "doc_id", + "value": "1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name": "doc_title", + "value": "Meeting notes" + }, + { + "name": "doc_type", + "value": "document" + }, + { + "name": "owner", + "value": "mary@worklytics.co" + }, + { + "name": "visibility", + "value": "shared_internally" + } + ] + } + ] + }, + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T16:01:12.100Z", + "uniqueQualifier": "6127585580824196361", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem2\"", + "actor": { + "callerType": "USER", + "email": "alice@worklytics.co", + "profileId": "117927411761819390794" + }, + "ipAddress": "203.0.113.42", + "ownerDomain": "worklytics.co", + "events": [ + { + "type": "acl_change", + "name": "change_user_access", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "doc_id", + "value": "1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name": "doc_title", + "value": "Meeting notes" + }, + { + "name": "doc_type", + "value": "document" + }, + { + "name": "owner", + "value": "mary@worklytics.co" + }, + { + "name": "target_user", + "value": "bob@worklytics.co" + }, + { + "name": "added_role", + "value": "can_edit" + }, + { + "name": "visibility", + "value": "shared_internally" + } + ] + } + ] + }, + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T16:15:00.000Z", + "uniqueQualifier": "6127585580824196362", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem3\"", + "actor": { + "callerType": "USER", + "email": "bob@worklytics.co", + "profileId": "100531288453445237356" + }, + "events": [ + { + "type": "access", + "name": "add_to_folder", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "doc_id", + "value": "1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name": "doc_title", + "value": "Meeting notes" + }, + { + "name": "destination_folder_id", + "value": "0Bxxxxxxxxxxxxxxxxxxxxx" + }, + { + "name": "destination_folder_title", + "value": "Shared with Finance" + }, + { + "name": "owner", + "value": "mary@worklytics.co" + }, + { + "name": "doc_type", + "value": "document" + } + ] + } + ] + } + ] +} diff --git a/docs/sources/google-workspace/gdrive-log/example-api-responses/sanitized/drive-activities.json b/docs/sources/google-workspace/gdrive-log/example-api-responses/sanitized/drive-activities.json new file mode 100644 index 0000000000..3a0199f4b6 --- /dev/null +++ b/docs/sources/google-workspace/gdrive-log/example-api-responses/sanitized/drive-activities.json @@ -0,0 +1,152 @@ +{ + "kind":"admin#reports#activities", + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogExampleEtag\"", + "nextPageToken":"A:1634053603289000:5470579845645479028:913471609310:C03v4gz0f", + "items":[ + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T15:46:45.286Z", + "uniqueQualifier":"6127585580824196360", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem1\"", + "actor":{ + "callerType":"USER", + "email":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co", + "profileId":"117927411761819390794" + }, + "ipAddress":"t~FF_NukhrsL6d49CqBcU9AAjHiGZYMHeLvoi9QuSKfE0", + "ownerDomain":"worklytics.co", + "events":[ + { + "type":"access", + "name":"edit", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + }, + { + "name":"owner_is_shared_drive", + "boolValue":false + }, + { + "name":"doc_id", + "value":"1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name":"doc_type", + "value":"document" + }, + { + "name":"owner", + "value":"t~SuLKx-wI7BY6ymwyzWhgNm1k2HNYbLCtZB2GQlqflXk@worklytics.co" + }, + { + "name":"visibility", + "value":"shared_internally" + } + ] + } + ] + }, + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T16:01:12.100Z", + "uniqueQualifier":"6127585580824196361", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem2\"", + "actor":{ + "callerType":"USER", + "email":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co", + "profileId":"117927411761819390794" + }, + "ipAddress":"t~FF_NukhrsL6d49CqBcU9AAjHiGZYMHeLvoi9QuSKfE0", + "ownerDomain":"worklytics.co", + "events":[ + { + "type":"acl_change", + "name":"change_user_access", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + }, + { + "name":"doc_id", + "value":"1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name":"doc_type", + "value":"document" + }, + { + "name":"owner", + "value":"t~SuLKx-wI7BY6ymwyzWhgNm1k2HNYbLCtZB2GQlqflXk@worklytics.co" + }, + { + "name":"target_user", + "value":"t~ANg5l6XVBpn-KyO-ivYDGQwZViEvM2uhytCnKiB0qr4@worklytics.co" + }, + { + "name":"added_role", + "value":"can_edit" + }, + { + "name":"visibility", + "value":"shared_internally" + } + ] + } + ] + }, + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T16:15:00.000Z", + "uniqueQualifier":"6127585580824196362", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem3\"", + "actor":{ + "callerType":"USER", + "email":"t~ANg5l6XVBpn-KyO-ivYDGQwZViEvM2uhytCnKiB0qr4@worklytics.co", + "profileId":"100531288453445237356" + }, + "events":[ + { + "type":"access", + "name":"add_to_folder", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + }, + { + "name":"doc_id", + "value":"1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name":"destination_folder_id", + "value":"0Bxxxxxxxxxxxxxxxxxxxxx" + }, + { + "name":"owner", + "value":"t~SuLKx-wI7BY6ymwyzWhgNm1k2HNYbLCtZB2GQlqflXk@worklytics.co" + }, + { + "name":"doc_type", + "value":"document" + } + ] + } + ] + } + ] +} diff --git a/docs/sources/google-workspace/gdrive-log/gdrive-log.yaml b/docs/sources/google-workspace/gdrive-log/gdrive-log.yaml new file mode 100644 index 0000000000..c9ef630095 --- /dev/null +++ b/docs/sources/google-workspace/gdrive-log/gdrive-log.yaml @@ -0,0 +1,16 @@ +--- +endpoints: + - pathTemplate: "/admin/reports/v1/activity/users/all/applications/drive" + transforms: + - ! + jsonPaths: + - "$..email" + - "$.items[*].events[*].parameters[?(@.name in ['owner','target_user','new_owner','delegating_principal'])].value" + encoding: "URL_SAFE_TOKEN" + - ! + jsonPaths: + - "$.items[*].ipAddress" + - "$.items[*].events[*].parameters[?(@.name == 'ip_address')].value" + - ! + jsonPaths: + - "$.items[*].events[*].parameters[?(!(@.name =~ /^owner|target_user|new_owner|delegating_principal|actor_is_collaborator_account|billable|primary_event|doc_id|doc_type|is_encrypted|originating_app_id|owner_is_shared_drive|owner_shared_drive_id|shared_drive_id|visibility|visibility_change|old_visibility|added_role|removed_role|requested_role|destination_folder_id|source_folder_id|copy_type|new_owner_is_shared_drive|new_owner_shared_drive_id|target_domain|membership_change_type|ip_address$/i))]" diff --git a/docs/sources/google-workspace/gdrive/README.md b/docs/sources/google-workspace/gdrive/README.md index 6756d1f38b..2e5d18671f 100644 --- a/docs/sources/google-workspace/gdrive/README.md +++ b/docs/sources/google-workspace/gdrive/README.md @@ -7,6 +7,8 @@ Please review the [Google Workspace™ README](../README.md) for general information applicable to all Google Workspace connectors. +This connector uses the Drive files API. For Drive events from the Workspace audit log, see [`gdrive-log`](../gdrive-log/README.md). + ## Required OAuth Scopes - `drive.metadata.readonly` diff --git a/infra/examples-dev/aws/google-workspace-variables.tf b/infra/examples-dev/aws/google-workspace-variables.tf index cf63f376b4..878cdd0a4e 100644 --- a/infra/examples-dev/aws/google-workspace-variables.tf +++ b/infra/examples-dev/aws/google-workspace-variables.tf @@ -67,7 +67,7 @@ variable "google_workspace_key_rotation_days" { locals { # tflint-ignore: terraform_unused_declarations - some_google_connector_enabled = (length(setintersection(var.enabled_connectors, ["gcal", "gdirectory", "gdrive", "gmail", "google-meet", "google-chat", "gemini-for-workspace"])) > 0) + some_google_connector_enabled = (length(setintersection(var.enabled_connectors, ["gcal", "gdirectory", "gdrive", "gdrive-log", "gmail", "google-meet", "google-chat", "gemini-for-workspace"])) > 0) validate_google_workspace_gcp_project_id = (var.google_workspace_gcp_project_id == null || var.google_workspace_gcp_project_id == "") && local.some_google_connector_enabled validate_google_workspace_gcp_project_id_message = "The google_workspace_gcp_project_id var should be populated if a Google Workspace connector is enabled." validate_google_workspace_gcp_project_id_check = regex( diff --git a/infra/examples-dev/gcp/google-workspace-variables.tf b/infra/examples-dev/gcp/google-workspace-variables.tf index cf63f376b4..878cdd0a4e 100644 --- a/infra/examples-dev/gcp/google-workspace-variables.tf +++ b/infra/examples-dev/gcp/google-workspace-variables.tf @@ -67,7 +67,7 @@ variable "google_workspace_key_rotation_days" { locals { # tflint-ignore: terraform_unused_declarations - some_google_connector_enabled = (length(setintersection(var.enabled_connectors, ["gcal", "gdirectory", "gdrive", "gmail", "google-meet", "google-chat", "gemini-for-workspace"])) > 0) + some_google_connector_enabled = (length(setintersection(var.enabled_connectors, ["gcal", "gdirectory", "gdrive", "gdrive-log", "gmail", "google-meet", "google-chat", "gemini-for-workspace"])) > 0) validate_google_workspace_gcp_project_id = (var.google_workspace_gcp_project_id == null || var.google_workspace_gcp_project_id == "") && local.some_google_connector_enabled validate_google_workspace_gcp_project_id_message = "The google_workspace_gcp_project_id var should be populated if a Google Workspace connector is enabled." validate_google_workspace_gcp_project_id_check = regex( diff --git a/infra/modules/worklytics-connector-specs/google-workspace.tf b/infra/modules/worklytics-connector-specs/google-workspace.tf index 61091d5752..9f24a6f821 100644 --- a/infra/modules/worklytics-connector-specs/google-workspace.tf +++ b/infra/modules/worklytics-connector-specs/google-workspace.tf @@ -96,6 +96,27 @@ locals { ], example_api_calls_user_to_impersonate : local.google_workspace_example_user }, + "gdrive-log" : { + source_kind : "gdrive-log", + availability : "beta", + enable_by_default : false + worklytics_connector_id : "gdrive-log-psoxy", + display_name : "Google Drive Audit Log" + apis_consumed : [ + "admin.googleapis.com" + ] + source_auth_strategy : "gcp_service_account_key" + target_host : "admin.googleapis.com" + oauth_scopes_needed : [ + "https://www.googleapis.com/auth/admin.reports.audit.readonly" + ] + environment_variables : {} + enable_side_output : false + example_api_calls : [ + "/admin/reports/v1/activity/users/all/applications/drive?maxResults=10" + ] + example_api_calls_user_to_impersonate : local.google_workspace_example_admin + }, "gmail" : { source_kind : "gmail", availability : "ga", diff --git a/java/core/src/main/java/co/worklytics/psoxy/rules/google/PrebuiltSanitizerRules.java b/java/core/src/main/java/co/worklytics/psoxy/rules/google/PrebuiltSanitizerRules.java index 44eefd0117..42484dcfe8 100644 --- a/java/core/src/main/java/co/worklytics/psoxy/rules/google/PrebuiltSanitizerRules.java +++ b/java/core/src/main/java/co/worklytics/psoxy/rules/google/PrebuiltSanitizerRules.java @@ -347,6 +347,46 @@ public class PrebuiltSanitizerRules { .build()) .build(); + static final Set GDRIVE_LOG_EVENT_PARAMETERS_PII = ImmutableSet.of( + "owner", + "target_user", + "new_owner", + "delegating_principal" + ); + + static final Set GDRIVE_LOG_EVENT_PARAMETERS_ALLOWED = ImmutableSet.builder() + .addAll(GDRIVE_LOG_EVENT_PARAMETERS_PII) + .add("actor_is_collaborator_account", "billable", "primary_event") + .add("doc_id", "doc_type", "is_encrypted", "originating_app_id") + .add("owner_is_shared_drive", "owner_shared_drive_id", "shared_drive_id") + .add("visibility", "visibility_change", "old_visibility") + .add("added_role", "removed_role", "requested_role") + .add("destination_folder_id", "source_folder_id") + .add("copy_type") + .add("new_owner_is_shared_drive", "new_owner_shared_drive_id") + .add("target_domain", "membership_change_type") + .add("ip_address") + .build(); + + static final RESTRules GDRIVE_LOG = Rules2.builder() + .endpoint(Endpoint.builder() + .pathTemplate("/admin/reports/v1/activity/users/all/applications/drive") + .transform(Transform.Pseudonymize.builder() + .jsonPath("$..email") + .jsonPath("$.items[*].events[*].parameters[?(@.name in ['owner','target_user','new_owner','delegating_principal'])].value") + .build()) + .transform(HashIp.builder() + .jsonPath("$.items[*].ipAddress") + .jsonPath("$.items[*].events[*].parameters[?(@.name == 'ip_address')].value") + .build()) + .transform(Transform.Redact.builder() + .jsonPath("$.items[*].events[*].parameters[?(!(@.name =~ /^" + + String.join("|", GDRIVE_LOG_EVENT_PARAMETERS_ALLOWED) + + "$/i))]") + .build()) + .build()) + .build(); + static final RESTRules GEMINI_IN_WORKSPACE_APPS = Rules2.load("sources/google-workspace/gemini-in-workspace-apps/gemini-in-workspace-apps.yaml"); static final RESTRules GEMINI_IN_WORKSPACE_APPS_NO_APP_IDS = @@ -357,6 +397,7 @@ public class PrebuiltSanitizerRules { .put("gdirectory", GDIRECTORY) .put("gdirectory" + ConfigRulesModule.NO_APP_IDS_SUFFIX, GDIRECTORY_WITHOUT_GOOGLE_IDS) .put("gdrive", GDRIVE) + .put("gdrive-log", GDRIVE_LOG) .put("gmail", GMAIL) .put("google-chat", GOOGLE_CHAT) .put("google-meet", GOOGLE_MEET) diff --git a/java/core/src/test/java/co/worklytics/psoxy/rules/google/GDriveLogTests.java b/java/core/src/test/java/co/worklytics/psoxy/rules/google/GDriveLogTests.java new file mode 100644 index 0000000000..4f322ad4b0 --- /dev/null +++ b/java/core/src/test/java/co/worklytics/psoxy/rules/google/GDriveLogTests.java @@ -0,0 +1,59 @@ +package co.worklytics.psoxy.rules.google; + +import co.worklytics.psoxy.rules.JavaRulesTestBaseCase; +import co.worklytics.psoxy.rules.RESTRules; +import lombok.Getter; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; + +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.stream.Stream; + +class GDriveLogTests extends JavaRulesTestBaseCase { + + @Getter + final RESTRules rulesUnderTest = PrebuiltSanitizerRules.GDRIVE_LOG; + + @Getter + final RulesTestSpec rulesTestSpec = RulesTestSpec.builder() + .sourceFamily("google-workspace") + .defaultScopeId("gapps") + .sourceKind("gdrive-log") + .build(); + + @SneakyThrows + @Test + void activities() { + String endpoint = "https://admin.googleapis.com/admin/reports/v1/activity/users/all/applications/drive"; + String jsonString = asJson("drive-activities.json"); + + Collection PII = Arrays.asList( + "alice@worklytics.co", + "mary@worklytics.co", + "bob@worklytics.co", + "203.0.113.42" + ); + assertNotSanitized(jsonString, PII); + + String sanitized = + sanitizer.sanitize("GET", new URL(endpoint), jsonString); + + assertPseudonymized(sanitized, PII); + + assertNotSanitized(jsonString, "Meeting notes", "Shared with Finance"); + assertRedacted(sanitized, "Meeting notes", "Shared with Finance"); + + assertUrlWithQueryParamsAllowed(endpoint); + assertUrlBlocked("https://admin.googleapis.com/admin/reports/v1/activity/users/all/applications/meet"); + assertUrlBlocked("https://admin.googleapis.com/admin/reports/v1/activity/users/all/applications/chat"); + assertUrlBlocked("https://www.googleapis.com/drive/v3/files"); + } + + public Stream getExamples() { + return Stream.of( + InvocationExample.of("https://admin.googleapis.com/admin/reports/v1/activity/users/all/applications/drive", "drive-activities.json") + ); + } +} From 10ff45c971c3257e80975b46a124bd996bee7bf2 Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Tue, 1 Sep 2026 19:40:42 -0700 Subject: [PATCH 2/3] Harden examples-dev ./apply for shared remote state and worktree-local test tooling. Check AWS/Google/Azure auth and the Terraform backend before apply, and install psoxy-test in this checkout when node_modules is missing so worktrees do not inherit a false "already installed" from remote state. Co-authored-by: Cursor --- infra/examples-dev/README.md | 2 + infra/examples-dev/aws/.gitignore | 3 - infra/examples-dev/aws/apply | 35 +--- infra/examples-dev/aws/az-auth | 1 + infra/examples-dev/gcp/apply | 35 +--- infra/modules/psoxy-test-tool/main.tf | 9 +- tools/examples-dev-apply-preflight.sh | 253 ++++++++++++++++++++++++++ tools/examples-dev-apply.sh | 38 ++++ tools/install-test-tool.sh | 46 +++-- 9 files changed, 347 insertions(+), 75 deletions(-) create mode 120000 infra/examples-dev/aws/az-auth create mode 100755 tools/examples-dev-apply-preflight.sh create mode 100755 tools/examples-dev-apply.sh diff --git a/infra/examples-dev/README.md b/infra/examples-dev/README.md index a48b579aaa..5ad4e9b525 100644 --- a/infra/examples-dev/README.md +++ b/infra/examples-dev/README.md @@ -4,3 +4,5 @@ Unlike the published customer example template repositories ([AWS](https://githu purposes, referencing Psoxy-provided modules locally, rather than the published versions in GitHub. Each example includes a `reset-example` symlink to [`tools/reset-example.sh`](../../tools/reset-example.sh) for resetting local IaC state during development (back up / recover `terraform.tfvars`, etc.). + +Use `./apply` (not a bare `terraform apply`) so auth and backend checks run first. That helper calls [`tools/examples-dev-apply-preflight.sh`](../../tools/examples-dev-apply-preflight.sh), which requires `terraform.tfvars` and `backend.tf`, confirms the initialized Terraform backend matches `backend.tf` (prefer a remote backend so worktrees share state), and checks AWS, Google, and Azure CLI auth only when that example actually needs them. A GCS backend needs Google ADC even on the AWS example. Microsoft 365 (`msft_tenant_id`) needs Azure CLI, preferably sandboxed via `./az-auth` / `./auth` into `.azure`. It also installs `tools/psoxy-test` npm deps in this worktree when they are missing; Terraform state from another worktree is not enough, because `node_modules` is local and gitignored. diff --git a/infra/examples-dev/aws/.gitignore b/infra/examples-dev/aws/.gitignore index c1fc7b7ddb..80a6839a17 100644 --- a/infra/examples-dev/aws/.gitignore +++ b/infra/examples-dev/aws/.gitignore @@ -28,6 +28,3 @@ */upgrade-terraform-modules */upgrade-terraform-modules.sh - -# helper scripts -az-auth \ No newline at end of file diff --git a/infra/examples-dev/aws/apply b/infra/examples-dev/aws/apply index 9dc7ef6449..d8c35d10c1 100755 --- a/infra/examples-dev/aws/apply +++ b/infra/examples-dev/aws/apply @@ -1,33 +1,10 @@ #!/bin/bash +# Apply this examples-dev Terraform configuration after auth/backend preflight. -# Use a local Azure CLI config directory if present to help prevent conflicts with other Azure tenants -if [ -d "${PWD}/.azure" ]; then - export AZURE_CONFIG_DIR="${PWD}/.azure" -fi +set -euo pipefail -printf "Running preflight auth checks...\n" +EXAMPLE_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${EXAMPLE_DIR}/../../.." && pwd)" -if ! aws sts get-caller-identity &> /dev/null; then - printf "\033[0;31mERROR: AWS CLI is not authenticated or token is expired.\033[0m\n" - printf "Please authenticate (e.g. via aws sso login or exporting credentials) before applying.\n" - exit 1 -fi - -if grep -E "^msft_tenant_id" terraform.tfvars &> /dev/null; then - if ! az account show &> /dev/null; then - printf "\033[0;31mERROR: Azure CLI is not authenticated but msft_tenant_id is set in terraform.tfvars.\033[0m\n" - printf "Please run './az-auth' to authenticate before applying.\n" - exit 1 - fi -fi - -# prompt if user wants to force_bundle or not; default to Y -read -p "Do you want to force rebuild of the bundle? (eg, have you made java code changes?) (Y/n): " force_bundle - -if [ "$force_bundle" == "y" ] || [ "$force_bundle" == "Y" ] || [ "$force_bundle" == "" ]; then - force_bundle="true" -else - force_bundle="false" -fi - -terraform apply -auto-approve -var="force_bundle=$force_bundle" +cd "$EXAMPLE_DIR" +exec "${REPO_ROOT}/tools/examples-dev-apply.sh" diff --git a/infra/examples-dev/aws/az-auth b/infra/examples-dev/aws/az-auth new file mode 120000 index 0000000000..ad24b8afda --- /dev/null +++ b/infra/examples-dev/aws/az-auth @@ -0,0 +1 @@ +../../../tools/az-auth.sh \ No newline at end of file diff --git a/infra/examples-dev/gcp/apply b/infra/examples-dev/gcp/apply index a40080d1e2..d8c35d10c1 100755 --- a/infra/examples-dev/gcp/apply +++ b/infra/examples-dev/gcp/apply @@ -1,33 +1,10 @@ #!/bin/bash +# Apply this examples-dev Terraform configuration after auth/backend preflight. -# Use a local Azure CLI config directory if present to help prevent conflicts with other Azure tenants -if [ -d "${PWD}/.azure" ]; then - export AZURE_CONFIG_DIR="${PWD}/.azure" -fi +set -euo pipefail -printf "Running preflight auth checks...\n" +EXAMPLE_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${EXAMPLE_DIR}/../../.." && pwd)" -if ! gcloud auth print-access-token &> /dev/null; then - printf "\033[0;31mERROR: Google Cloud CLI is not authenticated or token is expired.\033[0m\n" - printf "Please run 'gcloud auth login' and/or 'gcloud auth application-default login' before applying.\n" - exit 1 -fi - -if grep -E "^msft_tenant_id" terraform.tfvars &> /dev/null; then - if ! az account show &> /dev/null; then - printf "\033[0;31mERROR: Azure CLI is not authenticated but msft_tenant_id is set in terraform.tfvars.\033[0m\n" - printf "Please run './az-auth' to authenticate before applying.\n" - exit 1 - fi -fi - -# prompt if user wants to force_bundle or not; default to Y -read -p "Do you want to force rebuild of the bundle? (eg, have you made java code changes?) (Y/n): " force_bundle - -if [ "$force_bundle" == "y" ] || [ "$force_bundle" == "Y" ] || [ "$force_bundle" == "" ]; then - force_bundle="true" -else - force_bundle="false" -fi - -terraform apply -auto-approve -var="force_bundle=$force_bundle" +cd "$EXAMPLE_DIR" +exec "${REPO_ROOT}/tools/examples-dev-apply.sh" diff --git a/infra/modules/psoxy-test-tool/main.tf b/infra/modules/psoxy-test-tool/main.tf index 6551530a2f..87f52b2ccb 100644 --- a/infra/modules/psoxy-test-tool/main.tf +++ b/infra/modules/psoxy-test-tool/main.tf @@ -3,9 +3,14 @@ locals { test_tool_directory = "${var.path_to_tools}/psoxy-test" } -# installs test tool to your machine -# (no affect if no NPM, or if test tool not at expected location) +# installs test tool into this checkout (tools/psoxy-test/node_modules) +# (no effect if no NPM, or if test tool not at expected location) # conditional, as we don't want to depend on test tool +# +# local-exec is recorded in Terraform state. With a remote backend shared across +# git worktrees, a successful install on another checkout will skip this resource +# even when this worktree has no node_modules. examples-dev ./apply also runs +# tools/install-test-tool.sh against the current checkout to cover that. resource "null_resource" "install_test_tool" { count = fileexists("${var.path_to_tools}/install-test-tool.sh") ? 1 : 0 diff --git a/tools/examples-dev-apply-preflight.sh b/tools/examples-dev-apply-preflight.sh new file mode 100755 index 0000000000..6f28b21725 --- /dev/null +++ b/tools/examples-dev-apply-preflight.sh @@ -0,0 +1,253 @@ +#!/bin/bash +# Preflight for infra/examples-dev/{aws,gcp} ./apply. +# Verifies terraform.tfvars, the configured Terraform backend, and cloud CLI +# auth for AWS / Google / Azure as relevant to this example. +# +# Run from an examples-dev terraform directory. Exits 1 if a required check fails. + +set -euo pipefail + +COLORSCHEME_SH="$(cd "$(dirname "$0")" && pwd)/set-term-colorscheme.sh" +if [ -f "$COLORSCHEME_SH" ]; then + # shellcheck source=set-term-colorscheme.sh + source "$COLORSCHEME_SH" +else + ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m' +fi + +FAILED=0 + +fail() { + printf " ${ERR}%s${NC}\n" "$1" + FAILED=1 +} + +ok() { + printf " ${SUCCESS}%s${NC}\n" "$1" +} + +info() { + printf " ${INFO}%s${NC}\n" "$1" +} + +warn() { + printf " ${WARN}%s${NC}\n" "$1" +} + +# Uncommented terraform.tfvars assignment, first match; empty if missing. +tfvar_value() { + local name="$1" + local line + line="$(grep -E "^[[:space:]]*${name}[[:space:]]*=" terraform.tfvars 2>/dev/null | head -1 || true)" + if [ -z "$line" ]; then + return 0 + fi + printf '%s' "$line" | cut -d'=' -f2- | tr -d ' "' | xargs +} + +tfvar_set() { + local value + value="$(tfvar_value "$1")" + [ -n "$value" ] && [ "$value" != "null" ] +} + +# First uncommented backend type in backend.tf (local, gcs, s3, remote, ...). +backend_type_from_tf() { + if [ ! -f backend.tf ]; then + printf '' + return 0 + fi + grep -E '^[[:space:]]*backend "' backend.tf 2>/dev/null | head -1 | sed 's/.*backend "\([^"]*\)".*/\1/' || true +} + +initialized_backend_type() { + local state=".terraform/terraform.tfstate" + if [ ! -f "$state" ]; then + printf '' + return 0 + fi + if command -v python3 >/dev/null 2>&1; then + python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("backend",{}).get("type",""))' "$state" 2>/dev/null || true + else + grep -m1 '"type":' "$state" | sed 's/.*"type":[[:space:]]*"\([^"]*\)".*/\1/' || true + fi +} + +initialized_backend_bucket() { + local state=".terraform/terraform.tfstate" + if [ ! -f "$state" ]; then + printf '' + return 0 + fi + if command -v python3 >/dev/null 2>&1; then + python3 -c 'import json,sys; c=json.load(open(sys.argv[1])).get("backend",{}).get("config") or {}; print(c.get("bucket") or c.get("hostname") or "")' "$state" 2>/dev/null || true + fi +} + +initialized_backend_prefix() { + local state=".terraform/terraform.tfstate" + if [ ! -f "$state" ]; then + printf '' + return 0 + fi + if command -v python3 >/dev/null 2>&1; then + python3 -c 'import json,sys; c=json.load(open(sys.argv[1])).get("backend",{}).get("config") or {}; print(c.get("prefix") or c.get("key") or "")' "$state" 2>/dev/null || true + fi +} + +printf "${INFO}Preflight: Terraform config and cloud auth${NC}\n\n" + +if [ ! -f terraform.tfvars ]; then + fail "No terraform.tfvars in ${PWD}." + printf " Create one (./init) or copy your personal overlay into this worktree.\n" + exit 1 +fi +ok "terraform.tfvars: ${PWD}/terraform.tfvars" + +if [ ! -f backend.tf ]; then + fail "No backend.tf in ${PWD}." + exit 1 +fi + +BACKEND_TF="$(backend_type_from_tf)" +INIT_BACKEND="$(initialized_backend_type)" +INIT_BUCKET="$(initialized_backend_bucket)" +INIT_PREFIX="$(initialized_backend_prefix)" + +if [ -z "$BACKEND_TF" ]; then + fail "Could not parse an uncommented backend block from backend.tf." +else + info "backend.tf: ${BACKEND_TF}" +fi + +if [ -z "$INIT_BACKEND" ]; then + fail "Terraform is not initialized (.terraform/terraform.tfstate missing). Run ${CODE}terraform init${NC}." +else + if [ -n "$INIT_BUCKET" ]; then + ok "initialized backend: ${INIT_BACKEND} (${INIT_BUCKET} / ${INIT_PREFIX})" + else + info "initialized backend: ${INIT_BACKEND}" + fi +fi + +if [ -n "$BACKEND_TF" ] && [ -n "$INIT_BACKEND" ] && [ "$BACKEND_TF" != "$INIT_BACKEND" ]; then + fail "backend.tf is ${CODE}${BACKEND_TF}${NC} but this directory is initialized as ${CODE}${INIT_BACKEND}${NC}." + printf " Run ${CODE}terraform init -reconfigure${NC} so apply uses the backend in backend.tf.\n" +fi + +if [ "$BACKEND_TF" = "local" ] || [ "$INIT_BACKEND" = "local" ]; then + warn "Local Terraform state will not be shared across worktrees or machines." + printf " For a personal examples-dev overlay, point backend.tf at a remote bucket and run ${CODE}terraform init -reconfigure${NC}.\n" +fi + +NEED_AWS=0 +NEED_GOOGLE=0 +NEED_AZURE=0 + +if grep -q 'provider "aws"' ./*.tf 2>/dev/null || tfvar_set aws_account_id; then + NEED_AWS=1 +fi +if grep -q 'provider "google"' ./*.tf 2>/dev/null \ + || tfvar_set gcp_project_id \ + || tfvar_set google_workspace_gcp_project_id \ + || [ "$BACKEND_TF" = "gcs" ] \ + || [ "$INIT_BACKEND" = "gcs" ]; then + NEED_GOOGLE=1 +fi +if grep -q 'provider "azuread"' ./*.tf 2>/dev/null || grep -q 'provider "azurerm"' ./*.tf 2>/dev/null; then + if tfvar_set msft_tenant_id; then + NEED_AZURE=1 + fi +fi + +printf "\n" + +if [ "$NEED_AWS" -eq 1 ]; then + printf "AWS\n" + if ! command -v aws >/dev/null 2>&1; then + fail "aws CLI is not installed." + elif ! aws sts get-caller-identity >/dev/null 2>&1; then + fail "AWS CLI is not authenticated. Run ${CODE}aws sso login${NC} or export credentials." + else + CALLER="$(aws sts get-caller-identity --query Arn --output text 2>/dev/null || true)" + ok "authenticated as ${CALLER}" + ASSUME_ROLE="$(tfvar_value aws_assume_role_arn)" + if [ -n "$ASSUME_ROLE" ] && [ "$ASSUME_ROLE" != "null" ]; then + if aws sts assume-role --role-arn "$ASSUME_ROLE" --role-session-name "examples-dev-apply-preflight" >/dev/null 2>&1; then + ok "can assume ${ASSUME_ROLE}" + else + fail "cannot assume ${CODE}${ASSUME_ROLE}${NC} from terraform.tfvars. Check trust policy and current credentials." + fi + fi + fi + printf "\n" +fi + +if [ "$NEED_GOOGLE" -eq 1 ]; then + printf "Google Cloud\n" + if ! command -v gcloud >/dev/null 2>&1; then + fail "gcloud CLI is not installed." + else + if ! gcloud auth print-access-token >/dev/null 2>&1; then + fail "gcloud user credentials are missing or expired. Run ${CODE}gcloud auth login${NC}." + else + ACCOUNT="$(gcloud config get-value account 2>/dev/null || true)" + ok "gcloud user: ${ACCOUNT}" + fi + if ! gcloud auth application-default print-access-token >/dev/null 2>&1; then + fail "Application Default Credentials are missing or expired (Terraform Google provider / GCS backend). Run ${CODE}gcloud auth application-default login${NC}." + else + ok "application-default credentials are valid" + fi + PROJECT="$(tfvar_value gcp_project_id)" + if [ -z "$PROJECT" ]; then + PROJECT="$(tfvar_value google_workspace_gcp_project_id)" + fi + if [ -n "$PROJECT" ]; then + info "tfvars GCP project: ${PROJECT}" + fi + fi + printf "\n" +fi + +azure_auth_hint() { + if [ -e ./az-auth ]; then + printf './az-auth' + elif [ -e ./auth ]; then + printf './auth' + else + printf '../../../tools/az-auth.sh' + fi +} + +if [ "$NEED_AZURE" -eq 1 ]; then + printf "Azure\n" + AZ_HINT="$(azure_auth_hint)" + if [ -d "${PWD}/.azure" ]; then + export AZURE_CONFIG_DIR="${PWD}/.azure" + info "using AZURE_CONFIG_DIR=${AZURE_CONFIG_DIR}" + else + warn "No .azure directory; using the default Azure CLI profile (may be a different tenant)." + printf " Run ${CODE}%s${NC} to sandbox auth for this example's ${CODE}msft_tenant_id${NC}.\n" "$AZ_HINT" + fi + if ! command -v az >/dev/null 2>&1; then + fail "Azure CLI is not installed." + elif ! az account show >/dev/null 2>&1; then + fail "Azure CLI is not authenticated but ${CODE}msft_tenant_id${NC} is set. Run ${CODE}${AZ_HINT}${NC}." + else + AZ_TENANT="$(az account show --query tenantId -o tsv 2>/dev/null || true)" + TF_TENANT="$(tfvar_value msft_tenant_id)" + ok "authenticated; tenant ${AZ_TENANT}" + if [ -n "$TF_TENANT" ] && [ "$AZ_TENANT" != "$TF_TENANT" ]; then + fail "Azure tenant ${AZ_TENANT} does not match terraform.tfvars msft_tenant_id=${TF_TENANT}. Run ${AZ_HINT}." + fi + fi + printf "\n" +fi + +if [ "$FAILED" -ne 0 ]; then + printf "${ERR}Preflight failed.${NC} Fix the issues above before applying.\n" + exit 1 +fi + +printf "${SUCCESS}Preflight passed.${NC}\n" diff --git a/tools/examples-dev-apply.sh b/tools/examples-dev-apply.sh new file mode 100755 index 0000000000..7781d1ac1a --- /dev/null +++ b/tools/examples-dev-apply.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Shared apply for infra/examples-dev/{aws,gcp}. Invoked by each example's ./apply. + +set -euo pipefail + +EXAMPLE_DIR="$(pwd)" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +if [ -d "${EXAMPLE_DIR}/.azure" ]; then + export AZURE_CONFIG_DIR="${EXAMPLE_DIR}/.azure" +fi + +COLORSCHEME_SH="${REPO_ROOT}/tools/set-term-colorscheme.sh" +if [ -f "$COLORSCHEME_SH" ]; then + # shellcheck source=set-term-colorscheme.sh + source "$COLORSCHEME_SH" +else + ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m' +fi + +"${REPO_ROOT}/tools/examples-dev-apply-preflight.sh" + +# node_modules is worktree-local and gitignored. Shared remote Terraform state may +# already record null_resource.install_test_tool from another checkout, so apply +# would skip npm even when this worktree cannot run ./test-*.sh. +printf "\n${INFO}Test tool (this worktree)${NC}\n" +"${REPO_ROOT}/tools/install-test-tool.sh" "${REPO_ROOT}/tools" + +read -r -p "Do you want to force rebuild of the bundle? (eg, have you made java code changes?) (Y/n): " force_bundle + +if [ "${force_bundle:-}" = "y" ] || [ "${force_bundle:-}" = "Y" ] || [ "${force_bundle:-}" = "" ]; then + force_bundle="true" +else + force_bundle="false" +fi + +printf "\n${INFO}terraform apply -auto-approve -var=force_bundle=%s${NC}\n\n" "$force_bundle" +terraform apply -auto-approve -var="force_bundle=$force_bundle" diff --git a/tools/install-test-tool.sh b/tools/install-test-tool.sh index 1711559262..6d05d272c7 100755 --- a/tools/install-test-tool.sh +++ b/tools/install-test-tool.sh @@ -1,26 +1,48 @@ #!/bin/bash -# Install test tool, if npm available +# Install tools/psoxy-test npm deps if they are missing in this checkout. +# Safe to re-run: skips when node_modules already has the expected packages. +# Worktrees do not share node_modules; Terraform state from another worktree is not proof this checkout is installed. -PATH_TO_TOOLS=${1:-$(pwd)/tools} +set -euo pipefail + +PATH_TO_TOOLS="${1:-$(pwd)/tools}" COLORSCHEME_SH="$(dirname "$0")/set-term-colorscheme.sh" if [ -f "$COLORSCHEME_SH" ]; then - source "$COLORSCHEME_SH" + # shellcheck source=set-term-colorscheme.sh + source "$COLORSCHEME_SH" else - ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m' + ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m' fi TEST_TOOL_ROOT="${PATH_TO_TOOLS}/psoxy-test" +CHALK_PKG="${TEST_TOOL_ROOT}/node_modules/chalk/package.json" -if [ ! -d ${TEST_TOOL_ROOT} ]; then +if [ ! -d "${TEST_TOOL_ROOT}" ]; then printf "${ERR}No test tool source found at ${TEST_TOOL_ROOT}. Failed to install test tool.${NC}\n" - exit + exit 1 fi -if npm -v &> /dev/null ; then - printf "Installing ${INFO}psoxy-test${NC} tool ...\n" - npm --no-audit --no-fund --prefix "${TEST_TOOL_ROOT}" install - printf "Test tool ${SUCCESS}successfully${NC} installed at ${SUCCESS}${TEST_TOOL_ROOT}${NC}\n" -else - printf "${ERR}NPM / Node.JS not available; could not install test tool. We recommend installing Node.js ( https://nodejs.org/ LTS version preferred), then re-running this init script.${NC}\n" +test_tool_installed_here() { + [ -f "${CHALK_PKG}" ] +} + +if test_tool_installed_here; then + printf "psoxy-test already installed at ${SUCCESS}${TEST_TOOL_ROOT}${NC}\n" + exit 0 +fi + +if ! command -v npm >/dev/null 2>&1; then + printf "${ERR}NPM / Node.JS not available; could not install test tool. Install Node.js (https://nodejs.org/ LTS version preferred), then re-run.${NC}\n" + exit 1 fi + +printf "Installing ${INFO}psoxy-test${NC} into this checkout (${CODE}${TEST_TOOL_ROOT}${NC}) ...\n" +npm --no-audit --no-fund --prefix "${TEST_TOOL_ROOT}" install + +if ! test_tool_installed_here; then + printf "${ERR}npm install finished but ${CODE}chalk${NC} is still missing under ${CODE}${TEST_TOOL_ROOT}/node_modules${NC}.${NC}\n" + exit 1 +fi + +printf "Test tool ${SUCCESS}successfully${NC} installed at ${SUCCESS}${TEST_TOOL_ROOT}${NC}\n" From f70434ac66b5276c1f76c64d6c11c6fe628a68d2 Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Fri, 4 Sep 2026 10:13:54 -0700 Subject: [PATCH 3/3] Harden gdrive-log sanitization for resourceDetails, actor IDs, and client IPs. Drive audit events leak titles, emails, Gaia IDs, and residential IPs outside the event-parameter allowlist; hash those and expand the example fixtures to cover the extra event types. Co-authored-by: Cursor --- .../google-workspace/gdrive-log/README.md | 2 + .../original/drive-activities.json | 430 ++++++++++++++++++ .../sanitized/drive-activities.json | 367 ++++++++++++++- .../gdrive-log/gdrive-log.yaml | 13 +- .../rules/google/PrebuiltSanitizerRules.java | 17 +- .../psoxy/rules/google/GDriveLogTests.java | 40 +- 6 files changed, 861 insertions(+), 8 deletions(-) diff --git a/docs/sources/google-workspace/gdrive-log/README.md b/docs/sources/google-workspace/gdrive-log/README.md index 89347e4a48..7ea4783138 100644 --- a/docs/sources/google-workspace/gdrive-log/README.md +++ b/docs/sources/google-workspace/gdrive-log/README.md @@ -15,6 +15,8 @@ Relative to [`gdrive`](../gdrive/README.md), this source includes additional act The tradeoff is lookback. Google retains Drive audit-log events for about **6 months** (180 days). Older activity is not available from the Reports API, even if the files themselves still exist. Historical collaboration that predates that window is better covered by the files API connector. +Activity records can include the actor's client IP, which is often a residential address. Those values are hashed with the `hashIp` transform (emitted as `t~...` tokens), not passed through in the clear. + See Google's [Drive activity report](https://developers.google.com/workspace/admin/reports/v1/guides/manage-audit-drive) and [data retention](https://support.google.com/a/answer/7061566) documentation. ## Required OAuth Scopes diff --git a/docs/sources/google-workspace/gdrive-log/example-api-responses/original/drive-activities.json b/docs/sources/google-workspace/gdrive-log/example-api-responses/original/drive-activities.json index a6c1fb8895..23f4badf6d 100644 --- a/docs/sources/google-workspace/gdrive-log/example-api-responses/original/drive-activities.json +++ b/docs/sources/google-workspace/gdrive-log/example-api-responses/original/drive-activities.json @@ -19,6 +19,11 @@ }, "ipAddress": "203.0.113.42", "ownerDomain": "worklytics.co", + "networkInfo": { + "ipAsn": [13335], + "regionCode": "US", + "subdivisionCode": "US-CA" + }, "events": [ { "type": "access", @@ -28,6 +33,22 @@ "name": "primary_event", "boolValue": true }, + { + "name": "billable", + "boolValue": true + }, + { + "name": "actor_is_collaborator_account", + "boolValue": false + }, + { + "name": "is_encrypted", + "boolValue": false + }, + { + "name": "originating_app_id", + "value": "1066455591999" + }, { "name": "owner_is_shared_drive", "boolValue": false @@ -51,9 +72,46 @@ { "name": "visibility", "value": "shared_internally" + }, + { + "name": "ip_address", + "value": "203.0.113.42" } ] } + ], + "resourceDetails": [ + { + "id": "1FakeDriveItemId000000000000000000001", + "title": "4242424242.pdf", + "type": "DRIVE_ITEM", + "relation": "DRIVE_PRIMARY", + "appliedLabels": [ + { + "id": "labelIdConfidential", + "title": "Confidential", + "fieldValues": [ + { + "id": "fieldOwnerOrg", + "displayName": "Owning team", + "type": "text", + "textValue": "Finance" + } + ] + } + ], + "ownerDetails": { + "ownerType": "USER", + "ownerIdentity": [ + { + "userIdentity": { + "id": "0neiluseridxxxxx", + "userEmail": "neil@worklytics.co" + } + } + ] + } + } ] }, { @@ -105,9 +163,21 @@ "name": "added_role", "value": "can_edit" }, + { + "name": "old_visibility", + "value": "private" + }, { "name": "visibility", "value": "shared_internally" + }, + { + "name": "visibility_change", + "value": "shared_internally" + }, + { + "name": "recipients", + "multiValue": ["bob@worklytics.co"] } ] } @@ -144,6 +214,14 @@ "name": "doc_title", "value": "Meeting notes" }, + { + "name": "source_folder_id", + "value": "0Baaaaaaaaaaaaaaaaaaaaaa" + }, + { + "name": "source_folder_title", + "value": "My Drive" + }, { "name": "destination_folder_id", "value": "0Bxxxxxxxxxxxxxxxxxxxxx" @@ -163,6 +241,358 @@ ] } ] + }, + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T16:22:08.441Z", + "uniqueQualifier": "6127585580824196363", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem4\"", + "actor": { + "callerType": "USER", + "email": "bob@worklytics.co", + "profileId": "100531288453445237356" + }, + "ipAddress": "203.0.113.42", + "ownerDomain": "worklytics.co", + "events": [ + { + "type": "access", + "name": "view", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "doc_id", + "value": "1Q3PlanningSheetFakeId00000000000000000001" + }, + { + "name": "doc_title", + "value": "Q3 planning spreadsheet" + }, + { + "name": "doc_type", + "value": "spreadsheet" + }, + { + "name": "owner", + "value": "alice@worklytics.co" + }, + { + "name": "visibility", + "value": "shared_internally" + } + ] + } + ] + }, + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T16:40:11.002Z", + "uniqueQualifier": "6127585580824196364", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem5\"", + "actor": { + "callerType": "USER", + "email": "alice@worklytics.co", + "profileId": "117927411761819390794" + }, + "ipAddress": "203.0.113.42", + "ownerDomain": "worklytics.co", + "events": [ + { + "type": "access", + "name": "download", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "doc_id", + "value": "1CustomerListCsvFakeId0000000000000000002" + }, + { + "name": "doc_title", + "value": "Customer list.csv" + }, + { + "name": "doc_type", + "value": "msword" + }, + { + "name": "owner", + "value": "mary@worklytics.co" + }, + { + "name": "visibility", + "value": "private" + } + ] + } + ] + }, + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T17:05:33.900Z", + "uniqueQualifier": "6127585580824196365", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem6\"", + "actor": { + "callerType": "USER", + "email": "mary@worklytics.co", + "profileId": "100531288453445237356" + }, + "ownerDomain": "worklytics.co", + "events": [ + { + "type": "access", + "name": "rename", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "doc_id", + "value": "1BoardDeckFakeId00000000000000000000000003" + }, + { + "name": "doc_title", + "value": "FY26 compensation model" + }, + { + "name": "doc_type", + "value": "presentation" + }, + { + "name": "old_value", + "value": "Draft - compensation" + }, + { + "name": "new_value", + "value": "FY26 compensation model" + }, + { + "name": "owner", + "value": "mary@worklytics.co" + }, + { + "name": "visibility", + "value": "private" + } + ] + } + ] + }, + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T17:18:00.010Z", + "uniqueQualifier": "6127585580824196366", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem7\"", + "actor": { + "callerType": "USER", + "email": "bob@worklytics.co", + "profileId": "100531288453445237356" + }, + "ipAddress": "203.0.113.42", + "ownerDomain": "worklytics.co", + "events": [ + { + "type": "access", + "name": "create_comment", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "doc_id", + "value": "1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name": "doc_title", + "value": "Meeting notes" + }, + { + "name": "doc_type", + "value": "document" + }, + { + "name": "owner", + "value": "mary@worklytics.co" + }, + { + "name": "target_user", + "value": "alice@worklytics.co" + } + ] + } + ] + }, + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T17:31:44.777Z", + "uniqueQualifier": "6127585580824196367", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem8\"", + "actor": { + "callerType": "USER", + "email": "alice@worklytics.co", + "profileId": "117927411761819390794" + }, + "ownerDomain": "worklytics.co", + "events": [ + { + "type": "access", + "name": "copy", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "copy_type", + "value": "copy" + }, + { + "name": "doc_id", + "value": "1CopiedDocFakeId00000000000000000000000004" + }, + { + "name": "doc_title", + "value": "Copy of Meeting notes" + }, + { + "name": "doc_type", + "value": "document" + }, + { + "name": "owner", + "value": "alice@worklytics.co" + }, + { + "name": "destination_folder_id", + "value": "0Byyyyyyyyyyyyyyyyyyyyyyy" + }, + { + "name": "destination_folder_title", + "value": "Personal drafts" + } + ] + } + ] + }, + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T18:02:19.500Z", + "uniqueQualifier": "6127585580824196368", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem9\"", + "actor": { + "callerType": "USER", + "email": "alice@worklytics.co", + "profileId": "117927411761819390794" + }, + "ipAddress": "203.0.113.42", + "ownerDomain": "worklytics.co", + "events": [ + { + "type": "access", + "name": "search", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "query_type", + "value": "full_text" + }, + { + "name": "user_query", + "value": "salary review 2024" + }, + { + "name": "parsed_query", + "value": "salary review 2024" + } + ] + } + ] + }, + { + "kind": "admin#reports#activity", + "id": { + "time": "2021-10-12T18:15:01.000Z", + "uniqueQualifier": "6127585580824196369", + "applicationName": "drive", + "customerId": "C03v4gz0f" + }, + "etag": "\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem10\"", + "actor": { + "callerType": "USER", + "email": "mary@worklytics.co", + "profileId": "100531288453445237356" + }, + "ownerDomain": "worklytics.co", + "events": [ + { + "type": "acl_change", + "name": "change_owner", + "parameters": [ + { + "name": "primary_event", + "boolValue": true + }, + { + "name": "doc_id", + "value": "1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name": "doc_title", + "value": "Meeting notes" + }, + { + "name": "doc_type", + "value": "document" + }, + { + "name": "owner", + "value": "mary@worklytics.co" + }, + { + "name": "new_owner", + "value": "alice@worklytics.co" + }, + { + "name": "new_owner_is_shared_drive", + "boolValue": false + } + ] + } + ] } ] } diff --git a/docs/sources/google-workspace/gdrive-log/example-api-responses/sanitized/drive-activities.json b/docs/sources/google-workspace/gdrive-log/example-api-responses/sanitized/drive-activities.json index 3a0199f4b6..c19eaa7bfc 100644 --- a/docs/sources/google-workspace/gdrive-log/example-api-responses/sanitized/drive-activities.json +++ b/docs/sources/google-workspace/gdrive-log/example-api-responses/sanitized/drive-activities.json @@ -15,10 +15,17 @@ "actor":{ "callerType":"USER", "email":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co", - "profileId":"117927411761819390794" + "profileId":"t~Uya4owtCJeJfhhg3CYojl9Xjpe10yvY_5XOH14V7TxE" }, "ipAddress":"t~FF_NukhrsL6d49CqBcU9AAjHiGZYMHeLvoi9QuSKfE0", "ownerDomain":"worklytics.co", + "networkInfo":{ + "ipAsn":[ + 13335 + ], + "regionCode":"US", + "subdivisionCode":"US-CA" + }, "events":[ { "type":"access", @@ -28,6 +35,22 @@ "name":"primary_event", "boolValue":true }, + { + "name":"billable", + "boolValue":true + }, + { + "name":"actor_is_collaborator_account", + "boolValue":false + }, + { + "name":"is_encrypted", + "boolValue":false + }, + { + "name":"originating_app_id", + "value":"1066455591999" + }, { "name":"owner_is_shared_drive", "boolValue":false @@ -47,9 +70,31 @@ { "name":"visibility", "value":"shared_internally" + }, + { + "name":"ip_address", + "value":"t~FF_NukhrsL6d49CqBcU9AAjHiGZYMHeLvoi9QuSKfE0" } ] } + ], + "resourceDetails":[ + { + "id":"1FakeDriveItemId000000000000000000001", + "type":"DRIVE_ITEM", + "relation":"DRIVE_PRIMARY", + "ownerDetails":{ + "ownerType":"USER", + "ownerIdentity":[ + { + "userIdentity":{ + "id":"t~zzqNx5rfLMw4pjyuYGvGDIhcvcG4zaI7GrDNSZhjlEc", + "userEmail":"t~7LFvqPBD0TK__e9lJRtb9yQF6RdJOPf41SdhwZlBguI@worklytics.co" + } + } + ] + } + } ] }, { @@ -64,7 +109,7 @@ "actor":{ "callerType":"USER", "email":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co", - "profileId":"117927411761819390794" + "profileId":"t~Uya4owtCJeJfhhg3CYojl9Xjpe10yvY_5XOH14V7TxE" }, "ipAddress":"t~FF_NukhrsL6d49CqBcU9AAjHiGZYMHeLvoi9QuSKfE0", "ownerDomain":"worklytics.co", @@ -97,9 +142,17 @@ "name":"added_role", "value":"can_edit" }, + { + "name":"old_visibility", + "value":"private" + }, { "name":"visibility", "value":"shared_internally" + }, + { + "name":"visibility_change", + "value":"shared_internally" } ] } @@ -117,7 +170,7 @@ "actor":{ "callerType":"USER", "email":"t~ANg5l6XVBpn-KyO-ivYDGQwZViEvM2uhytCnKiB0qr4@worklytics.co", - "profileId":"100531288453445237356" + "profileId":"t~QC9uUiOogI34RablRTn07ze4n6xOTmznaaPCq9i2ZyE" }, "events":[ { @@ -132,6 +185,10 @@ "name":"doc_id", "value":"1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" }, + { + "name":"source_folder_id", + "value":"0Baaaaaaaaaaaaaaaaaaaaaa" + }, { "name":"destination_folder_id", "value":"0Bxxxxxxxxxxxxxxxxxxxxx" @@ -147,6 +204,310 @@ ] } ] + }, + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T16:22:08.441Z", + "uniqueQualifier":"6127585580824196363", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem4\"", + "actor":{ + "callerType":"USER", + "email":"t~ANg5l6XVBpn-KyO-ivYDGQwZViEvM2uhytCnKiB0qr4@worklytics.co", + "profileId":"t~QC9uUiOogI34RablRTn07ze4n6xOTmznaaPCq9i2ZyE" + }, + "ipAddress":"t~FF_NukhrsL6d49CqBcU9AAjHiGZYMHeLvoi9QuSKfE0", + "ownerDomain":"worklytics.co", + "events":[ + { + "type":"access", + "name":"view", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + }, + { + "name":"doc_id", + "value":"1Q3PlanningSheetFakeId00000000000000000001" + }, + { + "name":"doc_type", + "value":"spreadsheet" + }, + { + "name":"owner", + "value":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co" + }, + { + "name":"visibility", + "value":"shared_internally" + } + ] + } + ] + }, + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T16:40:11.002Z", + "uniqueQualifier":"6127585580824196364", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem5\"", + "actor":{ + "callerType":"USER", + "email":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co", + "profileId":"t~Uya4owtCJeJfhhg3CYojl9Xjpe10yvY_5XOH14V7TxE" + }, + "ipAddress":"t~FF_NukhrsL6d49CqBcU9AAjHiGZYMHeLvoi9QuSKfE0", + "ownerDomain":"worklytics.co", + "events":[ + { + "type":"access", + "name":"download", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + }, + { + "name":"doc_id", + "value":"1CustomerListCsvFakeId0000000000000000002" + }, + { + "name":"doc_type", + "value":"msword" + }, + { + "name":"owner", + "value":"t~SuLKx-wI7BY6ymwyzWhgNm1k2HNYbLCtZB2GQlqflXk@worklytics.co" + }, + { + "name":"visibility", + "value":"private" + } + ] + } + ] + }, + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T17:05:33.900Z", + "uniqueQualifier":"6127585580824196365", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem6\"", + "actor":{ + "callerType":"USER", + "email":"t~SuLKx-wI7BY6ymwyzWhgNm1k2HNYbLCtZB2GQlqflXk@worklytics.co", + "profileId":"t~QC9uUiOogI34RablRTn07ze4n6xOTmznaaPCq9i2ZyE" + }, + "ownerDomain":"worklytics.co", + "events":[ + { + "type":"access", + "name":"rename", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + }, + { + "name":"doc_id", + "value":"1BoardDeckFakeId00000000000000000000000003" + }, + { + "name":"doc_type", + "value":"presentation" + }, + { + "name":"owner", + "value":"t~SuLKx-wI7BY6ymwyzWhgNm1k2HNYbLCtZB2GQlqflXk@worklytics.co" + }, + { + "name":"visibility", + "value":"private" + } + ] + } + ] + }, + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T17:18:00.010Z", + "uniqueQualifier":"6127585580824196366", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem7\"", + "actor":{ + "callerType":"USER", + "email":"t~ANg5l6XVBpn-KyO-ivYDGQwZViEvM2uhytCnKiB0qr4@worklytics.co", + "profileId":"t~QC9uUiOogI34RablRTn07ze4n6xOTmznaaPCq9i2ZyE" + }, + "ipAddress":"t~FF_NukhrsL6d49CqBcU9AAjHiGZYMHeLvoi9QuSKfE0", + "ownerDomain":"worklytics.co", + "events":[ + { + "type":"access", + "name":"create_comment", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + }, + { + "name":"doc_id", + "value":"1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name":"doc_type", + "value":"document" + }, + { + "name":"owner", + "value":"t~SuLKx-wI7BY6ymwyzWhgNm1k2HNYbLCtZB2GQlqflXk@worklytics.co" + }, + { + "name":"target_user", + "value":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co" + } + ] + } + ] + }, + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T17:31:44.777Z", + "uniqueQualifier":"6127585580824196367", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem8\"", + "actor":{ + "callerType":"USER", + "email":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co", + "profileId":"t~Uya4owtCJeJfhhg3CYojl9Xjpe10yvY_5XOH14V7TxE" + }, + "ownerDomain":"worklytics.co", + "events":[ + { + "type":"access", + "name":"copy", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + }, + { + "name":"copy_type", + "value":"copy" + }, + { + "name":"doc_id", + "value":"1CopiedDocFakeId00000000000000000000000004" + }, + { + "name":"doc_type", + "value":"document" + }, + { + "name":"owner", + "value":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co" + }, + { + "name":"destination_folder_id", + "value":"0Byyyyyyyyyyyyyyyyyyyyyyy" + } + ] + } + ] + }, + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T18:02:19.500Z", + "uniqueQualifier":"6127585580824196368", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem9\"", + "actor":{ + "callerType":"USER", + "email":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co", + "profileId":"t~Uya4owtCJeJfhhg3CYojl9Xjpe10yvY_5XOH14V7TxE" + }, + "ipAddress":"t~FF_NukhrsL6d49CqBcU9AAjHiGZYMHeLvoi9QuSKfE0", + "ownerDomain":"worklytics.co", + "events":[ + { + "type":"access", + "name":"search", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + } + ] + } + ] + }, + { + "kind":"admin#reports#activity", + "id":{ + "time":"2021-10-12T18:15:01.000Z", + "uniqueQualifier":"6127585580824196369", + "applicationName":"drive", + "customerId":"C03v4gz0f" + }, + "etag":"\"g2-KWSBhdhEV1DJNuuX35hICepPpOEDe4lPvnwuxvf4/driveLogItem10\"", + "actor":{ + "callerType":"USER", + "email":"t~SuLKx-wI7BY6ymwyzWhgNm1k2HNYbLCtZB2GQlqflXk@worklytics.co", + "profileId":"t~QC9uUiOogI34RablRTn07ze4n6xOTmznaaPCq9i2ZyE" + }, + "ownerDomain":"worklytics.co", + "events":[ + { + "type":"acl_change", + "name":"change_owner", + "parameters":[ + { + "name":"primary_event", + "boolValue":true + }, + { + "name":"doc_id", + "value":"1DWuYM3ot_sAyEQqOz0xWJ9bVMSYzOmRNeBqbgtSwuK8" + }, + { + "name":"doc_type", + "value":"document" + }, + { + "name":"owner", + "value":"t~SuLKx-wI7BY6ymwyzWhgNm1k2HNYbLCtZB2GQlqflXk@worklytics.co" + }, + { + "name":"new_owner", + "value":"t~Qf4dLJ4jfqZLn9ef4VirvYjvOnRaVI5tf5oLnM65YOA@worklytics.co" + }, + { + "name":"new_owner_is_shared_drive", + "boolValue":false + } + ] + } + ] } ] } diff --git a/docs/sources/google-workspace/gdrive-log/gdrive-log.yaml b/docs/sources/google-workspace/gdrive-log/gdrive-log.yaml index c9ef630095..0769885c07 100644 --- a/docs/sources/google-workspace/gdrive-log/gdrive-log.yaml +++ b/docs/sources/google-workspace/gdrive-log/gdrive-log.yaml @@ -5,12 +5,23 @@ endpoints: - ! jsonPaths: - "$..email" + - "$..userEmail" + - "$..groupEmail" + - "$.items[*].actor.profileId" + - "$..userIdentity.id" + - "$..groupIdentity.id" - "$.items[*].events[*].parameters[?(@.name in ['owner','target_user','new_owner','delegating_principal'])].value" encoding: "URL_SAFE_TOKEN" - ! jsonPaths: - - "$.items[*].ipAddress" + - "$..ipAddress" - "$.items[*].events[*].parameters[?(@.name == 'ip_address')].value" + - "$.items[*].events[*].parameters[?(@.name == 'ip_address')].multiValue[*]" - ! jsonPaths: - "$.items[*].events[*].parameters[?(!(@.name =~ /^owner|target_user|new_owner|delegating_principal|actor_is_collaborator_account|billable|primary_event|doc_id|doc_type|is_encrypted|originating_app_id|owner_is_shared_drive|owner_shared_drive_id|shared_drive_id|visibility|visibility_change|old_visibility|added_role|removed_role|requested_role|destination_folder_id|source_folder_id|copy_type|new_owner_is_shared_drive|new_owner_shared_drive_id|target_domain|membership_change_type|ip_address$/i))]" + - "$.items[*].resourceDetails[*].title" + - "$.items[*].resourceDetails[*]..title" + - "$.items[*].resourceDetails[*]..displayName" + - "$.items[*].resourceDetails[*].appliedLabels" + - "$.items[*].events[*].sensitiveParameters" diff --git a/java/core/src/main/java/co/worklytics/psoxy/rules/google/PrebuiltSanitizerRules.java b/java/core/src/main/java/co/worklytics/psoxy/rules/google/PrebuiltSanitizerRules.java index 42484dcfe8..05e50581c6 100644 --- a/java/core/src/main/java/co/worklytics/psoxy/rules/google/PrebuiltSanitizerRules.java +++ b/java/core/src/main/java/co/worklytics/psoxy/rules/google/PrebuiltSanitizerRules.java @@ -373,16 +373,31 @@ public class PrebuiltSanitizerRules { .pathTemplate("/admin/reports/v1/activity/users/all/applications/drive") .transform(Transform.Pseudonymize.builder() .jsonPath("$..email") + .jsonPath("$..userEmail") + .jsonPath("$..groupEmail") + .jsonPath("$.items[*].actor.profileId") + .jsonPath("$..userIdentity.id") + .jsonPath("$..groupIdentity.id") .jsonPath("$.items[*].events[*].parameters[?(@.name in ['owner','target_user','new_owner','delegating_principal'])].value") .build()) .transform(HashIp.builder() - .jsonPath("$.items[*].ipAddress") + // Drive audit events include the actor's client IP, which is often a + // residential/home address. Hash (tokenize as t~...) rather than pass through. + .jsonPath("$..ipAddress") .jsonPath("$.items[*].events[*].parameters[?(@.name == 'ip_address')].value") + .jsonPath("$.items[*].events[*].parameters[?(@.name == 'ip_address')].multiValue[*]") .build()) .transform(Transform.Redact.builder() .jsonPath("$.items[*].events[*].parameters[?(!(@.name =~ /^" + String.join("|", GDRIVE_LOG_EVENT_PARAMETERS_ALLOWED) + "$/i))]") + // resourceDetails.title / labels / display names are content; user ids + // and emails are handled by the pseudonymize transform above + .jsonPath("$.items[*].resourceDetails[*].title") + .jsonPath("$.items[*].resourceDetails[*]..title") + .jsonPath("$.items[*].resourceDetails[*]..displayName") + .jsonPath("$.items[*].resourceDetails[*].appliedLabels") + .jsonPath("$.items[*].events[*].sensitiveParameters") .build()) .build()) .build(); diff --git a/java/core/src/test/java/co/worklytics/psoxy/rules/google/GDriveLogTests.java b/java/core/src/test/java/co/worklytics/psoxy/rules/google/GDriveLogTests.java index 4f322ad4b0..cdfe01b965 100644 --- a/java/core/src/test/java/co/worklytics/psoxy/rules/google/GDriveLogTests.java +++ b/java/core/src/test/java/co/worklytics/psoxy/rules/google/GDriveLogTests.java @@ -2,6 +2,7 @@ import co.worklytics.psoxy.rules.JavaRulesTestBaseCase; import co.worklytics.psoxy.rules.RESTRules; +import com.avaulta.gateway.rules.transforms.HashIp; import lombok.Getter; import lombok.SneakyThrows; import org.junit.jupiter.api.Test; @@ -33,17 +34,50 @@ void activities() { "alice@worklytics.co", "mary@worklytics.co", "bob@worklytics.co", - "203.0.113.42" + "neil@worklytics.co" ); assertNotSanitized(jsonString, PII); + assertNotSanitized(jsonString, "203.0.113.42"); String sanitized = sanitizer.sanitize("GET", new URL(endpoint), jsonString); assertPseudonymized(sanitized, PII); + assertTransformed(sanitized, Arrays.asList("203.0.113.42"), HashIp.builder().build()); + assertPseudonymized(sanitized, + "117927411761819390794", + "100531288453445237356", + "0neiluseridxxxxx" + ); - assertNotSanitized(jsonString, "Meeting notes", "Shared with Finance"); - assertRedacted(sanitized, "Meeting notes", "Shared with Finance"); + assertNotSanitized(jsonString, + "Meeting notes", + "Shared with Finance", + "Q3 planning spreadsheet", + "Customer list.csv", + "FY26 compensation model", + "Draft - compensation", + "Copy of Meeting notes", + "Personal drafts", + "salary review 2024", + "Confidential", + "Owning team", + "4242424242.pdf" + ); + assertRedacted(sanitized, + "Meeting notes", + "Shared with Finance", + "Q3 planning spreadsheet", + "Customer list.csv", + "FY26 compensation model", + "Draft - compensation", + "Copy of Meeting notes", + "Personal drafts", + "salary review 2024", + "Confidential", + "Owning team", + "4242424242.pdf" + ); assertUrlWithQueryParamsAllowed(endpoint); assertUrlBlocked("https://admin.googleapis.com/admin/reports/v1/activity/users/all/applications/meet");