PPR replace GCP SA keys with container ADC for GCP API calls

jobs/permanent/secured-party-notification/devops/vaults.gcp.env

@@ -24,7 +24,6 @@ NOTIFY_STATUS_RECIPIENTS="op://ppr/$APP_ENV/secured-party/NOTIFY_STATUS_RECIPIEN
 NOTIFY_STATUS_SUBJECT="op://ppr/$APP_ENV/secured-party/NOTIFY_STATUS_SUBJECT"
 NOTIFY_STATUS_BODY="op://ppr/$APP_ENV/secured-party/NOTIFY_STATUS_BODY"
 JOB_INTERVAL_HOURS="op://ppr/$APP_ENV/secured-party/JOB_INTERVAL_HOURS"
-GOOGLE_DEFAULT_SA="op://buckets/$APP_ENV/ppr-api/GOOGLE_DEFAULT_SA"
 GCP_CS_SA_SCOPES="op://buckets/$APP_ENV/ppr-api/GCP_CS_SA_SCOPES"
 GCP_CS_BUCKET_ID_MAIL="op://buckets/$APP_ENV/ppr-api/GCP_CS_BUCKET_ID_MAIL"
 VPC_CONNECTOR="op://CD/$APP_ENV/ppr-secured-party/VPC_CONNECTOR"

jobs/permanent/secured-party-notification/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "secured-party-notification"
-version = "0.0.1"
+version = "0.0.2"
 description = "PPR liens secured party nofifications on debtor changes."
 authors = ["thor wolpert <thor@daxiom.com>", "doug lovett <doug@daxiom.com>"]
 license = "BSD 3"

jobs/permanent/secured-party-notification/src/secured_party_notification/config.py

@@ -36,6 +36,8 @@ class Config(BaseConfig):
 
     AUTH_SVC_URL = f"{AUTH_API_URL + AUTH_API_VERSION}"
     NOTIFY_SVC_URL = f"{NOTIFY_API_URL + NOTIFY_API_VERSION}"
+    if not NOTIFY_SVC_URL.endswith("/notify"):
+        NOTIFY_SVC_URL += "/notify"
 
     JWT_OIDC_TOKEN_URL = os.getenv("JWT_OIDC_TOKEN_URL")
     # service accounts

jobs/permanent/secured-party-notification/src/secured_party_notification/services/document_storage/storage_service.py

@@ -43,7 +43,7 @@ def init_app(config: Config):
         """Set up the service"""
         bucket_id = config.GCP_CS_BUCKET_ID_MAIL
         credentials = GoogleAuthService.get_credentials()
-        storage_client = storage.Client(credentials=credentials)
+        storage_client = storage.Client(credentials=credentials) if credentials else storage.Client()
         GoogleStorageService.GCP_BUCKET = storage_client.bucket(bucket_id)
         GoogleStorageService.GCP_BUCKET_ID_MAIL = bucket_id
         GoogleStorageService.GCP_CREDENTIALS = credentials
@@ -112,10 +112,20 @@ def __call_cs_api(cls, method: str, name: str, data=None, content_type: str = CO
     @classmethod
     def __call_cs_api_link(cls, name: str, data=None, available_days: int = 1, content_type: str = CONTENT_TYPE_PDF):
         """Call the Cloud Storage API, returning a time-limited download link."""
-        blob = GoogleStorageService.GCP_BUCKET.blob(name)
+        creds = GoogleAuthService.get_cs_signed_credentials()
+        if not creds:
+            logger.warning(f"No credentials to create signed storage link for {name}")
+            return ""
+        s_client = storage.Client(credentials=creds)
+        bucket = s_client.bucket(cls.GCP_BUCKET_ID_MAIL)
+        blob = bucket.blob(name)
         if data:
             blob.upload_from_string(data=data, content_type=content_type)
         url = blob.generate_signed_url(
-            version="v4", expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), method="GET"
+            version="v4",
+            expiration=datetime.timedelta(days=available_days, hours=0, minutes=0),
+            method="GET",
+            service_account_email=creds.service_account_email,
+            access_token=creds.token,
         )
         return url

jobs/permanent/secured-party-notification/src/secured_party_notification/services/gcp_auth/auth_service.py

@@ -36,10 +36,6 @@ class GoogleAuthService:  # pylint: disable=too-few-public-methods
     service_account_info = None
     credentials = None
     report_api_audience = None
-    # Use service account env var if available.
-    if gcp_auth_key:
-        sa_bytes = bytes(gcp_auth_key, "utf-8")
-        service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8")))
 
     @staticmethod
     def init_app(config: Config):
@@ -49,11 +45,19 @@ def init_app(config: Config):
         if GoogleAuthService.gcp_auth_key:
             sa_bytes = bytes(GoogleAuthService.gcp_auth_key, "utf-8")
             GoogleAuthService.service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8")))
+            GoogleAuthService.credentials = service_account.Credentials.from_service_account_info(
+                GoogleAuthService.service_account_info, scopes=GoogleAuthService.gcp_sa_scopes
+            )
+        else:
+            logger.info("auth_service.init_app no SA info.")
         GoogleAuthService.report_api_audience = config.REPORT_API_AUDIENCE
 
     @classmethod
     def get_token(cls):
         """Generate an OAuth access token with cloud storage access."""
+        if not cls.gcp_auth_key or not cls.service_account_info:
+            return None
+
         if cls.credentials is None:
             cls.credentials = service_account.Credentials.from_service_account_info(
                 cls.service_account_info, scopes=cls.gcp_sa_scopes
@@ -76,9 +80,25 @@ def get_report_api_token(cls):
     @classmethod
     def get_credentials(cls):
         """Generate GCP auth credentials to pass to a GCP client."""
+        if not cls.gcp_auth_key or not cls.service_account_info:
+            return None
+
         if cls.credentials is None:
             cls.credentials = service_account.Credentials.from_service_account_info(
                 cls.service_account_info, scopes=cls.gcp_sa_scopes
             )
-        logger.info("Call successful: obtained credentials.")
+        logger.debug("Call successful: obtained credentials.")
         return cls.credentials
+
+    @classmethod
+    def get_cs_signed_credentials(cls):
+        """Extra steps for ADC cloud storage signed url - requires cert to sign."""
+        if cls.gcp_auth_key and cls.service_account_info:
+            return cls.get_credentials()
+
+        # Load default credentials
+        credentials, project = google.auth.default()  # pylint: disable=unused-variable; gcp api response
+        # Refresh credentials to ensure an access token is available
+        auth_request = google.auth.transport.requests.Request()
+        credentials.refresh(auth_request)
+        return credentials

ppr-api/devops/vaults.gcp.env

@@ -26,7 +26,6 @@ REPORT_TEMPLATE_PATH="op://API/$APP_ENV/report-api-gotenberg/REPORT_TEMPLATE_PAT
 REPORT_VERSION="op://ppr/$APP_ENV/ppr-api/REPORT_VERSION"
 REPORT_API_AUDIENCE="op://ppr/$APP_ENV/ppr-api/REPORT_API_AUDIENCE"
 REPORT_SEARCH_LIGHT="op://ppr/$APP_ENV/ppr-api/REPORT_SEARCH_LIGHT"
-GOOGLE_DEFAULT_SA="op://buckets/$APP_ENV/ppr-api/GOOGLE_DEFAULT_SA"
 GCP_CS_SA_SCOPES="op://buckets/$APP_ENV/ppr-api/GCP_CS_SA_SCOPES"
 GCP_CS_BUCKET_ID="op://buckets/$APP_ENV/ppr-api/GCP_CS_BUCKET_ID"
 GCP_CS_BUCKET_ID_REGISTRATION="op://buckets/$APP_ENV/ppr-api/GCP_CS_BUCKET_ID_REGISTRATION"

ppr-api/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ppr-api"
-version = "1.3.16"
+version = "1.3.17"
 description = ""
 authors = ["dlovett <doug@daxiom.com>"]
 license = "BSD 3"

ppr-api/src/ppr_api/callback/auth/token_service.py

@@ -46,9 +46,9 @@ class GoogleStorageTokenService(TokenService):  # pylint: disable=too-few-public
     service_account_info = None
     credentials = None
     # Use service account env var if available.
-    if gcp_auth_key:
-        sa_bytes = bytes(gcp_auth_key, "utf-8")
-        service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8")))
+    # if gcp_auth_key:
+    #    sa_bytes = bytes(gcp_auth_key, "utf-8")
+    #    service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8")))
     # Otherwise leave as none and use the service account attached to the Cloud service.
 
     @staticmethod
@@ -59,10 +59,18 @@ def init_app(app):
         if GoogleStorageTokenService.gcp_auth_key:
             sa_bytes = bytes(GoogleStorageTokenService.gcp_auth_key, "utf-8")
             GoogleStorageTokenService.service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8")))
+            GoogleStorageTokenService.credentials = service_account.Credentials.from_service_account_info(
+                GoogleStorageTokenService.service_account_info, scopes=GoogleStorageTokenService.gcp_sa_scopes
+            )
+        else:
+            logger.info("auth_service.init_app no SA info.")
 
     @classmethod
     def get_token(cls):
         """Generate an OAuth access token with cloud storage access."""
+        if not cls.gcp_auth_key or not cls.service_account_info:
+            return None
+
         if cls.credentials is None:
             cls.credentials = service_account.Credentials.from_service_account_info(
                 cls.service_account_info, scopes=cls.gcp_sa_scopes
@@ -75,11 +83,14 @@ def get_token(cls):
     @classmethod
     def get_credentials(cls):
         """Generate GCP auth credentials to pass to a GCP client."""
+        if not cls.gcp_auth_key or not cls.service_account_info:
+            return None
+
         if cls.credentials is None:
             cls.credentials = service_account.Credentials.from_service_account_info(
                 cls.service_account_info, scopes=cls.gcp_sa_scopes
             )
-        logger.info("Call successful: obtained credentials.")
+        logger.debug("Call successful: obtained credentials.")
         return cls.credentials
 
     @classmethod
@@ -94,3 +105,16 @@ def get_report_api_token(cls, rs_url: str = None):
         token = google.oauth2.id_token.fetch_id_token(auth_req, audience)
         logger.debug("Call successful: obtained token.")
         return token
+
+    @classmethod
+    def get_cs_signed_credentials(cls):
+        """Extra steps for ADC cloud storage signed url - requires cert to sign."""
+        if cls.gcp_auth_key and cls.service_account_info:
+            return cls.get_credentials()
+
+        # Load default credentials
+        credentials, project = google.auth.default()  # pylint: disable=unused-variable; gcp api response
+        # Refresh credentials to ensure an access token is available
+        auth_request = google.auth.transport.requests.Request()
+        credentials.refresh(auth_request)
+        return credentials

ppr-api/src/ppr_api/callback/document_storage/storage_service.py

@@ -153,7 +153,7 @@ def __call_cs_api(  # pylint: disable=too-many-arguments; just 1 more
     ):
         """Call the Cloud Storage API."""
         credentials = GoogleStorageTokenService.get_credentials()
-        storage_client = storage.Client(credentials=credentials)
+        storage_client = storage.Client(credentials=credentials) if credentials else storage.Client()
         bucket = storage_client.bucket(cls.__get_bucket_id(doc_type))
         blob = bucket.blob(name)
         if method == HTTP_POST:
@@ -171,14 +171,21 @@ def __call_cs_api(  # pylint: disable=too-many-arguments; just 1 more
     @classmethod
     def __call_cs_api_link(cls, name: str, data=None, doc_type: str = None, available_days: int = 1):
         """Call the Cloud Storage API, returning a time-limited download link."""
-        credentials = GoogleStorageTokenService.get_credentials()
+        credentials = GoogleStorageTokenService.get_cs_signed_credentials()
+        if not credentials:
+            logger.warning(f"No credentials to create signed storage link for {name}")
+            return ""
         storage_client = storage.Client(credentials=credentials)
         bucket = storage_client.bucket(cls.__get_bucket_id(doc_type))
         blob = bucket.blob(name)
         if data:
             media_type: str = CONTENT_TYPE_PDF
             blob.upload_from_string(data=data, content_type=media_type)
         url = blob.generate_signed_url(
-            version="v4", expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), method="GET"
+            version="v4",
+            expiration=datetime.timedelta(days=available_days, hours=0, minutes=0),
+            method="GET",
+            service_account_email=credentials.service_account_email,
+            access_token=credentials.token,
         )
         return url

ppr-api/src/ppr_api/resources/v1/searches.py

@@ -136,7 +136,6 @@ def put_searches(search_id: str):
     try:
         if search_id is None:
             return resource_utils.path_param_error_response("search ID")
-        search_id_int = int(search_id)
 
         # Quick check: must be staff or provide an account ID.
         account_id = resource_utils.get_account_id(request)
@@ -153,7 +152,7 @@ def put_searches(search_id: str):
         if not valid_format:
             return resource_utils.validation_error_response(errors, VAL_ERROR)
 
-        search_request = SearchRequest.find_by_id(search_id_int)
+        search_request = SearchRequest.find_by_id(int(search_id))
         if not search_request:
             return resource_utils.not_found_error_response("searchId", search_id)
 
@@ -177,7 +176,6 @@ def get_searches(search_id: str):
     try:
         if search_id is None:
             return resource_utils.path_param_error_response("search ID")
-        search_id_int = int(search_id)
 
         # Quick check: must be staff or provide an account ID.
         account_id = resource_utils.get_account_id(request)
@@ -188,7 +186,7 @@ def get_searches(search_id: str):
         if not authorized(account_id, jwt):
             return resource_utils.unauthorized_error_response(account_id)
 
-        search_request: SearchRequest = SearchRequest.find_by_id(search_id_int)
+        search_request: SearchRequest = SearchRequest.find_by_id(int(search_id))
         if not search_request or not search_request.search_result:
             return resource_utils.not_found_error_response("searchId", search_id)
         if search_request.search_result.is_payment_pending():
@@ -266,6 +264,8 @@ def build_staff_payment(req: request, account_id: str):
         "transactionType": TransactionTypes.SEARCH_STAFF_NO_FEE.value,
         "accountId": resource_utils.get_staff_account_id(req),
     }
+    if not payment_info.get("accountId"):
+        payment_info["accountId"] = account_id
     if is_bcol_help(account_id):
         return payment_info
 

ppr-api/src/ppr_api/services/queue_service.py

@@ -36,7 +36,9 @@ class GoogleQueueService:
     def init_app(app):
         """Set up the service"""
         credentials = GoogleStorageTokenService.get_credentials()
-        GoogleQueueService.publisher = pubsub_v1.PublisherClient(credentials=credentials)
+        GoogleQueueService.publisher = (
+            pubsub_v1.PublisherClient(credentials=credentials) if credentials else pubsub_v1.PublisherClient()
+        )
         project_id = str(app.config.get("GCP_PS_PROJECT_ID"))
         search_report_topic = str(app.config.get("GCP_PS_SEARCH_REPORT_TOPIC"))
         notification_topic = str(app.config.get("GCP_PS_NOTIFICATION_TOPIC"))

ppr-api/tests/unit/callback/test_storage_service.py

@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Google Storage token tests."""
-from ppr_api.callback.document_storage.storage_service import DocumentTypes, GoogleStorageService
 
 from flask import current_app
 
+from ppr_api.callback.document_storage.storage_service import DocumentTypes, GoogleStorageService
+from ppr_api.utils.logging import logger
+
 
 TEST_DOC_NAME = 'financing-statements_100348B.pdf'
 TEST_DATAFILE = 'tests/unit/callback/financing-statements_100348B.pdf'
@@ -52,7 +54,7 @@ def test_cs_save_document(session):
         data_file.close()
 
     response = GoogleStorageService.save_document(TEST_SAVE_DOC_NAME, raw_data)
-    print(response)
+    logger.debug(response)
     assert response
 
 
@@ -102,7 +104,7 @@ def test_cs_save_verification_document(session):
 
     response = GoogleStorageService.save_document(TEST_VERIFICATION_SAVE_DOC_NAME, raw_data,
                                                   DocumentTypes.VERIFICATION_MAIL)
-    print(response)
+    logger.debug(response)
     assert response
 
 
@@ -129,10 +131,23 @@ def test_cs_save_registration_document(session):
 
     response = GoogleStorageService.save_document(TEST_REGISTRATION_SAVE_DOC_NAME, raw_data,
                                                   DocumentTypes.REGISTRATION)
-    print(response)
+    logger.debug(response)
     assert response
 
 
+def test_cs_get_search_document_link(session):
+    """Assert that getting a document link from google cloud storage works as expected."""
+    if is_ci_testing():
+        return
+    download_link = GoogleStorageService.get_document_link("2025/12/23/search-results-report-4132437.pdf",
+                                                           DocumentTypes.SEARCH_RESULTS,
+                                                           7)
+    logger.debug(download_link)
+    assert download_link
+
+
 def is_ci_testing() -> bool:
-    """Check unit test environment: exclude most reports for CI testing."""
+    """Check unit test environment: exclude pub/sub for CI testing."""
+    if not current_app.config.get("GOOGLE_DEFAULT_SA"):
+        return True
     return  current_app.config.get("DEPLOYMENT_ENV", "testing") == "testing"

ppr-api/tests/unit/callback/test_token_service.py

@@ -12,17 +12,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Google Storage token tests."""
+from flask import current_app
+
 from ppr_api.callback.auth.token_service import GoogleStorageTokenService
+from ppr_api.utils.logging import logger
 
 
 def test_get_token(session, client, jwt):
     """Assert that config to get a google storage token works as expected."""
     token = GoogleStorageTokenService.get_token()
-    # print(token)
-    assert token
+    if current_app.config.get("GOOGLE_DEFAULT_SA"):
+        logger.debug(token)
+        assert token
+    else:
+        assert not token
 
 
 def test_get_credentials(session, client, jwt):
     """Assert that the configuration to get a google storage token works as expected (no exceptions)."""
     credentials = GoogleStorageTokenService.get_credentials()
-    assert credentials
+    if current_app.config.get("GOOGLE_DEFAULT_SA"):
+        assert credentials
+        assert credentials.token
+        assert credentials.service_account_email
+    else:
+        assert not credentials
+
+
+def test_get_cs_signed_credentials(session, client, jwt):
+    """Assert that the configuration to get a google storage token works as expected (no exceptions)."""
+    if current_app.config.get("GOOGLE_DEFAULT_SA"):
+        credentials = GoogleStorageTokenService.get_cs_signed_credentials()
+        assert credentials
+        assert credentials.token
+        assert credentials.service_account_email