diff --git a/jobs/permanent/secured-party-notification/devops/vaults.gcp.env b/jobs/permanent/secured-party-notification/devops/vaults.gcp.env index 4fe4d4ecb..b682d4c0f 100644 --- a/jobs/permanent/secured-party-notification/devops/vaults.gcp.env +++ b/jobs/permanent/secured-party-notification/devops/vaults.gcp.env @@ -24,7 +24,6 @@ NOTIFY_STATUS_RECIPIENTS="op://ppr/$APP_ENV/secured-party/NOTIFY_STATUS_RECIPIEN NOTIFY_STATUS_SUBJECT="op://ppr/$APP_ENV/secured-party/NOTIFY_STATUS_SUBJECT" NOTIFY_STATUS_BODY="op://ppr/$APP_ENV/secured-party/NOTIFY_STATUS_BODY" JOB_INTERVAL_HOURS="op://ppr/$APP_ENV/secured-party/JOB_INTERVAL_HOURS" -GOOGLE_DEFAULT_SA="op://buckets/$APP_ENV/ppr-api/GOOGLE_DEFAULT_SA" GCP_CS_SA_SCOPES="op://buckets/$APP_ENV/ppr-api/GCP_CS_SA_SCOPES" GCP_CS_BUCKET_ID_MAIL="op://buckets/$APP_ENV/ppr-api/GCP_CS_BUCKET_ID_MAIL" VPC_CONNECTOR="op://CD/$APP_ENV/ppr-secured-party/VPC_CONNECTOR" \ No newline at end of file diff --git a/jobs/permanent/secured-party-notification/pyproject.toml b/jobs/permanent/secured-party-notification/pyproject.toml index 7f6ae87d7..ea6524d7d 100644 --- a/jobs/permanent/secured-party-notification/pyproject.toml +++ b/jobs/permanent/secured-party-notification/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "secured-party-notification" -version = "0.0.1" +version = "0.0.2" description = "PPR liens secured party nofifications on debtor changes." authors = ["thor wolpert ", "doug lovett "] license = "BSD 3" diff --git a/jobs/permanent/secured-party-notification/src/secured_party_notification/config.py b/jobs/permanent/secured-party-notification/src/secured_party_notification/config.py index fed0bd076..7ec25c27e 100644 --- a/jobs/permanent/secured-party-notification/src/secured_party_notification/config.py +++ b/jobs/permanent/secured-party-notification/src/secured_party_notification/config.py @@ -36,6 +36,8 @@ class Config(BaseConfig): AUTH_SVC_URL = f"{AUTH_API_URL + AUTH_API_VERSION}" NOTIFY_SVC_URL = f"{NOTIFY_API_URL + NOTIFY_API_VERSION}" + if not NOTIFY_SVC_URL.endswith("/notify"): + NOTIFY_SVC_URL += "/notify" JWT_OIDC_TOKEN_URL = os.getenv("JWT_OIDC_TOKEN_URL") # service accounts diff --git a/jobs/permanent/secured-party-notification/src/secured_party_notification/services/document_storage/storage_service.py b/jobs/permanent/secured-party-notification/src/secured_party_notification/services/document_storage/storage_service.py index 2f57cc131..b98697993 100644 --- a/jobs/permanent/secured-party-notification/src/secured_party_notification/services/document_storage/storage_service.py +++ b/jobs/permanent/secured-party-notification/src/secured_party_notification/services/document_storage/storage_service.py @@ -43,7 +43,7 @@ def init_app(config: Config): """Set up the service""" bucket_id = config.GCP_CS_BUCKET_ID_MAIL credentials = GoogleAuthService.get_credentials() - storage_client = storage.Client(credentials=credentials) + storage_client = storage.Client(credentials=credentials) if credentials else storage.Client() GoogleStorageService.GCP_BUCKET = storage_client.bucket(bucket_id) GoogleStorageService.GCP_BUCKET_ID_MAIL = bucket_id GoogleStorageService.GCP_CREDENTIALS = credentials @@ -112,10 +112,20 @@ def __call_cs_api(cls, method: str, name: str, data=None, content_type: str = CO @classmethod def __call_cs_api_link(cls, name: str, data=None, available_days: int = 1, content_type: str = CONTENT_TYPE_PDF): """Call the Cloud Storage API, returning a time-limited download link.""" - blob = GoogleStorageService.GCP_BUCKET.blob(name) + creds = GoogleAuthService.get_cs_signed_credentials() + if not creds: + logger.warning(f"No credentials to create signed storage link for {name}") + return "" + s_client = storage.Client(credentials=creds) + bucket = s_client.bucket(cls.GCP_BUCKET_ID_MAIL) + blob = bucket.blob(name) if data: blob.upload_from_string(data=data, content_type=content_type) url = blob.generate_signed_url( - version="v4", expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), method="GET" + version="v4", + expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), + method="GET", + service_account_email=creds.service_account_email, + access_token=creds.token, ) return url diff --git a/jobs/permanent/secured-party-notification/src/secured_party_notification/services/gcp_auth/auth_service.py b/jobs/permanent/secured-party-notification/src/secured_party_notification/services/gcp_auth/auth_service.py index 4d62cd524..c9a56db20 100644 --- a/jobs/permanent/secured-party-notification/src/secured_party_notification/services/gcp_auth/auth_service.py +++ b/jobs/permanent/secured-party-notification/src/secured_party_notification/services/gcp_auth/auth_service.py @@ -36,10 +36,6 @@ class GoogleAuthService: # pylint: disable=too-few-public-methods service_account_info = None credentials = None report_api_audience = None - # Use service account env var if available. - if gcp_auth_key: - sa_bytes = bytes(gcp_auth_key, "utf-8") - service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8"))) @staticmethod def init_app(config: Config): @@ -49,11 +45,19 @@ def init_app(config: Config): if GoogleAuthService.gcp_auth_key: sa_bytes = bytes(GoogleAuthService.gcp_auth_key, "utf-8") GoogleAuthService.service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8"))) + GoogleAuthService.credentials = service_account.Credentials.from_service_account_info( + GoogleAuthService.service_account_info, scopes=GoogleAuthService.gcp_sa_scopes + ) + else: + logger.info("auth_service.init_app no SA info.") GoogleAuthService.report_api_audience = config.REPORT_API_AUDIENCE @classmethod def get_token(cls): """Generate an OAuth access token with cloud storage access.""" + if not cls.gcp_auth_key or not cls.service_account_info: + return None + if cls.credentials is None: cls.credentials = service_account.Credentials.from_service_account_info( cls.service_account_info, scopes=cls.gcp_sa_scopes @@ -76,9 +80,25 @@ def get_report_api_token(cls): @classmethod def get_credentials(cls): """Generate GCP auth credentials to pass to a GCP client.""" + if not cls.gcp_auth_key or not cls.service_account_info: + return None + if cls.credentials is None: cls.credentials = service_account.Credentials.from_service_account_info( cls.service_account_info, scopes=cls.gcp_sa_scopes ) - logger.info("Call successful: obtained credentials.") + logger.debug("Call successful: obtained credentials.") return cls.credentials + + @classmethod + def get_cs_signed_credentials(cls): + """Extra steps for ADC cloud storage signed url - requires cert to sign.""" + if cls.gcp_auth_key and cls.service_account_info: + return cls.get_credentials() + + # Load default credentials + credentials, project = google.auth.default() # pylint: disable=unused-variable; gcp api response + # Refresh credentials to ensure an access token is available + auth_request = google.auth.transport.requests.Request() + credentials.refresh(auth_request) + return credentials diff --git a/ppr-api/devops/vaults.gcp.env b/ppr-api/devops/vaults.gcp.env index af9d43bd5..fa4661a26 100644 --- a/ppr-api/devops/vaults.gcp.env +++ b/ppr-api/devops/vaults.gcp.env @@ -26,7 +26,6 @@ REPORT_TEMPLATE_PATH="op://API/$APP_ENV/report-api-gotenberg/REPORT_TEMPLATE_PAT REPORT_VERSION="op://ppr/$APP_ENV/ppr-api/REPORT_VERSION" REPORT_API_AUDIENCE="op://ppr/$APP_ENV/ppr-api/REPORT_API_AUDIENCE" REPORT_SEARCH_LIGHT="op://ppr/$APP_ENV/ppr-api/REPORT_SEARCH_LIGHT" -GOOGLE_DEFAULT_SA="op://buckets/$APP_ENV/ppr-api/GOOGLE_DEFAULT_SA" GCP_CS_SA_SCOPES="op://buckets/$APP_ENV/ppr-api/GCP_CS_SA_SCOPES" GCP_CS_BUCKET_ID="op://buckets/$APP_ENV/ppr-api/GCP_CS_BUCKET_ID" GCP_CS_BUCKET_ID_REGISTRATION="op://buckets/$APP_ENV/ppr-api/GCP_CS_BUCKET_ID_REGISTRATION" diff --git a/ppr-api/pyproject.toml b/ppr-api/pyproject.toml index 823df79ee..b5aaab017 100644 --- a/ppr-api/pyproject.toml +++ b/ppr-api/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ppr-api" -version = "1.3.16" +version = "1.3.17" description = "" authors = ["dlovett "] license = "BSD 3" diff --git a/ppr-api/src/ppr_api/callback/auth/token_service.py b/ppr-api/src/ppr_api/callback/auth/token_service.py index c51d584ef..e64d84810 100644 --- a/ppr-api/src/ppr_api/callback/auth/token_service.py +++ b/ppr-api/src/ppr_api/callback/auth/token_service.py @@ -46,9 +46,9 @@ class GoogleStorageTokenService(TokenService): # pylint: disable=too-few-public service_account_info = None credentials = None # Use service account env var if available. - if gcp_auth_key: - sa_bytes = bytes(gcp_auth_key, "utf-8") - service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8"))) + # if gcp_auth_key: + # sa_bytes = bytes(gcp_auth_key, "utf-8") + # service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8"))) # Otherwise leave as none and use the service account attached to the Cloud service. @staticmethod @@ -59,10 +59,18 @@ def init_app(app): if GoogleStorageTokenService.gcp_auth_key: sa_bytes = bytes(GoogleStorageTokenService.gcp_auth_key, "utf-8") GoogleStorageTokenService.service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8"))) + GoogleStorageTokenService.credentials = service_account.Credentials.from_service_account_info( + GoogleStorageTokenService.service_account_info, scopes=GoogleStorageTokenService.gcp_sa_scopes + ) + else: + logger.info("auth_service.init_app no SA info.") @classmethod def get_token(cls): """Generate an OAuth access token with cloud storage access.""" + if not cls.gcp_auth_key or not cls.service_account_info: + return None + if cls.credentials is None: cls.credentials = service_account.Credentials.from_service_account_info( cls.service_account_info, scopes=cls.gcp_sa_scopes @@ -75,11 +83,14 @@ def get_token(cls): @classmethod def get_credentials(cls): """Generate GCP auth credentials to pass to a GCP client.""" + if not cls.gcp_auth_key or not cls.service_account_info: + return None + if cls.credentials is None: cls.credentials = service_account.Credentials.from_service_account_info( cls.service_account_info, scopes=cls.gcp_sa_scopes ) - logger.info("Call successful: obtained credentials.") + logger.debug("Call successful: obtained credentials.") return cls.credentials @classmethod @@ -94,3 +105,16 @@ def get_report_api_token(cls, rs_url: str = None): token = google.oauth2.id_token.fetch_id_token(auth_req, audience) logger.debug("Call successful: obtained token.") return token + + @classmethod + def get_cs_signed_credentials(cls): + """Extra steps for ADC cloud storage signed url - requires cert to sign.""" + if cls.gcp_auth_key and cls.service_account_info: + return cls.get_credentials() + + # Load default credentials + credentials, project = google.auth.default() # pylint: disable=unused-variable; gcp api response + # Refresh credentials to ensure an access token is available + auth_request = google.auth.transport.requests.Request() + credentials.refresh(auth_request) + return credentials diff --git a/ppr-api/src/ppr_api/callback/document_storage/storage_service.py b/ppr-api/src/ppr_api/callback/document_storage/storage_service.py index ec33d3766..997d40603 100644 --- a/ppr-api/src/ppr_api/callback/document_storage/storage_service.py +++ b/ppr-api/src/ppr_api/callback/document_storage/storage_service.py @@ -153,7 +153,7 @@ def __call_cs_api( # pylint: disable=too-many-arguments; just 1 more ): """Call the Cloud Storage API.""" credentials = GoogleStorageTokenService.get_credentials() - storage_client = storage.Client(credentials=credentials) + storage_client = storage.Client(credentials=credentials) if credentials else storage.Client() bucket = storage_client.bucket(cls.__get_bucket_id(doc_type)) blob = bucket.blob(name) if method == HTTP_POST: @@ -171,7 +171,10 @@ def __call_cs_api( # pylint: disable=too-many-arguments; just 1 more @classmethod def __call_cs_api_link(cls, name: str, data=None, doc_type: str = None, available_days: int = 1): """Call the Cloud Storage API, returning a time-limited download link.""" - credentials = GoogleStorageTokenService.get_credentials() + credentials = GoogleStorageTokenService.get_cs_signed_credentials() + if not credentials: + logger.warning(f"No credentials to create signed storage link for {name}") + return "" storage_client = storage.Client(credentials=credentials) bucket = storage_client.bucket(cls.__get_bucket_id(doc_type)) blob = bucket.blob(name) @@ -179,6 +182,10 @@ def __call_cs_api_link(cls, name: str, data=None, doc_type: str = None, availabl media_type: str = CONTENT_TYPE_PDF blob.upload_from_string(data=data, content_type=media_type) url = blob.generate_signed_url( - version="v4", expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), method="GET" + version="v4", + expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), + method="GET", + service_account_email=credentials.service_account_email, + access_token=credentials.token, ) return url diff --git a/ppr-api/src/ppr_api/resources/v1/searches.py b/ppr-api/src/ppr_api/resources/v1/searches.py index 70b5283ce..f28100b1c 100644 --- a/ppr-api/src/ppr_api/resources/v1/searches.py +++ b/ppr-api/src/ppr_api/resources/v1/searches.py @@ -136,7 +136,6 @@ def put_searches(search_id: str): try: if search_id is None: return resource_utils.path_param_error_response("search ID") - search_id_int = int(search_id) # Quick check: must be staff or provide an account ID. account_id = resource_utils.get_account_id(request) @@ -153,7 +152,7 @@ def put_searches(search_id: str): if not valid_format: return resource_utils.validation_error_response(errors, VAL_ERROR) - search_request = SearchRequest.find_by_id(search_id_int) + search_request = SearchRequest.find_by_id(int(search_id)) if not search_request: return resource_utils.not_found_error_response("searchId", search_id) @@ -177,7 +176,6 @@ def get_searches(search_id: str): try: if search_id is None: return resource_utils.path_param_error_response("search ID") - search_id_int = int(search_id) # Quick check: must be staff or provide an account ID. account_id = resource_utils.get_account_id(request) @@ -188,7 +186,7 @@ def get_searches(search_id: str): if not authorized(account_id, jwt): return resource_utils.unauthorized_error_response(account_id) - search_request: SearchRequest = SearchRequest.find_by_id(search_id_int) + search_request: SearchRequest = SearchRequest.find_by_id(int(search_id)) if not search_request or not search_request.search_result: return resource_utils.not_found_error_response("searchId", search_id) if search_request.search_result.is_payment_pending(): @@ -266,6 +264,8 @@ def build_staff_payment(req: request, account_id: str): "transactionType": TransactionTypes.SEARCH_STAFF_NO_FEE.value, "accountId": resource_utils.get_staff_account_id(req), } + if not payment_info.get("accountId"): + payment_info["accountId"] = account_id if is_bcol_help(account_id): return payment_info diff --git a/ppr-api/src/ppr_api/services/queue_service.py b/ppr-api/src/ppr_api/services/queue_service.py index afff46d0c..1720a6475 100644 --- a/ppr-api/src/ppr_api/services/queue_service.py +++ b/ppr-api/src/ppr_api/services/queue_service.py @@ -36,7 +36,9 @@ class GoogleQueueService: def init_app(app): """Set up the service""" credentials = GoogleStorageTokenService.get_credentials() - GoogleQueueService.publisher = pubsub_v1.PublisherClient(credentials=credentials) + GoogleQueueService.publisher = ( + pubsub_v1.PublisherClient(credentials=credentials) if credentials else pubsub_v1.PublisherClient() + ) project_id = str(app.config.get("GCP_PS_PROJECT_ID")) search_report_topic = str(app.config.get("GCP_PS_SEARCH_REPORT_TOPIC")) notification_topic = str(app.config.get("GCP_PS_NOTIFICATION_TOPIC")) diff --git a/ppr-api/tests/unit/callback/test_storage_service.py b/ppr-api/tests/unit/callback/test_storage_service.py index 7029e3501..7c4b0d123 100644 --- a/ppr-api/tests/unit/callback/test_storage_service.py +++ b/ppr-api/tests/unit/callback/test_storage_service.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. """Google Storage token tests.""" -from ppr_api.callback.document_storage.storage_service import DocumentTypes, GoogleStorageService from flask import current_app +from ppr_api.callback.document_storage.storage_service import DocumentTypes, GoogleStorageService +from ppr_api.utils.logging import logger + TEST_DOC_NAME = 'financing-statements_100348B.pdf' TEST_DATAFILE = 'tests/unit/callback/financing-statements_100348B.pdf' @@ -52,7 +54,7 @@ def test_cs_save_document(session): data_file.close() response = GoogleStorageService.save_document(TEST_SAVE_DOC_NAME, raw_data) - print(response) + logger.debug(response) assert response @@ -102,7 +104,7 @@ def test_cs_save_verification_document(session): response = GoogleStorageService.save_document(TEST_VERIFICATION_SAVE_DOC_NAME, raw_data, DocumentTypes.VERIFICATION_MAIL) - print(response) + logger.debug(response) assert response @@ -129,10 +131,23 @@ def test_cs_save_registration_document(session): response = GoogleStorageService.save_document(TEST_REGISTRATION_SAVE_DOC_NAME, raw_data, DocumentTypes.REGISTRATION) - print(response) + logger.debug(response) assert response +def test_cs_get_search_document_link(session): + """Assert that getting a document link from google cloud storage works as expected.""" + if is_ci_testing(): + return + download_link = GoogleStorageService.get_document_link("2025/12/23/search-results-report-4132437.pdf", + DocumentTypes.SEARCH_RESULTS, + 7) + logger.debug(download_link) + assert download_link + + def is_ci_testing() -> bool: - """Check unit test environment: exclude most reports for CI testing.""" + """Check unit test environment: exclude pub/sub for CI testing.""" + if not current_app.config.get("GOOGLE_DEFAULT_SA"): + return True return current_app.config.get("DEPLOYMENT_ENV", "testing") == "testing" diff --git a/ppr-api/tests/unit/callback/test_token_service.py b/ppr-api/tests/unit/callback/test_token_service.py index dedcfe2d6..0932e3d27 100644 --- a/ppr-api/tests/unit/callback/test_token_service.py +++ b/ppr-api/tests/unit/callback/test_token_service.py @@ -12,17 +12,37 @@ # See the License for the specific language governing permissions and # limitations under the License. """Google Storage token tests.""" +from flask import current_app + from ppr_api.callback.auth.token_service import GoogleStorageTokenService +from ppr_api.utils.logging import logger def test_get_token(session, client, jwt): """Assert that config to get a google storage token works as expected.""" token = GoogleStorageTokenService.get_token() - # print(token) - assert token + if current_app.config.get("GOOGLE_DEFAULT_SA"): + logger.debug(token) + assert token + else: + assert not token def test_get_credentials(session, client, jwt): """Assert that the configuration to get a google storage token works as expected (no exceptions).""" credentials = GoogleStorageTokenService.get_credentials() - assert credentials + if current_app.config.get("GOOGLE_DEFAULT_SA"): + assert credentials + assert credentials.token + assert credentials.service_account_email + else: + assert not credentials + + +def test_get_cs_signed_credentials(session, client, jwt): + """Assert that the configuration to get a google storage token works as expected (no exceptions).""" + if current_app.config.get("GOOGLE_DEFAULT_SA"): + credentials = GoogleStorageTokenService.get_cs_signed_credentials() + assert credentials + assert credentials.token + assert credentials.service_account_email diff --git a/ppr-api/tests/unit/services/test_queue_service.py b/ppr-api/tests/unit/services/test_queue_service.py index 18b65141d..abb3e05c2 100644 --- a/ppr-api/tests/unit/services/test_queue_service.py +++ b/ppr-api/tests/unit/services/test_queue_service.py @@ -33,9 +33,8 @@ def test_publish_search_report(session): """Assert that enqueuing/publishing a search report event works as expected.""" - payload = TEST_PAYLOAD - test_env = current_app.config.get("DEPLOYMENT_ENV", "testing") - if test_env != "testing": + if not is_ci_testing(): + payload = TEST_PAYLOAD apikey = current_app.config.get('SUBSCRIPTION_API_KEY') if apikey: payload['apikey'] = apikey @@ -44,9 +43,8 @@ def test_publish_search_report(session): def test_publish_api_notification(session): """Assert that enqueuing/publishing an api notification event works as expected.""" - payload = TEST_PAYLOAD - test_env = current_app.config.get("DEPLOYMENT_ENV", "testing") - if test_env != "testing": + if not is_ci_testing(): + payload = TEST_PAYLOAD apikey = current_app.config.get('SUBSCRIPTION_API_KEY') if apikey: payload['apikey'] = apikey @@ -55,9 +53,8 @@ def test_publish_api_notification(session): def test_publish_verification_mail(session): """Assert that enqueuing/publishing an api verification statement mail event works as expected.""" - payload = TEST_PAYLOAD_VERIFICATION - test_env = current_app.config.get("DEPLOYMENT_ENV", "testing") - if test_env != "testing": + if not is_ci_testing(): + payload = TEST_PAYLOAD_VERIFICATION apikey = current_app.config.get('SUBSCRIPTION_API_KEY') if apikey: payload['apikey'] = apikey @@ -66,10 +63,16 @@ def test_publish_verification_mail(session): def test_publish_registration_report(session): """Assert that enqueuing/publishing a registration report event works as expected.""" - payload = TEST_PAYLOAD_REGISTRATION - test_env = current_app.config.get("DEPLOYMENT_ENV", "testing") - if test_env != "testing": + if not is_ci_testing(): + payload = TEST_PAYLOAD_REGISTRATION apikey = current_app.config.get('SUBSCRIPTION_API_KEY') if apikey: payload['apikey'] = apikey GoogleQueueService().publish_registration_report(payload) + + +def is_ci_testing() -> bool: + """Check unit test environment: exclude pub/sub for CI testing.""" + if not current_app.config.get("GOOGLE_DEFAULT_SA"): + return True + return current_app.config.get("DEPLOYMENT_ENV", "testing") == "testing"