fix(storage): resolve transport and retry issues

handwritten/storage/src/bucket.ts

@@ -3401,13 +3401,13 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
    * @returns {Promise<Bucket>}
    */
   async restore(options: RestoreOptions): Promise<Bucket> {
-    const bucket = await this.storageTransport.makeRequest<Bucket>({
+    const response = await this.storageTransport.makeRequest<Bucket>({
       method: 'POST',
       url: `${this.baseUrl}/${this.name}/restore`,
       queryParameters: options as unknown as StorageQueryParameters,
     });
 
-    return bucket as Bucket;
+    return response.data as Bucket;
   }
 
   makePrivate(

handwritten/storage/src/file.ts

@@ -349,6 +349,11 @@ export const STORAGE_POST_POLICY_BASE_URL = 'https://storage.googleapis.com';
  */
 const GS_URL_REGEXP = /^gs:\/\/([a-z0-9_.-]+)\/(.+)$/;
 
+/**
+ * @private
+ */
+const ENCRYPTION_ALGORITHM_AES256 = 'AES256';
+
 /**
  * @private
  * This regex will match compressible content types. These are primarily text/*, +json, +text, +xml content types.
@@ -1393,7 +1398,10 @@ class File extends ServiceObject<File, FileMetadata> {
     const headers = new Headers();
 
     if (this.encryptionKey !== undefined) {
-      headers.set('x-goog-copy-source-encryption-algorithm', 'AES256');
+      headers.set(
+        'x-goog-copy-source-encryption-algorithm',
+        ENCRYPTION_ALGORITHM_AES256,
+      );
       headers.set(
         'x-goog-copy-source-encryption-key',
         this.encryptionKeyBase64!,
@@ -1405,7 +1413,12 @@ class File extends ServiceObject<File, FileMetadata> {
     }
 
     if (newFile.encryptionKey !== undefined) {
-      this.setEncryptionKey(newFile.encryptionKey!);
+      headers.set('x-goog-encryption-algorithm', ENCRYPTION_ALGORITHM_AES256);
+      headers.set('x-goog-encryption-key', newFile.encryptionKeyBase64 || '');
+      headers.set(
+        'x-goog-encryption-key-sha256',
+        newFile.encryptionKeyHash || '',
+      );
     } else if (options.destinationKmsKeyName !== undefined) {
       query.destinationKmsKeyName = options.destinationKmsKeyName;
       delete options.destinationKmsKeyName;
@@ -1639,6 +1652,8 @@ class File extends ServiceObject<File, FileMetadata> {
       }
 
       const headers = response.headers;
+      const isStoredCompressed =
+        headers.get('x-goog-stored-content-encoding') === 'gzip';
       const isCompressed = headers.get('content-encoding') === 'gzip';
       const hashes: {crc32c?: string; md5?: string} = {};
 
@@ -1652,7 +1667,7 @@ class File extends ServiceObject<File, FileMetadata> {
 
       const transformStreams: Transform[] = [];
 
-      if (shouldRunValidation) {
+      if (shouldRunValidation && !isStoredCompressed) {
         // The x-goog-hash header should be set with a crc32c and md5 hash.
         // ex: headers.set('x-goog-hash', 'crc32c=xxxx,md5=xxxx')
         if (typeof headers.get('x-goog-hash') === 'string') {
@@ -1721,6 +1736,7 @@ class File extends ServiceObject<File, FileMetadata> {
       const headers = {
         'Accept-Encoding': 'gzip',
         'Cache-Control': 'no-store',
+        ...(this.encryptionKeyHeaders || {}),
       } as Headers;
 
       if (rangeRequest) {
@@ -1735,7 +1751,9 @@ class File extends ServiceObject<File, FileMetadata> {
         headers,
         queryParameters: query as unknown as StorageQueryParameters,
         responseType: 'stream',
-      };
+        decompress: options.decompress,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      } as any;
 
       if (options[GCCL_GCS_CMD_KEY]) {
         reqOpts[GCCL_GCS_CMD_KEY] = options[GCCL_GCS_CMD_KEY];
@@ -2426,6 +2444,18 @@ class File extends ServiceObject<File, FileMetadata> {
     }
   }
 
+  get encryptionKeyHeaders(): Record<string, string> | undefined {
+    if (!this.encryptionKey) {
+      return undefined;
+    }
+
+    return {
+      'x-goog-encryption-algorithm': ENCRYPTION_ALGORITHM_AES256,
+      'x-goog-encryption-key': this.encryptionKey.toString('base64'),
+      'x-goog-encryption-key-sha256': this.encryptionKeyHash || '',
+    };
+  }
+
   /**
    * The Storage API allows you to use a custom key for server-side encryption.
    *
@@ -2485,7 +2515,10 @@ class File extends ServiceObject<File, FileMetadata> {
     this.encryptionKeyInterceptor = {
       resolved: reqOpts => {
         reqOpts.headers = new Headers(reqOpts.headers || {});
-        reqOpts.headers.set('x-goog-encryption-algorithm', 'AES256');
+        reqOpts.headers.set(
+          'x-goog-encryption-algorithm',
+          ENCRYPTION_ALGORITHM_AES256,
+        );
         reqOpts.headers.set('x-goog-encryption-key', this.encryptionKeyBase64!);
         reqOpts.headers.set(
           'x-goog-encryption-key-sha256',
@@ -3315,7 +3348,11 @@ class File extends ServiceObject<File, FileMetadata> {
       undefined,
       callback,
     );
-    const url = `https://${this.storage.apiEndpoint}/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`;
+    const baseUrl = this.storage.apiEndpoint.startsWith('http') 
+      ? this.storage.apiEndpoint 
+      : `https://${this.storage.apiEndpoint}`;
+
+    const url = `${baseUrl}/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`;
 
     const gaxios = new Gaxios();
     const storageInterceptors = this.storage?.interceptors || [];
@@ -4025,12 +4062,12 @@ class File extends ServiceObject<File, FileMetadata> {
    * @returns {Promise<File>}
    */
   async restore(options: RestoreOptions): Promise<File> {
-    const file = await this.storageTransport.makeRequest<File>({
+    const response = await this.storageTransport.makeRequest<File>({
       method: 'POST',
       url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/restore`,
       queryParameters: options as unknown as StorageQueryParameters,
     });
-    return file as File;
+    return response.data as File;
   }
 
   rotateEncryptionKey(
@@ -4562,6 +4599,17 @@ class File extends ServiceObject<File, FileMetadata> {
       },
     ];
 
+    const headers: Record<string, string> = {};
+    if (this.encryptionKey) {
+      headers['x-goog-encryption-algorithm'] = ENCRYPTION_ALGORITHM_AES256;
+      headers['x-goog-encryption-key'] = this.encryptionKeyBase64!;
+      headers['x-goog-encryption-key-sha256'] = this.encryptionKeyHash!;
+    }
+    reqOpts.headers = {
+      ...reqOpts.headers,
+      ...headers,
+    };
+
     this.storageTransport
       .makeRequest(reqOpts as StorageRequestOptions, (err, body, resp) => {
         if (err) {

handwritten/storage/src/nodejs-common/service-object.ts

@@ -13,18 +13,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import {promisifyAll} from '@google-cloud/promisify';
-import {EventEmitter} from 'events';
-import {util} from './util.js';
-import {Bucket} from '../bucket.js';
-import {StorageRequestOptions, StorageTransport} from '../storage-transport.js';
+import { promisifyAll } from '@google-cloud/promisify';
+import { EventEmitter } from 'events';
+import { util } from './util.js';
+import { StorageRequestOptions, StorageTransport } from '../storage-transport.js';
 import {
   GaxiosError,
   GaxiosInterceptor,
   GaxiosOptionsPrepared,
   GaxiosResponse,
 } from 'gaxios';
 
+function isBucket(parent: any): boolean {
+  // TODO: remove any suppression during follow up PR to improve type safety.
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  return parent && typeof (parent as any).getFiles === 'function';
+}
+
 export type GetMetadataOptions = object;
 
 export type MetadataResponse<K> = [K, GaxiosResponse];
@@ -97,7 +102,7 @@ export interface InstanceResponseCallback<T> {
 }
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface CreateOptions {}
+export interface CreateOptions { }
 // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unused-vars
 export type CreateResponse<T> = any[];
 export interface CreateCallback<T> {
@@ -207,8 +212,8 @@ class ServiceObject<T, K extends BaseMetadata> extends EventEmitter {
             // The ServiceObject didn't redefine the method.
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             (this as any)[methodName] ===
-              // eslint-disable-next-line @typescript-eslint/no-explicit-any
-              (ServiceObject.prototype as any)[methodName] &&
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (ServiceObject.prototype as any)[methodName] &&
             // This method isn't wanted.
             !config.methods![methodName]
           );
@@ -293,8 +298,10 @@ class ServiceObject<T, K extends BaseMetadata> extends EventEmitter {
       (typeof this.methods.delete === 'object' && this.methods.delete) || {};
 
     let url = `${this.baseUrl}/${this.id}`;
-    if (this.parent instanceof Bucket) {
-      url = `${this.parent.baseUrl}/${this.parent.id}${url}`;
+    if (isBucket(this.parent)) {
+      // TODO: remove any suppression during follow up PR to improve type safety.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      url = `${this.parent.baseUrl}/${(this.parent as any).id}${url}`;
     }
 
     this.storageTransport
@@ -440,20 +447,39 @@ class ServiceObject<T, K extends BaseMetadata> extends EventEmitter {
       {};
 
     let url = `${this.baseUrl}/${this.id}`;
-    if (this.parent instanceof Bucket) {
-      url = `${this.parent.baseUrl}/${this.parent.id}${url}`;
+    if (isBucket(this.parent)) {
+      // TODO: remove any suppression during follow up PR to improve type safety.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      url = `${this.parent.baseUrl}/${(this.parent as any).id}${url}`;
     }
 
+    // TODO: remove any suppression during follow up PR to improve type safety.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const encryptionHeaders = (this as any).encryptionKeyHeaders || {};
+
+    const headers = {
+      ...encryptionHeaders,
+      ...methodConfig.reqOpts?.headers,
+      // TODO: remove any suppression during follow up PR to improve type safety.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      ...(options as any).headers,
+    };
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const query = { ...options } as any;
+    delete query.headers;
+
     this.storageTransport
       .makeRequest<K>(
         {
           method: 'GET',
           responseType: 'json',
           url,
           ...methodConfig.reqOpts,
+          headers,
           queryParameters: {
             ...methodConfig.reqOpts?.queryParameters,
-            ...options,
+            ...query,
           },
         },
         (err, data, resp) => {
@@ -498,8 +524,10 @@ class ServiceObject<T, K extends BaseMetadata> extends EventEmitter {
       {};
 
     let url = `${this.baseUrl}/${this.name}`;
-    if (this.parent instanceof Bucket) {
-      url = `${this.parent.baseUrl}/${this.parent.name}${url}`;
+    if (isBucket(this.parent)) {
+      // TODO: remove any suppression during follow up PR to improve type safety.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      url = `${this.parent.baseUrl}/${(this.parent as any).name}${url}`;
     }
 
     const body = Object.assign({}, methodConfig.reqOpts?.body, metadata);
@@ -529,6 +557,6 @@ class ServiceObject<T, K extends BaseMetadata> extends EventEmitter {
   }
 }
 
-promisifyAll(ServiceObject, {exclude: ['getRequestInterceptors']});
+promisifyAll(ServiceObject, { exclude: ['getRequestInterceptors'] });
 
-export {ServiceObject};
+export { ServiceObject };