feat(selectivity): add 'reason' to 'mapDependencyRelativePath' function

src/browser/cdp/selectivity/css-selectivity.ts

@@ -213,11 +213,9 @@ export class CSSSelectivity {
             };
         }
 
-        return setCachedSelectivityFile(
-            CacheType.CssSessionCache,
-            this._wdSessionId,
-            JSON.stringify(sessionCache),
-        ).catch(err => {
+        return setCachedSelectivityFile(CacheType.CssSessionCache, this._wdSessionId, JSON.stringify(sessionCache), {
+            overwrite: true,
+        }).catch(err => {
             debugSelectivity(`Couldn't save session cache for session '%s': %O`, this._wdSessionId, err);
         });
     }

src/browser/cdp/selectivity/fs-cache.ts

@@ -86,6 +86,7 @@ export const setCachedSelectivityFile = async (
     cacheType: CacheTypeValue,
     key: string,
     utf8Contents: string,
+    { overwrite = false }: { overwrite?: boolean } = {},
 ): Promise<void> => {
     if (!key) {
         throw new Error("Attepted to write cache with empty key");
@@ -96,7 +97,7 @@ export const setCachedSelectivityFile = async (
     const flagFilePath = cacheFilePath + SELECTIVITY_CACHE_READY_SUFFIX;
 
     // Cache was already written
-    if (await wasModifiedAfterProcessStart(flagFilePath)) {
+    if (!overwrite && (await wasModifiedAfterProcessStart(flagFilePath))) {
         return;
     }
 
@@ -106,7 +107,12 @@ export const setCachedSelectivityFile = async (
         .lock(flagFilePath, {
             stale: 5000,
             update: 1000,
-            retries: { minTimeout: 50, maxTimeout: 50, retries: 1 },
+            retries: {
+                factor: 2,
+                minTimeout: 50,
+                maxTimeout: 200,
+                retries: overwrite ? 30 : 1,
+            },
             realpath: false,
         })
         .catch(() => null);
@@ -117,7 +123,7 @@ export const setCachedSelectivityFile = async (
     }
 
     // Cache was written while trying to get lock
-    if (await wasModifiedAfterProcessStart(flagFilePath)) {
+    if (!overwrite && (await wasModifiedAfterProcessStart(flagFilePath))) {
         await releaseLock();
         return;
     }

src/browser/cdp/selectivity/index.ts

@@ -258,24 +258,16 @@ export const startSelectivity = async (browser: ExistingBrowser): Promise<StopSe
             return;
         }
 
-        const mapBrowserDepsRelativePath = mapDependencyRelativePath
-            ? (relativePath: string): string | boolean | void =>
-                  mapDependencyRelativePath({ scope: "browser", relativePath })
-            : null;
-
-        const mapTestplaneDepsRelativePath = mapDependencyRelativePath
-            ? (relativePath: string): string | boolean | void =>
-                  mapDependencyRelativePath({ scope: "testplane", relativePath })
-            : null;
-
         const testDependencyWriter = getTestDependenciesWriter(testDependenciesPath, compression);
         const browserDeps = transformSourceDependencies(
             { css: cssDependencies, js: jsDependencies, png: null },
-            mapBrowserDepsRelativePath,
+            mapDependencyRelativePath,
+            "browser",
         );
         const testplaneDeps = transformSourceDependencies(
             { css: null, js: getCollectedTestplaneJsDependencies(), png: getCollectedTestplanePngDependencies() },
-            mapTestplaneDepsRelativePath,
+            mapDependencyRelativePath,
+            "testplane",
         );
 
         process.send?.({

src/browser/cdp/selectivity/utils.ts

@@ -17,6 +17,11 @@ import type {
 import { WEBPACK_PROTOCOL } from "./constants";
 import { readJsonWithCompression } from "./json-utils";
 import type { Test } from "../../../types";
+import {
+    SelectivityDependencyReason,
+    SelectivityDependencyScope,
+    SelectivityMapDependencyRelativePathFn,
+} from "../../../config/types";
 
 /**
  * Tries to fetch text by url from node.js, then falls back to "fetch" from browser, if node.js fetch fails
@@ -224,15 +229,20 @@ export const transformSourceDependencies = (
         js: jsDependencies,
         png: pngDependencies,
     }: { css: Set<string> | null; js: Set<string> | null; png: Set<string> | null },
-    mapDependencyPathFn?: null | ((relativePath: string) => string | boolean | void),
+    mapDependencyPathFn: null | SelectivityMapDependencyRelativePathFn,
+    scope: SelectivityDependencyScope,
 ): NormalizedDependencies => {
     const nodeModulesLabel = "node_modules/";
     const cssSet: Set<string> = new Set();
     const jsSet: Set<string> = new Set();
     const modulesSet: Set<string> = new Set();
     const pngSet: Set<string> = new Set();
 
-    const classifyDependency = (dependency: string, typedResultSet: Set<string>): void => {
+    const classifyDependency = (
+        dependency: string,
+        typedResultSet: Set<string>,
+        reason: SelectivityDependencyReason,
+    ): void => {
         dependency = decodeURIComponent(softFileURLToPath(dependency));
 
         const protocol = getProtocol(dependency);
@@ -243,7 +253,13 @@ export const transformSourceDependencies = (
         }
 
         const initialDependencyRelativePath = path.posix.relative(path.posix.resolve(), path.posix.resolve(dependency));
-        const mapDependencyPathResult = mapDependencyPathFn ? mapDependencyPathFn(initialDependencyRelativePath) : true;
+        const mapDependencyPathResult = mapDependencyPathFn
+            ? mapDependencyPathFn({
+                  scope,
+                  reason,
+                  relativePath: initialDependencyRelativePath,
+              })
+            : true;
 
         if (!mapDependencyPathResult) {
             return;
@@ -285,19 +301,25 @@ export const transformSourceDependencies = (
 
     if (cssDependencies) {
         for (const cssDependency of cssDependencies.values()) {
-            classifyDependency(cssDependency, cssSet);
+            // Testplane can't have css dependencies
+            classifyDependency(cssDependency, cssSet, "browser-css-import");
         }
     }
 
     if (jsDependencies) {
         for (const jsDependency of jsDependencies.values()) {
-            classifyDependency(jsDependency, jsSet);
+            classifyDependency(
+                jsDependency,
+                jsSet,
+                scope === "browser" ? "browser-js-coverage" : "testplane-js-import",
+            );
         }
     }
 
     if (pngDependencies) {
         for (const pngDependency of pngDependencies.values()) {
-            classifyDependency(pngDependency, pngSet);
+            // Browser png dependencies are not tracked
+            classifyDependency(pngDependency, pngSet, "testplane-assert-view-reference");
         }
     }
 

src/config/types.ts

@@ -306,14 +306,22 @@ export type StateOpts = {
     keepFile?: boolean;
 };
 
+export type SelectivityDependencyScope = "browser" | "testplane";
+export type SelectivityDependencyReason =
+    | "browser-css-import"
+    | "browser-js-coverage"
+    | "testplane-js-import"
+    | "testplane-assert-view-reference";
+
 /**
  * @param {Object} dependency - Object with dependency scope and posix relative path
  * @param {"browser"|"testplane"|string} dependency.scope - Dependency scope
  * @param {string} dependency.relativePath - POSIX relative path
  * @returns mapped POSIX relative path, "true", if should be untouched, or "falsy", if should be ignored
  */
 export type SelectivityMapDependencyRelativePathFn = (dependency: {
-    scope: "browser" | "testplane" | (string & NonNullable<unknown>);
+    scope: SelectivityDependencyScope;
+    reason: SelectivityDependencyReason;
     relativePath: string;
 }) => string | boolean | void;
 

test/src/browser/cdp/selectivity/fs-cache.ts

@@ -276,7 +276,7 @@ describe("CDP/Selectivity/FsCache", () => {
             assert.calledWith(lockfileStub.lock, `/tmp/${SELECTIVITY_CACHE_DIRECTIRY}/thash-of-${key}-ready`, {
                 stale: 5000,
                 update: 1000,
-                retries: { minTimeout: 50, maxTimeout: 50, retries: 1 },
+                retries: { factor: 2, minTimeout: 50, maxTimeout: 200, retries: 1 },
                 realpath: false,
             });
             assert.calledWith(fsStub.writeFile, `/tmp/${SELECTIVITY_CACHE_DIRECTIRY}/thash-of-${key}`, content, {
@@ -424,5 +424,117 @@ describe("CDP/Selectivity/FsCache", () => {
             assert.calledWith(fsStub.writeFile, `/tmp/${SELECTIVITY_CACHE_DIRECTIRY}/cs${key}-ready`, "");
             assert.notCalled(getMD5Stub);
         });
+
+        describe("overwrite option", () => {
+            it("should write cache even if it already exists when overwrite is true", async () => {
+                const key = "test-key";
+                const cacheType = CacheType.TestFile;
+                const content = "new content";
+                const futureTime = Date.now() + 10000;
+                const releaseLockStub = sandbox.stub().resolves();
+
+                fsStub.stat.resolves({ mtimeMs: futureTime });
+                lockfileStub.lock.resolves(releaseLockStub);
+
+                await setCachedSelectivityFile(cacheType, key, content, { overwrite: true });
+
+                assert.calledOnce(fsStub.ensureDir);
+                assert.calledOnce(lockfileStub.lock);
+                assert.calledWith(fsStub.writeFile, `/tmp/${SELECTIVITY_CACHE_DIRECTIRY}/thash-of-${key}`, content, {
+                    encoding: "utf8",
+                });
+                assert.calledWith(fsStub.writeFile, `/tmp/${SELECTIVITY_CACHE_DIRECTIRY}/thash-of-${key}-ready`, "");
+                assert.calledOnce(releaseLockStub);
+            });
+
+            it("should not write cache if it already exists when overwrite is false", async () => {
+                const key = "test-key";
+                const cacheType = CacheType.TestFile;
+                const content = "new content";
+                const futureTime = Date.now() + 10000;
+
+                fsStub.stat.resolves({ mtimeMs: futureTime });
+
+                await setCachedSelectivityFile(cacheType, key, content, { overwrite: false });
+
+                assert.notCalled(fsStub.ensureDir);
+                assert.notCalled(lockfileStub.lock);
+                assert.notCalled(fsStub.writeFile);
+            });
+
+            it("should use 10 lock retries when overwrite is true", async () => {
+                const key = "test-key";
+                const cacheType = CacheType.TestFile;
+                const content = "test content";
+                const pastTime = 0;
+                const releaseLockStub = sandbox.stub().resolves();
+
+                fsStub.stat.resolves({ mtimeMs: pastTime });
+                lockfileStub.lock.resolves(releaseLockStub);
+
+                await setCachedSelectivityFile(cacheType, key, content, { overwrite: true });
+
+                assert.calledWith(lockfileStub.lock, `/tmp/${SELECTIVITY_CACHE_DIRECTIRY}/thash-of-${key}-ready`, {
+                    stale: 5000,
+                    update: 1000,
+                    retries: { factor: 2, minTimeout: 50, maxTimeout: 200, retries: 30 },
+                    realpath: false,
+                });
+            });
+
+            it("should use 1 lock retry when overwrite is false", async () => {
+                const key = "test-key";
+                const cacheType = CacheType.TestFile;
+                const content = "test content";
+                const pastTime = 0;
+                const releaseLockStub = sandbox.stub().resolves();
+
+                fsStub.stat.resolves({ mtimeMs: pastTime });
+                lockfileStub.lock.resolves(releaseLockStub);
+
+                await setCachedSelectivityFile(cacheType, key, content, { overwrite: false });
+
+                assert.calledWith(lockfileStub.lock, `/tmp/${SELECTIVITY_CACHE_DIRECTIRY}/thash-of-${key}-ready`, {
+                    stale: 5000,
+                    update: 1000,
+                    retries: { factor: 2, minTimeout: 50, maxTimeout: 200, retries: 1 },
+                    realpath: false,
+                });
+            });
+
+            it("should write cache even if it was created while acquiring lock when overwrite is true", async () => {
+                const key = "test-key";
+                const cacheType = CacheType.Asset;
+                const content = "new content";
+                const pastTime = 0;
+                const futureTime = Date.now() + 10000;
+                const releaseLockStub = sandbox.stub().resolves();
+
+                fsStub.stat.onCall(0).resolves({ mtimeMs: pastTime }); // Flag not ready
+                fsStub.stat.onCall(1).resolves({ mtimeMs: futureTime }); // Cache file exists after lock
+                lockfileStub.lock.resolves(releaseLockStub);
+
+                await setCachedSelectivityFile(cacheType, key, content, { overwrite: true });
+
+                assert.calledWith(fsStub.writeFile, `/tmp/${SELECTIVITY_CACHE_DIRECTIRY}/ahash-of-${key}`, content, {
+                    encoding: "utf8",
+                });
+                assert.calledWith(fsStub.writeFile, `/tmp/${SELECTIVITY_CACHE_DIRECTIRY}/ahash-of-${key}-ready`, "");
+                assert.calledOnce(releaseLockStub);
+            });
+
+            it("should default overwrite to false when options object is not provided", async () => {
+                const key = "test-key";
+                const cacheType = CacheType.TestFile;
+                const content = "test content";
+                const futureTime = Date.now() + 10000;
+
+                fsStub.stat.resolves({ mtimeMs: futureTime });
+
+                await setCachedSelectivityFile(cacheType, key, content);
+
+                assert.notCalled(fsStub.writeFile);
+            });
+        });
     });
 });