diff --git a/src/browser/calibrator.ts b/src/browser/calibrator.ts index 870621428..bed954cdf 100644 --- a/src/browser/calibrator.ts +++ b/src/browser/calibrator.ts @@ -4,31 +4,21 @@ import looksSame from "looks-same"; import { CoreError } from "./core-error"; import { ExistingBrowser } from "./existing-browser"; import type { Image } from "../image"; - -const DIRECTION = { FORWARD: "forward", REVERSE: "reverse" } as const; +import { Coord, Length, Rect, XBand, getHeight, getIntersection, getWidth } from "./isomorphic"; +import * as logger from "../utils/logger"; +import os from "node:os"; interface BrowserFeatures { needsCompatLib: boolean; pixelRatio: number; - innerWidth: number; + innerWidth: Length<"css", "x">; } export interface CalibrationResult extends BrowserFeatures { - top: number; - left: number; + viewportArea: Rect<"image", "device">; usePixelRatio: boolean; } -interface ViewportStart { - x: number; - y: number; -} - -interface ImageAnalysisResult { - viewportStart: ViewportStart; - colorLength?: number; -} - export class Calibrator { private _cache: Record; private _script: string; @@ -49,9 +39,16 @@ export class Calibrator { const { innerWidth, pixelRatio } = features; const hasPixelRatio = Boolean(pixelRatio && pixelRatio > 1.0); - const imageFeatures = await this._analyzeImage(image, { calculateColorLength: hasPixelRatio }); + const imageFeatures = await this._findMarkerAreaInImage(image); if (!imageFeatures) { + const screenshotPath = path.join(os.tmpdir(), "testplane-calibration-page.png"); + await image.save(screenshotPath); + logger.error( + "Could not calibrate, because marker area was not found. See calibration page screenshot for details: " + + screenshotPath, + ); + await image.save(screenshotPath); throw new CoreError( "Could not calibrate. This could be due to calibration page has failed to open properly", ); @@ -59,25 +56,47 @@ export class Calibrator { const calibratedFeatures: CalibrationResult = { ...features, - top: imageFeatures.viewportStart.y, - left: imageFeatures.viewportStart.x, - usePixelRatio: hasPixelRatio && imageFeatures.colorLength! > innerWidth, + viewportArea: imageFeatures, + usePixelRatio: hasPixelRatio && imageFeatures.width > innerWidth, }; this._cache[browser.id] = calibratedFeatures; return calibratedFeatures; } - private async _analyzeImage( - image: Image, - params: { calculateColorLength?: boolean }, - ): Promise { + private async _findMarkerAreaInImage(image: Image): Promise | null> { const imageHeight = (await image.getSize()).height; - for (let y = 0; y < imageHeight; y++) { - const result = await analyzeRow(y, image, params); + let topPart: Rect<"image", "device"> | null = null; + + for (let y = 0 as Coord<"image", "device", "y">; y < imageHeight; y++) { + const result = await findMarkerXBandInRow(y, image); if (result) { - return result; + topPart = { + top: y, + left: result.left, + width: result.width, + height: getHeight(y, imageHeight as Coord<"image", "device", "y">), + }; + break; + } + } + + if (topPart === null) { + return null; + } + + for (let y = (imageHeight - 1) as Coord<"image", "device", "y">; y >= 0; y--) { + const result = await findMarkerXBandInRow(y, image); + if (result) { + const bottomPart = { + top: 0, + left: result.left, + width: result.width, + height: getHeight(0 as Coord<"image", "device", "y">, y), + }; + + return getIntersection(topPart, bottomPart); } } @@ -85,59 +104,65 @@ export class Calibrator { } } -async function analyzeRow( - row: number, +async function findMarkerXBandInRow( + row: Coord<"image", "device", "y">, image: Image, - params: { calculateColorLength?: boolean } = {}, -): Promise { - const markerStart = await findMarkerInRow(row, image, DIRECTION.FORWARD); +): Promise | null> { + const markerStart = await findMarkerStartInRow(row, image); - if (markerStart === -1) { + if (markerStart === null) { return null; } - const result: ImageAnalysisResult = { viewportStart: { x: markerStart, y: row } }; + const markerEnd = await findMarkerEndInRow(row, image); - if (!params.calculateColorLength) { - return result; + if (markerEnd === null) { + return null; } - const markerEnd = await findMarkerInRow(row, image, DIRECTION.REVERSE); - const colorLength = markerEnd - markerStart + 1; - - return { ...result, colorLength }; + return { + left: markerStart, + width: getWidth(markerStart, markerEnd), + }; } -async function findMarkerInRow(row: number, image: Image, searchDirection: "forward" | "reverse"): Promise { - const imageWidth = (await image.getSize()).width; +async function isMarkerColorAtPoint( + image: Image, + x: Coord<"image", "device", "x">, + y: Coord<"image", "device", "y">, +): Promise { const searchColor = { R: 148, G: 250, B: 0 }; + const color = await image.getRGB(x, y); - if (searchDirection === DIRECTION.REVERSE) { - return searchReverse_(); - } else { - return searchForward_(); - } + return looksSame.colors(color, searchColor); +} - async function searchForward_(): Promise { - for (let x = 0; x < imageWidth; x++) { - if (await compare_(x)) { - return x; - } +async function findMarkerStartInRow( + row: Coord<"image", "device", "y">, + image: Image, +): Promise | null> { + const imageWidth = (await image.getSize()).width; + + for (let x = 0 as Coord<"image", "device", "x">; x < imageWidth; x++) { + if (await isMarkerColorAtPoint(image, x, row)) { + return x; } - return -1; } - async function searchReverse_(): Promise { - for (let x = imageWidth - 1; x >= 0; x--) { - if (await compare_(x)) { - return x; - } + return null; +} + +async function findMarkerEndInRow( + row: Coord<"image", "device", "y">, + image: Image, +): Promise | null> { + const imageWidth = (await image.getSize()).width; + + for (let x = (imageWidth - 1) as Coord<"image", "device", "x">; x >= 0; x--) { + if (await isMarkerColorAtPoint(image, x, row)) { + return x; } - return -1; } - async function compare_(x: number): Promise { - const color = await image.getRGB(x, row); - return looksSame.colors(color, searchColor); - } + return null; } diff --git a/src/browser/camera/index.ts b/src/browser/camera/index.ts index 79f9a770b..d9fc7e184 100644 --- a/src/browser/camera/index.ts +++ b/src/browser/camera/index.ts @@ -1,34 +1,39 @@ +import os from "node:os"; +import path from "node:path"; import _ from "lodash"; +import makeDebug from "debug"; + import { Image } from "../../image"; import * as utils from "./utils"; -import makeDebug from "debug"; +import * as logger from "../../utils/logger"; +import { + getIntersection, + type Coord, + type Point, + type Rect, + type Size, + prettyRect, + prettySize, + prettyPoint, +} from "../isomorphic/geometry"; +import { NEW_ISSUE_LINK } from "../../constants/help"; const debug = makeDebug("testplane:screenshots:camera"); -export interface ImageArea { - left: number; - top: number; - width: number; - height: number; -} - export type ScreenshotMode = "fullpage" | "viewport" | "auto"; -export interface PageMeta { - viewport: ImageArea; - documentHeight: number; - documentWidth: number; -} - -interface Calibration { - left: number; - top: number; +export interface CaptureViewportImageOpts { + viewportOffset: Point<"page", "device">; + viewportSize: Size<"device">; + /** Delay before taking the screenshot, in milliseconds. */ + screenshotDelay?: number; } export class Camera { private _screenshotMode: ScreenshotMode; private _takeScreenshot: () => Promise; - private _calibration: Calibration | null; + private _calibratedArea: Rect<"image", "device"> | null; + private _debugTmpDir: string | null = null; static create(screenshotMode: ScreenshotMode, takeScreenshot: () => Promise): Camera { return new this(screenshotMode, takeScreenshot); @@ -37,23 +42,42 @@ export class Camera { constructor(screenshotMode: ScreenshotMode, takeScreenshot: () => Promise) { this._screenshotMode = screenshotMode; this._takeScreenshot = takeScreenshot; - this._calibration = null; + this._calibratedArea = null; + + if (process.env.TESTPLANE_DEBUG_SCREENSHOTS) { + this._debugTmpDir = path.join( + os.tmpdir(), + `testplane-camera-viewports-${Math.random().toString(36).slice(2)}`, + ); + console.log("Debug camera images will be saved to: ", this._debugTmpDir); + } } - calibrate(calibration: Calibration): void { - this._calibration = calibration; + calibrate(calibratedArea: Rect<"image", "device">): void { + debug("Setting calibrated area: %O", calibratedArea); + this._calibratedArea = calibratedArea; } - /** @param viewport - Current state of the viewport. Top/left denote scroll offsets, width/height denote viewport size. */ - async captureViewportImage(viewport?: ImageArea): Promise { + async captureViewportImage(opts?: CaptureViewportImageOpts): Promise { + if (opts?.screenshotDelay) { + await new Promise(resolve => setTimeout(resolve, opts.screenshotDelay)); + } + const base64 = await this._takeScreenshot(); const image = Image.fromBase64(base64); - const { width, height } = await image.getSize(); - const imageArea: ImageArea = { left: 0, top: 0, width, height }; + const { width, height } = (await image.getSize()) as Size<"device">; + const imageArea: Rect<"image", "device"> = { + left: 0 as Coord<"image", "device", "x">, + top: 0 as Coord<"image", "device", "y">, + width, + height, + }; + + const calibratedArea = this._cropAreaToCalibratedArea(imageArea); - const calibratedArea = this._calibrateArea(imageArea); - const viewportCroppedArea = this._cropAreaToViewport(calibratedArea, viewport); + const viewportCroppedArea = this._cropAreaToViewport(calibratedArea, { width, height }, opts); + await utils.saveViewportImageForDebugIfNeeded(image, calibratedArea, this._debugTmpDir); if (viewportCroppedArea.width !== width || viewportCroppedArea.height !== height) { await image.crop(viewportCroppedArea); @@ -62,38 +86,73 @@ export class Camera { return image; } - private _calibrateArea(imageArea: ImageArea): ImageArea { - if (!this._calibration) { + private _cropAreaToCalibratedArea(imageArea: Rect<"image", "device">): Rect<"image", "device"> { + if (!this._calibratedArea) { return imageArea; } - const { left, top } = this._calibration; + const intersection = getIntersection(imageArea, this._calibratedArea); + if (intersection === null) { + logger.warn( + `No intersection found between image area and calibrated viewport area, falling back to original image area.\n` + + `imageArea: ${prettyRect(imageArea)}, calibratedViewportArea: ${prettyRect( + this._calibratedArea, + )}\n` + + `This likely means Testplane incorrectly determined area free of system UI elements. You can let us know at ${NEW_ISSUE_LINK}, providing this log and browser used.`, + ); + + return imageArea; + } - return { left, top, width: imageArea.width - left, height: imageArea.height - top }; + return intersection; } /* On some browsers, e.g. older firefox versions, the screenshot returned by the browser can be the whole page (even beyond the viewport, potentially spanning thousands of pixels down). This function is used to detect such cases and crop the image to the viewport, always. */ - private _cropAreaToViewport(imageArea: ImageArea, viewport?: ImageArea): ImageArea { - if (!viewport) { - return imageArea; + private _cropAreaToViewport( + imageAreaToCrop: Rect<"image", "device">, + originalImageSize: Size<"device">, + opts?: CaptureViewportImageOpts, + ): Rect<"image", "device"> { + if (!opts?.viewportSize || !opts?.viewportOffset) { + return imageAreaToCrop; } - const isFullPage = utils.isFullPage(imageArea, viewport, this._screenshotMode); - const cropArea = _.clone(viewport); + const isFullPage = utils.isFullPage( + imageAreaToCrop, + originalImageSize, + this._calibratedArea ?? imageAreaToCrop, + this._screenshotMode, + ); + const cropArea = { ...opts.viewportSize, ...opts.viewportOffset }; if (!isFullPage) { - _.extend(cropArea, { top: 0, left: 0 }); + return imageAreaToCrop; } debug( - "cropping area to viewport. imageArea: %O, viewport: %O, cropArea: %O, isFullPage: %s", - imageArea, - viewport, + "cropping area to viewport.\n imageArea: %O\n viewportSize: %O\n viewportOffset: %O\n cropArea: %O\n isFullPage: %s\n screenshotMode: %s\n documentSize: %O", + imageAreaToCrop, + opts.viewportSize, + opts.viewportOffset, cropArea, isFullPage, + this._screenshotMode, ); - return utils.getIntersection(imageArea, cropArea); + const result = getIntersection(imageAreaToCrop, cropArea); + if (result === null) { + logger.warn( + `No intersection found between image area and viewport area, falling back to original image area.\n` + + `imageArea: ${prettyRect(imageAreaToCrop)},\n` + + `viewportSize: ${prettySize(opts.viewportSize)},\n` + + `viewportOffset: ${prettyPoint(opts.viewportOffset)}\n` + + `This likely means Testplane incorrectly determined whether returned image is full page and viewport state. You can let us know at ${NEW_ISSUE_LINK}, providing this log and browser used.`, + ); + + return imageAreaToCrop; + } + + return result; } } diff --git a/src/browser/camera/utils.ts b/src/browser/camera/utils.ts index 37a53ef5b..af8c8dd19 100644 --- a/src/browser/camera/utils.ts +++ b/src/browser/camera/utils.ts @@ -1,26 +1,56 @@ -import { ImageArea, ScreenshotMode } from "."; +import path from "path"; +import fs from "fs"; +import { ScreenshotMode } from "."; +import { Image } from "../../image"; +import { Rect, Size, getBottom } from "../isomorphic/geometry"; +import { saveViewportImageWithDebugRects } from "../screen-shooter/composite-image/debug-utils"; + +export const isFullPage = ( + imageSize: Rect<"image", "device">, + viewportSize: Size<"device">, + calibratedArea: Rect<"image", "device">, + screenshotMode: ScreenshotMode, +): boolean => { + // "system ui" is something like status bar on safari mobile, or address bar at the bottom + const systemUiHeight = calibratedArea.top + (imageSize.height - getBottom(calibratedArea)); -export const isFullPage = (imageArea: ImageArea, viewport: ImageArea, screenshotMode: ScreenshotMode): boolean => { switch (screenshotMode) { case "fullpage": return true; case "viewport": return false; case "auto": - return imageArea.height > viewport.height || imageArea.width > viewport.width; + return imageSize.height > viewportSize.height + systemUiHeight; } }; -export const getIntersection = (area1: ImageArea, area2: ImageArea): ImageArea => { - const left = Math.max(area1.left, area2.left); - const top = Math.max(area1.top, area2.top); - const right = Math.min(area1.left + area1.width, area2.left + area2.width); - const bottom = Math.min(area1.top + area1.height, area2.top + area2.height); +export async function saveViewportImageForDebugIfNeeded( + viewportImage: Image, + viewportCroppedArea: Rect<"image", "device">, + debugDir: string | null, +): Promise { + if (!process.env.TESTPLANE_DEBUG_SCREENSHOTS || !debugDir) { + return; + } - return { - left, - top, - width: Math.max(0, right - left), - height: Math.max(0, bottom - top), - }; -}; + try { + fs.mkdirSync(debugDir, { recursive: true }); + + const timestamp = String(Date.now()).padStart(13, "0"); + const randomId = Math.random().toString(36).substring(2, 8); + const outputPath = path.join(debugDir, `viewport-${timestamp}-${randomId}.png`); + + await saveViewportImageWithDebugRects( + viewportImage, + [ + { + rect: viewportCroppedArea as unknown as Rect<"viewport", "device">, + color: { r: 0, g: 255, b: 0, a: 255 }, + }, + ], + outputPath, + ); + } catch (error) { + console.warn("Failed to save camera viewport debug image: %O", error); + } +} diff --git a/src/browser/client-scripts/calibrate.js b/src/browser/client-scripts/calibrate.js index 5646953df..3eb443ff7 100644 --- a/src/browser/client-scripts/calibrate.js +++ b/src/browser/client-scripts/calibrate.js @@ -28,7 +28,18 @@ bodyStyle.border = 0; } - bodyStyle.backgroundColor = "#96fa00"; + // For example of how this looks, see https://github.com/gemini-testing/testplane/pull/1239 + bodyStyle.backgroundColor = "#ff0000"; + + var fullPageElement = document.createElement("div"); + fullPageElement.style.width = "100vw"; + fullPageElement.style.height = "100vh"; + fullPageElement.style.position = "fixed"; + fullPageElement.style.top = "0"; + fullPageElement.style.left = "0"; + fullPageElement.style.zIndex = "999999"; + fullPageElement.style.backgroundColor = "#96fa00"; + document.body.appendChild(fullPageElement); } function hasCSS3Selectors() { diff --git a/src/browser/existing-browser.ts b/src/browser/existing-browser.ts index 443bb1e16..05ef3c3e6 100644 --- a/src/browser/existing-browser.ts +++ b/src/browser/existing-browser.ts @@ -180,12 +180,12 @@ export class ExistingBrowser extends Browser { return this._config.automationProtocol === WEBDRIVER_PROTOCOL; } - async captureViewportImage(viewport?: ImageArea, screenshotDelay?: number): Promise { + async captureViewportImage(opts?: CaptureViewportImageOpts, screenshotDelay?: number): Promise { if (screenshotDelay) { await new Promise(resolve => setTimeout(resolve, screenshotDelay)); } - return this._camera.captureViewportImage(viewport); + return this._camera.captureViewportImage(opts); } scrollBy(params: ScrollByParams): Promise { @@ -469,7 +469,7 @@ export class ExistingBrowser extends Browser { return calibrator.calibrate(this).then(calibration => { this._calibration = calibration; - this._camera.calibrate(calibration); + this._camera.calibrate(calibration.viewportArea); }); } @@ -506,4 +506,8 @@ export class ExistingBrowser extends Browser { get cdp(): CDP | null { return this._cdp; } + + get camera(): Camera { + return this._camera; + } } diff --git a/test/src/browser/calibrator.js b/test/src/browser/calibrator.js index a973a4a8e..42ed97d45 100644 --- a/test/src/browser/calibrator.js +++ b/test/src/browser/calibrator.js @@ -44,8 +44,8 @@ describe("calibrator", () => { const result = await calibrator.calibrate(browser); - assert.match(result.top, 2); - assert.match(result.left, 2); + assert.match(result.viewportArea.top, 2); + assert.match(result.viewportArea.left, 2); }); }); @@ -74,10 +74,11 @@ describe("calibrator", () => { const result = await calibrator.calibrate(browser); - await calibrator.calibrate(browser); + const cachedResult = await calibrator.calibrate(browser); - assert.match(result.top, 2); - assert.match(result.left, 2); + assert.equal(cachedResult, result); + assert.match(result.viewportArea.top, 2); + assert.match(result.viewportArea.left, 2); }); it("should fail on broken calibration page", () => { diff --git a/test/src/browser/camera/index.js b/test/src/browser/camera/index.js index 450d9a159..ce39ed118 100644 --- a/test/src/browser/camera/index.js +++ b/test/src/browser/camera/index.js @@ -7,13 +7,16 @@ describe("browser/camera", () => { const sandbox = sinon.createSandbox(); let Camera; let isFullPageStub; + let getIntersectionStub; let image; beforeEach(() => { isFullPageStub = sinon.stub(); + getIntersectionStub = sinon.stub().callsFake((_, area) => area); Camera = proxyquire("src/browser/camera", { "./utils": { isFullPage: isFullPageStub, + getIntersection: getIntersectionStub, }, }).Camera; @@ -42,7 +45,7 @@ describe("browser/camera", () => { const camera = Camera.create(null, sinon.stub().resolves()); image.getSize.resolves({ width: 10, height: 10 }); - camera.calibrate({ top: 6, left: 4 }); + camera.calibrate({ top: 6, left: 4, width: 10, height: 10 }); await camera.captureViewportImage(); assert.calledOnceWith(image.crop, { @@ -55,7 +58,9 @@ describe("browser/camera", () => { }); describe("crop to viewport", () => { - let viewport; + let viewportOffset; + let viewportSize; + let opts; const mkCamera_ = browserOptions => { const screenshotMode = (browserOptions || {}).screenshotMode || "auto"; @@ -63,12 +68,18 @@ describe("browser/camera", () => { }; beforeEach(() => { - viewport = { + viewportOffset = { left: 1, top: 1, + }; + viewportSize = { width: 100, height: 100, }; + opts = { + viewportOffset, + viewportSize, + }; }); it("should not crop to viewport if page disposition was not passed", async () => { @@ -80,37 +91,35 @@ describe("browser/camera", () => { it("should crop fullPage image with viewport value if page disposition was set", async () => { isFullPageStub.returns(true); - await mkCamera_({ screenshotMode: "fullPage" }).captureViewportImage(viewport); + await mkCamera_({ screenshotMode: "fullpage" }).captureViewportImage(opts); - assert.calledOnceWith(image.crop, viewport); + assert.calledOnceWith(image.crop, { + ...viewportSize, + ...viewportOffset, + }); }); it("should use viewportOffset for fullPage image crop if provided", async () => { isFullPageStub.returns(true); - viewport.top = 10; - viewport.left = 20; + viewportOffset.top = 10; + viewportOffset.left = 20; - await mkCamera_({ screenshotMode: "fullPage" }).captureViewportImage(viewport); + await mkCamera_({ screenshotMode: "fullpage" }).captureViewportImage(opts); assert.calledOnceWith(image.crop, { top: 10, left: 20, - width: viewport.width, - height: viewport.height, + width: viewportSize.width, + height: viewportSize.height, }); }); - it("should crop not fullPage image to the left and right", async () => { + it("should not crop not fullPage image", async () => { isFullPageStub.returns(false); - await mkCamera_({ screenshotMode: "viewport" }).captureViewportImage(viewport); + await mkCamera_({ screenshotMode: "viewport" }).captureViewportImage(opts); - assert.calledOnceWith(image.crop, { - left: 0, - top: 0, - height: viewport.height, - width: viewport.width, - }); + assert.notCalled(image.crop); }); }); });