From 554b2d3cd093f89b9494efc249aad1de3d36329f Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Thu, 8 Jan 2026 12:22:11 -0600 Subject: [PATCH 01/21] Add goldctl dependency on unit tests --- .ci.yaml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.ci.yaml b/.ci.yaml index c036774df2e..b286a784aaf 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -137,6 +137,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] add_recipes_cq: "true" target_file: dart_unit_tests.yaml channel: master @@ -147,6 +151,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] target_file: dart_unit_tests.yaml channel: master version_file: flutter_master.version @@ -156,6 +164,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] target_file: dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -165,6 +177,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] target_file: dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -174,6 +190,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] add_recipes_cq: "true" target_file: web_dart_unit_tests.yaml channel: master @@ -184,6 +204,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] target_file: web_dart_unit_tests.yaml channel: master version_file: flutter_master.version @@ -193,6 +217,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] target_file: web_dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -202,6 +230,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] target_file: web_dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -212,6 +244,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] add_recipes_cq: "true" target_file: web_dart_unit_tests_wasm.yaml channel: master @@ -222,6 +258,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] target_file: web_dart_unit_tests_wasm.yaml channel: master version_file: flutter_master.version @@ -952,6 +992,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] target_file: windows_dart_unit_tests.yaml channel: master version_file: flutter_master.version @@ -961,6 +1005,10 @@ targets: recipe: packages/packages timeout: 60 properties: + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] target_file: windows_dart_unit_tests.yaml channel: master version_file: flutter_master.version From eecf01486927f7dc5fd2cd88041e8204a6d096b2 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Thu, 8 Jan 2026 15:51:11 -0600 Subject: [PATCH 02/21] WIP - add gold integration to flutter/packages --- .../two_dimensional_scrollables/pubspec.yaml | 2 + .../test/flutter_test_config.dart | 22 + .../test/goldens/goldens.dart | 21 + .../test/goldens_io.dart | 5 + .../test/goldens_web.dart | 9 + script/flutter_goldens/README.md | 11 + .../flutter_goldens/lib/flutter_goldens.dart | 697 ++++++++++ script/flutter_goldens/lib/skia_client.dart | 518 +++++++ script/flutter_goldens/pubspec.yaml | 15 + .../test/comparator_selection_test.dart | 111 ++ .../test/flutter_goldens_test.dart | 1230 +++++++++++++++++ .../flutter_goldens/test/json_templates.dart | 94 ++ 12 files changed, 2735 insertions(+) create mode 100644 packages/two_dimensional_scrollables/test/flutter_test_config.dart create mode 100644 packages/two_dimensional_scrollables/test/goldens/goldens.dart create mode 100644 packages/two_dimensional_scrollables/test/goldens_io.dart create mode 100644 packages/two_dimensional_scrollables/test/goldens_web.dart create mode 100644 script/flutter_goldens/README.md create mode 100644 script/flutter_goldens/lib/flutter_goldens.dart create mode 100644 script/flutter_goldens/lib/skia_client.dart create mode 100644 script/flutter_goldens/pubspec.yaml create mode 100644 script/flutter_goldens/test/comparator_selection_test.dart create mode 100644 script/flutter_goldens/test/flutter_goldens_test.dart create mode 100644 script/flutter_goldens/test/json_templates.dart diff --git a/packages/two_dimensional_scrollables/pubspec.yaml b/packages/two_dimensional_scrollables/pubspec.yaml index 3942be48404..bc710d0d62f 100644 --- a/packages/two_dimensional_scrollables/pubspec.yaml +++ b/packages/two_dimensional_scrollables/pubspec.yaml @@ -13,6 +13,8 @@ dependencies: sdk: flutter dev_dependencies: + flutter_goldens: + path: ../../script/flutter_goldens flutter_test: sdk: flutter diff --git a/packages/two_dimensional_scrollables/test/flutter_test_config.dart b/packages/two_dimensional_scrollables/test/flutter_test_config.dart new file mode 100644 index 00000000000..f8faa41b4f2 --- /dev/null +++ b/packages/two_dimensional_scrollables/test/flutter_test_config.dart @@ -0,0 +1,22 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + + +// Initial testing of instance only, not for merging. +// Plenty to do next (if this works): +// - verify service accounts used by Luci in pre/post submit tests in flutter/packages, auth works +// - Get new Gold frontend up +// - document how to enable golden file testing for a package +// - update flutter/cocoon to add flutter-gold check for triage + +import 'dart:async'; + +import 'goldens_io.dart' + if (dart.library.js_interop) 'goldens_web.dart' + as flutter_goldens; + +Future testExecutable(FutureOr Function() testMain) { + // Enable golden file testing using Skia Gold. + return flutter_goldens.testExecutable(testMain, namePrefix: 'two_dimensional_scrollables'); +} diff --git a/packages/two_dimensional_scrollables/test/goldens/goldens.dart b/packages/two_dimensional_scrollables/test/goldens/goldens.dart new file mode 100644 index 00000000000..27dd3c77ed5 --- /dev/null +++ b/packages/two_dimensional_scrollables/test/goldens/goldens.dart @@ -0,0 +1,21 @@ +// Copyright 2013 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + testWidgets('Inconsequential golden test', (WidgetTester tester) async { + // The test validates the Flutter Gold integration. Any changes to the + // golden file can be approved at any time. + await tester.pumpWidget(RepaintBoundary(child: Container(color: const Color(0xAFF61145)))); + + await tester.pumpAndSettle(); + await expectLater( + find.byType(RepaintBoundary), + matchesGoldenFile('inconsequential_golden_file.png'), + ); + }); +} diff --git a/packages/two_dimensional_scrollables/test/goldens_io.dart b/packages/two_dimensional_scrollables/test/goldens_io.dart new file mode 100644 index 00000000000..d552235d690 --- /dev/null +++ b/packages/two_dimensional_scrollables/test/goldens_io.dart @@ -0,0 +1,5 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'package:flutter_goldens/flutter_goldens.dart' show testExecutable; diff --git a/packages/two_dimensional_scrollables/test/goldens_web.dart b/packages/two_dimensional_scrollables/test/goldens_web.dart new file mode 100644 index 00000000000..1b4a8a2af31 --- /dev/null +++ b/packages/two_dimensional_scrollables/test/goldens_web.dart @@ -0,0 +1,9 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +// package:flutter_goldens is not used as part of the test process for web. +Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async => + testMain(); diff --git a/script/flutter_goldens/README.md b/script/flutter_goldens/README.md new file mode 100644 index 00000000000..2187c8e28f6 --- /dev/null +++ b/script/flutter_goldens/README.md @@ -0,0 +1,11 @@ +This package is an internal implementation detail for our testing +infrastructure. It enables the framework to use the Skia Gold +infrastructure for tracking golden image tests. + +See also: + + * https://skia.org/docs/dev/testing/skiagold/ + * https://flutter-gold.skia.org/ + * [Writing a golden file test for package flutter] + +[Writing a golden file test for package flutter]: /docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart new file mode 100644 index 00000000000..895cb2e89e6 --- /dev/null +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -0,0 +1,697 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// @docImport 'dart:io'; +library; + +import 'dart:async' show FutureOr; +import 'dart:io' as io show HttpClient, OSError, SocketException; + +import 'package:file/file.dart'; +import 'package:file/local.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:platform/platform.dart'; +import 'package:process/process.dart'; + +import 'skia_client.dart'; +export 'skia_client.dart'; + +// If you are here trying to figure out how to use golden files for Flutter +// repos, consider reading this documentation: +// https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md + +// If you are trying to debug this package, you may like to use the golden test +// titled "Inconsequential golden test" in this file: +// TODO(Piinks): Add canary test somewhere. + +// const String _kFlutterRootKey = 'FLUTTER_ROOT'; + +bool _isMainBranch(String? branch) { + return branch == 'main' || branch == 'master'; +} + +/// Main method that can be used in a `flutter_test_config.dart` file to set +/// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that +/// works for the current test. _Which_ [FlutterGoldenFileComparator] is +/// instantiated is based on the current testing environment. +/// +/// When set, the `namePrefix` is prepended to the names of all gold images. +/// +/// This function assumes the [goldenFileComparator] has been set to a +/// [LocalFileComparator], which happens in the bootstrap code used when running +/// tests using `flutter test`. This should not be called when running a test +/// using `flutter run`, as in that environment, the [goldenFileComparator] is a +/// [TrivialComparator]. +/// +/// An [HttpClient] is created when this method is called. That client is used +/// to communicate with the Skia Gold servers. Any [HttpOverrides] set in this +/// will affect whether this is effective or not. For example, if the current +/// override provides a mock client that always fails, then all calls to gold +/// comparison functions will fail. +Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async { + assert( + goldenFileComparator is LocalFileComparator, + 'The flutter_goldens package should be used from a flutter_test_config.dart ' + 'file, which is only invoked when using "flutter test". The "flutter test" ' + 'bootstrap logic sets "goldenFileComparator" to a LocalFileComparator. It ' + 'appears in this instance however that the "goldenFileComparator" is a ' + '${goldenFileComparator.runtimeType}.\n' + 'See also: https://flutter.dev/to/flutter-test-docs', + ); + const Platform platform = LocalPlatform(); + const FileSystem fs = LocalFileSystem(); + const ProcessManager process = LocalProcessManager(); + final httpClient = io.HttpClient(); + if (FlutterPostSubmitFileComparator.isForEnvironment(platform)) { + goldenFileComparator = await FlutterPostSubmitFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + platform: platform, + namePrefix: namePrefix, + log: print, + fs: fs, + process: process, + httpClient: httpClient, + ); + } else if (FlutterPreSubmitFileComparator.isForEnvironment(platform)) { + goldenFileComparator = await FlutterPreSubmitFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + platform: platform, + namePrefix: namePrefix, + log: print, + fs: fs, + process: process, + httpClient: httpClient, + ); + } else if (FlutterSkippingFileComparator.isForEnvironment(platform)) { + goldenFileComparator = FlutterSkippingFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + 'Golden file testing is not executed on LUCI environments outside of ' + 'flutter, or in test shards that are not configured for using goldctl.', + platform: platform, + namePrefix: namePrefix, + log: print, + fs: fs, + process: process, + httpClient: httpClient, + ); + } else { + goldenFileComparator = await FlutterLocalFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + platform: platform, + log: print, + fs: fs, + process: process, + httpClient: httpClient, + ); + } + await testMain(); +} + +/// Abstract base class golden file comparator specific to the `flutter/packages` +/// repository. +/// +/// Golden file testing for the `flutter/flutter` repository is handled by three +/// different [FlutterGoldenFileComparator]s, depending on the current testing +/// environment. +/// +/// * The [FlutterPostSubmitFileComparator] is utilized during post-submit +/// testing, after a pull request has landed on the master branch. This +/// comparator uses the [SkiaGoldClient] and the `goldctl` tool to upload +/// tests to the [Flutter Packages Gold dashboard](https://flutter-packages-gold.skia.org). +/// Flutter Gold manages the master golden files for the packages that use +/// matchesGoldenFile in the `flutter/packages` repository. +/// +/// * The [FlutterPreSubmitFileComparator] is utilized in pre-submit testing, +/// before a pull request lands on the main branch. This +/// comparator uses the [SkiaGoldClient] to execute tryjobs, allowing +/// contributors to view and check in visual differences before landing the +/// change. +/// +/// * The [FlutterLocalFileComparator] is used for local development testing. +/// This comparator will use the [SkiaGoldClient] to request baseline images +/// from [Flutter Packages Gold](https://flutter-packages-gold.skia.org) and +/// manually compare pixels. If a difference is detected, this comparator will +/// generate failure output illustrating the found difference. If a baseline +/// is not found for a given test image, it will consider it a new test and +/// output the new image for verification. +/// +/// The [FlutterSkippingFileComparator] is utilized to skip tests outside +/// of the appropriate environments described above. Currently, some +/// packages or environments do not execute golden file testing, and as such do +/// not require a comparator. This comparator is also used when an internet +/// connection is unavailable. +abstract class FlutterGoldenFileComparator extends GoldenFileComparator { + /// Creates a [FlutterGoldenFileComparator] that will resolve golden file + /// URIs relative to the specified [basedir], and retrieve golden baselines + /// using the [skiaClient]. The [basedir] is used for writing and accessing + /// information and files for interacting with the [skiaClient]. When testing + /// locally, the [basedir] will also contain any diffs from failed tests, or + /// goldens generated from newly introduced tests. + @visibleForTesting + FlutterGoldenFileComparator( + this.basedir, + this.skiaClient, { + required this.fs, + required this.platform, + this.namePrefix, + required this.log, + }); + + /// The directory to which golden file URIs will be resolved in [compare] and + /// [update]. + final Uri basedir; + + /// A client for uploading image tests and making baseline requests to the + /// Flutter Gold Dashboard. + final SkiaGoldClient skiaClient; + + /// The file system used to perform file access. + final FileSystem fs; + + /// The environment (current working directory, identity of the OS, + /// environment variables, etc). + final Platform platform; + + /// The prefix that is added to all golden names. + final String? namePrefix; + + /// The logging function to use when reporting messages to the console. + final LogCallback log; + + @override + Future update(Uri golden, Uint8List imageBytes) async { + final File goldenFile = getGoldenFile(golden); + await goldenFile.parent.create(recursive: true); + await goldenFile.writeAsBytes(imageBytes, flush: true); + } + + @override + Uri getTestUri(Uri key, int? version) => key; + + /// Calculate the appropriate basedir for the current test context. + /// + /// The optional [suffix] argument is used by the + /// [FlutterPostSubmitFileComparator] and the [FlutterPreSubmitFileComparator]. + /// These [FlutterGoldenFileComparator]s randomize their base directories to + /// maintain thread safety while using the `goldctl` tool. + @protected + @visibleForTesting + static Directory getBaseDirectory( + LocalFileComparator defaultComparator, { + required Platform platform, + String? suffix, + required FileSystem fs, + }) { + // final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]); + final Directory comparisonRoot = switch (suffix) { + // null => flutterRoot.childDirectory(fs.path.join('bin', 'cache', 'pkg', 'skia_goldens')), + null => fs.directory(defaultComparator.basedir).childDirectory('skia_goldens'), + _ => fs.systemTempDirectory.createTempSync(suffix), + }; + return comparisonRoot; //.childDirectory(fs.path.relative(testPath, from: flutterRoot.path)); + } + + /// Returns the golden [File] identified by the given [Uri]. + @protected + File getGoldenFile(Uri uri) { + final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path); + print(goldenFile); + return goldenFile; + } + + /// Prepends the golden URL with the library name that encloses the current + /// test. + Uri _addPrefix(Uri golden) { + // Ensure the Uri ends in .png as the SkiaClient expects + assert( + golden.toString().split('.').last == 'png', + 'Golden files in the Flutter framework must end with the file extension ' + '.png.', + ); + print('basedir: $basedir'); + return Uri.parse( + [ + ?namePrefix, + basedir.pathSegments[1], // ~/packages/ + golden.toString(), + ].join('.'), + ); + } +} + +/// A [FlutterGoldenFileComparator] for testing golden images with Skia Gold in +/// post-submit. +/// +/// For testing across all platforms, the [SkiaGoldClient] is used to upload +/// images for framework-related golden tests and process results. +/// +/// See also: +/// +/// * [GoldenFileComparator], the abstract class that +/// [FlutterGoldenFileComparator] implements. +/// * [FlutterPreSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images before changes are +/// merged into the master branch. +/// * [FlutterLocalFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images locally on your +/// current machine. +class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { + /// Creates a [FlutterPostSubmitFileComparator] that will test golden file + /// images against Skia Gold. + /// + /// The [fs] parameter is useful in tests, where the default + /// file system can be replaced by mock instances. + FlutterPostSubmitFileComparator( + super.basedir, + super.skiaClient, { + required super.fs, + required super.platform, + super.namePrefix, + required super.log, + }); + + /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative + /// path resolution of the provided `localFileComparator`. + /// + /// The [goldens] parameter is visible for testing purposes only. + static Future fromLocalFileComparator({ + SkiaGoldClient? goldens, + required LocalFileComparator localFileComparator, + required Platform platform, + String? namePrefix, + required LogCallback log, + required FileSystem fs, + required ProcessManager process, + required io.HttpClient httpClient, + }) async { + print('Creating Postsubmit'); + final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory( + localFileComparator, + platform: platform, + suffix: 'flutter_goldens_postsubmit.', + fs: fs, + ); + print(baseDirectory); + baseDirectory.createSync(recursive: true); + + goldens ??= SkiaGoldClient( + baseDirectory, + log: log, + platform: platform, + fs: fs, + process: process, + httpClient: httpClient, + ); + await goldens.auth(); + return FlutterPostSubmitFileComparator( + baseDirectory.uri, + goldens, + platform: platform, + namePrefix: namePrefix, + log: log, + fs: fs, + ); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + await skiaClient.imgtestInit(); + golden = _addPrefix(golden); + await update(golden, imageBytes); + final File goldenFile = getGoldenFile(golden); + try { + return await skiaClient.imgtestAdd(golden.path, goldenFile); + } on SkiaException catch (e) { + // Convert SkiaException -> TestFailure so that this class implements the + // contract of GoldenFileComparator, and matchesGoldenFile() converts the + // TestFailure into a standard reported test error (with a better stack + // trace, for example). + // + // https://github.com/flutter/flutter/issues/162621 + throw TestFailure('$e'); + } + } + + /// Decides based on the current environment if goldens tests should be + /// executed through Skia Gold. + static bool isForEnvironment(Platform platform) { + final bool luciPostSubmit = + platform.environment.containsKey('SWARMING_TASK_ID') && + platform.environment.containsKey('GOLDCTL') + // Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator]. + && + !platform.environment.containsKey('GOLD_TRYJOB') + // Only run on main branch. + && + _isMainBranch(platform.environment['GIT_BRANCH']); + return luciPostSubmit; + } +} + +/// A [FlutterGoldenFileComparator] for testing golden images before changes are +/// merged into the master branch. The comparator executes tryjobs using the +/// [SkiaGoldClient]. +/// +/// See also: +/// +/// * [GoldenFileComparator], the abstract class that +/// [FlutterGoldenFileComparator] implements. +/// * [FlutterPostSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold +/// dashboard in post-submit. +/// * [FlutterLocalFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images locally on your +/// current machine. +class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { + /// Creates a [FlutterPreSubmitFileComparator] that will test golden file + /// images against baselines requested from Flutter Gold. + /// + /// The [fs] parameter is useful in tests, where the default + /// file system can be replaced by mock instances. + FlutterPreSubmitFileComparator( + super.basedir, + super.skiaClient, { + required super.fs, + required super.platform, + super.namePrefix, + required super.log, + }); + + /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the + /// relative path resolution of the default [goldenFileComparator]. + /// + /// The [goldens] parameter is visible for testing purposes only. + static Future fromLocalFileComparator({ + SkiaGoldClient? goldens, + required LocalFileComparator localFileComparator, + required Platform platform, + Directory? testBasedir, + String? namePrefix, + required LogCallback log, + required FileSystem fs, + required ProcessManager process, + required io.HttpClient httpClient, + }) async { + print('Creating presubmit'); + final Directory baseDirectory = + testBasedir ?? + FlutterGoldenFileComparator.getBaseDirectory( + localFileComparator, + platform: platform, + suffix: 'flutter_goldens_presubmit.', + fs: fs, + ); + print(baseDirectory); + if (!baseDirectory.existsSync()) { + baseDirectory.createSync(recursive: true); + } + + goldens ??= SkiaGoldClient( + baseDirectory, + platform: platform, + log: log, + fs: fs, + process: process, + httpClient: httpClient, + ); + + await goldens.auth(); + return FlutterPreSubmitFileComparator( + baseDirectory.uri, + goldens, + platform: platform, + namePrefix: namePrefix, + log: log, + fs: fs, + ); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + await skiaClient.tryjobInit(); + golden = _addPrefix(golden); + await update(golden, imageBytes); + final File goldenFile = getGoldenFile(golden); + + await skiaClient.tryjobAdd(golden.path, goldenFile); + + // This will always return true since golden file test failures are managed + // in pre-submit checks by the flutter-gold status check. + return true; + } + + /// Decides based on the current environment if goldens tests should be + /// executed as pre-submit tests with Skia Gold. + static bool isForEnvironment(Platform platform) { + final bool luciPreSubmit = + platform.environment.containsKey('SWARMING_TASK_ID') && + platform.environment.containsKey('GOLDCTL') && + platform.environment.containsKey('GOLD_TRYJOB') + // Only run on the main branch + && + _isMainBranch(platform.environment['GIT_BRANCH']); + return luciPreSubmit; + } +} + +/// A [FlutterGoldenFileComparator] for testing conditions that do not execute +/// golden file tests. +/// +/// Currently, this comparator is used on Luci environments when executing tests +/// outside of the flutter/flutter repository. +/// +/// See also: +/// +/// * [FlutterPostSubmitFileComparator], another [FlutterGoldenFileComparator] +/// that tests golden images through Skia Gold. +/// * [FlutterPreSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images before changes are +/// merged into the master branch. +/// * [FlutterLocalFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images locally on your +/// current machine. +class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { + /// Creates a [FlutterSkippingFileComparator] that will skip tests that + /// are not in the right environment for golden file testing. + FlutterSkippingFileComparator( + super.basedir, + super.skiaClient, + this.reason, { + super.namePrefix, + required super.platform, + required super.log, + required super.fs, + }); + + /// Describes the reason for using the [FlutterSkippingFileComparator]. + final String reason; + + /// Creates a new [FlutterSkippingFileComparator] that mirrors the + /// relative path resolution of the given [localFileComparator]. + static FlutterSkippingFileComparator fromLocalFileComparator( + String reason, { + required LocalFileComparator localFileComparator, + String? namePrefix, + required Platform platform, + required LogCallback log, + required FileSystem fs, + required ProcessManager process, + required io.HttpClient httpClient, + }) { + print('Creating skip'); + final Uri basedir = localFileComparator.basedir; + final skiaClient = SkiaGoldClient( + fs.directory(basedir), + platform: platform, + log: log, + fs: fs, + process: process, + httpClient: httpClient, + ); + return FlutterSkippingFileComparator( + basedir, + skiaClient, + reason, + namePrefix: namePrefix, + platform: platform, + log: log, + fs: fs, + ); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + log('Skipping "$golden" test: $reason'); + return true; + } + + @override + Future update(Uri golden, Uint8List imageBytes) async {} + + /// Decides, based on the current environment, if this comparator should be + /// used. + /// + /// If we are in a CI environment, i.e. LUCI, but are not using the other + /// comparators, we skip. Otherwise we would fallback to the local comparator, + /// for which failures cannot be resolved in a CI environment. + static bool isForEnvironment(Platform platform) { + return platform.environment.containsKey('SWARMING_TASK_ID'); + } +} + +/// A [FlutterGoldenFileComparator] for testing golden images locally on your +/// current machine. +/// +/// This comparator utilizes the [SkiaGoldClient] to request baseline images for +/// the given device under test for comparison. This comparator is initialized +/// when conditions for all other [FlutterGoldenFileComparator]s have not been +/// met, see the `isForEnvironment` method for each one listed below. +/// +/// The [FlutterLocalFileComparator] is intended to run on local machines and +/// serve as a smoke test during development. As such, it will not be able to +/// detect unintended changes on environments other than the currently executing +/// machine, until they are tested using the [FlutterPreSubmitFileComparator]. +/// +/// See also: +/// +/// * [GoldenFileComparator], the abstract class that +/// [FlutterGoldenFileComparator] implements. +/// * [FlutterPostSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold +/// dashboard. +/// * [FlutterPreSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images before changes are +/// merged into the master branch. +/// * [FlutterSkippingFileComparator], another +/// [FlutterGoldenFileComparator] that controls post-submit testing +/// conditions that do not execute golden file tests. +class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalComparisonOutput { + /// Creates a [FlutterLocalFileComparator] that will test golden file + /// images against baselines requested from Flutter Gold. + /// + /// The [fs] parameter is useful in tests, where the default + /// file system can be replaced by mock instances. + FlutterLocalFileComparator( + super.basedir, + super.skiaClient, { + required super.fs, + required super.platform, + required super.log, + }); + + /// Creates a new [FlutterLocalFileComparator] that mirrors the + /// relative path resolution of the given [localFileComparator]. + /// + /// The [goldens] and [baseDirectory] parameters are + /// visible for testing purposes only. + static Future fromLocalFileComparator({ + SkiaGoldClient? goldens, + required LocalFileComparator localFileComparator, + required Platform platform, + Directory? baseDirectory, + required LogCallback log, + required FileSystem fs, + required ProcessManager process, + required io.HttpClient httpClient, + }) async { + print('Creating Local'); + baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory( + localFileComparator, + platform: platform, + fs: fs, + ); + print('**'); + print(baseDirectory); + print('**'); + + if (!baseDirectory.existsSync()) { + baseDirectory.createSync(recursive: true); + } + + goldens ??= SkiaGoldClient( + baseDirectory, + platform: platform, + log: log, + fs: fs, + process: process, + httpClient: httpClient, + ); + try { + // Check if we can reach Gold. + await goldens.getExpectationForTest(''); + } on io.OSError catch (_) { + return FlutterSkippingFileComparator( + baseDirectory.uri, + goldens, + 'OSError occurred, could not reach Gold. ' + 'Switching to FlutterSkippingGoldenFileComparator.', + platform: platform, + log: log, + fs: fs, + ); + } on io.SocketException catch (_) { + return FlutterSkippingFileComparator( + baseDirectory.uri, + goldens, + 'SocketException occurred, could not reach Gold. ' + 'Switching to FlutterSkippingGoldenFileComparator.', + platform: platform, + log: log, + fs: fs, + ); + } on FormatException catch (_) { + return FlutterSkippingFileComparator( + baseDirectory.uri, + goldens, + 'FormatException occurred, could not reach Gold. ' + 'Switching to FlutterSkippingGoldenFileComparator.', + platform: platform, + log: log, + fs: fs, + ); + } + + return FlutterLocalFileComparator( + baseDirectory.uri, + goldens, + platform: platform, + log: log, + fs: fs, + ); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + golden = _addPrefix(golden); + final String testName = skiaClient.cleanTestName(golden.path); + late String? testExpectation; + testExpectation = await skiaClient.getExpectationForTest(testName); + + if (testExpectation == null || testExpectation.isEmpty) { + log( + 'No expectations provided by Skia Gold for test: $golden. ' + 'This may be a new test. If this is an unexpected result, check ' + 'https://flutter-packages-gold.skia.org.\n' + 'Validate image output found at $basedir', + ); + update(golden, imageBytes); + return true; + } + + ComparisonResult result; + final List goldenBytes = await skiaClient.getImageBytes(testExpectation); + + result = await GoldenFileComparator.compareLists(imageBytes, goldenBytes); + + if (result.passed) { + result.dispose(); + return true; + } + + final String error = await generateFailureOutput(result, golden, basedir); + result.dispose(); + throw FlutterError(error); + } +} diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart new file mode 100644 index 00000000000..d9b760f4c21 --- /dev/null +++ b/script/flutter_goldens/lib/skia_client.dart @@ -0,0 +1,518 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// @docImport 'flutter_goldens.dart'; +library; + +import 'dart:convert'; +import 'dart:io' as io; + +import 'package:crypto/crypto.dart'; +import 'package:file/file.dart'; +import 'package:path/path.dart' as path; +import 'package:platform/platform.dart'; +import 'package:process/process.dart'; + +// If you are here trying to figure out how to use golden files in relevant +// Flutter repos, consider reading this wiki page: +// https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md + +// const String _kFlutterRootKey = 'FLUTTER_ROOT'; +const String _kGoldctlKey = 'GOLDCTL'; +const String _kTestBrowserKey = 'CHROME_EXECUTABLE'; + +/// Signature of callbacks used to inject [print] replacements. +typedef LogCallback = void Function(String); + +/// Exception thrown when an error is returned from the [SkiaGoldClient]. +class SkiaException implements Exception { + /// Creates a new `SkiaException` with a required error [message]. + const SkiaException(this.message); + + /// A message describing the error. + final String message; + + /// Returns a description of the Skia exception. + /// + /// The description always contains the [message]. + @override + String toString() => 'SkiaException: $message'; +} + +/// A client for uploading image tests and making baseline requests to the +/// Flutter Packages Gold Dashboard. +class SkiaGoldClient { + /// Creates a [SkiaGoldClient] with the given [workDirectory] and [Platform]. + /// + /// All other parameters are optional. They may be provided in tests to + /// override the defaults for [fs], [process], and [httpClient]. + SkiaGoldClient( + this.workDirectory, { + required this.fs, + required this.process, + required this.platform, + required this.httpClient, + required this.log, + }); + + /// The file system to use for storing the local clone of the repository. + /// + /// This is useful in tests, where a local file system (the default) can be + /// replaced by a memory file system. + final FileSystem fs; + + /// The environment (current working directory, identity of the OS, + /// environment variables, etc). + final Platform platform; + + /// A controller for launching sub-processes. + /// + /// This is useful in tests, where the real process manager (the default) can + /// be replaced by a mock process manager that doesn't really create + /// sub-processes. + final ProcessManager process; + + /// A client for making Http requests to the Flutter Packages Gold dashboard. + final io.HttpClient httpClient; + + /// The local [Directory] within the comparison root for the current test + /// context. In this directory, the client will create image and JSON files + /// for the goldctl tool to use. + /// + /// This is informed by [FlutterGoldenFileComparator.basedir]. It cannot be + /// null. + final Directory workDirectory; + + /// The logging function to use when reporting messages to the console. + final LogCallback log; + + /// The local [Directory] where the Flutter repository is hosted. + /// + /// Uses the [fs] file system. + // Directory get _flutterRoot => fs.directory(platform.environment[_kFlutterRootKey]); + + /// The path to the local [Directory] where the goldctl tool is hosted. + /// + /// Uses the [platform] environment in this implementation. + String get _goldctl => platform.environment[_kGoldctlKey]!; + + /// Prepares the local work space for golden file testing and calls the + /// goldctl `auth` command. + /// + /// This ensures that the goldctl tool is authorized and ready for testing. + /// Used by the [FlutterPostSubmitFileComparator] and the + /// [FlutterPreSubmitFileComparator]. + Future auth() async { + if (await clientIsAuthorized()) { + return; + } + final authCommand = [ + _goldctl, + 'auth', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--luci', + ]; + + final io.ProcessResult result = await process.run(authCommand); + + if (result.exitCode != 0) { + final buf = StringBuffer() + ..writeln('Skia Gold authorization failed.') + ..writeln( + 'Luci environments authenticate using the file provided ' + 'by LUCI_CONTEXT. There may be an error with this file or Gold ' + 'authentication.', + ) + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}'); + throw SkiaException(buf.toString()); + } + } + + /// Signals if this client is initialized for uploading images to the Gold + /// service. + /// + /// Since Flutter framework tests are executed in parallel, and in random + /// order, this will signal is this instance of the Gold client has been + /// initialized. + bool _initialized = false; + + /// Executes the `imgtest init` command in the goldctl tool. + /// + /// The `imgtest` command collects and uploads test results to the Skia Gold + /// backend, the `init` argument initializes the current test. Used by the + /// [FlutterPostSubmitFileComparator]. + Future imgtestInit() async { + // This client has already been initialized + if (_initialized) { + return; + } + + final File keys = workDirectory.childFile('keys.json'); + final File failures = workDirectory.childFile('failures.json'); + + await keys.writeAsString(_getKeysJSON()); + await failures.create(); + final String commitHash = await _getCurrentCommit(); + print(commitHash); + + final imgtestInitCommand = [ + _goldctl, + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--commit', + commitHash, + '--keys-file', + keys.path, + '--failure-file', + failures.path, + '--passfail', + ]; + + if (imgtestInitCommand.contains(null)) { + final buf = StringBuffer() + ..writeln('A null argument was provided for Skia Gold imgtest init.') + ..writeln('Please confirm the settings of your golden file test.') + ..writeln('Arguments provided:'); + imgtestInitCommand.forEach(buf.writeln); + throw SkiaException(buf.toString()); + } + + final io.ProcessResult result = await process.run(imgtestInitCommand); + + if (result.exitCode != 0) { + _initialized = false; + final buf = StringBuffer() + ..writeln('Skia Gold imgtest init failed.') + ..writeln('An error occurred when initializing golden file test with ') + ..writeln('goldctl.') + ..writeln() + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}'); + throw SkiaException(buf.toString()); + } + _initialized = true; + } + + /// Executes the `imgtest add` command in the goldctl tool. + /// + /// The `imgtest` command collects and uploads test results to the Skia Gold + /// backend, the `add` argument uploads the current image test. A response is + /// returned from the invocation of this command that indicates a pass or fail + /// result. + /// + /// The [testName] and [goldenFile] parameters reference the current + /// comparison being evaluated by the [FlutterPostSubmitFileComparator]. + Future imgtestAdd(String testName, File goldenFile) async { + final imgtestCommand = [ + _goldctl, + 'imgtest', + 'add', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--test-name', + cleanTestName(testName), + '--png-file', + goldenFile.path, + '--passfail', + ]; + + final io.ProcessResult result = await process.run(imgtestCommand); + + if (result.exitCode != 0) { + // If an unapproved image has made it to post-submit, throw to close the + // tree. + String? resultContents; + final File resultFile = workDirectory.childFile(fs.path.join('result-state.json')); + if (await resultFile.exists()) { + resultContents = await resultFile.readAsString(); + } + + final buf = StringBuffer() + ..writeln('Skia Gold received an unapproved image in post-submit ') + ..writeln('testing. Golden file images in flutter/flutter are triaged ') + ..writeln('in pre-submit during code review for the given PR.') + ..writeln() + ..writeln('Visit https://flutter-gold.skia.org/ to view and approve ') + ..writeln('the image(s), or revert the associated change. For more ') + ..writeln('information, visit the wiki: ') + ..writeln( + 'https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md', + ) + ..writeln() + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}') + ..writeln() + ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}'); + throw SkiaException(buf.toString()); + } + + return true; + } + + /// Signals if this client is initialized for uploading tryjobs to the Gold + /// service. + /// + /// Since Flutter framework tests are executed in parallel, and in random + /// order, this will signal is this instance of the Gold client has been + /// initialized for tryjobs. + bool _tryjobInitialized = false; + + /// Executes the `imgtest init` command in the goldctl tool for tryjobs. + /// + /// The `imgtest` command collects and uploads test results to the Skia Gold + /// backend, the `init` argument initializes the current tryjob. Used by the + /// [FlutterPreSubmitFileComparator]. + Future tryjobInit() async { + // This client has already been initialized + if (_tryjobInitialized) { + return; + } + + final File keys = workDirectory.childFile('keys.json'); + final File failures = workDirectory.childFile('failures.json'); + + await keys.writeAsString(_getKeysJSON()); + await failures.create(); + final String commitHash = await _getCurrentCommit(); + print(commitHash); + + final imgtestInitCommand = [ + _goldctl, + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--commit', + commitHash, + '--keys-file', + keys.path, + '--failure-file', + failures.path, + '--passfail', + '--crs', + 'github', + '--patchset_id', + commitHash, + ...getCIArguments(), + ]; + + if (imgtestInitCommand.contains(null)) { + final buf = StringBuffer() + ..writeln('A null argument was provided for Skia Gold tryjob init.') + ..writeln('Please confirm the settings of your golden file test.') + ..writeln('Arguments provided:'); + imgtestInitCommand.forEach(buf.writeln); + throw SkiaException(buf.toString()); + } + + final io.ProcessResult result = await process.run(imgtestInitCommand); + + if (result.exitCode != 0) { + _tryjobInitialized = false; + final buf = StringBuffer() + ..writeln('Skia Gold tryjobInit failure.') + ..writeln('An error occurred when initializing golden file tryjob with ') + ..writeln('goldctl.') + ..writeln() + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}'); + throw SkiaException(buf.toString()); + } + _tryjobInitialized = true; + } + + /// Executes the `imgtest add` command in the goldctl tool for tryjobs. + /// + /// The `imgtest` command collects and uploads test results to the Skia Gold + /// backend, the `add` argument uploads the current image test. A response is + /// returned from the invocation of this command that indicates a pass or fail + /// result for the tryjob. + /// + /// The [testName] and [goldenFile] parameters reference the current + /// comparison being evaluated by the [FlutterPreSubmitFileComparator]. + /// + /// If the tryjob fails due to pixel differences, the method will succeed + /// as the failure will be triaged in the 'Flutter Gold' dashboard, and the + /// `stdout` will contain the failure message; otherwise will return `null`. + Future tryjobAdd(String testName, File goldenFile) async { + final imgtestCommand = [ + _goldctl, + 'imgtest', + 'add', + '--work-dir', + workDirectory.childDirectory('temp').path, + '--test-name', + cleanTestName(testName), + '--png-file', + goldenFile.path, + ]; + + final io.ProcessResult result = await process.run(imgtestCommand); + + final resultStdout = result.stdout.toString(); + if (result.exitCode != 0 && + !(resultStdout.contains('Untriaged') || resultStdout.contains('negative image'))) { + String? resultContents; + final File resultFile = workDirectory.childFile(fs.path.join('result-state.json')); + if (await resultFile.exists()) { + resultContents = await resultFile.readAsString(); + } + final buf = StringBuffer() + ..writeln('Unexpected Gold tryjobAdd failure.') + ..writeln('Tryjob execution for golden file test $testName failed for') + ..writeln('a reason unrelated to pixel comparison.') + ..writeln() + ..writeln('Debug information for Gold --------------------------------') + ..writeln('stdout: ${result.stdout}') + ..writeln('stderr: ${result.stderr}') + ..writeln() + ..writeln() + ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}'); + throw SkiaException(buf.toString()); + } + return result.exitCode == 0 ? null : resultStdout; + } + + /// Returns the latest positive digest for the given test known to Flutter + /// Packages Gold at head. + Future getExpectationForTest(String testName) async { + late String? expectation; + final String traceID = getTraceID(testName); + final Uri requestForExpectations = Uri.parse( + 'https://flutter-packages-gold.skia.org/json/v2/latestpositivedigest/$traceID', + ); + late String rawResponse; + try { + final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations); + final io.HttpClientResponse response = await request.close(); + rawResponse = await utf8.decodeStream(response); + final dynamic jsonResponse = json.decode(rawResponse); + if (jsonResponse is! Map) { + throw const FormatException('Skia gold expectations do not match expected format.'); + } + expectation = jsonResponse['digest'] as String?; + } on FormatException catch (error) { + log( + 'Formatting error detected requesting expectations from Flutter Gold.\n' + 'error: $error\n' + 'url: $requestForExpectations\n' + 'response: $rawResponse', + ); + rethrow; + } + return expectation; + } + + /// Returns a list of bytes representing the golden image retrieved from the + /// Flutter Packages Gold dashboard. + /// + /// The provided image hash represents an expectation from Flutter Packages Gold. + Future> getImageBytes(String imageHash) async { + final imageBytes = []; + final Uri requestForImage = Uri.parse( + 'https://flutter-packages-gold.skia.org/img/images/$imageHash.png', + ); + final io.HttpClientRequest request = await httpClient.getUrl(requestForImage); + final io.HttpClientResponse response = await request.close(); + await response.forEach((List bytes) => imageBytes.addAll(bytes)); + return imageBytes; + } + + /// Returns the current commit hash of the Flutter repository. + Future _getCurrentCommit() async { + if (!workDirectory.existsSync()) { + throw SkiaException('The SkiaClient.workDirectory could not be found: $workDirectory\n'); + } else { + final io.ProcessResult revParse = await process.run([ + 'git', + 'rev-parse', + 'HEAD', + ], workingDirectory: workDirectory.path); + if (revParse.exitCode != 0) { + throw const SkiaException('Current commit of Flutter can not be found.'); + } + return (revParse.stdout as String).trim(); + } + } + + /// Returns a JSON String with keys value pairs used to uniquely identify the + /// configuration that generated the given golden file. + /// + /// Currently, the only key value pairs being tracked is the platform the + /// image was rendered on, and for web tests, the browser the image was + /// rendered on. + String _getKeysJSON() { + final keys = { + 'Platform': platform.operatingSystem, + 'CI': 'luci', + 'Web' : _isBrowserTest, + }; + return json.encode(keys); + } + + /// Removes the file extension from the [fileName] to represent the test name + /// properly. + String cleanTestName(String fileName) { + return fileName.split(path.extension(fileName))[0]; + } + + /// Returns a boolean value to prevent the client from re-authorizing itself + /// for multiple tests. + Future clientIsAuthorized() async { + final File authFile = workDirectory.childFile(fs.path.join('temp', 'auth_opt.json')); + + if (await authFile.exists()) { + final String contents = await authFile.readAsString(); + final decoded = json.decode(contents) as Map; + return !(decoded['GSUtil'] as bool); + } + return false; + } + + /// Returns a list of arguments for initializing a tryjob based on the testing + /// environment. + List getCIArguments() { + final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']!.split('/').last; + final List refs = platform.environment['GOLD_TRYJOB']!.split('/'); + final String pullRequest = refs[refs.length - 2]; + + return ['--changelist', pullRequest, '--cis', 'buildbucket', '--jobid', jobId]; + } + + bool get _isBrowserTest { + return platform.environment[_kTestBrowserKey] != null; + } + + /// Returns a trace id based on the current testing environment to lookup + /// the latest positive digest on Flutter Gold with a hex-encoded md5 hash of + /// the image keys. + String getTraceID(String testName) { + final parameters = { + 'CI': 'luci', + 'Platform': platform.operatingSystem, + 'Web': _isBrowserTest.toString(), + 'name': testName, + 'source_type': 'flutter packages', + }; + final sorted = {}; + for (final String key in parameters.keys.toList()..sort()) { + sorted[key] = parameters[key]; + } + final String jsonTrace = json.encode(sorted); + final md5Sum = md5.convert(utf8.encode(jsonTrace)).toString(); + return md5Sum; + } +} diff --git a/script/flutter_goldens/pubspec.yaml b/script/flutter_goldens/pubspec.yaml new file mode 100644 index 00000000000..0280270025a --- /dev/null +++ b/script/flutter_goldens/pubspec.yaml @@ -0,0 +1,15 @@ +name: flutter_goldens + +environment: + sdk: ^3.9.0-0 + +dependencies: + crypto: any + file: any + flutter: + sdk: flutter + flutter_test: + sdk: flutter + path: any + platform: any + process: any diff --git a/script/flutter_goldens/test/comparator_selection_test.dart b/script/flutter_goldens/test/comparator_selection_test.dart new file mode 100644 index 00000000000..6057e42cc27 --- /dev/null +++ b/script/flutter_goldens/test/comparator_selection_test.dart @@ -0,0 +1,111 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_goldens/flutter_goldens.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:platform/platform.dart'; + +enum _Comparator { post, pre, skip, local } + +_Comparator _testRecommendations({ + bool hasLuci = false, + bool hasGold = false, + bool hasTryJob = false, + String branch = 'main', + String os = 'macos', +}) { + final Platform platform = FakePlatform( + environment: { + if (hasLuci) 'SWARMING_TASK_ID': '8675309', + if (hasGold) 'GOLDCTL': 'goldctl', + if (hasTryJob) 'GOLD_TRYJOB': 'git/ref/12345/head', + 'GIT_BRANCH': branch, + }, + operatingSystem: os, + ); + if (FlutterPostSubmitFileComparator.isForEnvironment(platform)) { + return _Comparator.post; + } + if (FlutterPreSubmitFileComparator.isForEnvironment(platform)) { + return _Comparator.pre; + } + if (FlutterSkippingFileComparator.isForEnvironment(platform)) { + return _Comparator.skip; + } + return _Comparator.local; +} + +void main() { + test('Comparator recommendations - main branch', () { + // If we're running locally (no CI), use a local comparator. + expect(_testRecommendations(), _Comparator.local); + expect(_testRecommendations(hasGold: true), _Comparator.local); + + // If we don't have gold but are on CI, we skip regardless. + expect(_testRecommendations(hasLuci: true), _Comparator.skip); + expect(_testRecommendations(hasLuci: true, hasTryJob: true), _Comparator.skip); + + // On Luci, with Gold, post-submit. Flutter root and LUCI variables should have no effect. + expect(_testRecommendations(hasGold: true, hasLuci: true), _Comparator.post); + + // On Luci, with Gold, pre-submit. Flutter root and LUCI variables should have no effect. + expect(_testRecommendations(hasGold: true, hasLuci: true, hasTryJob: true), _Comparator.pre); + }); + + test('Comparator recommendations - release branch', () { + // If we're running locally (no CI), use a local comparator. + expect(_testRecommendations(branch: 'flutter-3.16-candidate.0'), _Comparator.local); + + expect( + _testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true), + _Comparator.local, + ); + + // If we don't have gold but are on CI, we skip regardless. + expect( + _testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true), + _Comparator.skip, + ); + expect( + _testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true, hasTryJob: true), + _Comparator.skip, + ); + + // On Luci, with Gold, post-submit. Flutter root and LUCI variables should have no effect. Branch should make us skip. + expect( + _testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true), + _Comparator.skip, + ); + + // On Luci, with Gold, pre-submit. Flutter root and LUCI variables should have no effect. Branch should make us skip. + expect( + _testRecommendations( + branch: 'flutter-3.16-candidate.0', + hasGold: true, + hasLuci: true, + hasTryJob: true, + ), + _Comparator.skip, + ); + }); + + test('Comparator recommendations - Linux', () { + // If we're running locally (no CI), use a local comparator. + expect(_testRecommendations(os: 'linux'), _Comparator.local); + expect(_testRecommendations(os: 'linux', hasGold: true), _Comparator.local); + + // If we don't have gold but are on CI, we skip regardless. + expect(_testRecommendations(os: 'linux', hasLuci: true), _Comparator.skip); + expect(_testRecommendations(os: 'linux', hasLuci: true, hasTryJob: true), _Comparator.skip); + + // On Luci, with Gold, post-submit. Flutter root has no effect. + expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true), _Comparator.post); + + // On Luci, with Gold, pre-submit. Flutter root should have no effect. + expect( + _testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasTryJob: true), + _Comparator.pre, + ); + }); +} diff --git a/script/flutter_goldens/test/flutter_goldens_test.dart b/script/flutter_goldens/test/flutter_goldens_test.dart new file mode 100644 index 00000000000..5c37af2e82c --- /dev/null +++ b/script/flutter_goldens/test/flutter_goldens_test.dart @@ -0,0 +1,1230 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// See also dev/automated_tests/flutter_test/flutter_gold_test.dart + +import 'dart:convert'; +import 'dart:io' hide Directory; + +import 'package:file/file.dart'; +import 'package:file/memory.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_goldens/flutter_goldens.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as path; +import 'package:platform/platform.dart'; +import 'package:process/process.dart'; + +import 'json_templates.dart'; + +// TODO(ianh): make sure all constructors order their arguments in a manner consistent with the defined parameter order + +const String _kFlutterRoot = '/flutter'; + +// 1x1 transparent pixel +const List _kTestPngBytes = [ + 137, + 80, + 78, + 71, + 13, + 10, + 26, + 10, + 0, + 0, + 0, + 13, + 73, + 72, + 68, + 82, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 8, + 6, + 0, + 0, + 0, + 31, + 21, + 196, + 137, + 0, + 0, + 0, + 11, + 73, + 68, + 65, + 84, + 120, + 1, + 99, + 97, + 0, + 2, + 0, + 0, + 25, + 0, + 5, + 144, + 240, + 54, + 245, + 0, + 0, + 0, + 0, + 73, + 69, + 78, + 68, + 174, + 66, + 96, + 130, +]; + +void main() { + group('SkiaGoldClient', () { + + test('auth performs minimal work if already authorized', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + final File authFile = fs.file('/workDirectory/temp/auth_opt.json') + ..createSync(recursive: true); + authFile.writeAsStringSync(authTemplate()); + process.fallbackProcessResult = ProcessResult(123, 0, '', ''); + await skiaClient.auth(); + + expect(process.workingDirectories, isEmpty); + }); + + test('gsutil is checked when authorization file is present', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + final File authFile = fs.file('/workDirectory/temp/auth_opt.json') + ..createSync(recursive: true); + authFile.writeAsStringSync(authTemplate(gsutil: true)); + expect(await skiaClient.clientIsAuthorized(), isFalse); + }); + + test('throws for error state from auth', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + environment: { + 'GOLD_SERVICE_ACCOUNT': 'Service Account', + 'GOLDCTL': 'goldctl', + }, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect(skiaClient.auth(), throwsException); + }); + + test('throws for error state from init', () { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + const gitInvocation = RunInvocation(['git', 'rev-parse', 'HEAD'], '/flutter'); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + '/workDirectory/temp', + '--commit', + '12345678', + '--keys-file', + '/workDirectory/keys.json', + '--failure-file', + '/workDirectory/failures.json', + '--passfail', + ], null); + process.processResults[gitInvocation] = ProcessResult(12345678, 0, '12345678', ''); + process.processResults[goldctlInvocation] = ProcessResult( + 123, + 1, + 'Expected failure', + 'Expected failure', + ); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect(skiaClient.imgtestInit(), throwsException); + }); + + test('Only calls init once', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + const gitInvocation = RunInvocation(['git', 'rev-parse', 'HEAD'], '/flutter'); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + '/workDirectory/temp', + '--commit', + '1234', + '--keys-file', + '/workDirectory/keys.json', + '--failure-file', + '/workDirectory/failures.json', + '--passfail', + ], null); + process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); + process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + // First call + await skiaClient.imgtestInit(); + + // Remove fake process result. + // If the init call is executed again, the fallback process will throw. + process.processResults.remove(goldctlInvocation); + + // Second call + await skiaClient.imgtestInit(); + }); + + test('Only calls tryjob init once', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'SWARMING_TASK_ID': '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB': 'refs/pull/49815/head', + }, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + const gitInvocation = RunInvocation(['git', 'rev-parse', 'HEAD'], '/flutter'); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'init', + '--instance', + 'flutter', + '--work-dir', + '/workDirectory/temp', + '--commit', + '1234', + '--keys-file', + '/workDirectory/keys.json', + '--failure-file', + '/workDirectory/failures.json', + '--passfail', + '--crs', + 'github', + '--patchset_id', + '1234', + '--changelist', + '49815', + '--cis', + 'buildbucket', + '--jobid', + '8885996262141582672', + ], null); + process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); + process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + // First call + await skiaClient.tryjobInit(); + + // Remove fake process result. + // If the init call is executed again, the fallback process will throw. + process.processResults.remove(goldctlInvocation); + + // Second call + await skiaClient.tryjobInit(); + }); + + test('throws for error state from imgtestAdd', () { + final fs = MemoryFileSystem(); + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + final platform = FakePlatform( + environment: {'GOLDCTL': 'goldctl'}, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'add', + '--work-dir', + '/workDirectory/temp', + '--test-name', + 'golden_file_test', + '--png-file', + '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], null); + process.processResults[goldctlInvocation] = ProcessResult( + 123, + 1, + 'Expected failure', + 'Expected failure', + ); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect(skiaClient.imgtestAdd('golden_file_test', goldenFile), throwsException); + }); + + test('correctly inits tryjob for luci', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'SWARMING_TASK_ID': '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB': 'refs/pull/49815/head', + }, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + final List ciArguments = skiaClient.getCIArguments(); + + expect( + ciArguments, + equals([ + '--changelist', + '49815', + '--cis', + 'buildbucket', + '--jobid', + '8885996262141582672', + ]), + ); + }); + + test('Creates traceID correctly', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'SWARMING_TASK_ID': '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB': 'refs/pull/49815/head', + }, + operatingSystem: 'linux', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + expect(skiaClient.getTraceID('flutter.golden.1'), equals('ae18c7a6aa48e0685525dfe8fdf79003')); + }); + + test('Creates traceID correctly - Browser', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'SWARMING_TASK_ID': '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB': 'refs/pull/49815/head', + 'FLUTTER_TEST_BROWSER': 'chrome', + }, + operatingSystem: 'linux', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + + expect(skiaClient.getTraceID('flutter.golden.1'), equals('e9d5c296c48e7126808520e9cc191243')); + }); + + test('Creates traceID correctly - locally - should defer to luci traceID', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + expect(skiaClient.getTraceID('flutter.golden.1'), equals('9968695b9ae78cdb77cbb2be621ca2d6')); + }); + + test('throws for error state from imgtestAdd', () { + final fs = MemoryFileSystem(); + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + final platform = FakePlatform( + environment: {'GOLDCTL': 'goldctl'}, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'add', + '--work-dir', + '/workDirectory/temp', + '--test-name', + 'golden_file_test', + '--png-file', + '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], null); + process.processResults[goldctlInvocation] = ProcessResult( + 123, + 1, + 'Expected failure', + 'Expected failure', + ); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect( + skiaClient.imgtestAdd('golden_file_test', goldenFile), + throwsA( + isA().having( + (SkiaException error) => error.message, + 'message', + contains('result-state.json'), + ), + ), + ); + }); + + test('throws for error state from tryjobAdd', () { + final fs = MemoryFileSystem(); + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + final platform = FakePlatform( + environment: {'GOLDCTL': 'goldctl'}, + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + const goldctlInvocation = RunInvocation([ + 'goldctl', + 'imgtest', + 'add', + '--work-dir', + '/workDirectory/temp', + '--test-name', + 'golden_file_test', + '--png-file', + '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], null); + process.processResults[goldctlInvocation] = ProcessResult( + 123, + 1, + 'Expected failure', + 'Expected failure', + ); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + expect( + skiaClient.tryjobAdd('golden_file_test', goldenFile), + throwsA( + isA().having( + (SkiaException error) => error.message, + 'message', + contains('result-state.json'), + ), + ), + ); + }); + + group('Request Handling', () { + test('image bytes are processed properly', () async { + const expectation = '55109a4bed52acc780530f7a9aeff6c0'; + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + ); + final Uri imageUrl = Uri.parse('https://flutter-gold.skia.org/img/images/$expectation.png'); + final fakeImageRequest = FakeHttpClientRequest(); + final fakeImageResponse = FakeHttpImageResponse(imageResponseTemplate()); + + fakeHttpClient.request = fakeImageRequest; + fakeImageRequest.response = fakeImageResponse; + + final List masterBytes = await skiaClient.getImageBytes(expectation); + + expect(fakeHttpClient.lastUri, imageUrl); + expect(masterBytes, equals(_kTestPngBytes)); + }); + }); + }); + + group('FlutterGoldenFileComparator', () { + test('calculates the basedir correctly from defaultComparator for local testing', () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final defaultComparator = FakeLocalFileComparator(); + final Directory root = fs.directory('/') + ..createSync(recursive: true); + defaultComparator.basedir = root.childDirectory('baz').uri; + final Directory basedir = FlutterGoldenFileComparator.getBaseDirectory( + defaultComparator, + platform: platform, + fs: fs, + ); + expect(basedir.uri, fs.directory('/baz/skia_goldens').uri); + }); + + test('ignores version number', () { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/')..createSync(recursive: true); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + FakeSkiaGoldClient(), + fs: fs, + platform: platform, + log: log.add, + ); + final Uri key = comparator.getTestUri(Uri.parse('foo.png'), 1); + expect(key, Uri.parse('foo.png')); + expect(log, isEmpty); + }); + + test('adds namePrefix', () async { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + const packageName = 'sidedishes'; + const namePrefix = 'tomatosalad'; + const fileName = 'lettuce.png'; + final fakeSkiaClient = FakeSkiaGoldClient(); + final Directory basedir = fs.directory('$packageName/test/') + ..createSync(recursive: true); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + namePrefix: namePrefix, + log: log.add, + ); + await comparator.compare(Uint8List.fromList(_kTestPngBytes), Uri.parse(fileName)); + expect(fakeSkiaClient.testNames.single, '$namePrefix.$packageName.$fileName'); + expect(log, isEmpty); + }); + + group('Post-Submit', () { + test('asserts .png format', () async { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); + await expectLater( + () async { + return comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1'), + ); + }, + throwsA( + isA().having( + (AssertionError error) => error.toString(), + 'description', + contains( + 'Golden files in the Flutter framework must end with the file ' + 'extension .png.', + ), + ), + ), + ); + expect(log, isEmpty); + }); + + test('calls init during compare', () { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); + expect(fakeSkiaClient.initCalls, 0); + comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + expect(fakeSkiaClient.initCalls, 1); + expect(log, isEmpty); + }); + + test('does not call init in during construction', () { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + expect(fakeSkiaClient.initCalls, 0); + FlutterPostSubmitFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator(Uri.parse('/test'), pathStyle: path.Style.posix), + platform: platform, + goldens: fakeSkiaClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(fakeSkiaClient.initCalls, 0); + }); + + test('reports a failure as a TestFailure', () async { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( + basedir.uri, + ThrowsOnImgTestAddSkiaClient( + message: 'Skia Gold received an unapproved image in post-submit', + ), + fs: fs, + platform: platform, + log: log.add, + ); + await expectLater( + () async { + return comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + }, + throwsA( + isA().having( + (TestFailure error) => error.toString(), + 'description', + contains('Skia Gold received an unapproved image in post-submit'), + ), + ), + ); + expect(log, isEmpty); + }); + }); + + group('Pre-Submit', () { + test('asserts .png format', () async { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterPreSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); + await expectLater( + () async { + return comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1'), + ); + }, + throwsA( + isA().having( + (AssertionError error) => error.toString(), + 'description', + contains( + 'Golden files in the Flutter framework must end with the file ' + 'extension .png.', + ), + ), + ), + ); + expect(log, isEmpty); + }); + + test('calls init during compare', () { + final log = []; + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterPreSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); + expect(fakeSkiaClient.tryInitCalls, 0); + comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + expect(fakeSkiaClient.tryInitCalls, 1); + expect(log, isEmpty); + }); + + test('does not call init in during construction', () { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + expect(fakeSkiaClient.tryInitCalls, 0); + FlutterPostSubmitFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator(Uri.parse('/test'), pathStyle: path.Style.posix), + platform: platform, + goldens: fakeSkiaClient, + log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(fakeSkiaClient.tryInitCalls, 0); + }); + }); + + group('Local', () { + test('asserts .png format', () async { + final log = []; + final fs = MemoryFileSystem(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterLocalFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: FakePlatform( + operatingSystem: 'macos', + ), + log: log.add, + ); + const hash = '55109a4bed52acc780530f7a9aeff6c0'; + fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; + fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; + fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = + 'flutter.golden_test.1'; + await expectLater( + () async { + return comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1'), + ); + }, + throwsA( + isA().having( + (AssertionError error) => error.toString(), + 'description', + contains( + 'Golden files in the Flutter framework must end with the file ' + 'extension .png.', + ), + ), + ), + ); + expect(log, isEmpty); + }); + + test('passes when bytes match', () async { + final log = []; + final fs = MemoryFileSystem(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + final FlutterGoldenFileComparator comparator = FlutterLocalFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: FakePlatform( + operatingSystem: 'macos', + ), + log: log.add, + ); + const hash = '55109a4bed52acc780530f7a9aeff6c0'; + fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; + fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; + fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = + 'flutter.golden_test.1'; + expect( + await comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ), + isTrue, + ); + expect(log, isEmpty); + }); + + test( + 'returns FlutterSkippingGoldenFileComparator when network connection is unavailable', + () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + ); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final fakeSkiaClient = FakeSkiaGoldClient(); + + const hash = '55109a4bed52acc780530f7a9aeff6c0'; + fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; + fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; + fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = + 'flutter.golden_test.1'; + final fakeDirectory = FakeDirectory(); + fakeDirectory.existsSyncValue = true; + fakeDirectory.uri = Uri.parse('/flutter'); + + fakeSkiaClient.getExpectationForTestThrowable = const OSError("Can't reach Gold"); + final FlutterGoldenFileComparator comparator1 = + await FlutterLocalFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator( + Uri.parse('/test'), + pathStyle: path.Style.posix, + ), + platform: platform, + goldens: fakeSkiaClient, + baseDirectory: fakeDirectory, + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(comparator1.runtimeType, FlutterSkippingFileComparator); + + fakeSkiaClient.getExpectationForTestThrowable = const SocketException("Can't reach Gold"); + final FlutterGoldenFileComparator comparator2 = + await FlutterLocalFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator( + Uri.parse('/test'), + pathStyle: path.Style.posix, + ), + platform: platform, + goldens: fakeSkiaClient, + baseDirectory: fakeDirectory, + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(comparator2.runtimeType, FlutterSkippingFileComparator); + + fakeSkiaClient.getExpectationForTestThrowable = const FormatException("Can't reach Gold"); + final FlutterGoldenFileComparator comparator3 = + await FlutterLocalFileComparator.fromLocalFileComparator( + localFileComparator: LocalFileComparator( + Uri.parse('/test'), + pathStyle: path.Style.posix, + ), + platform: platform, + goldens: fakeSkiaClient, + baseDirectory: fakeDirectory, + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), + fs: fs, + process: FakeProcessManager(), + httpClient: FakeHttpClient(), + ); + expect(comparator3.runtimeType, FlutterSkippingFileComparator); + + // reset property or it will carry on to other tests + fakeSkiaClient.getExpectationForTestThrowable = null; + }, + ); + }); + }); +} + +@immutable +class RunInvocation { + const RunInvocation(this.command, this.workingDirectory); + + final List command; + final String? workingDirectory; + + @override + int get hashCode => Object.hash(Object.hashAll(command), workingDirectory); + + bool _commandEquals(List other) { + if (other == command) { + return true; + } + if (other.length != command.length) { + return false; + } + for (var index = 0; index < other.length; index += 1) { + if (other[index] != command[index]) { + return false; + } + } + return true; + } + + @override + bool operator ==(Object other) { + if (other.runtimeType != runtimeType) { + return false; + } + return other is RunInvocation && + _commandEquals(other.command) && + other.workingDirectory == workingDirectory; + } + + @override + String toString() => '$command ($workingDirectory)'; +} + +class FakeProcessManager extends Fake implements ProcessManager { + Map processResults = {}; + + /// Used if [processResults] does not contain a matching invocation. + ProcessResult? fallbackProcessResult; + + final List workingDirectories = []; + + @override + Future run( + List command, { + String? workingDirectory, + Map? environment, + bool includeParentEnvironment = true, + bool runInShell = false, + Encoding? stdoutEncoding = systemEncoding, + Encoding? stderrEncoding = systemEncoding, + }) async { + workingDirectories.add(workingDirectory); + final ProcessResult? result = + processResults[RunInvocation(command.cast(), workingDirectory)]; + if (result == null && fallbackProcessResult == null) { + printOnFailure( + 'ProcessManager.run was called with $command ($workingDirectory) unexpectedly - $processResults.', + ); + fail('See above.'); + } + return result ?? fallbackProcessResult!; + } +} + +// See also dev/automated_tests/flutter_test/flutter_gold_test.dart +class FakeSkiaGoldClient extends Fake implements SkiaGoldClient { + Map expectationForTestValues = {}; + Exception? getExpectationForTestThrowable; + @override + Future getExpectationForTest(String testName) async { + if (getExpectationForTestThrowable != null) { + throw getExpectationForTestThrowable!; + } + return expectationForTestValues[testName] ?? ''; + } + + @override + Future auth() async {} + + final List testNames = []; + + int initCalls = 0; + @override + Future imgtestInit() async => initCalls += 1; + @override + Future imgtestAdd(String testName, File goldenFile) async { + testNames.add(testName); + return true; + } + + int tryInitCalls = 0; + @override + Future tryjobInit() async => tryInitCalls += 1; + @override + Future tryjobAdd(String testName, File goldenFile) async => null; + + Map> imageBytesValues = >{}; + @override + Future> getImageBytes(String imageHash) async => imageBytesValues[imageHash]!; + + Map cleanTestNameValues = {}; + @override + String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? ''; +} + +class ThrowsOnImgTestAddSkiaClient extends Fake implements SkiaGoldClient { + ThrowsOnImgTestAddSkiaClient({required this.message}); + final String message; + + @override + Future imgtestInit() async { + // Assume this function works. + } + + @override + Future imgtestAdd(String testName, File goldenFile) { + throw SkiaException(message); + } +} + +class FakeLocalFileComparator extends Fake implements LocalFileComparator { + @override + late Uri basedir; +} + +class FakeDirectory extends Fake implements Directory { + late bool existsSyncValue; + @override + bool existsSync() => existsSyncValue; + + @override + late Uri uri; +} + +class FakeHttpClient extends Fake implements HttpClient { + late Uri lastUri; + late FakeHttpClientRequest request; + + @override + Future getUrl(Uri url) async { + lastUri = url; + return request; + } +} + +class FakeHttpClientRequest extends Fake implements HttpClientRequest { + late FakeHttpImageResponse response; + + @override + Future close() async { + return response; + } +} + +class FakeHttpImageResponse extends Fake implements HttpClientResponse { + FakeHttpImageResponse(this.response); + + final List> response; + + @override + Future forEach(void Function(List element) action) async { + response.forEach(action); + } +} diff --git a/script/flutter_goldens/test/json_templates.dart b/script/flutter_goldens/test/json_templates.dart new file mode 100644 index 00000000000..5a55826e30d --- /dev/null +++ b/script/flutter_goldens/test/json_templates.dart @@ -0,0 +1,94 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Json response template for the contents of the auth_opt.json file created by +/// goldctl. +String authTemplate({bool gsutil = false}) { + return ''' + { + "Luci":false, + "ServiceAccount":"${gsutil ? '' : '/packages/flutter/test/widgets/serviceAccount.json'}", + "GSUtil":$gsutil + } + '''; +} + +/// Json response template for Skia Gold image request: +/// https://flutter-gold.skia.org/img/images/{imageHash}.png +List> imageResponseTemplate() { + return >[ + [ + 137, + 80, + 78, + 71, + 13, + 10, + 26, + 10, + 0, + 0, + 0, + 13, + 73, + 72, + 68, + 82, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 8, + 6, + 0, + 0, + 0, + 31, + 21, + 196, + 137, + 0, + ], + [ + 0, + 0, + 11, + 73, + 68, + 65, + 84, + 120, + 1, + 99, + 97, + 0, + 2, + 0, + 0, + 25, + 0, + 5, + 144, + 240, + 54, + 245, + 0, + 0, + 0, + 0, + 73, + 69, + 78, + 68, + 174, + 66, + 96, + 130, + ], + ]; +} From 83c8701094d54b1b1a2e424aaefb8c0bdeff4acd Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Thu, 8 Jan 2026 15:52:50 -0600 Subject: [PATCH 03/21] Logs for debug --- script/flutter_goldens/lib/flutter_goldens.dart | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index 895cb2e89e6..a6bd83b48ce 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -230,6 +230,7 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { 'Golden files in the Flutter framework must end with the file extension ' '.png.', ); + print('_addPrefix, golden: $golden'); print('basedir: $basedir'); return Uri.parse( [ @@ -319,6 +320,7 @@ class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { Future compare(Uint8List imageBytes, Uri golden) async { await skiaClient.imgtestInit(); golden = _addPrefix(golden); + print('prefixed golden: $golden'); await update(golden, imageBytes); final File goldenFile = getGoldenFile(golden); try { @@ -432,6 +434,7 @@ class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { Future compare(Uint8List imageBytes, Uri golden) async { await skiaClient.tryjobInit(); golden = _addPrefix(golden); + print('prefixed golden: $golden'); await update(golden, imageBytes); final File goldenFile = getGoldenFile(golden); @@ -665,6 +668,7 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC @override Future compare(Uint8List imageBytes, Uri golden) async { golden = _addPrefix(golden); + print('prefixed golden: $golden'); final String testName = skiaClient.cleanTestName(golden.path); late String? testExpectation; testExpectation = await skiaClient.getExpectationForTest(testName); From afd2a5eca281f084c8c31f25d6af6365325b0c7a Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Thu, 8 Jan 2026 16:02:26 -0600 Subject: [PATCH 04/21] ++ --- script/flutter_goldens/README.md | 2 +- script/flutter_goldens/lib/flutter_goldens.dart | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/script/flutter_goldens/README.md b/script/flutter_goldens/README.md index 2187c8e28f6..acbd025acb0 100644 --- a/script/flutter_goldens/README.md +++ b/script/flutter_goldens/README.md @@ -5,7 +5,7 @@ infrastructure for tracking golden image tests. See also:  * https://skia.org/docs/dev/testing/skiagold/ - * https://flutter-gold.skia.org/ + * https://flutter-packages-gold.skia.org/ * [Writing a golden file test for package flutter] [Writing a golden file test for package flutter]: /docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index a6bd83b48ce..1b48570cfeb 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -138,7 +138,7 @@ Future testExecutable(FutureOr Function() testMain, {String? namePre /// output the new image for verification. /// /// The [FlutterSkippingFileComparator] is utilized to skip tests outside -/// of the appropriate environments described above. Currently, some +/// of the appropriate environments described above. Currently, some /// packages or environments do not execute golden file testing, and as such do /// not require a comparator. This comparator is also used when an internet /// connection is unavailable. @@ -235,7 +235,6 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { return Uri.parse( [ ?namePrefix, - basedir.pathSegments[1], // ~/packages/ golden.toString(), ].join('.'), ); From 6db8d569a862e9b763392eb12867be86cc7a58e7 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 24 Feb 2026 15:04:06 -0600 Subject: [PATCH 05/21] Reshuffle --- packages/material_ui/pubspec.yaml | 5 ++++- .../test/flutter_test_config.dart | 0 .../test/goldens/goldens_test.dart} | 3 +-- .../test/goldens_io.dart | 0 .../test/goldens_web.dart | 0 packages/two_dimensional_scrollables/pubspec.yaml | 2 -- script/flutter_goldens/README.md | 4 ++-- script/flutter_goldens/lib/flutter_goldens.dart | 1 - script/flutter_goldens/pubspec.yaml | 1 + 9 files changed, 8 insertions(+), 8 deletions(-) rename packages/{two_dimensional_scrollables => material_ui}/test/flutter_test_config.dart (100%) rename packages/{two_dimensional_scrollables/test/goldens/goldens.dart => material_ui/test/goldens/goldens_test.dart} (89%) rename packages/{two_dimensional_scrollables => material_ui}/test/goldens_io.dart (100%) rename packages/{two_dimensional_scrollables => material_ui}/test/goldens_web.dart (100%) diff --git a/packages/material_ui/pubspec.yaml b/packages/material_ui/pubspec.yaml index 1700a387845..4626a6fd00f 100644 --- a/packages/material_ui/pubspec.yaml +++ b/packages/material_ui/pubspec.yaml @@ -1,6 +1,7 @@ name: material_ui description: The official Flutter Material UI Library, implementing Google's Material Design design system. -version: 0.0.1 +version: 0.0.2 +publish_to: none repository: https://github.com/flutter/packages/tree/main/packages/material_ui issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A%20material%20design%22 @@ -13,6 +14,8 @@ dependencies: sdk: flutter dev_dependencies: + flutter_goldens: + path: ../../script/flutter_goldens flutter_test: sdk: flutter diff --git a/packages/two_dimensional_scrollables/test/flutter_test_config.dart b/packages/material_ui/test/flutter_test_config.dart similarity index 100% rename from packages/two_dimensional_scrollables/test/flutter_test_config.dart rename to packages/material_ui/test/flutter_test_config.dart diff --git a/packages/two_dimensional_scrollables/test/goldens/goldens.dart b/packages/material_ui/test/goldens/goldens_test.dart similarity index 89% rename from packages/two_dimensional_scrollables/test/goldens/goldens.dart rename to packages/material_ui/test/goldens/goldens_test.dart index 27dd3c77ed5..bc048c5862d 100644 --- a/packages/two_dimensional_scrollables/test/goldens/goldens.dart +++ b/packages/material_ui/test/goldens/goldens_test.dart @@ -2,9 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'package:flutter/material.dart'; -import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:material_ui/material_ui.dart'; void main() { testWidgets('Inconsequential golden test', (WidgetTester tester) async { diff --git a/packages/two_dimensional_scrollables/test/goldens_io.dart b/packages/material_ui/test/goldens_io.dart similarity index 100% rename from packages/two_dimensional_scrollables/test/goldens_io.dart rename to packages/material_ui/test/goldens_io.dart diff --git a/packages/two_dimensional_scrollables/test/goldens_web.dart b/packages/material_ui/test/goldens_web.dart similarity index 100% rename from packages/two_dimensional_scrollables/test/goldens_web.dart rename to packages/material_ui/test/goldens_web.dart diff --git a/packages/two_dimensional_scrollables/pubspec.yaml b/packages/two_dimensional_scrollables/pubspec.yaml index bc710d0d62f..3942be48404 100644 --- a/packages/two_dimensional_scrollables/pubspec.yaml +++ b/packages/two_dimensional_scrollables/pubspec.yaml @@ -13,8 +13,6 @@ dependencies: sdk: flutter dev_dependencies: - flutter_goldens: - path: ../../script/flutter_goldens flutter_test: sdk: flutter diff --git a/script/flutter_goldens/README.md b/script/flutter_goldens/README.md index acbd025acb0..24b37a4d51e 100644 --- a/script/flutter_goldens/README.md +++ b/script/flutter_goldens/README.md @@ -1,5 +1,5 @@ This package is an internal implementation detail for our testing -infrastructure. It enables the framework to use the Skia Gold +infrastructure. It enables packages to use the Skia Gold infrastructure for tracking golden image tests. See also: @@ -8,4 +8,4 @@ See also: * https://flutter-packages-gold.skia.org/ * [Writing a golden file test for package flutter] -[Writing a golden file test for package flutter]: /docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md +[Writing a golden file test for package flutter]: https://github.com/flutter/flutter/blob/master/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index 1b48570cfeb..a35e1b0ec96 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -24,7 +24,6 @@ export 'skia_client.dart'; // If you are trying to debug this package, you may like to use the golden test // titled "Inconsequential golden test" in this file: -// TODO(Piinks): Add canary test somewhere. // const String _kFlutterRootKey = 'FLUTTER_ROOT'; diff --git a/script/flutter_goldens/pubspec.yaml b/script/flutter_goldens/pubspec.yaml index 0280270025a..b24f5e6deb7 100644 --- a/script/flutter_goldens/pubspec.yaml +++ b/script/flutter_goldens/pubspec.yaml @@ -1,4 +1,5 @@ name: flutter_goldens +publish_to: none environment: sdk: ^3.9.0-0 From b1db62b65cfd6f95f03fc3a5eeb8cb543ee119e9 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 24 Feb 2026 15:31:02 -0600 Subject: [PATCH 06/21] Fix commit logic --- script/flutter_goldens/lib/skia_client.dart | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index d9b760f4c21..41d73c13a57 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -18,7 +18,7 @@ import 'package:process/process.dart'; // Flutter repos, consider reading this wiki page: // https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md -// const String _kFlutterRootKey = 'FLUTTER_ROOT'; +const String _kPackagesRootKey = 'PWD'; const String _kGoldctlKey = 'GOLDCTL'; const String _kTestBrowserKey = 'CHROME_EXECUTABLE'; @@ -87,10 +87,10 @@ class SkiaGoldClient { /// The logging function to use when reporting messages to the console. final LogCallback log; - /// The local [Directory] where the Flutter repository is hosted. + /// The local [Directory] where the packages repository is hosted. /// /// Uses the [fs] file system. - // Directory get _flutterRoot => fs.directory(platform.environment[_kFlutterRootKey]); + Directory get _packagesRoot => fs.directory(path.join(platform.environment[_kPackagesRootKey]!, 'packages')); /// The path to the local [Directory] where the goldctl tool is hosted. /// @@ -431,20 +431,23 @@ class SkiaGoldClient { return imageBytes; } + /// Returns the current commit hash of the Flutter repository. /// Returns the current commit hash of the Flutter repository. Future _getCurrentCommit() async { - if (!workDirectory.existsSync()) { - throw SkiaException('The SkiaClient.workDirectory could not be found: $workDirectory\n'); + if (!_packagesRoot.existsSync()) { + throw SkiaException('Flutter root could not be found: $_packagesRoot\n'); } else { final io.ProcessResult revParse = await process.run([ 'git', 'rev-parse', 'HEAD', - ], workingDirectory: workDirectory.path); + ], workingDirectory: _packagesRoot.path); if (revParse.exitCode != 0) { throw const SkiaException('Current commit of Flutter can not be found.'); } - return (revParse.stdout as String).trim(); + final String commit = (revParse.stdout as String).trim(); + print(commit); + return commit; } } From c8d5656832658194d6eb5693d3a07d3cbfa3e5a1 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 24 Feb 2026 15:53:47 -0600 Subject: [PATCH 07/21] ++ --- script/flutter_goldens/lib/skia_client.dart | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index 41d73c13a57..a1e96c88aab 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -87,11 +87,6 @@ class SkiaGoldClient { /// The logging function to use when reporting messages to the console. final LogCallback log; - /// The local [Directory] where the packages repository is hosted. - /// - /// Uses the [fs] file system. - Directory get _packagesRoot => fs.directory(path.join(platform.environment[_kPackagesRootKey]!, 'packages')); - /// The path to the local [Directory] where the goldctl tool is hosted. /// /// Uses the [platform] environment in this implementation. @@ -431,24 +426,21 @@ class SkiaGoldClient { return imageBytes; } - /// Returns the current commit hash of the Flutter repository. - /// Returns the current commit hash of the Flutter repository. + /// Returns the current commit hash of the packages repository. Future _getCurrentCommit() async { - if (!_packagesRoot.existsSync()) { - throw SkiaException('Flutter root could not be found: $_packagesRoot\n'); - } else { - final io.ProcessResult revParse = await process.run([ + print(workDirectory.path); + final io.ProcessResult revParse = await process.run([ 'git', 'rev-parse', 'HEAD', - ], workingDirectory: _packagesRoot.path); + ], workingDirectory: workDirectory.path); if (revParse.exitCode != 0) { - throw const SkiaException('Current commit of Flutter can not be found.'); + throw const SkiaException('Current commit of flutter/packages can not be found.'); } final String commit = (revParse.stdout as String).trim(); print(commit); return commit; - } + } /// Returns a JSON String with keys value pairs used to uniquely identify the From 71f20169273647655c5647ecea351b44fee6c937 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 24 Feb 2026 16:36:41 -0600 Subject: [PATCH 08/21] ++ --- script/flutter_goldens/lib/skia_client.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index a1e96c88aab..bb557b30abe 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -18,7 +18,7 @@ import 'package:process/process.dart'; // Flutter repos, consider reading this wiki page: // https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md -const String _kPackagesRootKey = 'PWD'; +const String _kPWDKey = 'PWD'; const String _kGoldctlKey = 'GOLDCTL'; const String _kTestBrowserKey = 'CHROME_EXECUTABLE'; @@ -428,12 +428,12 @@ class SkiaGoldClient { /// Returns the current commit hash of the packages repository. Future _getCurrentCommit() async { - print(workDirectory.path); + print(path.join(platform.environment[_kPWDRootKey]!, 'packages')); final io.ProcessResult revParse = await process.run([ 'git', 'rev-parse', 'HEAD', - ], workingDirectory: workDirectory.path); + ], workingDirectory: path.join(platform.environment[_kPWDRootKey]!, 'packages')); if (revParse.exitCode != 0) { throw const SkiaException('Current commit of flutter/packages can not be found.'); } From ca7f637687af783f164c80d36ff60571510fb958 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 24 Feb 2026 16:47:53 -0600 Subject: [PATCH 09/21] Tweaking... --- packages/material_ui/test/flutter_test_config.dart | 2 +- script/flutter_goldens/lib/flutter_goldens.dart | 2 +- script/flutter_goldens/lib/skia_client.dart | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/material_ui/test/flutter_test_config.dart b/packages/material_ui/test/flutter_test_config.dart index f8faa41b4f2..ac0ed96dd87 100644 --- a/packages/material_ui/test/flutter_test_config.dart +++ b/packages/material_ui/test/flutter_test_config.dart @@ -18,5 +18,5 @@ import 'goldens_io.dart' Future testExecutable(FutureOr Function() testMain) { // Enable golden file testing using Skia Gold. - return flutter_goldens.testExecutable(testMain, namePrefix: 'two_dimensional_scrollables'); + return flutter_goldens.testExecutable(testMain, namePrefix: 'material_ui'); } diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index a35e1b0ec96..1b07835f17e 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -23,7 +23,7 @@ export 'skia_client.dart'; // https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md // If you are trying to debug this package, you may like to use the golden test -// titled "Inconsequential golden test" in this file: +// titled "Inconsequential golden test" in this file: packages/material_ui/test/goldens/goldens_test.dart // const String _kFlutterRootKey = 'FLUTTER_ROOT'; diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index bb557b30abe..d0de4ec3b89 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -428,12 +428,12 @@ class SkiaGoldClient { /// Returns the current commit hash of the packages repository. Future _getCurrentCommit() async { - print(path.join(platform.environment[_kPWDRootKey]!, 'packages')); + print(path.join(platform.environment[_kPWDKey]!, 'packages')); final io.ProcessResult revParse = await process.run([ 'git', 'rev-parse', 'HEAD', - ], workingDirectory: path.join(platform.environment[_kPWDRootKey]!, 'packages')); + ], workingDirectory: path.join(platform.environment[_kPWDKey]!, 'packages')); if (revParse.exitCode != 0) { throw const SkiaException('Current commit of flutter/packages can not be found.'); } From 21461b05eda064e46b1715d0e8a3629ead9fc06e Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 24 Feb 2026 17:42:27 -0600 Subject: [PATCH 10/21] Debug tryjobInit --- script/flutter_goldens/lib/skia_client.dart | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index d0de4ec3b89..988d380fbb9 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -279,7 +279,6 @@ class SkiaGoldClient { await keys.writeAsString(_getKeysJSON()); await failures.create(); final String commitHash = await _getCurrentCommit(); - print(commitHash); final imgtestInitCommand = [ _goldctl, @@ -311,6 +310,7 @@ class SkiaGoldClient { imgtestInitCommand.forEach(buf.writeln); throw SkiaException(buf.toString()); } + print(imgtestInitCommand); final io.ProcessResult result = await process.run(imgtestInitCommand); @@ -428,7 +428,6 @@ class SkiaGoldClient { /// Returns the current commit hash of the packages repository. Future _getCurrentCommit() async { - print(path.join(platform.environment[_kPWDKey]!, 'packages')); final io.ProcessResult revParse = await process.run([ 'git', 'rev-parse', @@ -437,9 +436,7 @@ class SkiaGoldClient { if (revParse.exitCode != 0) { throw const SkiaException('Current commit of flutter/packages can not be found.'); } - final String commit = (revParse.stdout as String).trim(); - print(commit); - return commit; + return (revParse.stdout as String).trim(); } @@ -455,6 +452,7 @@ class SkiaGoldClient { 'CI': 'luci', 'Web' : _isBrowserTest, }; + print(json.encode(keys)); return json.encode(keys); } From da71cb3bd9b4017f6dec702da5107fbbe035dcbc Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 24 Feb 2026 18:48:36 -0600 Subject: [PATCH 11/21] Revert, what happened? --- script/flutter_goldens/lib/skia_client.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index 988d380fbb9..1a1beb0eb83 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -428,6 +428,7 @@ class SkiaGoldClient { /// Returns the current commit hash of the packages repository. Future _getCurrentCommit() async { + print(path.join(platform.environment[_kPWDKey]!, 'packages')); final io.ProcessResult revParse = await process.run([ 'git', 'rev-parse', @@ -436,7 +437,9 @@ class SkiaGoldClient { if (revParse.exitCode != 0) { throw const SkiaException('Current commit of flutter/packages can not be found.'); } - return (revParse.stdout as String).trim(); + final String commit = (revParse.stdout as String).trim(); + print(commit); + return commit; } From f19d61d5ac82cad0837b840ec6351f6fbb2ef392 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Wed, 25 Feb 2026 11:19:44 -0600 Subject: [PATCH 12/21] Update to use sdk path --- script/flutter_goldens/lib/skia_client.dart | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index 1a1beb0eb83..4f6480bc7d7 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -18,7 +18,7 @@ import 'package:process/process.dart'; // Flutter repos, consider reading this wiki page: // https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md -const String _kPWDKey = 'PWD'; +const String _kSDKKey = 'SDK_CHECKOUT_PATH'; const String _kGoldctlKey = 'GOLDCTL'; const String _kTestBrowserKey = 'CHROME_EXECUTABLE'; @@ -428,12 +428,13 @@ class SkiaGoldClient { /// Returns the current commit hash of the packages repository. Future _getCurrentCommit() async { - print(path.join(platform.environment[_kPWDKey]!, 'packages')); + final String cleanPath = path.normalize(platform.environment[_kSDKKey]!); + print(path.join(path.dirname(cleanPath), 'packages')); final io.ProcessResult revParse = await process.run([ 'git', 'rev-parse', 'HEAD', - ], workingDirectory: path.join(platform.environment[_kPWDKey]!, 'packages')); + ], workingDirectory: path.join(path.dirname(cleanPath), 'packages')); if (revParse.exitCode != 0) { throw const SkiaException('Current commit of flutter/packages can not be found.'); } @@ -453,7 +454,7 @@ class SkiaGoldClient { final keys = { 'Platform': platform.operatingSystem, 'CI': 'luci', - 'Web' : _isBrowserTest, + 'Web' : _isBrowserTest.toString(), }; print(json.encode(keys)); return json.encode(keys); From 4584c5173911c6e7ccae3b274ea9bffebe270f42 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 2 Mar 2026 17:11:31 -0600 Subject: [PATCH 13/21] Iterating --- packages/cupertino_ui/pubspec.yaml | 5 +++- .../test/flutter_test_config.dart | 14 ++++++++++ .../test/goldens/goldens_test.dart | 27 +++++++++++++++++++ packages/cupertino_ui/test/goldens_io.dart | 5 ++++ packages/cupertino_ui/test/goldens_web.dart | 9 +++++++ .../material_ui/test/flutter_test_config.dart | 8 ------ .../test/goldens/goldens_test.dart | 2 +- 7 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 packages/cupertino_ui/test/flutter_test_config.dart create mode 100644 packages/cupertino_ui/test/goldens/goldens_test.dart create mode 100644 packages/cupertino_ui/test/goldens_io.dart create mode 100644 packages/cupertino_ui/test/goldens_web.dart diff --git a/packages/cupertino_ui/pubspec.yaml b/packages/cupertino_ui/pubspec.yaml index df913bbfc04..fd5575ddb15 100644 --- a/packages/cupertino_ui/pubspec.yaml +++ b/packages/cupertino_ui/pubspec.yaml @@ -1,6 +1,7 @@ name: cupertino_ui description: The official Flutter Cupertino Design Library, implementing the iOS design system. -version: 0.0.1 +version: 0.0.2 +publish_to: none repository: https://github.com/flutter/packages/tree/main/packages/cupertino_ui issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A%20cupertino%22 @@ -13,6 +14,8 @@ dependencies: sdk: flutter dev_dependencies: + flutter_goldens: + path: ../../script/flutter_goldens flutter_test: sdk: flutter diff --git a/packages/cupertino_ui/test/flutter_test_config.dart b/packages/cupertino_ui/test/flutter_test_config.dart new file mode 100644 index 00000000000..c1f9aea6e76 --- /dev/null +++ b/packages/cupertino_ui/test/flutter_test_config.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'goldens_io.dart' + if (dart.library.js_interop) 'goldens_web.dart' + as flutter_goldens; + +Future testExecutable(FutureOr Function() testMain) { + // Enable golden file testing using Skia Gold. + return flutter_goldens.testExecutable(testMain, namePrefix: 'cupertino_ui'); +} diff --git a/packages/cupertino_ui/test/goldens/goldens_test.dart b/packages/cupertino_ui/test/goldens/goldens_test.dart new file mode 100644 index 00000000000..e7977dae646 --- /dev/null +++ b/packages/cupertino_ui/test/goldens/goldens_test.dart @@ -0,0 +1,27 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter/cupertino.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + testWidgets('Inconsequential golden test', (WidgetTester tester) async { + // The test validates the Flutter Gold integration. Any changes to the + // golden file can be approved at any time. + await tester.pumpWidget( + const CupertinoApp( + home: Center( + child: Text('Cupertino Goldens'), + ), + ), + ); + + await tester.pumpAndSettle(); + await expectLater( + find.byType(CupertinoApp), + matchesGoldenFile('inconsequential_golden_file.png'), + ); + }, skip: kIsWeb); +} diff --git a/packages/cupertino_ui/test/goldens_io.dart b/packages/cupertino_ui/test/goldens_io.dart new file mode 100644 index 00000000000..d552235d690 --- /dev/null +++ b/packages/cupertino_ui/test/goldens_io.dart @@ -0,0 +1,5 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'package:flutter_goldens/flutter_goldens.dart' show testExecutable; diff --git a/packages/cupertino_ui/test/goldens_web.dart b/packages/cupertino_ui/test/goldens_web.dart new file mode 100644 index 00000000000..1b4a8a2af31 --- /dev/null +++ b/packages/cupertino_ui/test/goldens_web.dart @@ -0,0 +1,9 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +// package:flutter_goldens is not used as part of the test process for web. +Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async => + testMain(); diff --git a/packages/material_ui/test/flutter_test_config.dart b/packages/material_ui/test/flutter_test_config.dart index ac0ed96dd87..5414abd6f3b 100644 --- a/packages/material_ui/test/flutter_test_config.dart +++ b/packages/material_ui/test/flutter_test_config.dart @@ -2,14 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. - -// Initial testing of instance only, not for merging. -// Plenty to do next (if this works): -// - verify service accounts used by Luci in pre/post submit tests in flutter/packages, auth works -// - Get new Gold frontend up -// - document how to enable golden file testing for a package -// - update flutter/cocoon to add flutter-gold check for triage - import 'dart:async'; import 'goldens_io.dart' diff --git a/packages/material_ui/test/goldens/goldens_test.dart b/packages/material_ui/test/goldens/goldens_test.dart index bc048c5862d..8aea3ada4e7 100644 --- a/packages/material_ui/test/goldens/goldens_test.dart +++ b/packages/material_ui/test/goldens/goldens_test.dart @@ -16,5 +16,5 @@ void main() { find.byType(RepaintBoundary), matchesGoldenFile('inconsequential_golden_file.png'), ); - }); + }, skip: kIsWeb); } From 94aade53d5c6b7314357a15a71e29ee6ca6219db Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 23 Mar 2026 10:59:03 -0500 Subject: [PATCH 14/21] ++ --- .../test/goldens/goldens_test.dart | 2 + .../flutter_goldens/lib/flutter_goldens.dart | 116 ++-- script/flutter_goldens/lib/skia_client.dart | 83 ++- .../test/comparator_selection_test.dart | 49 +- .../test/flutter_goldens_test.dart | 589 +++++++++++------- 5 files changed, 524 insertions(+), 315 deletions(-) diff --git a/packages/material_ui/test/goldens/goldens_test.dart b/packages/material_ui/test/goldens/goldens_test.dart index 8aea3ada4e7..09b4fdf971a 100644 --- a/packages/material_ui/test/goldens/goldens_test.dart +++ b/packages/material_ui/test/goldens/goldens_test.dart @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'package:flutter/cupertino.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:material_ui/material_ui.dart'; diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index 1b07835f17e..aa5569820b3 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -49,7 +49,10 @@ bool _isMainBranch(String? branch) { /// will affect whether this is effective or not. For example, if the current /// override provides a mock client that always fails, then all calls to gold /// comparison functions will fail. -Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async { +Future testExecutable( + FutureOr Function() testMain, { + String? namePrefix, +}) async { assert( goldenFileComparator is LocalFileComparator, 'The flutter_goldens package should be used from a flutter_test_config.dart ' @@ -64,25 +67,27 @@ Future testExecutable(FutureOr Function() testMain, {String? namePre const ProcessManager process = LocalProcessManager(); final httpClient = io.HttpClient(); if (FlutterPostSubmitFileComparator.isForEnvironment(platform)) { - goldenFileComparator = await FlutterPostSubmitFileComparator.fromLocalFileComparator( - localFileComparator: goldenFileComparator as LocalFileComparator, - platform: platform, - namePrefix: namePrefix, - log: print, - fs: fs, - process: process, - httpClient: httpClient, - ); + goldenFileComparator = + await FlutterPostSubmitFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + platform: platform, + namePrefix: namePrefix, + log: (String _) {}, + fs: fs, + process: process, + httpClient: httpClient, + ); } else if (FlutterPreSubmitFileComparator.isForEnvironment(platform)) { - goldenFileComparator = await FlutterPreSubmitFileComparator.fromLocalFileComparator( - localFileComparator: goldenFileComparator as LocalFileComparator, - platform: platform, - namePrefix: namePrefix, - log: print, - fs: fs, - process: process, - httpClient: httpClient, - ); + goldenFileComparator = + await FlutterPreSubmitFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + platform: platform, + namePrefix: namePrefix, + log: (String _) {}, + fs: fs, + process: process, + httpClient: httpClient, + ); } else if (FlutterSkippingFileComparator.isForEnvironment(platform)) { goldenFileComparator = FlutterSkippingFileComparator.fromLocalFileComparator( localFileComparator: goldenFileComparator as LocalFileComparator, @@ -90,20 +95,21 @@ Future testExecutable(FutureOr Function() testMain, {String? namePre 'flutter, or in test shards that are not configured for using goldctl.', platform: platform, namePrefix: namePrefix, - log: print, + log: (String _) {}, fs: fs, process: process, httpClient: httpClient, ); } else { - goldenFileComparator = await FlutterLocalFileComparator.fromLocalFileComparator( - localFileComparator: goldenFileComparator as LocalFileComparator, - platform: platform, - log: print, - fs: fs, - process: process, - httpClient: httpClient, - ); + goldenFileComparator = + await FlutterLocalFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + platform: platform, + log: (String _) {}, + fs: fs, + process: process, + httpClient: httpClient, + ); } await testMain(); } @@ -206,7 +212,8 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { // final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]); final Directory comparisonRoot = switch (suffix) { // null => flutterRoot.childDirectory(fs.path.join('bin', 'cache', 'pkg', 'skia_goldens')), - null => fs.directory(defaultComparator.basedir).childDirectory('skia_goldens'), + null => + fs.directory(defaultComparator.basedir).childDirectory('skia_goldens'), _ => fs.systemTempDirectory.createTempSync(suffix), }; return comparisonRoot; //.childDirectory(fs.path.relative(testPath, from: flutterRoot.path)); @@ -216,7 +223,7 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { @protected File getGoldenFile(Uri uri) { final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path); - print(goldenFile); + return goldenFile; } @@ -229,14 +236,8 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { 'Golden files in the Flutter framework must end with the file extension ' '.png.', ); - print('_addPrefix, golden: $golden'); - print('basedir: $basedir'); - return Uri.parse( - [ - ?namePrefix, - golden.toString(), - ].join('.'), - ); + + return Uri.parse([?namePrefix, golden.toString()].join('.')); } } @@ -285,14 +286,14 @@ class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { required ProcessManager process, required io.HttpClient httpClient, }) async { - print('Creating Postsubmit'); - final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory( - localFileComparator, - platform: platform, - suffix: 'flutter_goldens_postsubmit.', - fs: fs, - ); - print(baseDirectory); + final Directory baseDirectory = + FlutterGoldenFileComparator.getBaseDirectory( + localFileComparator, + platform: platform, + suffix: 'flutter_goldens_postsubmit.', + fs: fs, + ); + baseDirectory.createSync(recursive: true); goldens ??= SkiaGoldClient( @@ -318,7 +319,7 @@ class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { Future compare(Uint8List imageBytes, Uri golden) async { await skiaClient.imgtestInit(); golden = _addPrefix(golden); - print('prefixed golden: $golden'); + await update(golden, imageBytes); final File goldenFile = getGoldenFile(golden); try { @@ -394,7 +395,6 @@ class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { required ProcessManager process, required io.HttpClient httpClient, }) async { - print('Creating presubmit'); final Directory baseDirectory = testBasedir ?? FlutterGoldenFileComparator.getBaseDirectory( @@ -403,7 +403,7 @@ class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { suffix: 'flutter_goldens_presubmit.', fs: fs, ); - print(baseDirectory); + if (!baseDirectory.existsSync()) { baseDirectory.createSync(recursive: true); } @@ -432,7 +432,7 @@ class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { Future compare(Uint8List imageBytes, Uri golden) async { await skiaClient.tryjobInit(); golden = _addPrefix(golden); - print('prefixed golden: $golden'); + await update(golden, imageBytes); final File goldenFile = getGoldenFile(golden); @@ -501,7 +501,6 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { required ProcessManager process, required io.HttpClient httpClient, }) { - print('Creating skip'); final Uri basedir = localFileComparator.basedir; final skiaClient = SkiaGoldClient( fs.directory(basedir), @@ -568,7 +567,8 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { /// * [FlutterSkippingFileComparator], another /// [FlutterGoldenFileComparator] that controls post-submit testing /// conditions that do not execute golden file tests. -class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalComparisonOutput { +class FlutterLocalFileComparator extends FlutterGoldenFileComparator + with LocalComparisonOutput { /// Creates a [FlutterLocalFileComparator] that will test golden file /// images against baselines requested from Flutter Gold. /// @@ -597,15 +597,11 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC required ProcessManager process, required io.HttpClient httpClient, }) async { - print('Creating Local'); baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory( localFileComparator, platform: platform, fs: fs, ); - print('**'); - print(baseDirectory); - print('**'); if (!baseDirectory.existsSync()) { baseDirectory.createSync(recursive: true); @@ -666,7 +662,7 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC @override Future compare(Uint8List imageBytes, Uri golden) async { golden = _addPrefix(golden); - print('prefixed golden: $golden'); + final String testName = skiaClient.cleanTestName(golden.path); late String? testExpectation; testExpectation = await skiaClient.getExpectationForTest(testName); @@ -678,12 +674,14 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC 'https://flutter-packages-gold.skia.org.\n' 'Validate image output found at $basedir', ); - update(golden, imageBytes); + await update(golden, imageBytes); return true; } ComparisonResult result; - final List goldenBytes = await skiaClient.getImageBytes(testExpectation); + final List goldenBytes = await skiaClient.getImageBytes( + testExpectation, + ); result = await GoldenFileComparator.compareLists(imageBytes, goldenBytes); diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index 4f6480bc7d7..90b9c25404e 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -152,7 +152,6 @@ class SkiaGoldClient { await keys.writeAsString(_getKeysJSON()); await failures.create(); final String commitHash = await _getCurrentCommit(); - print(commitHash); final imgtestInitCommand = [ _goldctl, @@ -226,7 +225,9 @@ class SkiaGoldClient { // If an unapproved image has made it to post-submit, throw to close the // tree. String? resultContents; - final File resultFile = workDirectory.childFile(fs.path.join('result-state.json')); + final File resultFile = workDirectory.childFile( + fs.path.join('result-state.json'), + ); if (await resultFile.exists()) { resultContents = await resultFile.readAsString(); } @@ -247,7 +248,9 @@ class SkiaGoldClient { ..writeln('stdout: ${result.stdout}') ..writeln('stderr: ${result.stderr}') ..writeln() - ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}'); + ..writeln( + 'result-state.json: ${resultContents ?? 'No result file found.'}', + ); throw SkiaException(buf.toString()); } @@ -310,7 +313,6 @@ class SkiaGoldClient { imgtestInitCommand.forEach(buf.writeln); throw SkiaException(buf.toString()); } - print(imgtestInitCommand); final io.ProcessResult result = await process.run(imgtestInitCommand); @@ -318,7 +320,9 @@ class SkiaGoldClient { _tryjobInitialized = false; final buf = StringBuffer() ..writeln('Skia Gold tryjobInit failure.') - ..writeln('An error occurred when initializing golden file tryjob with ') + ..writeln( + 'An error occurred when initializing golden file tryjob with ', + ) ..writeln('goldctl.') ..writeln() ..writeln('Debug information for Gold --------------------------------') @@ -359,9 +363,12 @@ class SkiaGoldClient { final resultStdout = result.stdout.toString(); if (result.exitCode != 0 && - !(resultStdout.contains('Untriaged') || resultStdout.contains('negative image'))) { + !(resultStdout.contains('Untriaged') || + resultStdout.contains('negative image'))) { String? resultContents; - final File resultFile = workDirectory.childFile(fs.path.join('result-state.json')); + final File resultFile = workDirectory.childFile( + fs.path.join('result-state.json'), + ); if (await resultFile.exists()) { resultContents = await resultFile.readAsString(); } @@ -375,7 +382,9 @@ class SkiaGoldClient { ..writeln('stderr: ${result.stderr}') ..writeln() ..writeln() - ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}'); + ..writeln( + 'result-state.json: ${resultContents ?? 'No result file found.'}', + ); throw SkiaException(buf.toString()); } return result.exitCode == 0 ? null : resultStdout; @@ -391,12 +400,16 @@ class SkiaGoldClient { ); late String rawResponse; try { - final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations); + final io.HttpClientRequest request = await httpClient.getUrl( + requestForExpectations, + ); final io.HttpClientResponse response = await request.close(); rawResponse = await utf8.decodeStream(response); final dynamic jsonResponse = json.decode(rawResponse); if (jsonResponse is! Map) { - throw const FormatException('Skia gold expectations do not match expected format.'); + throw const FormatException( + 'Skia gold expectations do not match expected format.', + ); } expectation = jsonResponse['digest'] as String?; } on FormatException catch (error) { @@ -420,7 +433,9 @@ class SkiaGoldClient { final Uri requestForImage = Uri.parse( 'https://flutter-packages-gold.skia.org/img/images/$imageHash.png', ); - final io.HttpClientRequest request = await httpClient.getUrl(requestForImage); + final io.HttpClientRequest request = await httpClient.getUrl( + requestForImage, + ); final io.HttpClientResponse response = await request.close(); await response.forEach((List bytes) => imageBytes.addAll(bytes)); return imageBytes; @@ -429,19 +444,20 @@ class SkiaGoldClient { /// Returns the current commit hash of the packages repository. Future _getCurrentCommit() async { final String cleanPath = path.normalize(platform.environment[_kSDKKey]!); - print(path.join(path.dirname(cleanPath), 'packages')); + final io.ProcessResult revParse = await process.run([ - 'git', - 'rev-parse', - 'HEAD', - ], workingDirectory: path.join(path.dirname(cleanPath), 'packages')); - if (revParse.exitCode != 0) { - throw const SkiaException('Current commit of flutter/packages can not be found.'); - } - final String commit = (revParse.stdout as String).trim(); - print(commit); - return commit; - + 'git', + 'rev-parse', + 'HEAD', + ], workingDirectory: path.join(path.dirname(cleanPath), 'packages')); + if (revParse.exitCode != 0) { + throw const SkiaException( + 'Current commit of flutter/packages can not be found.', + ); + } + final String commit = (revParse.stdout as String).trim(); + + return commit; } /// Returns a JSON String with keys value pairs used to uniquely identify the @@ -454,9 +470,9 @@ class SkiaGoldClient { final keys = { 'Platform': platform.operatingSystem, 'CI': 'luci', - 'Web' : _isBrowserTest.toString(), + 'Web': _isBrowserTest.toString(), }; - print(json.encode(keys)); + return json.encode(keys); } @@ -469,7 +485,9 @@ class SkiaGoldClient { /// Returns a boolean value to prevent the client from re-authorizing itself /// for multiple tests. Future clientIsAuthorized() async { - final File authFile = workDirectory.childFile(fs.path.join('temp', 'auth_opt.json')); + final File authFile = workDirectory.childFile( + fs.path.join('temp', 'auth_opt.json'), + ); if (await authFile.exists()) { final String contents = await authFile.readAsString(); @@ -482,11 +500,20 @@ class SkiaGoldClient { /// Returns a list of arguments for initializing a tryjob based on the testing /// environment. List getCIArguments() { - final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']!.split('/').last; + final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']! + .split('/') + .last; final List refs = platform.environment['GOLD_TRYJOB']!.split('/'); final String pullRequest = refs[refs.length - 2]; - return ['--changelist', pullRequest, '--cis', 'buildbucket', '--jobid', jobId]; + return [ + '--changelist', + pullRequest, + '--cis', + 'buildbucket', + '--jobid', + jobId, + ]; } bool get _isBrowserTest { diff --git a/script/flutter_goldens/test/comparator_selection_test.dart b/script/flutter_goldens/test/comparator_selection_test.dart index 6057e42cc27..792d0c61d7f 100644 --- a/script/flutter_goldens/test/comparator_selection_test.dart +++ b/script/flutter_goldens/test/comparator_selection_test.dart @@ -44,18 +44,30 @@ void main() { // If we don't have gold but are on CI, we skip regardless. expect(_testRecommendations(hasLuci: true), _Comparator.skip); - expect(_testRecommendations(hasLuci: true, hasTryJob: true), _Comparator.skip); + expect( + _testRecommendations(hasLuci: true, hasTryJob: true), + _Comparator.skip, + ); // On Luci, with Gold, post-submit. Flutter root and LUCI variables should have no effect. - expect(_testRecommendations(hasGold: true, hasLuci: true), _Comparator.post); + expect( + _testRecommendations(hasGold: true, hasLuci: true), + _Comparator.post, + ); // On Luci, with Gold, pre-submit. Flutter root and LUCI variables should have no effect. - expect(_testRecommendations(hasGold: true, hasLuci: true, hasTryJob: true), _Comparator.pre); + expect( + _testRecommendations(hasGold: true, hasLuci: true, hasTryJob: true), + _Comparator.pre, + ); }); test('Comparator recommendations - release branch', () { // If we're running locally (no CI), use a local comparator. - expect(_testRecommendations(branch: 'flutter-3.16-candidate.0'), _Comparator.local); + expect( + _testRecommendations(branch: 'flutter-3.16-candidate.0'), + _Comparator.local, + ); expect( _testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true), @@ -68,13 +80,21 @@ void main() { _Comparator.skip, ); expect( - _testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true, hasTryJob: true), + _testRecommendations( + branch: 'flutter-3.16-candidate.0', + hasLuci: true, + hasTryJob: true, + ), _Comparator.skip, ); // On Luci, with Gold, post-submit. Flutter root and LUCI variables should have no effect. Branch should make us skip. expect( - _testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true), + _testRecommendations( + branch: 'flutter-3.16-candidate.0', + hasGold: true, + hasLuci: true, + ), _Comparator.skip, ); @@ -97,14 +117,25 @@ void main() { // If we don't have gold but are on CI, we skip regardless. expect(_testRecommendations(os: 'linux', hasLuci: true), _Comparator.skip); - expect(_testRecommendations(os: 'linux', hasLuci: true, hasTryJob: true), _Comparator.skip); + expect( + _testRecommendations(os: 'linux', hasLuci: true, hasTryJob: true), + _Comparator.skip, + ); // On Luci, with Gold, post-submit. Flutter root has no effect. - expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true), _Comparator.post); + expect( + _testRecommendations(os: 'linux', hasGold: true, hasLuci: true), + _Comparator.post, + ); // On Luci, with Gold, pre-submit. Flutter root should have no effect. expect( - _testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasTryJob: true), + _testRecommendations( + os: 'linux', + hasGold: true, + hasLuci: true, + hasTryJob: true, + ), _Comparator.pre, ); }); diff --git a/script/flutter_goldens/test/flutter_goldens_test.dart b/script/flutter_goldens/test/flutter_goldens_test.dart index 5c37af2e82c..8288fac9492 100644 --- a/script/flutter_goldens/test/flutter_goldens_test.dart +++ b/script/flutter_goldens/test/flutter_goldens_test.dart @@ -96,23 +96,25 @@ const List _kTestPngBytes = [ void main() { group('SkiaGoldClient', () { - test('auth performs minimal work if already authorized', () async { final fs = MemoryFileSystem(); final platform = FakePlatform( operatingSystem: 'macos', + environment: {}, ); final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); final File authFile = fs.file('/workDirectory/temp/auth_opt.json') ..createSync(recursive: true); @@ -127,18 +129,21 @@ void main() { final fs = MemoryFileSystem(); final platform = FakePlatform( operatingSystem: 'macos', + environment: {}, ); final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); final File authFile = fs.file('/workDirectory/temp/auth_opt.json') ..createSync(recursive: true); @@ -158,17 +163,24 @@ void main() { final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + process.fallbackProcessResult = ProcessResult( + 123, + 1, + 'Fallback failure', + 'Fallback failure', + ); expect(skiaClient.auth(), throwsException); }); @@ -177,21 +189,31 @@ void main() { final fs = MemoryFileSystem(); final platform = FakePlatform( operatingSystem: 'macos', + environment: { + 'SDK_CHECKOUT_PATH': '/flutter', + 'GOLDCTL': 'goldctl', + }, ); final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); - const gitInvocation = RunInvocation(['git', 'rev-parse', 'HEAD'], '/flutter'); + const gitInvocation = RunInvocation([ + 'git', + 'rev-parse', + 'HEAD', + ], '/packages'); const goldctlInvocation = RunInvocation([ 'goldctl', 'imgtest', @@ -208,14 +230,25 @@ void main() { '/workDirectory/failures.json', '--passfail', ], null); - process.processResults[gitInvocation] = ProcessResult(12345678, 0, '12345678', ''); + + process.processResults[gitInvocation] = ProcessResult( + 12345678, + 0, + '12345678', + '', + ); process.processResults[goldctlInvocation] = ProcessResult( 123, 1, 'Expected failure', 'Expected failure', ); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + process.fallbackProcessResult = ProcessResult( + 123, + 1, + 'Fallback failure', + 'Fallback failure', + ); expect(skiaClient.imgtestInit(), throwsException); }); @@ -224,21 +257,31 @@ void main() { final fs = MemoryFileSystem(); final platform = FakePlatform( operatingSystem: 'macos', + environment: { + 'SDK_CHECKOUT_PATH': '/flutter', + 'GOLDCTL': 'goldctl', + }, ); final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); - const gitInvocation = RunInvocation(['git', 'rev-parse', 'HEAD'], '/flutter'); + const gitInvocation = RunInvocation([ + 'git', + 'rev-parse', + 'HEAD', + ], '/packages'); const goldctlInvocation = RunInvocation([ 'goldctl', 'imgtest', @@ -255,9 +298,25 @@ void main() { '/workDirectory/failures.json', '--passfail', ], null); - process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); - process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + process.processResults[gitInvocation] = ProcessResult( + 1234, + 0, + '1234', + '', + ); + process.processResults[goldctlInvocation] = ProcessResult( + 5678, + 0, + '5678', + '', + ); + process.fallbackProcessResult = ProcessResult( + 123, + 1, + 'Fallback failure', + 'Fallback failure', + ); // First call await skiaClient.imgtestInit(); @@ -276,25 +335,33 @@ void main() { environment: { 'GOLDCTL': 'goldctl', 'SWARMING_TASK_ID': '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'LOGDOG_STREAM_PREFIX': + 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', 'GOLD_TRYJOB': 'refs/pull/49815/head', + 'SDK_CHECKOUT_PATH': '/flutter', }, operatingSystem: 'macos', ); final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); - const gitInvocation = RunInvocation(['git', 'rev-parse', 'HEAD'], '/flutter'); + const gitInvocation = RunInvocation([ + 'git', + 'rev-parse', + 'HEAD', + ], '/packages'); const goldctlInvocation = RunInvocation([ 'goldctl', 'imgtest', @@ -321,9 +388,25 @@ void main() { '--jobid', '8885996262141582672', ], null); - process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); - process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + process.processResults[gitInvocation] = ProcessResult( + 1234, + 0, + '1234', + '', + ); + process.processResults[goldctlInvocation] = ProcessResult( + 5678, + 0, + '5678', + '', + ); + process.fallbackProcessResult = ProcessResult( + 123, + 1, + 'Fallback failure', + 'Fallback failure', + ); // First call await skiaClient.tryjobInit(); @@ -338,8 +421,9 @@ void main() { test('throws for error state from imgtestAdd', () { final fs = MemoryFileSystem(); - final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') - ..createSync(recursive: true); + final File goldenFile = fs.file( + '/workDirectory/temp/golden_file_test.png', + )..createSync(recursive: true); final platform = FakePlatform( environment: {'GOLDCTL': 'goldctl'}, operatingSystem: 'macos', @@ -347,14 +431,16 @@ void main() { final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); const goldctlInvocation = RunInvocation([ 'goldctl', @@ -374,9 +460,17 @@ void main() { 'Expected failure', 'Expected failure', ); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + process.fallbackProcessResult = ProcessResult( + 123, + 1, + 'Fallback failure', + 'Fallback failure', + ); - expect(skiaClient.imgtestAdd('golden_file_test', goldenFile), throwsException); + expect( + skiaClient.imgtestAdd('golden_file_test', goldenFile), + throwsException, + ); }); test('correctly inits tryjob for luci', () async { @@ -385,7 +479,8 @@ void main() { environment: { 'GOLDCTL': 'goldctl', 'SWARMING_TASK_ID': '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'LOGDOG_STREAM_PREFIX': + 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', 'GOLD_TRYJOB': 'refs/pull/49815/head', }, operatingSystem: 'macos', @@ -393,14 +488,16 @@ void main() { final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); final List ciArguments = skiaClient.getCIArguments(); @@ -424,7 +521,8 @@ void main() { environment: { 'GOLDCTL': 'goldctl', 'SWARMING_TASK_ID': '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'LOGDOG_STREAM_PREFIX': + 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', 'GOLD_TRYJOB': 'refs/pull/49815/head', }, operatingSystem: 'linux', @@ -432,17 +530,22 @@ void main() { final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); - expect(skiaClient.getTraceID('flutter.golden.1'), equals('ae18c7a6aa48e0685525dfe8fdf79003')); + expect( + skiaClient.getTraceID('flutter.golden.1'), + equals('9b44cb4464826eef24cbb8095407edc8'), + ); }); test('Creates traceID correctly - Browser', () async { @@ -451,52 +554,69 @@ void main() { environment: { 'GOLDCTL': 'goldctl', 'SWARMING_TASK_ID': '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'LOGDOG_STREAM_PREFIX': + 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', 'GOLD_TRYJOB': 'refs/pull/49815/head', - 'FLUTTER_TEST_BROWSER': 'chrome', + + 'CHROME_EXECUTABLE': 'chrome', }, operatingSystem: 'linux', ); final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); - expect(skiaClient.getTraceID('flutter.golden.1'), equals('e9d5c296c48e7126808520e9cc191243')); - }); - - test('Creates traceID correctly - locally - should defer to luci traceID', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + expect( + skiaClient.getTraceID('flutter.golden.1'), + equals('2592d057e5ea8dbc8ac4e8851090695a'), ); - expect(skiaClient.getTraceID('flutter.golden.1'), equals('9968695b9ae78cdb77cbb2be621ca2d6')); }); + test( + 'Creates traceID correctly - locally - should defer to luci traceID', + () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform( + operatingSystem: 'macos', + environment: {}, + ); + final process = FakeProcessManager(); + final fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); + final skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), + ); + expect( + skiaClient.getTraceID('flutter.golden.1'), + equals('198543ea507122f8fafde25f946bccb0'), + ); + }, + ); + test('throws for error state from imgtestAdd', () { final fs = MemoryFileSystem(); - final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') - ..createSync(recursive: true); + final File goldenFile = fs.file( + '/workDirectory/temp/golden_file_test.png', + )..createSync(recursive: true); final platform = FakePlatform( environment: {'GOLDCTL': 'goldctl'}, operatingSystem: 'macos', @@ -504,14 +624,16 @@ void main() { final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); const goldctlInvocation = RunInvocation([ 'goldctl', @@ -531,7 +653,12 @@ void main() { 'Expected failure', 'Expected failure', ); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + process.fallbackProcessResult = ProcessResult( + 123, + 1, + 'Fallback failure', + 'Fallback failure', + ); expect( skiaClient.imgtestAdd('golden_file_test', goldenFile), @@ -547,8 +674,9 @@ void main() { test('throws for error state from tryjobAdd', () { final fs = MemoryFileSystem(); - final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') - ..createSync(recursive: true); + final File goldenFile = fs.file( + '/workDirectory/temp/golden_file_test.png', + )..createSync(recursive: true); final platform = FakePlatform( environment: {'GOLDCTL': 'goldctl'}, operatingSystem: 'macos', @@ -556,14 +684,16 @@ void main() { final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), ); const goldctlInvocation = RunInvocation([ 'goldctl', @@ -583,7 +713,12 @@ void main() { 'Expected failure', 'Expected failure', ); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + process.fallbackProcessResult = ProcessResult( + 123, + 1, + 'Fallback failure', + 'Fallback failure', + ); expect( skiaClient.tryjobAdd('golden_file_test', goldenFile), throwsA( @@ -600,29 +735,35 @@ void main() { test('image bytes are processed properly', () async { const expectation = '55109a4bed52acc780530f7a9aeff6c0'; final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); final process = FakeProcessManager(); final fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true); + final Directory workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); final skiaClient = SkiaGoldClient( workDirectory, fs: fs, process: process, platform: platform, httpClient: fakeHttpClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), + ); + final Uri imageUrl = Uri.parse( + 'https://flutter-packages-gold.skia.org/img/images/$expectation.png', ); - final Uri imageUrl = Uri.parse('https://flutter-gold.skia.org/img/images/$expectation.png'); final fakeImageRequest = FakeHttpClientRequest(); - final fakeImageResponse = FakeHttpImageResponse(imageResponseTemplate()); + final fakeImageResponse = FakeHttpImageResponse( + imageResponseTemplate(), + ); fakeHttpClient.request = fakeImageRequest; fakeImageRequest.response = fakeImageResponse; - final List masterBytes = await skiaClient.getImageBytes(expectation); + final List masterBytes = await skiaClient.getImageBytes( + expectation, + ); expect(fakeHttpClient.lastUri, imageUrl); expect(masterBytes, equals(_kTestPngBytes)); @@ -631,39 +772,39 @@ void main() { }); group('FlutterGoldenFileComparator', () { - test('calculates the basedir correctly from defaultComparator for local testing', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final defaultComparator = FakeLocalFileComparator(); - final Directory root = fs.directory('/') - ..createSync(recursive: true); - defaultComparator.basedir = root.childDirectory('baz').uri; - final Directory basedir = FlutterGoldenFileComparator.getBaseDirectory( - defaultComparator, - platform: platform, - fs: fs, - ); - expect(basedir.uri, fs.directory('/baz/skia_goldens').uri); - }); + test( + 'calculates the basedir correctly from defaultComparator for local testing', + () async { + final fs = MemoryFileSystem(); + final platform = FakePlatform(operatingSystem: 'macos'); + fs.directory(_kFlutterRoot).createSync(recursive: true); + final defaultComparator = FakeLocalFileComparator(); + final Directory root = fs.directory('/')..createSync(recursive: true); + defaultComparator.basedir = root.childDirectory('baz').uri; + final Directory basedir = FlutterGoldenFileComparator.getBaseDirectory( + defaultComparator, + platform: platform, + fs: fs, + ); + expect(basedir.uri, fs.directory('/baz/skia_goldens').uri); + }, + ); test('ignores version number', () { final log = []; final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory basedir = fs.directory('flutter/test/library/')..createSync(recursive: true); - final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( - basedir.uri, - FakeSkiaGoldClient(), - fs: fs, - platform: platform, - log: log.add, - ); + final Directory basedir = fs.directory('flutter/test/library/') + ..createSync(recursive: true); + final FlutterGoldenFileComparator comparator = + FlutterPostSubmitFileComparator( + basedir.uri, + FakeSkiaGoldClient(), + fs: fs, + platform: platform, + log: log.add, + ); final Uri key = comparator.getTestUri(Uri.parse('foo.png'), 1); expect(key, Uri.parse('foo.png')); expect(log, isEmpty); @@ -672,9 +813,7 @@ void main() { test('adds namePrefix', () async { final log = []; final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); const packageName = 'sidedishes'; const namePrefix = 'tomatosalad'; @@ -682,16 +821,20 @@ void main() { final fakeSkiaClient = FakeSkiaGoldClient(); final Directory basedir = fs.directory('$packageName/test/') ..createSync(recursive: true); - final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - namePrefix: namePrefix, - log: log.add, + final FlutterGoldenFileComparator comparator = + FlutterPostSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + namePrefix: namePrefix, + log: log.add, + ); + await comparator.compare( + Uint8List.fromList(_kTestPngBytes), + Uri.parse(fileName), ); - await comparator.compare(Uint8List.fromList(_kTestPngBytes), Uri.parse(fileName)); - expect(fakeSkiaClient.testNames.single, '$namePrefix.$packageName.$fileName'); + expect(fakeSkiaClient.testNames.single, '$namePrefix.$fileName'); expect(log, isEmpty); }); @@ -699,20 +842,19 @@ void main() { test('asserts .png format', () async { final log = []; final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final Directory basedir = fs.directory('flutter/test/library/') ..createSync(recursive: true); final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - log: log.add, - ); + final FlutterGoldenFileComparator comparator = + FlutterPostSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); await expectLater( () async { return comparator.compare( @@ -737,20 +879,19 @@ void main() { test('calls init during compare', () { final log = []; final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final Directory basedir = fs.directory('flutter/test/library/') ..createSync(recursive: true); final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - log: log.add, - ); + final FlutterGoldenFileComparator comparator = + FlutterPostSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); expect(fakeSkiaClient.initCalls, 0); comparator.compare( Uint8List.fromList(_kTestPngBytes), @@ -762,17 +903,19 @@ void main() { test('does not call init in during construction', () { final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final fakeSkiaClient = FakeSkiaGoldClient(); expect(fakeSkiaClient.initCalls, 0); FlutterPostSubmitFileComparator.fromLocalFileComparator( - localFileComparator: LocalFileComparator(Uri.parse('/test'), pathStyle: path.Style.posix), + localFileComparator: LocalFileComparator( + Uri.parse('/test'), + pathStyle: path.Style.posix, + ), platform: platform, goldens: fakeSkiaClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), fs: fs, process: FakeProcessManager(), httpClient: FakeHttpClient(), @@ -783,21 +926,21 @@ void main() { test('reports a failure as a TestFailure', () async { final log = []; final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final Directory basedir = fs.directory('flutter/test/library/') ..createSync(recursive: true); - final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( - basedir.uri, - ThrowsOnImgTestAddSkiaClient( - message: 'Skia Gold received an unapproved image in post-submit', - ), - fs: fs, - platform: platform, - log: log.add, - ); + final FlutterGoldenFileComparator comparator = + FlutterPostSubmitFileComparator( + basedir.uri, + ThrowsOnImgTestAddSkiaClient( + message: + 'Skia Gold received an unapproved image in post-submit', + ), + fs: fs, + platform: platform, + log: log.add, + ); await expectLater( () async { return comparator.compare( @@ -821,20 +964,19 @@ void main() { test('asserts .png format', () async { final log = []; final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final Directory basedir = fs.directory('flutter/test/library/') ..createSync(recursive: true); final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = FlutterPreSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - log: log.add, - ); + final FlutterGoldenFileComparator comparator = + FlutterPreSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); await expectLater( () async { return comparator.compare( @@ -859,20 +1001,19 @@ void main() { test('calls init during compare', () { final log = []; final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final Directory basedir = fs.directory('flutter/test/library/') ..createSync(recursive: true); final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = FlutterPreSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - log: log.add, - ); + final FlutterGoldenFileComparator comparator = + FlutterPreSubmitFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: platform, + log: log.add, + ); expect(fakeSkiaClient.tryInitCalls, 0); comparator.compare( Uint8List.fromList(_kTestPngBytes), @@ -884,17 +1025,19 @@ void main() { test('does not call init in during construction', () { final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final fakeSkiaClient = FakeSkiaGoldClient(); expect(fakeSkiaClient.tryInitCalls, 0); FlutterPostSubmitFileComparator.fromLocalFileComparator( - localFileComparator: LocalFileComparator(Uri.parse('/test'), pathStyle: path.Style.posix), + localFileComparator: LocalFileComparator( + Uri.parse('/test'), + pathStyle: path.Style.posix, + ), platform: platform, goldens: fakeSkiaClient, - log: (String message) => fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => + fail('skia gold client printed unexpected output: "$message"'), fs: fs, process: FakeProcessManager(), httpClient: FakeHttpClient(), @@ -911,19 +1054,18 @@ void main() { final Directory basedir = fs.directory('flutter/test/library/') ..createSync(recursive: true); final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = FlutterLocalFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: FakePlatform( - operatingSystem: 'macos', - ), - log: log.add, - ); + final FlutterGoldenFileComparator comparator = + FlutterLocalFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: FakePlatform(operatingSystem: 'macos'), + log: log.add, + ); const hash = '55109a4bed52acc780530f7a9aeff6c0'; fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; - fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = + fakeSkiaClient.cleanTestNameValues['flutter.golden_test.1.png'] = 'flutter.golden_test.1'; await expectLater( () async { @@ -953,19 +1095,18 @@ void main() { final Directory basedir = fs.directory('flutter/test/library/') ..createSync(recursive: true); final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = FlutterLocalFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: FakePlatform( - operatingSystem: 'macos', - ), - log: log.add, - ); + final FlutterGoldenFileComparator comparator = + FlutterLocalFileComparator( + basedir.uri, + fakeSkiaClient, + fs: fs, + platform: FakePlatform(operatingSystem: 'macos'), + log: log.add, + ); const hash = '55109a4bed52acc780530f7a9aeff6c0'; fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; - fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = + fakeSkiaClient.cleanTestNameValues['flutter.golden_test.1.png'] = 'flutter.golden_test.1'; expect( await comparator.compare( @@ -981,22 +1122,23 @@ void main() { 'returns FlutterSkippingGoldenFileComparator when network connection is unavailable', () async { final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - ); + final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final fakeSkiaClient = FakeSkiaGoldClient(); const hash = '55109a4bed52acc780530f7a9aeff6c0'; - fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; + fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = + hash; fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; - fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = + fakeSkiaClient.cleanTestNameValues['flutter.golden_test.1.png'] = 'flutter.golden_test.1'; final fakeDirectory = FakeDirectory(); fakeDirectory.existsSyncValue = true; fakeDirectory.uri = Uri.parse('/flutter'); - fakeSkiaClient.getExpectationForTestThrowable = const OSError("Can't reach Gold"); + fakeSkiaClient.getExpectationForTestThrowable = const OSError( + "Can't reach Gold", + ); final FlutterGoldenFileComparator comparator1 = await FlutterLocalFileComparator.fromLocalFileComparator( localFileComparator: LocalFileComparator( @@ -1006,15 +1148,18 @@ void main() { platform: platform, goldens: fakeSkiaClient, baseDirectory: fakeDirectory, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => fail( + 'skia gold client printed unexpected output: "$message"', + ), fs: fs, process: FakeProcessManager(), httpClient: FakeHttpClient(), ); expect(comparator1.runtimeType, FlutterSkippingFileComparator); - fakeSkiaClient.getExpectationForTestThrowable = const SocketException("Can't reach Gold"); + fakeSkiaClient.getExpectationForTestThrowable = const SocketException( + "Can't reach Gold", + ); final FlutterGoldenFileComparator comparator2 = await FlutterLocalFileComparator.fromLocalFileComparator( localFileComparator: LocalFileComparator( @@ -1024,15 +1169,18 @@ void main() { platform: platform, goldens: fakeSkiaClient, baseDirectory: fakeDirectory, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => fail( + 'skia gold client printed unexpected output: "$message"', + ), fs: fs, process: FakeProcessManager(), httpClient: FakeHttpClient(), ); expect(comparator2.runtimeType, FlutterSkippingFileComparator); - fakeSkiaClient.getExpectationForTestThrowable = const FormatException("Can't reach Gold"); + fakeSkiaClient.getExpectationForTestThrowable = const FormatException( + "Can't reach Gold", + ); final FlutterGoldenFileComparator comparator3 = await FlutterLocalFileComparator.fromLocalFileComparator( localFileComparator: LocalFileComparator( @@ -1042,8 +1190,9 @@ void main() { platform: platform, goldens: fakeSkiaClient, baseDirectory: fakeDirectory, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), + log: (String message) => fail( + 'skia gold client printed unexpected output: "$message"', + ), fs: fs, process: FakeProcessManager(), httpClient: FakeHttpClient(), @@ -1098,7 +1247,8 @@ class RunInvocation { } class FakeProcessManager extends Fake implements ProcessManager { - Map processResults = {}; + Map processResults = + {}; /// Used if [processResults] does not contain a matching invocation. ProcessResult? fallbackProcessResult; @@ -1162,7 +1312,8 @@ class FakeSkiaGoldClient extends Fake implements SkiaGoldClient { Map> imageBytesValues = >{}; @override - Future> getImageBytes(String imageHash) async => imageBytesValues[imageHash]!; + Future> getImageBytes(String imageHash) async => + imageBytesValues[imageHash]!; Map cleanTestNameValues = {}; @override From 16f25dfad80ffa146b63632963e012ac61a887dc Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Thu, 16 Apr 2026 15:08:39 -0500 Subject: [PATCH 15/21] Bump goldctl, format, fix headers --- .ci.yaml | 24 +++++++++---------- .../test/flutter_test_config.dart | 2 +- .../test/goldens/goldens_test.dart | 8 ++----- packages/cupertino_ui/test/goldens_io.dart | 2 +- packages/cupertino_ui/test/goldens_web.dart | 8 ++++--- .../material_ui/test/flutter_test_config.dart | 2 +- .../test/goldens/goldens_test.dart | 4 +++- packages/material_ui/test/goldens_io.dart | 2 +- packages/material_ui/test/goldens_web.dart | 8 ++++--- 9 files changed, 31 insertions(+), 29 deletions(-) diff --git a/.ci.yaml b/.ci.yaml index b286a784aaf..be15f8b9934 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -139,7 +139,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] add_recipes_cq: "true" target_file: dart_unit_tests.yaml @@ -153,7 +153,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] target_file: dart_unit_tests.yaml channel: master @@ -166,7 +166,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] target_file: dart_unit_tests.yaml channel: stable @@ -179,7 +179,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] target_file: dart_unit_tests.yaml channel: stable @@ -192,7 +192,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] add_recipes_cq: "true" target_file: web_dart_unit_tests.yaml @@ -206,7 +206,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] target_file: web_dart_unit_tests.yaml channel: master @@ -219,7 +219,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] target_file: web_dart_unit_tests.yaml channel: stable @@ -232,7 +232,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] target_file: web_dart_unit_tests.yaml channel: stable @@ -246,7 +246,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] add_recipes_cq: "true" target_file: web_dart_unit_tests_wasm.yaml @@ -260,7 +260,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] target_file: web_dart_unit_tests_wasm.yaml channel: master @@ -994,7 +994,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] target_file: windows_dart_unit_tests.yaml channel: master @@ -1007,7 +1007,7 @@ targets: properties: dependencies: >- [ - {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} ] target_file: windows_dart_unit_tests.yaml channel: master diff --git a/packages/cupertino_ui/test/flutter_test_config.dart b/packages/cupertino_ui/test/flutter_test_config.dart index c1f9aea6e76..38dacb7c0b1 100644 --- a/packages/cupertino_ui/test/flutter_test_config.dart +++ b/packages/cupertino_ui/test/flutter_test_config.dart @@ -1,4 +1,4 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. diff --git a/packages/cupertino_ui/test/goldens/goldens_test.dart b/packages/cupertino_ui/test/goldens/goldens_test.dart index e7977dae646..d099e7ac14b 100644 --- a/packages/cupertino_ui/test/goldens/goldens_test.dart +++ b/packages/cupertino_ui/test/goldens/goldens_test.dart @@ -1,4 +1,4 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -11,11 +11,7 @@ void main() { // The test validates the Flutter Gold integration. Any changes to the // golden file can be approved at any time. await tester.pumpWidget( - const CupertinoApp( - home: Center( - child: Text('Cupertino Goldens'), - ), - ), + const CupertinoApp(home: Center(child: Text('Cupertino Goldens'))), ); await tester.pumpAndSettle(); diff --git a/packages/cupertino_ui/test/goldens_io.dart b/packages/cupertino_ui/test/goldens_io.dart index d552235d690..001d3149471 100644 --- a/packages/cupertino_ui/test/goldens_io.dart +++ b/packages/cupertino_ui/test/goldens_io.dart @@ -1,4 +1,4 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. diff --git a/packages/cupertino_ui/test/goldens_web.dart b/packages/cupertino_ui/test/goldens_web.dart index 1b4a8a2af31..80945b782d2 100644 --- a/packages/cupertino_ui/test/goldens_web.dart +++ b/packages/cupertino_ui/test/goldens_web.dart @@ -1,9 +1,11 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; // package:flutter_goldens is not used as part of the test process for web. -Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async => - testMain(); +Future testExecutable( + FutureOr Function() testMain, { + String? namePrefix, +}) async => testMain(); diff --git a/packages/material_ui/test/flutter_test_config.dart b/packages/material_ui/test/flutter_test_config.dart index 5414abd6f3b..1ceb2ea7052 100644 --- a/packages/material_ui/test/flutter_test_config.dart +++ b/packages/material_ui/test/flutter_test_config.dart @@ -1,4 +1,4 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. diff --git a/packages/material_ui/test/goldens/goldens_test.dart b/packages/material_ui/test/goldens/goldens_test.dart index 09b4fdf971a..be773a42a84 100644 --- a/packages/material_ui/test/goldens/goldens_test.dart +++ b/packages/material_ui/test/goldens/goldens_test.dart @@ -11,7 +11,9 @@ void main() { testWidgets('Inconsequential golden test', (WidgetTester tester) async { // The test validates the Flutter Gold integration. Any changes to the // golden file can be approved at any time. - await tester.pumpWidget(RepaintBoundary(child: Container(color: const Color(0xAFF61145)))); + await tester.pumpWidget( + RepaintBoundary(child: Container(color: const Color(0xAFF61145))), + ); await tester.pumpAndSettle(); await expectLater( diff --git a/packages/material_ui/test/goldens_io.dart b/packages/material_ui/test/goldens_io.dart index d552235d690..001d3149471 100644 --- a/packages/material_ui/test/goldens_io.dart +++ b/packages/material_ui/test/goldens_io.dart @@ -1,4 +1,4 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. diff --git a/packages/material_ui/test/goldens_web.dart b/packages/material_ui/test/goldens_web.dart index 1b4a8a2af31..80945b782d2 100644 --- a/packages/material_ui/test/goldens_web.dart +++ b/packages/material_ui/test/goldens_web.dart @@ -1,9 +1,11 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; // package:flutter_goldens is not used as part of the test process for web. -Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async => - testMain(); +Future testExecutable( + FutureOr Function() testMain, { + String? namePrefix, +}) async => testMain(); From 6542c4b9a568e4cd29b4fa4db22736f23f37caa7 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Thu, 16 Apr 2026 17:03:22 -0500 Subject: [PATCH 16/21] Set up CI support --- .ci/scripts/analyze_flutter_goldens.sh | 8 ++++++++ .ci/scripts/flutter_goldens_format.sh | 8 ++++++++ .ci/scripts/flutter_goldens_tests.sh | 8 ++++++++ .ci/scripts/prepare_tool.sh | 3 +++ .ci/targets/analyze.yaml | 2 ++ .ci/targets/repo_checks.yaml | 4 ++++ .ci/targets/repo_tools_tests.yaml | 2 ++ script/flutter_goldens/lib/skia_client.dart | 6 +++--- 8 files changed, 38 insertions(+), 3 deletions(-) create mode 100755 .ci/scripts/analyze_flutter_goldens.sh create mode 100755 .ci/scripts/flutter_goldens_format.sh create mode 100755 .ci/scripts/flutter_goldens_tests.sh diff --git a/.ci/scripts/analyze_flutter_goldens.sh b/.ci/scripts/analyze_flutter_goldens.sh new file mode 100755 index 00000000000..d36446ec0b6 --- /dev/null +++ b/.ci/scripts/analyze_flutter_goldens.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright 2013 The Flutter Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +set -e + +cd script/flutter_goldens +flutter analyze --fatal-infos diff --git a/.ci/scripts/flutter_goldens_format.sh b/.ci/scripts/flutter_goldens_format.sh new file mode 100755 index 00000000000..6a92ca0fa05 --- /dev/null +++ b/.ci/scripts/flutter_goldens_format.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright 2013 The Flutter Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +set -e + +cd script/flutter_goldens +dart format --set-exit-if-changed . diff --git a/.ci/scripts/flutter_goldens_tests.sh b/.ci/scripts/flutter_goldens_tests.sh new file mode 100755 index 00000000000..51ebc3b727e --- /dev/null +++ b/.ci/scripts/flutter_goldens_tests.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright 2013 The Flutter Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +set -e + +cd script/flutter_goldens +flutter test diff --git a/.ci/scripts/prepare_tool.sh b/.ci/scripts/prepare_tool.sh index 33646d56e7c..29f2b38ce32 100755 --- a/.ci/scripts/prepare_tool.sh +++ b/.ci/scripts/prepare_tool.sh @@ -10,3 +10,6 @@ git branch main origin/main cd script/tool dart pub get + +cd ../flutter_goldens +flutter pub get diff --git a/.ci/targets/analyze.yaml b/.ci/targets/analyze.yaml index 92f5ebec7b2..3d4f4716cff 100644 --- a/.ci/targets/analyze.yaml +++ b/.ci/targets/analyze.yaml @@ -4,6 +4,8 @@ tasks: infra_step: true # Note infra steps failing prevents "always" from running. - name: analyze repo tools script: .ci/scripts/analyze_repo_tools.sh + - name: analyze flutter_goldens + script: .ci/scripts/analyze_flutter_goldens.sh - name: download Dart deps script: .ci/scripts/tool_runner.sh args: ["fetch-deps"] diff --git a/.ci/targets/repo_checks.yaml b/.ci/targets/repo_checks.yaml index 92446835113..be7cd9204c7 100644 --- a/.ci/targets/repo_checks.yaml +++ b/.ci/targets/repo_checks.yaml @@ -13,6 +13,10 @@ tasks: script: .ci/scripts/plugin_tools_tests.sh - name: tool format script: .ci/scripts/plugin_tools_format.sh + - name: flutter_goldens unit tests + script: .ci/scripts/flutter_goldens_tests.sh + - name: flutter_goldens format + script: .ci/scripts/flutter_goldens_format.sh - name: format script: .ci/scripts/tool_runner.sh # Skip Swift formatting on Linux builders. diff --git a/.ci/targets/repo_tools_tests.yaml b/.ci/targets/repo_tools_tests.yaml index bd80daeb3fb..d484e268e97 100644 --- a/.ci/targets/repo_tools_tests.yaml +++ b/.ci/targets/repo_tools_tests.yaml @@ -4,3 +4,5 @@ tasks: infra_step: true # Note infra steps failing prevents "always" from running. - name: tool unit tests script: .ci/scripts/plugin_tools_tests.sh + - name: flutter_goldens unit tests + script: .ci/scripts/flutter_goldens_tests.sh diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index 90b9c25404e..1ee16ca9759 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -228,7 +228,7 @@ class SkiaGoldClient { final File resultFile = workDirectory.childFile( fs.path.join('result-state.json'), ); - if (await resultFile.exists()) { + if (resultFile.existsSync()) { resultContents = await resultFile.readAsString(); } @@ -369,7 +369,7 @@ class SkiaGoldClient { final File resultFile = workDirectory.childFile( fs.path.join('result-state.json'), ); - if (await resultFile.exists()) { + if (resultFile.existsSync()) { resultContents = await resultFile.readAsString(); } final buf = StringBuffer() @@ -489,7 +489,7 @@ class SkiaGoldClient { fs.path.join('temp', 'auth_opt.json'), ); - if (await authFile.exists()) { + if (authFile.existsSync()) { final String contents = await authFile.readAsString(); final decoded = json.decode(contents) as Map; return !(decoded['GSUtil'] as bool); From cf69c3a0e471b5e889da25217cf798399fa08834 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Thu, 16 Apr 2026 17:05:43 -0500 Subject: [PATCH 17/21] Fix headers in flutter_goldens --- script/flutter_goldens/lib/flutter_goldens.dart | 2 +- script/flutter_goldens/lib/skia_client.dart | 2 +- script/flutter_goldens/test/comparator_selection_test.dart | 2 +- script/flutter_goldens/test/flutter_goldens_test.dart | 2 +- script/flutter_goldens/test/json_templates.dart | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index aa5569820b3..ff23c319392 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -1,4 +1,4 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index 1ee16ca9759..71abf3e9c2b 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -1,4 +1,4 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. diff --git a/script/flutter_goldens/test/comparator_selection_test.dart b/script/flutter_goldens/test/comparator_selection_test.dart index 792d0c61d7f..b79995cbccf 100644 --- a/script/flutter_goldens/test/comparator_selection_test.dart +++ b/script/flutter_goldens/test/comparator_selection_test.dart @@ -1,4 +1,4 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. diff --git a/script/flutter_goldens/test/flutter_goldens_test.dart b/script/flutter_goldens/test/flutter_goldens_test.dart index 8288fac9492..53d534e90a7 100644 --- a/script/flutter_goldens/test/flutter_goldens_test.dart +++ b/script/flutter_goldens/test/flutter_goldens_test.dart @@ -1,4 +1,4 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. diff --git a/script/flutter_goldens/test/json_templates.dart b/script/flutter_goldens/test/json_templates.dart index 5a55826e30d..ee6c98a674a 100644 --- a/script/flutter_goldens/test/json_templates.dart +++ b/script/flutter_goldens/test/json_templates.dart @@ -1,4 +1,4 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. +// Copyright 2013 The Flutter Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. From b4272cd0303ed2bd1e539646d7989b3f344bb70f Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 27 Apr 2026 15:33:07 -0500 Subject: [PATCH 18/21] Updates --- .ci.yaml | 24 ----- .../test/flutter_test_config.dart | 2 +- .../material_ui/test/flutter_test_config.dart | 2 +- .../flutter_goldens/lib/flutter_goldens.dart | 23 +++++ script/flutter_goldens/lib/skia_client.dart | 16 ++-- script/flutter_goldens/pubspec.yaml | 1 + .../test/flutter_goldens_test.dart | 92 ++++++++++--------- 7 files changed, 83 insertions(+), 77 deletions(-) diff --git a/.ci.yaml b/.ci.yaml index be15f8b9934..da44560344f 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -190,10 +190,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] add_recipes_cq: "true" target_file: web_dart_unit_tests.yaml channel: master @@ -204,10 +200,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] target_file: web_dart_unit_tests.yaml channel: master version_file: flutter_master.version @@ -217,10 +209,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] target_file: web_dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -230,10 +218,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] target_file: web_dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -244,10 +228,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] add_recipes_cq: "true" target_file: web_dart_unit_tests_wasm.yaml channel: master @@ -258,10 +238,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] target_file: web_dart_unit_tests_wasm.yaml channel: master version_file: flutter_master.version diff --git a/packages/cupertino_ui/test/flutter_test_config.dart b/packages/cupertino_ui/test/flutter_test_config.dart index 38dacb7c0b1..89496e15a77 100644 --- a/packages/cupertino_ui/test/flutter_test_config.dart +++ b/packages/cupertino_ui/test/flutter_test_config.dart @@ -10,5 +10,5 @@ import 'goldens_io.dart' Future testExecutable(FutureOr Function() testMain) { // Enable golden file testing using Skia Gold. - return flutter_goldens.testExecutable(testMain, namePrefix: 'cupertino_ui'); + return flutter_goldens.testExecutable(testMain); } diff --git a/packages/material_ui/test/flutter_test_config.dart b/packages/material_ui/test/flutter_test_config.dart index 1ceb2ea7052..89496e15a77 100644 --- a/packages/material_ui/test/flutter_test_config.dart +++ b/packages/material_ui/test/flutter_test_config.dart @@ -10,5 +10,5 @@ import 'goldens_io.dart' Future testExecutable(FutureOr Function() testMain) { // Enable golden file testing using Skia Gold. - return flutter_goldens.testExecutable(testMain, namePrefix: 'material_ui'); + return flutter_goldens.testExecutable(testMain); } diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index ff23c319392..4ee7cfbc2ea 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -14,6 +14,7 @@ import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:platform/platform.dart'; import 'package:process/process.dart'; +import 'package:yaml/yaml.dart'; import 'skia_client.dart'; export 'skia_client.dart'; @@ -66,6 +67,9 @@ Future testExecutable( const FileSystem fs = LocalFileSystem(); const ProcessManager process = LocalProcessManager(); final httpClient = io.HttpClient(); + + namePrefix ??= FlutterGoldenFileComparator.getPackageName(fs); + if (FlutterPostSubmitFileComparator.isForEnvironment(platform)) { goldenFileComparator = await FlutterPostSubmitFileComparator.fromLocalFileComparator( @@ -227,6 +231,25 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { return goldenFile; } + /// Extracts the package name from the nearest `pubspec.yaml` file. + @visibleForTesting + static String? getPackageName(FileSystem fs) { + Directory current = fs.currentDirectory; + while (current.path != current.parent.path) { + final File pubspec = current.childFile('pubspec.yaml'); + if (pubspec.existsSync()) { + try { + final YamlMap yaml = loadYaml(pubspec.readAsStringSync()) as YamlMap; + return yaml['name'] as String?; + } catch (e) { + // Ignore parsing errors and keep looking + } + } + current = current.parent; + } + return null; + } + /// Prepends the golden URL with the library name that encloses the current /// test. Uri _addPrefix(Uri golden) { diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart index 71abf3e9c2b..16ce29833ee 100644 --- a/script/flutter_goldens/lib/skia_client.dart +++ b/script/flutter_goldens/lib/skia_client.dart @@ -20,7 +20,6 @@ import 'package:process/process.dart'; const String _kSDKKey = 'SDK_CHECKOUT_PATH'; const String _kGoldctlKey = 'GOLDCTL'; -const String _kTestBrowserKey = 'CHROME_EXECUTABLE'; /// Signature of callbacks used to inject [print] replacements. typedef LogCallback = void Function(String); @@ -449,7 +448,7 @@ class SkiaGoldClient { 'git', 'rev-parse', 'HEAD', - ], workingDirectory: path.join(path.dirname(cleanPath), 'packages')); + ], workingDirectory: cleanPath); if (revParse.exitCode != 0) { throw const SkiaException( 'Current commit of flutter/packages can not be found.', @@ -463,14 +462,13 @@ class SkiaGoldClient { /// Returns a JSON String with keys value pairs used to uniquely identify the /// configuration that generated the given golden file. /// - /// Currently, the only key value pairs being tracked is the platform the - /// image was rendered on, and for web tests, the browser the image was - /// rendered on. + /// Currently, the key value pairs being tracked are the platform the + /// image was rendered on and the Flutter channel the test was run on. String _getKeysJSON() { final keys = { 'Platform': platform.operatingSystem, 'CI': 'luci', - 'Web': _isBrowserTest.toString(), + 'Channel': _channel, }; return json.encode(keys); @@ -516,8 +514,8 @@ class SkiaGoldClient { ]; } - bool get _isBrowserTest { - return platform.environment[_kTestBrowserKey] != null; + String get _channel { + return platform.environment['CHANNEL'] ?? 'stable'; } /// Returns a trace id based on the current testing environment to lookup @@ -527,7 +525,7 @@ class SkiaGoldClient { final parameters = { 'CI': 'luci', 'Platform': platform.operatingSystem, - 'Web': _isBrowserTest.toString(), + 'Channel': _channel, 'name': testName, 'source_type': 'flutter packages', }; diff --git a/script/flutter_goldens/pubspec.yaml b/script/flutter_goldens/pubspec.yaml index b24f5e6deb7..6ac1f887f6a 100644 --- a/script/flutter_goldens/pubspec.yaml +++ b/script/flutter_goldens/pubspec.yaml @@ -14,3 +14,4 @@ dependencies: path: any platform: any process: any + yaml: any diff --git a/script/flutter_goldens/test/flutter_goldens_test.dart b/script/flutter_goldens/test/flutter_goldens_test.dart index 53d534e90a7..46f8b6f7840 100644 --- a/script/flutter_goldens/test/flutter_goldens_test.dart +++ b/script/flutter_goldens/test/flutter_goldens_test.dart @@ -18,8 +18,6 @@ import 'package:process/process.dart'; import 'json_templates.dart'; -// TODO(ianh): make sure all constructors order their arguments in a manner consistent with the defined parameter order - const String _kFlutterRoot = '/flutter'; // 1x1 transparent pixel @@ -213,7 +211,7 @@ void main() { 'git', 'rev-parse', 'HEAD', - ], '/packages'); + ], '/flutter'); const goldctlInvocation = RunInvocation([ 'goldctl', 'imgtest', @@ -281,7 +279,7 @@ void main() { 'git', 'rev-parse', 'HEAD', - ], '/packages'); + ], '/flutter'); const goldctlInvocation = RunInvocation([ 'goldctl', 'imgtest', @@ -361,7 +359,7 @@ void main() { 'git', 'rev-parse', 'HEAD', - ], '/packages'); + ], '/flutter'); const goldctlInvocation = RunInvocation([ 'goldctl', 'imgtest', @@ -544,42 +542,7 @@ void main() { expect( skiaClient.getTraceID('flutter.golden.1'), - equals('9b44cb4464826eef24cbb8095407edc8'), - ); - }); - - test('Creates traceID correctly - Browser', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - environment: { - 'GOLDCTL': 'goldctl', - 'SWARMING_TASK_ID': '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX': - 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', - 'GOLD_TRYJOB': 'refs/pull/49815/head', - - 'CHROME_EXECUTABLE': 'chrome', - }, - operatingSystem: 'linux', - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - - expect( - skiaClient.getTraceID('flutter.golden.1'), - equals('2592d057e5ea8dbc8ac4e8851090695a'), + equals('abe4ba07d57982f282adcd425aa8581f'), ); }); @@ -607,7 +570,7 @@ void main() { ); expect( skiaClient.getTraceID('flutter.golden.1'), - equals('198543ea507122f8fafde25f946bccb0'), + equals('405ca6a70c598037ab019d85f35f8357'), ); }, ); @@ -1204,6 +1167,51 @@ void main() { }, ); }); + + group('_getPackageName', () { + test('extracts name from pubspec.yaml', () { + final fs = MemoryFileSystem(); + final Directory packageDir = fs.directory('/my_package') + ..createSync(recursive: true); + packageDir.childFile('pubspec.yaml').writeAsStringSync('name: my_package_name\n'); + fs.currentDirectory = packageDir; + + expect(FlutterGoldenFileComparator.getPackageName(fs) +, 'my_package_name'); + }); + + test('traverses upwards to find pubspec.yaml', () { + final fs = MemoryFileSystem(); + final Directory packageDir = fs.directory('/my_package') + ..createSync(recursive: true); + packageDir.childFile('pubspec.yaml').writeAsStringSync('name: my_package_name\n'); + final Directory testDir = packageDir.childDirectory('test')..createSync(recursive: true); + fs.currentDirectory = testDir; + + expect(FlutterGoldenFileComparator.getPackageName(fs) +, 'my_package_name'); + }); + + test('returns null if no pubspec.yaml is found', () { + final fs = MemoryFileSystem(); + final Directory someDir = fs.directory('/some/dir')..createSync(recursive: true); + fs.currentDirectory = someDir; + + expect(FlutterGoldenFileComparator.getPackageName(fs) +, isNull); + }); + + test('handles invalid yaml gracefully', () { + final fs = MemoryFileSystem(); + final Directory packageDir = fs.directory('/my_package') + ..createSync(recursive: true); + packageDir.childFile('pubspec.yaml').writeAsStringSync('invalid: yaml: : :'); + fs.currentDirectory = packageDir; + + expect(FlutterGoldenFileComparator.getPackageName(fs) +, isNull); + }); + }); }); } From 18d4b8e75cacfc43fa3798dc8946b873234de81b Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 5 May 2026 09:18:03 -0500 Subject: [PATCH 19/21] Skipping comparator for migration --- .ci.yaml | 24 - .../flutter_goldens/lib/flutter_goldens.dart | 553 +------ script/flutter_goldens/lib/skia_client.dart | 540 ------- .../test/comparator_selection_test.dart | 142 -- .../test/flutter_goldens_test.dart | 1343 +---------------- .../flutter_goldens/test/json_templates.dart | 94 -- 6 files changed, 39 insertions(+), 2657 deletions(-) delete mode 100644 script/flutter_goldens/lib/skia_client.dart delete mode 100644 script/flutter_goldens/test/comparator_selection_test.dart delete mode 100644 script/flutter_goldens/test/json_templates.dart diff --git a/.ci.yaml b/.ci.yaml index da44560344f..c036774df2e 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -137,10 +137,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] add_recipes_cq: "true" target_file: dart_unit_tests.yaml channel: master @@ -151,10 +147,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] target_file: dart_unit_tests.yaml channel: master version_file: flutter_master.version @@ -164,10 +156,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] target_file: dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -177,10 +165,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] target_file: dart_unit_tests.yaml channel: stable version_file: flutter_stable.version @@ -968,10 +952,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] target_file: windows_dart_unit_tests.yaml channel: master version_file: flutter_master.version @@ -981,10 +961,6 @@ targets: recipe: packages/packages timeout: 60 properties: - dependencies: >- - [ - {"dependency": "goldctl", "version": "git_revision:031e93819017b95c3f2dfe463189c7b8d02f2f83"} - ] target_file: windows_dart_unit_tests.yaml channel: master version_file: flutter_master.version diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index 4ee7cfbc2ea..e981328e8c9 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -6,32 +6,13 @@ library; import 'dart:async' show FutureOr; -import 'dart:io' as io show HttpClient, OSError, SocketException; import 'package:file/file.dart'; import 'package:file/local.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; -import 'package:platform/platform.dart'; -import 'package:process/process.dart'; import 'package:yaml/yaml.dart'; -import 'skia_client.dart'; -export 'skia_client.dart'; - -// If you are here trying to figure out how to use golden files for Flutter -// repos, consider reading this documentation: -// https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md - -// If you are trying to debug this package, you may like to use the golden test -// titled "Inconsequential golden test" in this file: packages/material_ui/test/goldens/goldens_test.dart - -// const String _kFlutterRootKey = 'FLUTTER_ROOT'; - -bool _isMainBranch(String? branch) { - return branch == 'main' || branch == 'master'; -} - /// Main method that can be used in a `flutter_test_config.dart` file to set /// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that /// works for the current test. _Which_ [FlutterGoldenFileComparator] is @@ -63,132 +44,48 @@ Future testExecutable( '${goldenFileComparator.runtimeType}.\n' 'See also: https://flutter.dev/to/flutter-test-docs', ); - const Platform platform = LocalPlatform(); const FileSystem fs = LocalFileSystem(); - const ProcessManager process = LocalProcessManager(); - final httpClient = io.HttpClient(); namePrefix ??= FlutterGoldenFileComparator.getPackageName(fs); - if (FlutterPostSubmitFileComparator.isForEnvironment(platform)) { - goldenFileComparator = - await FlutterPostSubmitFileComparator.fromLocalFileComparator( - localFileComparator: goldenFileComparator as LocalFileComparator, - platform: platform, - namePrefix: namePrefix, - log: (String _) {}, - fs: fs, - process: process, - httpClient: httpClient, - ); - } else if (FlutterPreSubmitFileComparator.isForEnvironment(platform)) { - goldenFileComparator = - await FlutterPreSubmitFileComparator.fromLocalFileComparator( - localFileComparator: goldenFileComparator as LocalFileComparator, - platform: platform, - namePrefix: namePrefix, - log: (String _) {}, - fs: fs, - process: process, - httpClient: httpClient, - ); - } else if (FlutterSkippingFileComparator.isForEnvironment(platform)) { - goldenFileComparator = FlutterSkippingFileComparator.fromLocalFileComparator( - localFileComparator: goldenFileComparator as LocalFileComparator, - 'Golden file testing is not executed on LUCI environments outside of ' - 'flutter, or in test shards that are not configured for using goldctl.', - platform: platform, - namePrefix: namePrefix, - log: (String _) {}, - fs: fs, - process: process, - httpClient: httpClient, - ); - } else { - goldenFileComparator = - await FlutterLocalFileComparator.fromLocalFileComparator( - localFileComparator: goldenFileComparator as LocalFileComparator, - platform: platform, - log: (String _) {}, - fs: fs, - process: process, - httpClient: httpClient, - ); - } + goldenFileComparator = FlutterSkippingFileComparator.fromLocalFileComparator( + localFileComparator: goldenFileComparator as LocalFileComparator, + 'Golden file testing is currently skipped.', + namePrefix: namePrefix, + fs: fs, + ); await testMain(); } /// Abstract base class golden file comparator specific to the `flutter/packages` /// repository. /// -/// Golden file testing for the `flutter/flutter` repository is handled by three -/// different [FlutterGoldenFileComparator]s, depending on the current testing -/// environment. -/// -/// * The [FlutterPostSubmitFileComparator] is utilized during post-submit -/// testing, after a pull request has landed on the master branch. This -/// comparator uses the [SkiaGoldClient] and the `goldctl` tool to upload -/// tests to the [Flutter Packages Gold dashboard](https://flutter-packages-gold.skia.org). -/// Flutter Gold manages the master golden files for the packages that use -/// matchesGoldenFile in the `flutter/packages` repository. -/// -/// * The [FlutterPreSubmitFileComparator] is utilized in pre-submit testing, -/// before a pull request lands on the main branch. This -/// comparator uses the [SkiaGoldClient] to execute tryjobs, allowing -/// contributors to view and check in visual differences before landing the -/// change. -/// -/// * The [FlutterLocalFileComparator] is used for local development testing. -/// This comparator will use the [SkiaGoldClient] to request baseline images -/// from [Flutter Packages Gold](https://flutter-packages-gold.skia.org) and -/// manually compare pixels. If a difference is detected, this comparator will -/// generate failure output illustrating the found difference. If a baseline -/// is not found for a given test image, it will consider it a new test and -/// output the new image for verification. -/// /// The [FlutterSkippingFileComparator] is utilized to skip tests outside -/// of the appropriate environments described above. Currently, some -/// packages or environments do not execute golden file testing, and as such do -/// not require a comparator. This comparator is also used when an internet -/// connection is unavailable. +/// of the appropriate environments. Currently, some packages or environments +/// do not execute golden file testing, and as such do not require a +/// comparator. This comparator is also used when an internet connection is unavailable. abstract class FlutterGoldenFileComparator extends GoldenFileComparator { /// Creates a [FlutterGoldenFileComparator] that will resolve golden file - /// URIs relative to the specified [basedir], and retrieve golden baselines - /// using the [skiaClient]. The [basedir] is used for writing and accessing - /// information and files for interacting with the [skiaClient]. When testing - /// locally, the [basedir] will also contain any diffs from failed tests, or - /// goldens generated from newly introduced tests. + /// URIs relative to the specified [basedir]. When testing locally, the + /// [basedir] will also contain any diffs from failed tests, or goldens + /// generated from newly introduced tests. @visibleForTesting FlutterGoldenFileComparator( - this.basedir, - this.skiaClient, { + this.basedir, { required this.fs, - required this.platform, this.namePrefix, - required this.log, }); /// The directory to which golden file URIs will be resolved in [compare] and /// [update]. final Uri basedir; - /// A client for uploading image tests and making baseline requests to the - /// Flutter Gold Dashboard. - final SkiaGoldClient skiaClient; - /// The file system used to perform file access. final FileSystem fs; - /// The environment (current working directory, identity of the OS, - /// environment variables, etc). - final Platform platform; - /// The prefix that is added to all golden names. final String? namePrefix; - /// The logging function to use when reporting messages to the console. - final LogCallback log; - @override Future update(Uri golden, Uint8List imageBytes) async { final File goldenFile = getGoldenFile(golden); @@ -209,18 +106,15 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { @visibleForTesting static Directory getBaseDirectory( LocalFileComparator defaultComparator, { - required Platform platform, String? suffix, required FileSystem fs, }) { - // final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]); final Directory comparisonRoot = switch (suffix) { - // null => flutterRoot.childDirectory(fs.path.join('bin', 'cache', 'pkg', 'skia_goldens')), null => fs.directory(defaultComparator.basedir).childDirectory('skia_goldens'), _ => fs.systemTempDirectory.createTempSync(suffix), }; - return comparisonRoot; //.childDirectory(fs.path.relative(testPath, from: flutterRoot.path)); + return comparisonRoot; } /// Returns the golden [File] identified by the given [Uri]. @@ -239,7 +133,7 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { final File pubspec = current.childFile('pubspec.yaml'); if (pubspec.existsSync()) { try { - final YamlMap yaml = loadYaml(pubspec.readAsStringSync()) as YamlMap; + final yaml = loadYaml(pubspec.readAsStringSync()) as YamlMap; return yaml['name'] as String?; } catch (e) { // Ignore parsing errors and keep looking @@ -249,235 +143,6 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { } return null; } - - /// Prepends the golden URL with the library name that encloses the current - /// test. - Uri _addPrefix(Uri golden) { - // Ensure the Uri ends in .png as the SkiaClient expects - assert( - golden.toString().split('.').last == 'png', - 'Golden files in the Flutter framework must end with the file extension ' - '.png.', - ); - - return Uri.parse([?namePrefix, golden.toString()].join('.')); - } -} - -/// A [FlutterGoldenFileComparator] for testing golden images with Skia Gold in -/// post-submit. -/// -/// For testing across all platforms, the [SkiaGoldClient] is used to upload -/// images for framework-related golden tests and process results. -/// -/// See also: -/// -/// * [GoldenFileComparator], the abstract class that -/// [FlutterGoldenFileComparator] implements. -/// * [FlutterPreSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images before changes are -/// merged into the master branch. -/// * [FlutterLocalFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images locally on your -/// current machine. -class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { - /// Creates a [FlutterPostSubmitFileComparator] that will test golden file - /// images against Skia Gold. - /// - /// The [fs] parameter is useful in tests, where the default - /// file system can be replaced by mock instances. - FlutterPostSubmitFileComparator( - super.basedir, - super.skiaClient, { - required super.fs, - required super.platform, - super.namePrefix, - required super.log, - }); - - /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative - /// path resolution of the provided `localFileComparator`. - /// - /// The [goldens] parameter is visible for testing purposes only. - static Future fromLocalFileComparator({ - SkiaGoldClient? goldens, - required LocalFileComparator localFileComparator, - required Platform platform, - String? namePrefix, - required LogCallback log, - required FileSystem fs, - required ProcessManager process, - required io.HttpClient httpClient, - }) async { - final Directory baseDirectory = - FlutterGoldenFileComparator.getBaseDirectory( - localFileComparator, - platform: platform, - suffix: 'flutter_goldens_postsubmit.', - fs: fs, - ); - - baseDirectory.createSync(recursive: true); - - goldens ??= SkiaGoldClient( - baseDirectory, - log: log, - platform: platform, - fs: fs, - process: process, - httpClient: httpClient, - ); - await goldens.auth(); - return FlutterPostSubmitFileComparator( - baseDirectory.uri, - goldens, - platform: platform, - namePrefix: namePrefix, - log: log, - fs: fs, - ); - } - - @override - Future compare(Uint8List imageBytes, Uri golden) async { - await skiaClient.imgtestInit(); - golden = _addPrefix(golden); - - await update(golden, imageBytes); - final File goldenFile = getGoldenFile(golden); - try { - return await skiaClient.imgtestAdd(golden.path, goldenFile); - } on SkiaException catch (e) { - // Convert SkiaException -> TestFailure so that this class implements the - // contract of GoldenFileComparator, and matchesGoldenFile() converts the - // TestFailure into a standard reported test error (with a better stack - // trace, for example). - // - // https://github.com/flutter/flutter/issues/162621 - throw TestFailure('$e'); - } - } - - /// Decides based on the current environment if goldens tests should be - /// executed through Skia Gold. - static bool isForEnvironment(Platform platform) { - final bool luciPostSubmit = - platform.environment.containsKey('SWARMING_TASK_ID') && - platform.environment.containsKey('GOLDCTL') - // Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator]. - && - !platform.environment.containsKey('GOLD_TRYJOB') - // Only run on main branch. - && - _isMainBranch(platform.environment['GIT_BRANCH']); - return luciPostSubmit; - } -} - -/// A [FlutterGoldenFileComparator] for testing golden images before changes are -/// merged into the master branch. The comparator executes tryjobs using the -/// [SkiaGoldClient]. -/// -/// See also: -/// -/// * [GoldenFileComparator], the abstract class that -/// [FlutterGoldenFileComparator] implements. -/// * [FlutterPostSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold -/// dashboard in post-submit. -/// * [FlutterLocalFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images locally on your -/// current machine. -class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { - /// Creates a [FlutterPreSubmitFileComparator] that will test golden file - /// images against baselines requested from Flutter Gold. - /// - /// The [fs] parameter is useful in tests, where the default - /// file system can be replaced by mock instances. - FlutterPreSubmitFileComparator( - super.basedir, - super.skiaClient, { - required super.fs, - required super.platform, - super.namePrefix, - required super.log, - }); - - /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the - /// relative path resolution of the default [goldenFileComparator]. - /// - /// The [goldens] parameter is visible for testing purposes only. - static Future fromLocalFileComparator({ - SkiaGoldClient? goldens, - required LocalFileComparator localFileComparator, - required Platform platform, - Directory? testBasedir, - String? namePrefix, - required LogCallback log, - required FileSystem fs, - required ProcessManager process, - required io.HttpClient httpClient, - }) async { - final Directory baseDirectory = - testBasedir ?? - FlutterGoldenFileComparator.getBaseDirectory( - localFileComparator, - platform: platform, - suffix: 'flutter_goldens_presubmit.', - fs: fs, - ); - - if (!baseDirectory.existsSync()) { - baseDirectory.createSync(recursive: true); - } - - goldens ??= SkiaGoldClient( - baseDirectory, - platform: platform, - log: log, - fs: fs, - process: process, - httpClient: httpClient, - ); - - await goldens.auth(); - return FlutterPreSubmitFileComparator( - baseDirectory.uri, - goldens, - platform: platform, - namePrefix: namePrefix, - log: log, - fs: fs, - ); - } - - @override - Future compare(Uint8List imageBytes, Uri golden) async { - await skiaClient.tryjobInit(); - golden = _addPrefix(golden); - - await update(golden, imageBytes); - final File goldenFile = getGoldenFile(golden); - - await skiaClient.tryjobAdd(golden.path, goldenFile); - - // This will always return true since golden file test failures are managed - // in pre-submit checks by the flutter-gold status check. - return true; - } - - /// Decides based on the current environment if goldens tests should be - /// executed as pre-submit tests with Skia Gold. - static bool isForEnvironment(Platform platform) { - final bool luciPreSubmit = - platform.environment.containsKey('SWARMING_TASK_ID') && - platform.environment.containsKey('GOLDCTL') && - platform.environment.containsKey('GOLD_TRYJOB') - // Only run on the main branch - && - _isMainBranch(platform.environment['GIT_BRANCH']); - return luciPreSubmit; - } } /// A [FlutterGoldenFileComparator] for testing conditions that do not execute @@ -501,11 +166,8 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { /// are not in the right environment for golden file testing. FlutterSkippingFileComparator( super.basedir, - super.skiaClient, this.reason, { super.namePrefix, - required super.platform, - required super.log, required super.fs, }); @@ -518,203 +180,20 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { String reason, { required LocalFileComparator localFileComparator, String? namePrefix, - required Platform platform, - required LogCallback log, required FileSystem fs, - required ProcessManager process, - required io.HttpClient httpClient, }) { final Uri basedir = localFileComparator.basedir; - final skiaClient = SkiaGoldClient( - fs.directory(basedir), - platform: platform, - log: log, - fs: fs, - process: process, - httpClient: httpClient, - ); return FlutterSkippingFileComparator( basedir, - skiaClient, reason, namePrefix: namePrefix, - platform: platform, - log: log, fs: fs, ); } @override - Future compare(Uint8List imageBytes, Uri golden) async { - log('Skipping "$golden" test: $reason'); - return true; - } + Future compare(Uint8List imageBytes, Uri golden) async => true; @override Future update(Uri golden, Uint8List imageBytes) async {} - - /// Decides, based on the current environment, if this comparator should be - /// used. - /// - /// If we are in a CI environment, i.e. LUCI, but are not using the other - /// comparators, we skip. Otherwise we would fallback to the local comparator, - /// for which failures cannot be resolved in a CI environment. - static bool isForEnvironment(Platform platform) { - return platform.environment.containsKey('SWARMING_TASK_ID'); - } -} - -/// A [FlutterGoldenFileComparator] for testing golden images locally on your -/// current machine. -/// -/// This comparator utilizes the [SkiaGoldClient] to request baseline images for -/// the given device under test for comparison. This comparator is initialized -/// when conditions for all other [FlutterGoldenFileComparator]s have not been -/// met, see the `isForEnvironment` method for each one listed below. -/// -/// The [FlutterLocalFileComparator] is intended to run on local machines and -/// serve as a smoke test during development. As such, it will not be able to -/// detect unintended changes on environments other than the currently executing -/// machine, until they are tested using the [FlutterPreSubmitFileComparator]. -/// -/// See also: -/// -/// * [GoldenFileComparator], the abstract class that -/// [FlutterGoldenFileComparator] implements. -/// * [FlutterPostSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold -/// dashboard. -/// * [FlutterPreSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images before changes are -/// merged into the master branch. -/// * [FlutterSkippingFileComparator], another -/// [FlutterGoldenFileComparator] that controls post-submit testing -/// conditions that do not execute golden file tests. -class FlutterLocalFileComparator extends FlutterGoldenFileComparator - with LocalComparisonOutput { - /// Creates a [FlutterLocalFileComparator] that will test golden file - /// images against baselines requested from Flutter Gold. - /// - /// The [fs] parameter is useful in tests, where the default - /// file system can be replaced by mock instances. - FlutterLocalFileComparator( - super.basedir, - super.skiaClient, { - required super.fs, - required super.platform, - required super.log, - }); - - /// Creates a new [FlutterLocalFileComparator] that mirrors the - /// relative path resolution of the given [localFileComparator]. - /// - /// The [goldens] and [baseDirectory] parameters are - /// visible for testing purposes only. - static Future fromLocalFileComparator({ - SkiaGoldClient? goldens, - required LocalFileComparator localFileComparator, - required Platform platform, - Directory? baseDirectory, - required LogCallback log, - required FileSystem fs, - required ProcessManager process, - required io.HttpClient httpClient, - }) async { - baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory( - localFileComparator, - platform: platform, - fs: fs, - ); - - if (!baseDirectory.existsSync()) { - baseDirectory.createSync(recursive: true); - } - - goldens ??= SkiaGoldClient( - baseDirectory, - platform: platform, - log: log, - fs: fs, - process: process, - httpClient: httpClient, - ); - try { - // Check if we can reach Gold. - await goldens.getExpectationForTest(''); - } on io.OSError catch (_) { - return FlutterSkippingFileComparator( - baseDirectory.uri, - goldens, - 'OSError occurred, could not reach Gold. ' - 'Switching to FlutterSkippingGoldenFileComparator.', - platform: platform, - log: log, - fs: fs, - ); - } on io.SocketException catch (_) { - return FlutterSkippingFileComparator( - baseDirectory.uri, - goldens, - 'SocketException occurred, could not reach Gold. ' - 'Switching to FlutterSkippingGoldenFileComparator.', - platform: platform, - log: log, - fs: fs, - ); - } on FormatException catch (_) { - return FlutterSkippingFileComparator( - baseDirectory.uri, - goldens, - 'FormatException occurred, could not reach Gold. ' - 'Switching to FlutterSkippingGoldenFileComparator.', - platform: platform, - log: log, - fs: fs, - ); - } - - return FlutterLocalFileComparator( - baseDirectory.uri, - goldens, - platform: platform, - log: log, - fs: fs, - ); - } - - @override - Future compare(Uint8List imageBytes, Uri golden) async { - golden = _addPrefix(golden); - - final String testName = skiaClient.cleanTestName(golden.path); - late String? testExpectation; - testExpectation = await skiaClient.getExpectationForTest(testName); - - if (testExpectation == null || testExpectation.isEmpty) { - log( - 'No expectations provided by Skia Gold for test: $golden. ' - 'This may be a new test. If this is an unexpected result, check ' - 'https://flutter-packages-gold.skia.org.\n' - 'Validate image output found at $basedir', - ); - await update(golden, imageBytes); - return true; - } - - ComparisonResult result; - final List goldenBytes = await skiaClient.getImageBytes( - testExpectation, - ); - - result = await GoldenFileComparator.compareLists(imageBytes, goldenBytes); - - if (result.passed) { - result.dispose(); - return true; - } - - final String error = await generateFailureOutput(result, golden, basedir); - result.dispose(); - throw FlutterError(error); - } } diff --git a/script/flutter_goldens/lib/skia_client.dart b/script/flutter_goldens/lib/skia_client.dart deleted file mode 100644 index 16ce29833ee..00000000000 --- a/script/flutter_goldens/lib/skia_client.dart +++ /dev/null @@ -1,540 +0,0 @@ -// Copyright 2013 The Flutter Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -/// @docImport 'flutter_goldens.dart'; -library; - -import 'dart:convert'; -import 'dart:io' as io; - -import 'package:crypto/crypto.dart'; -import 'package:file/file.dart'; -import 'package:path/path.dart' as path; -import 'package:platform/platform.dart'; -import 'package:process/process.dart'; - -// If you are here trying to figure out how to use golden files in relevant -// Flutter repos, consider reading this wiki page: -// https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md - -const String _kSDKKey = 'SDK_CHECKOUT_PATH'; -const String _kGoldctlKey = 'GOLDCTL'; - -/// Signature of callbacks used to inject [print] replacements. -typedef LogCallback = void Function(String); - -/// Exception thrown when an error is returned from the [SkiaGoldClient]. -class SkiaException implements Exception { - /// Creates a new `SkiaException` with a required error [message]. - const SkiaException(this.message); - - /// A message describing the error. - final String message; - - /// Returns a description of the Skia exception. - /// - /// The description always contains the [message]. - @override - String toString() => 'SkiaException: $message'; -} - -/// A client for uploading image tests and making baseline requests to the -/// Flutter Packages Gold Dashboard. -class SkiaGoldClient { - /// Creates a [SkiaGoldClient] with the given [workDirectory] and [Platform]. - /// - /// All other parameters are optional. They may be provided in tests to - /// override the defaults for [fs], [process], and [httpClient]. - SkiaGoldClient( - this.workDirectory, { - required this.fs, - required this.process, - required this.platform, - required this.httpClient, - required this.log, - }); - - /// The file system to use for storing the local clone of the repository. - /// - /// This is useful in tests, where a local file system (the default) can be - /// replaced by a memory file system. - final FileSystem fs; - - /// The environment (current working directory, identity of the OS, - /// environment variables, etc). - final Platform platform; - - /// A controller for launching sub-processes. - /// - /// This is useful in tests, where the real process manager (the default) can - /// be replaced by a mock process manager that doesn't really create - /// sub-processes. - final ProcessManager process; - - /// A client for making Http requests to the Flutter Packages Gold dashboard. - final io.HttpClient httpClient; - - /// The local [Directory] within the comparison root for the current test - /// context. In this directory, the client will create image and JSON files - /// for the goldctl tool to use. - /// - /// This is informed by [FlutterGoldenFileComparator.basedir]. It cannot be - /// null. - final Directory workDirectory; - - /// The logging function to use when reporting messages to the console. - final LogCallback log; - - /// The path to the local [Directory] where the goldctl tool is hosted. - /// - /// Uses the [platform] environment in this implementation. - String get _goldctl => platform.environment[_kGoldctlKey]!; - - /// Prepares the local work space for golden file testing and calls the - /// goldctl `auth` command. - /// - /// This ensures that the goldctl tool is authorized and ready for testing. - /// Used by the [FlutterPostSubmitFileComparator] and the - /// [FlutterPreSubmitFileComparator]. - Future auth() async { - if (await clientIsAuthorized()) { - return; - } - final authCommand = [ - _goldctl, - 'auth', - '--work-dir', - workDirectory.childDirectory('temp').path, - '--luci', - ]; - - final io.ProcessResult result = await process.run(authCommand); - - if (result.exitCode != 0) { - final buf = StringBuffer() - ..writeln('Skia Gold authorization failed.') - ..writeln( - 'Luci environments authenticate using the file provided ' - 'by LUCI_CONTEXT. There may be an error with this file or Gold ' - 'authentication.', - ) - ..writeln('Debug information for Gold --------------------------------') - ..writeln('stdout: ${result.stdout}') - ..writeln('stderr: ${result.stderr}'); - throw SkiaException(buf.toString()); - } - } - - /// Signals if this client is initialized for uploading images to the Gold - /// service. - /// - /// Since Flutter framework tests are executed in parallel, and in random - /// order, this will signal is this instance of the Gold client has been - /// initialized. - bool _initialized = false; - - /// Executes the `imgtest init` command in the goldctl tool. - /// - /// The `imgtest` command collects and uploads test results to the Skia Gold - /// backend, the `init` argument initializes the current test. Used by the - /// [FlutterPostSubmitFileComparator]. - Future imgtestInit() async { - // This client has already been initialized - if (_initialized) { - return; - } - - final File keys = workDirectory.childFile('keys.json'); - final File failures = workDirectory.childFile('failures.json'); - - await keys.writeAsString(_getKeysJSON()); - await failures.create(); - final String commitHash = await _getCurrentCommit(); - - final imgtestInitCommand = [ - _goldctl, - 'imgtest', - 'init', - '--instance', - 'flutter', - '--work-dir', - workDirectory.childDirectory('temp').path, - '--commit', - commitHash, - '--keys-file', - keys.path, - '--failure-file', - failures.path, - '--passfail', - ]; - - if (imgtestInitCommand.contains(null)) { - final buf = StringBuffer() - ..writeln('A null argument was provided for Skia Gold imgtest init.') - ..writeln('Please confirm the settings of your golden file test.') - ..writeln('Arguments provided:'); - imgtestInitCommand.forEach(buf.writeln); - throw SkiaException(buf.toString()); - } - - final io.ProcessResult result = await process.run(imgtestInitCommand); - - if (result.exitCode != 0) { - _initialized = false; - final buf = StringBuffer() - ..writeln('Skia Gold imgtest init failed.') - ..writeln('An error occurred when initializing golden file test with ') - ..writeln('goldctl.') - ..writeln() - ..writeln('Debug information for Gold --------------------------------') - ..writeln('stdout: ${result.stdout}') - ..writeln('stderr: ${result.stderr}'); - throw SkiaException(buf.toString()); - } - _initialized = true; - } - - /// Executes the `imgtest add` command in the goldctl tool. - /// - /// The `imgtest` command collects and uploads test results to the Skia Gold - /// backend, the `add` argument uploads the current image test. A response is - /// returned from the invocation of this command that indicates a pass or fail - /// result. - /// - /// The [testName] and [goldenFile] parameters reference the current - /// comparison being evaluated by the [FlutterPostSubmitFileComparator]. - Future imgtestAdd(String testName, File goldenFile) async { - final imgtestCommand = [ - _goldctl, - 'imgtest', - 'add', - '--work-dir', - workDirectory.childDirectory('temp').path, - '--test-name', - cleanTestName(testName), - '--png-file', - goldenFile.path, - '--passfail', - ]; - - final io.ProcessResult result = await process.run(imgtestCommand); - - if (result.exitCode != 0) { - // If an unapproved image has made it to post-submit, throw to close the - // tree. - String? resultContents; - final File resultFile = workDirectory.childFile( - fs.path.join('result-state.json'), - ); - if (resultFile.existsSync()) { - resultContents = await resultFile.readAsString(); - } - - final buf = StringBuffer() - ..writeln('Skia Gold received an unapproved image in post-submit ') - ..writeln('testing. Golden file images in flutter/flutter are triaged ') - ..writeln('in pre-submit during code review for the given PR.') - ..writeln() - ..writeln('Visit https://flutter-gold.skia.org/ to view and approve ') - ..writeln('the image(s), or revert the associated change. For more ') - ..writeln('information, visit the wiki: ') - ..writeln( - 'https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md', - ) - ..writeln() - ..writeln('Debug information for Gold --------------------------------') - ..writeln('stdout: ${result.stdout}') - ..writeln('stderr: ${result.stderr}') - ..writeln() - ..writeln( - 'result-state.json: ${resultContents ?? 'No result file found.'}', - ); - throw SkiaException(buf.toString()); - } - - return true; - } - - /// Signals if this client is initialized for uploading tryjobs to the Gold - /// service. - /// - /// Since Flutter framework tests are executed in parallel, and in random - /// order, this will signal is this instance of the Gold client has been - /// initialized for tryjobs. - bool _tryjobInitialized = false; - - /// Executes the `imgtest init` command in the goldctl tool for tryjobs. - /// - /// The `imgtest` command collects and uploads test results to the Skia Gold - /// backend, the `init` argument initializes the current tryjob. Used by the - /// [FlutterPreSubmitFileComparator]. - Future tryjobInit() async { - // This client has already been initialized - if (_tryjobInitialized) { - return; - } - - final File keys = workDirectory.childFile('keys.json'); - final File failures = workDirectory.childFile('failures.json'); - - await keys.writeAsString(_getKeysJSON()); - await failures.create(); - final String commitHash = await _getCurrentCommit(); - - final imgtestInitCommand = [ - _goldctl, - 'imgtest', - 'init', - '--instance', - 'flutter', - '--work-dir', - workDirectory.childDirectory('temp').path, - '--commit', - commitHash, - '--keys-file', - keys.path, - '--failure-file', - failures.path, - '--passfail', - '--crs', - 'github', - '--patchset_id', - commitHash, - ...getCIArguments(), - ]; - - if (imgtestInitCommand.contains(null)) { - final buf = StringBuffer() - ..writeln('A null argument was provided for Skia Gold tryjob init.') - ..writeln('Please confirm the settings of your golden file test.') - ..writeln('Arguments provided:'); - imgtestInitCommand.forEach(buf.writeln); - throw SkiaException(buf.toString()); - } - - final io.ProcessResult result = await process.run(imgtestInitCommand); - - if (result.exitCode != 0) { - _tryjobInitialized = false; - final buf = StringBuffer() - ..writeln('Skia Gold tryjobInit failure.') - ..writeln( - 'An error occurred when initializing golden file tryjob with ', - ) - ..writeln('goldctl.') - ..writeln() - ..writeln('Debug information for Gold --------------------------------') - ..writeln('stdout: ${result.stdout}') - ..writeln('stderr: ${result.stderr}'); - throw SkiaException(buf.toString()); - } - _tryjobInitialized = true; - } - - /// Executes the `imgtest add` command in the goldctl tool for tryjobs. - /// - /// The `imgtest` command collects and uploads test results to the Skia Gold - /// backend, the `add` argument uploads the current image test. A response is - /// returned from the invocation of this command that indicates a pass or fail - /// result for the tryjob. - /// - /// The [testName] and [goldenFile] parameters reference the current - /// comparison being evaluated by the [FlutterPreSubmitFileComparator]. - /// - /// If the tryjob fails due to pixel differences, the method will succeed - /// as the failure will be triaged in the 'Flutter Gold' dashboard, and the - /// `stdout` will contain the failure message; otherwise will return `null`. - Future tryjobAdd(String testName, File goldenFile) async { - final imgtestCommand = [ - _goldctl, - 'imgtest', - 'add', - '--work-dir', - workDirectory.childDirectory('temp').path, - '--test-name', - cleanTestName(testName), - '--png-file', - goldenFile.path, - ]; - - final io.ProcessResult result = await process.run(imgtestCommand); - - final resultStdout = result.stdout.toString(); - if (result.exitCode != 0 && - !(resultStdout.contains('Untriaged') || - resultStdout.contains('negative image'))) { - String? resultContents; - final File resultFile = workDirectory.childFile( - fs.path.join('result-state.json'), - ); - if (resultFile.existsSync()) { - resultContents = await resultFile.readAsString(); - } - final buf = StringBuffer() - ..writeln('Unexpected Gold tryjobAdd failure.') - ..writeln('Tryjob execution for golden file test $testName failed for') - ..writeln('a reason unrelated to pixel comparison.') - ..writeln() - ..writeln('Debug information for Gold --------------------------------') - ..writeln('stdout: ${result.stdout}') - ..writeln('stderr: ${result.stderr}') - ..writeln() - ..writeln() - ..writeln( - 'result-state.json: ${resultContents ?? 'No result file found.'}', - ); - throw SkiaException(buf.toString()); - } - return result.exitCode == 0 ? null : resultStdout; - } - - /// Returns the latest positive digest for the given test known to Flutter - /// Packages Gold at head. - Future getExpectationForTest(String testName) async { - late String? expectation; - final String traceID = getTraceID(testName); - final Uri requestForExpectations = Uri.parse( - 'https://flutter-packages-gold.skia.org/json/v2/latestpositivedigest/$traceID', - ); - late String rawResponse; - try { - final io.HttpClientRequest request = await httpClient.getUrl( - requestForExpectations, - ); - final io.HttpClientResponse response = await request.close(); - rawResponse = await utf8.decodeStream(response); - final dynamic jsonResponse = json.decode(rawResponse); - if (jsonResponse is! Map) { - throw const FormatException( - 'Skia gold expectations do not match expected format.', - ); - } - expectation = jsonResponse['digest'] as String?; - } on FormatException catch (error) { - log( - 'Formatting error detected requesting expectations from Flutter Gold.\n' - 'error: $error\n' - 'url: $requestForExpectations\n' - 'response: $rawResponse', - ); - rethrow; - } - return expectation; - } - - /// Returns a list of bytes representing the golden image retrieved from the - /// Flutter Packages Gold dashboard. - /// - /// The provided image hash represents an expectation from Flutter Packages Gold. - Future> getImageBytes(String imageHash) async { - final imageBytes = []; - final Uri requestForImage = Uri.parse( - 'https://flutter-packages-gold.skia.org/img/images/$imageHash.png', - ); - final io.HttpClientRequest request = await httpClient.getUrl( - requestForImage, - ); - final io.HttpClientResponse response = await request.close(); - await response.forEach((List bytes) => imageBytes.addAll(bytes)); - return imageBytes; - } - - /// Returns the current commit hash of the packages repository. - Future _getCurrentCommit() async { - final String cleanPath = path.normalize(platform.environment[_kSDKKey]!); - - final io.ProcessResult revParse = await process.run([ - 'git', - 'rev-parse', - 'HEAD', - ], workingDirectory: cleanPath); - if (revParse.exitCode != 0) { - throw const SkiaException( - 'Current commit of flutter/packages can not be found.', - ); - } - final String commit = (revParse.stdout as String).trim(); - - return commit; - } - - /// Returns a JSON String with keys value pairs used to uniquely identify the - /// configuration that generated the given golden file. - /// - /// Currently, the key value pairs being tracked are the platform the - /// image was rendered on and the Flutter channel the test was run on. - String _getKeysJSON() { - final keys = { - 'Platform': platform.operatingSystem, - 'CI': 'luci', - 'Channel': _channel, - }; - - return json.encode(keys); - } - - /// Removes the file extension from the [fileName] to represent the test name - /// properly. - String cleanTestName(String fileName) { - return fileName.split(path.extension(fileName))[0]; - } - - /// Returns a boolean value to prevent the client from re-authorizing itself - /// for multiple tests. - Future clientIsAuthorized() async { - final File authFile = workDirectory.childFile( - fs.path.join('temp', 'auth_opt.json'), - ); - - if (authFile.existsSync()) { - final String contents = await authFile.readAsString(); - final decoded = json.decode(contents) as Map; - return !(decoded['GSUtil'] as bool); - } - return false; - } - - /// Returns a list of arguments for initializing a tryjob based on the testing - /// environment. - List getCIArguments() { - final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']! - .split('/') - .last; - final List refs = platform.environment['GOLD_TRYJOB']!.split('/'); - final String pullRequest = refs[refs.length - 2]; - - return [ - '--changelist', - pullRequest, - '--cis', - 'buildbucket', - '--jobid', - jobId, - ]; - } - - String get _channel { - return platform.environment['CHANNEL'] ?? 'stable'; - } - - /// Returns a trace id based on the current testing environment to lookup - /// the latest positive digest on Flutter Gold with a hex-encoded md5 hash of - /// the image keys. - String getTraceID(String testName) { - final parameters = { - 'CI': 'luci', - 'Platform': platform.operatingSystem, - 'Channel': _channel, - 'name': testName, - 'source_type': 'flutter packages', - }; - final sorted = {}; - for (final String key in parameters.keys.toList()..sort()) { - sorted[key] = parameters[key]; - } - final String jsonTrace = json.encode(sorted); - final md5Sum = md5.convert(utf8.encode(jsonTrace)).toString(); - return md5Sum; - } -} diff --git a/script/flutter_goldens/test/comparator_selection_test.dart b/script/flutter_goldens/test/comparator_selection_test.dart deleted file mode 100644 index b79995cbccf..00000000000 --- a/script/flutter_goldens/test/comparator_selection_test.dart +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2013 The Flutter Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter_goldens/flutter_goldens.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:platform/platform.dart'; - -enum _Comparator { post, pre, skip, local } - -_Comparator _testRecommendations({ - bool hasLuci = false, - bool hasGold = false, - bool hasTryJob = false, - String branch = 'main', - String os = 'macos', -}) { - final Platform platform = FakePlatform( - environment: { - if (hasLuci) 'SWARMING_TASK_ID': '8675309', - if (hasGold) 'GOLDCTL': 'goldctl', - if (hasTryJob) 'GOLD_TRYJOB': 'git/ref/12345/head', - 'GIT_BRANCH': branch, - }, - operatingSystem: os, - ); - if (FlutterPostSubmitFileComparator.isForEnvironment(platform)) { - return _Comparator.post; - } - if (FlutterPreSubmitFileComparator.isForEnvironment(platform)) { - return _Comparator.pre; - } - if (FlutterSkippingFileComparator.isForEnvironment(platform)) { - return _Comparator.skip; - } - return _Comparator.local; -} - -void main() { - test('Comparator recommendations - main branch', () { - // If we're running locally (no CI), use a local comparator. - expect(_testRecommendations(), _Comparator.local); - expect(_testRecommendations(hasGold: true), _Comparator.local); - - // If we don't have gold but are on CI, we skip regardless. - expect(_testRecommendations(hasLuci: true), _Comparator.skip); - expect( - _testRecommendations(hasLuci: true, hasTryJob: true), - _Comparator.skip, - ); - - // On Luci, with Gold, post-submit. Flutter root and LUCI variables should have no effect. - expect( - _testRecommendations(hasGold: true, hasLuci: true), - _Comparator.post, - ); - - // On Luci, with Gold, pre-submit. Flutter root and LUCI variables should have no effect. - expect( - _testRecommendations(hasGold: true, hasLuci: true, hasTryJob: true), - _Comparator.pre, - ); - }); - - test('Comparator recommendations - release branch', () { - // If we're running locally (no CI), use a local comparator. - expect( - _testRecommendations(branch: 'flutter-3.16-candidate.0'), - _Comparator.local, - ); - - expect( - _testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true), - _Comparator.local, - ); - - // If we don't have gold but are on CI, we skip regardless. - expect( - _testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true), - _Comparator.skip, - ); - expect( - _testRecommendations( - branch: 'flutter-3.16-candidate.0', - hasLuci: true, - hasTryJob: true, - ), - _Comparator.skip, - ); - - // On Luci, with Gold, post-submit. Flutter root and LUCI variables should have no effect. Branch should make us skip. - expect( - _testRecommendations( - branch: 'flutter-3.16-candidate.0', - hasGold: true, - hasLuci: true, - ), - _Comparator.skip, - ); - - // On Luci, with Gold, pre-submit. Flutter root and LUCI variables should have no effect. Branch should make us skip. - expect( - _testRecommendations( - branch: 'flutter-3.16-candidate.0', - hasGold: true, - hasLuci: true, - hasTryJob: true, - ), - _Comparator.skip, - ); - }); - - test('Comparator recommendations - Linux', () { - // If we're running locally (no CI), use a local comparator. - expect(_testRecommendations(os: 'linux'), _Comparator.local); - expect(_testRecommendations(os: 'linux', hasGold: true), _Comparator.local); - - // If we don't have gold but are on CI, we skip regardless. - expect(_testRecommendations(os: 'linux', hasLuci: true), _Comparator.skip); - expect( - _testRecommendations(os: 'linux', hasLuci: true, hasTryJob: true), - _Comparator.skip, - ); - - // On Luci, with Gold, post-submit. Flutter root has no effect. - expect( - _testRecommendations(os: 'linux', hasGold: true, hasLuci: true), - _Comparator.post, - ); - - // On Luci, with Gold, pre-submit. Flutter root should have no effect. - expect( - _testRecommendations( - os: 'linux', - hasGold: true, - hasLuci: true, - hasTryJob: true, - ), - _Comparator.pre, - ); - }); -} diff --git a/script/flutter_goldens/test/flutter_goldens_test.dart b/script/flutter_goldens/test/flutter_goldens_test.dart index 46f8b6f7840..094ffb4590a 100644 --- a/script/flutter_goldens/test/flutter_goldens_test.dart +++ b/script/flutter_goldens/test/flutter_goldens_test.dart @@ -4,1386 +4,89 @@ // See also dev/automated_tests/flutter_test/flutter_gold_test.dart -import 'dart:convert'; -import 'dart:io' hide Directory; - import 'package:file/file.dart'; import 'package:file/memory.dart'; -import 'package:flutter/foundation.dart'; import 'package:flutter_goldens/flutter_goldens.dart'; import 'package:flutter_test/flutter_test.dart'; -import 'package:path/path.dart' as path; -import 'package:platform/platform.dart'; -import 'package:process/process.dart'; - -import 'json_templates.dart'; const String _kFlutterRoot = '/flutter'; -// 1x1 transparent pixel -const List _kTestPngBytes = [ - 137, - 80, - 78, - 71, - 13, - 10, - 26, - 10, - 0, - 0, - 0, - 13, - 73, - 72, - 68, - 82, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 8, - 6, - 0, - 0, - 0, - 31, - 21, - 196, - 137, - 0, - 0, - 0, - 11, - 73, - 68, - 65, - 84, - 120, - 1, - 99, - 97, - 0, - 2, - 0, - 0, - 25, - 0, - 5, - 144, - 240, - 54, - 245, - 0, - 0, - 0, - 0, - 73, - 69, - 78, - 68, - 174, - 66, - 96, - 130, -]; - void main() { - group('SkiaGoldClient', () { - test('auth performs minimal work if already authorized', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - environment: {}, - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - final File authFile = fs.file('/workDirectory/temp/auth_opt.json') - ..createSync(recursive: true); - authFile.writeAsStringSync(authTemplate()); - process.fallbackProcessResult = ProcessResult(123, 0, '', ''); - await skiaClient.auth(); - - expect(process.workingDirectories, isEmpty); - }); - - test('gsutil is checked when authorization file is present', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - environment: {}, - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - final File authFile = fs.file('/workDirectory/temp/auth_opt.json') - ..createSync(recursive: true); - authFile.writeAsStringSync(authTemplate(gsutil: true)); - expect(await skiaClient.clientIsAuthorized(), isFalse); - }); - - test('throws for error state from auth', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - environment: { - 'GOLD_SERVICE_ACCOUNT': 'Service Account', - 'GOLDCTL': 'goldctl', - }, - operatingSystem: 'macos', - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - - process.fallbackProcessResult = ProcessResult( - 123, - 1, - 'Fallback failure', - 'Fallback failure', - ); - - expect(skiaClient.auth(), throwsException); - }); - - test('throws for error state from init', () { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - environment: { - 'SDK_CHECKOUT_PATH': '/flutter', - 'GOLDCTL': 'goldctl', - }, - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - - const gitInvocation = RunInvocation([ - 'git', - 'rev-parse', - 'HEAD', - ], '/flutter'); - const goldctlInvocation = RunInvocation([ - 'goldctl', - 'imgtest', - 'init', - '--instance', - 'flutter', - '--work-dir', - '/workDirectory/temp', - '--commit', - '12345678', - '--keys-file', - '/workDirectory/keys.json', - '--failure-file', - '/workDirectory/failures.json', - '--passfail', - ], null); - - process.processResults[gitInvocation] = ProcessResult( - 12345678, - 0, - '12345678', - '', - ); - process.processResults[goldctlInvocation] = ProcessResult( - 123, - 1, - 'Expected failure', - 'Expected failure', - ); - process.fallbackProcessResult = ProcessResult( - 123, - 1, - 'Fallback failure', - 'Fallback failure', - ); - - expect(skiaClient.imgtestInit(), throwsException); - }); - - test('Only calls init once', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - environment: { - 'SDK_CHECKOUT_PATH': '/flutter', - 'GOLDCTL': 'goldctl', - }, - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - - const gitInvocation = RunInvocation([ - 'git', - 'rev-parse', - 'HEAD', - ], '/flutter'); - const goldctlInvocation = RunInvocation([ - 'goldctl', - 'imgtest', - 'init', - '--instance', - 'flutter', - '--work-dir', - '/workDirectory/temp', - '--commit', - '1234', - '--keys-file', - '/workDirectory/keys.json', - '--failure-file', - '/workDirectory/failures.json', - '--passfail', - ], null); - - process.processResults[gitInvocation] = ProcessResult( - 1234, - 0, - '1234', - '', - ); - process.processResults[goldctlInvocation] = ProcessResult( - 5678, - 0, - '5678', - '', - ); - process.fallbackProcessResult = ProcessResult( - 123, - 1, - 'Fallback failure', - 'Fallback failure', - ); - - // First call - await skiaClient.imgtestInit(); - - // Remove fake process result. - // If the init call is executed again, the fallback process will throw. - process.processResults.remove(goldctlInvocation); - - // Second call - await skiaClient.imgtestInit(); - }); - - test('Only calls tryjob init once', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - environment: { - 'GOLDCTL': 'goldctl', - 'SWARMING_TASK_ID': '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX': - 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', - 'GOLD_TRYJOB': 'refs/pull/49815/head', - 'SDK_CHECKOUT_PATH': '/flutter', - }, - operatingSystem: 'macos', - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - - const gitInvocation = RunInvocation([ - 'git', - 'rev-parse', - 'HEAD', - ], '/flutter'); - const goldctlInvocation = RunInvocation([ - 'goldctl', - 'imgtest', - 'init', - '--instance', - 'flutter', - '--work-dir', - '/workDirectory/temp', - '--commit', - '1234', - '--keys-file', - '/workDirectory/keys.json', - '--failure-file', - '/workDirectory/failures.json', - '--passfail', - '--crs', - 'github', - '--patchset_id', - '1234', - '--changelist', - '49815', - '--cis', - 'buildbucket', - '--jobid', - '8885996262141582672', - ], null); - - process.processResults[gitInvocation] = ProcessResult( - 1234, - 0, - '1234', - '', - ); - process.processResults[goldctlInvocation] = ProcessResult( - 5678, - 0, - '5678', - '', - ); - process.fallbackProcessResult = ProcessResult( - 123, - 1, - 'Fallback failure', - 'Fallback failure', - ); - - // First call - await skiaClient.tryjobInit(); - - // Remove fake process result. - // If the init call is executed again, the fallback process will throw. - process.processResults.remove(goldctlInvocation); - - // Second call - await skiaClient.tryjobInit(); - }); - - test('throws for error state from imgtestAdd', () { - final fs = MemoryFileSystem(); - final File goldenFile = fs.file( - '/workDirectory/temp/golden_file_test.png', - )..createSync(recursive: true); - final platform = FakePlatform( - environment: {'GOLDCTL': 'goldctl'}, - operatingSystem: 'macos', - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - const goldctlInvocation = RunInvocation([ - 'goldctl', - 'imgtest', - 'add', - '--work-dir', - '/workDirectory/temp', - '--test-name', - 'golden_file_test', - '--png-file', - '/workDirectory/temp/golden_file_test.png', - '--passfail', - ], null); - process.processResults[goldctlInvocation] = ProcessResult( - 123, - 1, - 'Expected failure', - 'Expected failure', - ); - process.fallbackProcessResult = ProcessResult( - 123, - 1, - 'Fallback failure', - 'Fallback failure', - ); - - expect( - skiaClient.imgtestAdd('golden_file_test', goldenFile), - throwsException, - ); - }); - - test('correctly inits tryjob for luci', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - environment: { - 'GOLDCTL': 'goldctl', - 'SWARMING_TASK_ID': '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX': - 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', - 'GOLD_TRYJOB': 'refs/pull/49815/head', - }, - operatingSystem: 'macos', - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - - final List ciArguments = skiaClient.getCIArguments(); - - expect( - ciArguments, - equals([ - '--changelist', - '49815', - '--cis', - 'buildbucket', - '--jobid', - '8885996262141582672', - ]), - ); - }); - - test('Creates traceID correctly', () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - environment: { - 'GOLDCTL': 'goldctl', - 'SWARMING_TASK_ID': '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX': - 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', - 'GOLD_TRYJOB': 'refs/pull/49815/head', - }, - operatingSystem: 'linux', - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - - expect( - skiaClient.getTraceID('flutter.golden.1'), - equals('abe4ba07d57982f282adcd425aa8581f'), - ); - }); - - test( - 'Creates traceID correctly - locally - should defer to luci traceID', - () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform( - operatingSystem: 'macos', - environment: {}, - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - expect( - skiaClient.getTraceID('flutter.golden.1'), - equals('405ca6a70c598037ab019d85f35f8357'), - ); - }, - ); - - test('throws for error state from imgtestAdd', () { - final fs = MemoryFileSystem(); - final File goldenFile = fs.file( - '/workDirectory/temp/golden_file_test.png', - )..createSync(recursive: true); - final platform = FakePlatform( - environment: {'GOLDCTL': 'goldctl'}, - operatingSystem: 'macos', - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - const goldctlInvocation = RunInvocation([ - 'goldctl', - 'imgtest', - 'add', - '--work-dir', - '/workDirectory/temp', - '--test-name', - 'golden_file_test', - '--png-file', - '/workDirectory/temp/golden_file_test.png', - '--passfail', - ], null); - process.processResults[goldctlInvocation] = ProcessResult( - 123, - 1, - 'Expected failure', - 'Expected failure', - ); - process.fallbackProcessResult = ProcessResult( - 123, - 1, - 'Fallback failure', - 'Fallback failure', - ); - - expect( - skiaClient.imgtestAdd('golden_file_test', goldenFile), - throwsA( - isA().having( - (SkiaException error) => error.message, - 'message', - contains('result-state.json'), - ), - ), - ); - }); - - test('throws for error state from tryjobAdd', () { - final fs = MemoryFileSystem(); - final File goldenFile = fs.file( - '/workDirectory/temp/golden_file_test.png', - )..createSync(recursive: true); - final platform = FakePlatform( - environment: {'GOLDCTL': 'goldctl'}, - operatingSystem: 'macos', - ); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - const goldctlInvocation = RunInvocation([ - 'goldctl', - 'imgtest', - 'add', - '--work-dir', - '/workDirectory/temp', - '--test-name', - 'golden_file_test', - '--png-file', - '/workDirectory/temp/golden_file_test.png', - '--passfail', - ], null); - process.processResults[goldctlInvocation] = ProcessResult( - 123, - 1, - 'Expected failure', - 'Expected failure', - ); - process.fallbackProcessResult = ProcessResult( - 123, - 1, - 'Fallback failure', - 'Fallback failure', - ); - expect( - skiaClient.tryjobAdd('golden_file_test', goldenFile), - throwsA( - isA().having( - (SkiaException error) => error.message, - 'message', - contains('result-state.json'), - ), - ), - ); - }); - - group('Request Handling', () { - test('image bytes are processed properly', () async { - const expectation = '55109a4bed52acc780530f7a9aeff6c0'; - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - final process = FakeProcessManager(); - final fakeHttpClient = FakeHttpClient(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - final skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - ); - final Uri imageUrl = Uri.parse( - 'https://flutter-packages-gold.skia.org/img/images/$expectation.png', - ); - final fakeImageRequest = FakeHttpClientRequest(); - final fakeImageResponse = FakeHttpImageResponse( - imageResponseTemplate(), - ); - - fakeHttpClient.request = fakeImageRequest; - fakeImageRequest.response = fakeImageResponse; - - final List masterBytes = await skiaClient.getImageBytes( - expectation, - ); - - expect(fakeHttpClient.lastUri, imageUrl); - expect(masterBytes, equals(_kTestPngBytes)); - }); - }); - }); - group('FlutterGoldenFileComparator', () { test( 'calculates the basedir correctly from defaultComparator for local testing', () async { final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); fs.directory(_kFlutterRoot).createSync(recursive: true); final defaultComparator = FakeLocalFileComparator(); final Directory root = fs.directory('/')..createSync(recursive: true); defaultComparator.basedir = root.childDirectory('baz').uri; final Directory basedir = FlutterGoldenFileComparator.getBaseDirectory( defaultComparator, - platform: platform, fs: fs, ); expect(basedir.uri, fs.directory('/baz/skia_goldens').uri); }, ); - test('ignores version number', () { - final log = []; - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory basedir = fs.directory('flutter/test/library/') - ..createSync(recursive: true); - final FlutterGoldenFileComparator comparator = - FlutterPostSubmitFileComparator( - basedir.uri, - FakeSkiaGoldClient(), - fs: fs, - platform: platform, - log: log.add, - ); - final Uri key = comparator.getTestUri(Uri.parse('foo.png'), 1); - expect(key, Uri.parse('foo.png')); - expect(log, isEmpty); - }); - - test('adds namePrefix', () async { - final log = []; - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - const packageName = 'sidedishes'; - const namePrefix = 'tomatosalad'; - const fileName = 'lettuce.png'; - final fakeSkiaClient = FakeSkiaGoldClient(); - final Directory basedir = fs.directory('$packageName/test/') - ..createSync(recursive: true); - final FlutterGoldenFileComparator comparator = - FlutterPostSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - namePrefix: namePrefix, - log: log.add, - ); - await comparator.compare( - Uint8List.fromList(_kTestPngBytes), - Uri.parse(fileName), - ); - expect(fakeSkiaClient.testNames.single, '$namePrefix.$fileName'); - expect(log, isEmpty); - }); - - group('Post-Submit', () { - test('asserts .png format', () async { - final log = []; - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory basedir = fs.directory('flutter/test/library/') - ..createSync(recursive: true); - final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = - FlutterPostSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - log: log.add, - ); - await expectLater( - () async { - return comparator.compare( - Uint8List.fromList(_kTestPngBytes), - Uri.parse('flutter.golden_test.1'), - ); - }, - throwsA( - isA().having( - (AssertionError error) => error.toString(), - 'description', - contains( - 'Golden files in the Flutter framework must end with the file ' - 'extension .png.', - ), - ), - ), - ); - expect(log, isEmpty); - }); - - test('calls init during compare', () { - final log = []; - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory basedir = fs.directory('flutter/test/library/') - ..createSync(recursive: true); - final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = - FlutterPostSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - log: log.add, - ); - expect(fakeSkiaClient.initCalls, 0); - comparator.compare( - Uint8List.fromList(_kTestPngBytes), - Uri.parse('flutter.golden_test.1.png'), - ); - expect(fakeSkiaClient.initCalls, 1); - expect(log, isEmpty); - }); - - test('does not call init in during construction', () { - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final fakeSkiaClient = FakeSkiaGoldClient(); - expect(fakeSkiaClient.initCalls, 0); - FlutterPostSubmitFileComparator.fromLocalFileComparator( - localFileComparator: LocalFileComparator( - Uri.parse('/test'), - pathStyle: path.Style.posix, - ), - platform: platform, - goldens: fakeSkiaClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - fs: fs, - process: FakeProcessManager(), - httpClient: FakeHttpClient(), - ); - expect(fakeSkiaClient.initCalls, 0); - }); - - test('reports a failure as a TestFailure', () async { - final log = []; - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory basedir = fs.directory('flutter/test/library/') - ..createSync(recursive: true); - final FlutterGoldenFileComparator comparator = - FlutterPostSubmitFileComparator( - basedir.uri, - ThrowsOnImgTestAddSkiaClient( - message: - 'Skia Gold received an unapproved image in post-submit', - ), - fs: fs, - platform: platform, - log: log.add, - ); - await expectLater( - () async { - return comparator.compare( - Uint8List.fromList(_kTestPngBytes), - Uri.parse('flutter.golden_test.1.png'), - ); - }, - throwsA( - isA().having( - (TestFailure error) => error.toString(), - 'description', - contains('Skia Gold received an unapproved image in post-submit'), - ), - ), - ); - expect(log, isEmpty); - }); - }); - - group('Pre-Submit', () { - test('asserts .png format', () async { - final log = []; - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory basedir = fs.directory('flutter/test/library/') - ..createSync(recursive: true); - final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = - FlutterPreSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - log: log.add, - ); - await expectLater( - () async { - return comparator.compare( - Uint8List.fromList(_kTestPngBytes), - Uri.parse('flutter.golden_test.1'), - ); - }, - throwsA( - isA().having( - (AssertionError error) => error.toString(), - 'description', - contains( - 'Golden files in the Flutter framework must end with the file ' - 'extension .png.', - ), - ), - ), - ); - expect(log, isEmpty); - }); - - test('calls init during compare', () { - final log = []; - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory basedir = fs.directory('flutter/test/library/') - ..createSync(recursive: true); - final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = - FlutterPreSubmitFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: platform, - log: log.add, - ); - expect(fakeSkiaClient.tryInitCalls, 0); - comparator.compare( - Uint8List.fromList(_kTestPngBytes), - Uri.parse('flutter.golden_test.1.png'), - ); - expect(fakeSkiaClient.tryInitCalls, 1); - expect(log, isEmpty); - }); - - test('does not call init in during construction', () { - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final fakeSkiaClient = FakeSkiaGoldClient(); - expect(fakeSkiaClient.tryInitCalls, 0); - FlutterPostSubmitFileComparator.fromLocalFileComparator( - localFileComparator: LocalFileComparator( - Uri.parse('/test'), - pathStyle: path.Style.posix, - ), - platform: platform, - goldens: fakeSkiaClient, - log: (String message) => - fail('skia gold client printed unexpected output: "$message"'), - fs: fs, - process: FakeProcessManager(), - httpClient: FakeHttpClient(), - ); - expect(fakeSkiaClient.tryInitCalls, 0); - }); - }); - - group('Local', () { - test('asserts .png format', () async { - final log = []; - final fs = MemoryFileSystem(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory basedir = fs.directory('flutter/test/library/') - ..createSync(recursive: true); - final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = - FlutterLocalFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: FakePlatform(operatingSystem: 'macos'), - log: log.add, - ); - const hash = '55109a4bed52acc780530f7a9aeff6c0'; - fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; - fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; - fakeSkiaClient.cleanTestNameValues['flutter.golden_test.1.png'] = - 'flutter.golden_test.1'; - await expectLater( - () async { - return comparator.compare( - Uint8List.fromList(_kTestPngBytes), - Uri.parse('flutter.golden_test.1'), - ); - }, - throwsA( - isA().having( - (AssertionError error) => error.toString(), - 'description', - contains( - 'Golden files in the Flutter framework must end with the file ' - 'extension .png.', - ), - ), - ), - ); - expect(log, isEmpty); - }); - - test('passes when bytes match', () async { - final log = []; - final fs = MemoryFileSystem(); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final Directory basedir = fs.directory('flutter/test/library/') - ..createSync(recursive: true); - final fakeSkiaClient = FakeSkiaGoldClient(); - final FlutterGoldenFileComparator comparator = - FlutterLocalFileComparator( - basedir.uri, - fakeSkiaClient, - fs: fs, - platform: FakePlatform(operatingSystem: 'macos'), - log: log.add, - ); - const hash = '55109a4bed52acc780530f7a9aeff6c0'; - fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; - fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; - fakeSkiaClient.cleanTestNameValues['flutter.golden_test.1.png'] = - 'flutter.golden_test.1'; - expect( - await comparator.compare( - Uint8List.fromList(_kTestPngBytes), - Uri.parse('flutter.golden_test.1.png'), - ), - isTrue, - ); - expect(log, isEmpty); - }); - - test( - 'returns FlutterSkippingGoldenFileComparator when network connection is unavailable', - () async { - final fs = MemoryFileSystem(); - final platform = FakePlatform(operatingSystem: 'macos'); - fs.directory(_kFlutterRoot).createSync(recursive: true); - final fakeSkiaClient = FakeSkiaGoldClient(); - - const hash = '55109a4bed52acc780530f7a9aeff6c0'; - fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = - hash; - fakeSkiaClient.imageBytesValues[hash] = _kTestPngBytes; - fakeSkiaClient.cleanTestNameValues['flutter.golden_test.1.png'] = - 'flutter.golden_test.1'; - final fakeDirectory = FakeDirectory(); - fakeDirectory.existsSyncValue = true; - fakeDirectory.uri = Uri.parse('/flutter'); - - fakeSkiaClient.getExpectationForTestThrowable = const OSError( - "Can't reach Gold", - ); - final FlutterGoldenFileComparator comparator1 = - await FlutterLocalFileComparator.fromLocalFileComparator( - localFileComparator: LocalFileComparator( - Uri.parse('/test'), - pathStyle: path.Style.posix, - ), - platform: platform, - goldens: fakeSkiaClient, - baseDirectory: fakeDirectory, - log: (String message) => fail( - 'skia gold client printed unexpected output: "$message"', - ), - fs: fs, - process: FakeProcessManager(), - httpClient: FakeHttpClient(), - ); - expect(comparator1.runtimeType, FlutterSkippingFileComparator); - - fakeSkiaClient.getExpectationForTestThrowable = const SocketException( - "Can't reach Gold", - ); - final FlutterGoldenFileComparator comparator2 = - await FlutterLocalFileComparator.fromLocalFileComparator( - localFileComparator: LocalFileComparator( - Uri.parse('/test'), - pathStyle: path.Style.posix, - ), - platform: platform, - goldens: fakeSkiaClient, - baseDirectory: fakeDirectory, - log: (String message) => fail( - 'skia gold client printed unexpected output: "$message"', - ), - fs: fs, - process: FakeProcessManager(), - httpClient: FakeHttpClient(), - ); - expect(comparator2.runtimeType, FlutterSkippingFileComparator); - - fakeSkiaClient.getExpectationForTestThrowable = const FormatException( - "Can't reach Gold", - ); - final FlutterGoldenFileComparator comparator3 = - await FlutterLocalFileComparator.fromLocalFileComparator( - localFileComparator: LocalFileComparator( - Uri.parse('/test'), - pathStyle: path.Style.posix, - ), - platform: platform, - goldens: fakeSkiaClient, - baseDirectory: fakeDirectory, - log: (String message) => fail( - 'skia gold client printed unexpected output: "$message"', - ), - fs: fs, - process: FakeProcessManager(), - httpClient: FakeHttpClient(), - ); - expect(comparator3.runtimeType, FlutterSkippingFileComparator); - - // reset property or it will carry on to other tests - fakeSkiaClient.getExpectationForTestThrowable = null; - }, - ); - }); - group('_getPackageName', () { test('extracts name from pubspec.yaml', () { final fs = MemoryFileSystem(); final Directory packageDir = fs.directory('/my_package') ..createSync(recursive: true); - packageDir.childFile('pubspec.yaml').writeAsStringSync('name: my_package_name\n'); + packageDir + .childFile('pubspec.yaml') + .writeAsStringSync('name: my_package_name\n'); fs.currentDirectory = packageDir; - expect(FlutterGoldenFileComparator.getPackageName(fs) -, 'my_package_name'); + expect( + FlutterGoldenFileComparator.getPackageName(fs), + 'my_package_name', + ); }); test('traverses upwards to find pubspec.yaml', () { final fs = MemoryFileSystem(); final Directory packageDir = fs.directory('/my_package') ..createSync(recursive: true); - packageDir.childFile('pubspec.yaml').writeAsStringSync('name: my_package_name\n'); - final Directory testDir = packageDir.childDirectory('test')..createSync(recursive: true); + packageDir + .childFile('pubspec.yaml') + .writeAsStringSync('name: my_package_name\n'); + final Directory testDir = packageDir.childDirectory('test') + ..createSync(recursive: true); fs.currentDirectory = testDir; - expect(FlutterGoldenFileComparator.getPackageName(fs) -, 'my_package_name'); + expect( + FlutterGoldenFileComparator.getPackageName(fs), + 'my_package_name', + ); }); test('returns null if no pubspec.yaml is found', () { final fs = MemoryFileSystem(); - final Directory someDir = fs.directory('/some/dir')..createSync(recursive: true); + final Directory someDir = fs.directory('/some/dir') + ..createSync(recursive: true); fs.currentDirectory = someDir; - expect(FlutterGoldenFileComparator.getPackageName(fs) -, isNull); + expect(FlutterGoldenFileComparator.getPackageName(fs), isNull); }); test('handles invalid yaml gracefully', () { final fs = MemoryFileSystem(); final Directory packageDir = fs.directory('/my_package') ..createSync(recursive: true); - packageDir.childFile('pubspec.yaml').writeAsStringSync('invalid: yaml: : :'); + packageDir + .childFile('pubspec.yaml') + .writeAsStringSync('invalid: yaml: : :'); fs.currentDirectory = packageDir; - expect(FlutterGoldenFileComparator.getPackageName(fs) -, isNull); + expect(FlutterGoldenFileComparator.getPackageName(fs), isNull); }); }); }); } -@immutable -class RunInvocation { - const RunInvocation(this.command, this.workingDirectory); - - final List command; - final String? workingDirectory; - - @override - int get hashCode => Object.hash(Object.hashAll(command), workingDirectory); - - bool _commandEquals(List other) { - if (other == command) { - return true; - } - if (other.length != command.length) { - return false; - } - for (var index = 0; index < other.length; index += 1) { - if (other[index] != command[index]) { - return false; - } - } - return true; - } - - @override - bool operator ==(Object other) { - if (other.runtimeType != runtimeType) { - return false; - } - return other is RunInvocation && - _commandEquals(other.command) && - other.workingDirectory == workingDirectory; - } - - @override - String toString() => '$command ($workingDirectory)'; -} - -class FakeProcessManager extends Fake implements ProcessManager { - Map processResults = - {}; - - /// Used if [processResults] does not contain a matching invocation. - ProcessResult? fallbackProcessResult; - - final List workingDirectories = []; - - @override - Future run( - List command, { - String? workingDirectory, - Map? environment, - bool includeParentEnvironment = true, - bool runInShell = false, - Encoding? stdoutEncoding = systemEncoding, - Encoding? stderrEncoding = systemEncoding, - }) async { - workingDirectories.add(workingDirectory); - final ProcessResult? result = - processResults[RunInvocation(command.cast(), workingDirectory)]; - if (result == null && fallbackProcessResult == null) { - printOnFailure( - 'ProcessManager.run was called with $command ($workingDirectory) unexpectedly - $processResults.', - ); - fail('See above.'); - } - return result ?? fallbackProcessResult!; - } -} - -// See also dev/automated_tests/flutter_test/flutter_gold_test.dart -class FakeSkiaGoldClient extends Fake implements SkiaGoldClient { - Map expectationForTestValues = {}; - Exception? getExpectationForTestThrowable; - @override - Future getExpectationForTest(String testName) async { - if (getExpectationForTestThrowable != null) { - throw getExpectationForTestThrowable!; - } - return expectationForTestValues[testName] ?? ''; - } - - @override - Future auth() async {} - - final List testNames = []; - - int initCalls = 0; - @override - Future imgtestInit() async => initCalls += 1; - @override - Future imgtestAdd(String testName, File goldenFile) async { - testNames.add(testName); - return true; - } - - int tryInitCalls = 0; - @override - Future tryjobInit() async => tryInitCalls += 1; - @override - Future tryjobAdd(String testName, File goldenFile) async => null; - - Map> imageBytesValues = >{}; - @override - Future> getImageBytes(String imageHash) async => - imageBytesValues[imageHash]!; - - Map cleanTestNameValues = {}; - @override - String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? ''; -} - -class ThrowsOnImgTestAddSkiaClient extends Fake implements SkiaGoldClient { - ThrowsOnImgTestAddSkiaClient({required this.message}); - final String message; - - @override - Future imgtestInit() async { - // Assume this function works. - } - - @override - Future imgtestAdd(String testName, File goldenFile) { - throw SkiaException(message); - } -} - class FakeLocalFileComparator extends Fake implements LocalFileComparator { @override late Uri basedir; } - -class FakeDirectory extends Fake implements Directory { - late bool existsSyncValue; - @override - bool existsSync() => existsSyncValue; - - @override - late Uri uri; -} - -class FakeHttpClient extends Fake implements HttpClient { - late Uri lastUri; - late FakeHttpClientRequest request; - - @override - Future getUrl(Uri url) async { - lastUri = url; - return request; - } -} - -class FakeHttpClientRequest extends Fake implements HttpClientRequest { - late FakeHttpImageResponse response; - - @override - Future close() async { - return response; - } -} - -class FakeHttpImageResponse extends Fake implements HttpClientResponse { - FakeHttpImageResponse(this.response); - - final List> response; - - @override - Future forEach(void Function(List element) action) async { - response.forEach(action); - } -} diff --git a/script/flutter_goldens/test/json_templates.dart b/script/flutter_goldens/test/json_templates.dart deleted file mode 100644 index ee6c98a674a..00000000000 --- a/script/flutter_goldens/test/json_templates.dart +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2013 The Flutter Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -/// Json response template for the contents of the auth_opt.json file created by -/// goldctl. -String authTemplate({bool gsutil = false}) { - return ''' - { - "Luci":false, - "ServiceAccount":"${gsutil ? '' : '/packages/flutter/test/widgets/serviceAccount.json'}", - "GSUtil":$gsutil - } - '''; -} - -/// Json response template for Skia Gold image request: -/// https://flutter-gold.skia.org/img/images/{imageHash}.png -List> imageResponseTemplate() { - return >[ - [ - 137, - 80, - 78, - 71, - 13, - 10, - 26, - 10, - 0, - 0, - 0, - 13, - 73, - 72, - 68, - 82, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 8, - 6, - 0, - 0, - 0, - 31, - 21, - 196, - 137, - 0, - ], - [ - 0, - 0, - 11, - 73, - 68, - 65, - 84, - 120, - 1, - 99, - 97, - 0, - 2, - 0, - 0, - 25, - 0, - 5, - 144, - 240, - 54, - 245, - 0, - 0, - 0, - 0, - 73, - 69, - 78, - 68, - 174, - 66, - 96, - 130, - ], - ]; -} From e35263c2446626f96a89e5f0a8bd999f7d3038fe Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 5 May 2026 09:23:52 -0500 Subject: [PATCH 20/21] Updates --- .../flutter_goldens/lib/flutter_goldens.dart | 25 ------------------- .../test/flutter_goldens_test.dart | 18 +++++++++++++ 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index e981328e8c9..5a621e7557c 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -25,12 +25,6 @@ import 'package:yaml/yaml.dart'; /// tests using `flutter test`. This should not be called when running a test /// using `flutter run`, as in that environment, the [goldenFileComparator] is a /// [TrivialComparator]. -/// -/// An [HttpClient] is created when this method is called. That client is used -/// to communicate with the Skia Gold servers. Any [HttpOverrides] set in this -/// will affect whether this is effective or not. For example, if the current -/// override provides a mock client that always fails, then all calls to gold -/// comparison functions will fail. Future testExecutable( FutureOr Function() testMain, { String? namePrefix, @@ -97,11 +91,6 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { Uri getTestUri(Uri key, int? version) => key; /// Calculate the appropriate basedir for the current test context. - /// - /// The optional [suffix] argument is used by the - /// [FlutterPostSubmitFileComparator] and the [FlutterPreSubmitFileComparator]. - /// These [FlutterGoldenFileComparator]s randomize their base directories to - /// maintain thread safety while using the `goldctl` tool. @protected @visibleForTesting static Directory getBaseDirectory( @@ -147,20 +136,6 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { /// A [FlutterGoldenFileComparator] for testing conditions that do not execute /// golden file tests. -/// -/// Currently, this comparator is used on Luci environments when executing tests -/// outside of the flutter/flutter repository. -/// -/// See also: -/// -/// * [FlutterPostSubmitFileComparator], another [FlutterGoldenFileComparator] -/// that tests golden images through Skia Gold. -/// * [FlutterPreSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images before changes are -/// merged into the master branch. -/// * [FlutterLocalFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images locally on your -/// current machine. class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { /// Creates a [FlutterSkippingFileComparator] that will skip tests that /// are not in the right environment for golden file testing. diff --git a/script/flutter_goldens/test/flutter_goldens_test.dart b/script/flutter_goldens/test/flutter_goldens_test.dart index 094ffb4590a..9a8f329a78d 100644 --- a/script/flutter_goldens/test/flutter_goldens_test.dart +++ b/script/flutter_goldens/test/flutter_goldens_test.dart @@ -4,6 +4,8 @@ // See also dev/automated_tests/flutter_test/flutter_gold_test.dart +import 'dart:typed_data'; + import 'package:file/file.dart'; import 'package:file/memory.dart'; import 'package:flutter_goldens/flutter_goldens.dart'; @@ -84,6 +86,22 @@ void main() { }); }); }); + + group('FlutterSkippingFileComparator', () { + test('compare returns true', () async { + final fs = MemoryFileSystem(); + final comparator = FlutterSkippingFileComparator( + Uri.parse('/basedir'), + 'reason', + fs: fs, + ); + final bool result = await comparator.compare( + Uint8List.fromList([1, 2, 3]), + Uri.parse('golden.png'), + ); + expect(result, isTrue); + }); + }); } class FakeLocalFileComparator extends Fake implements LocalFileComparator { From d6317c8b7834c761224e8bcb83de64bd2af749d6 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Tue, 5 May 2026 10:46:42 -0500 Subject: [PATCH 21/21] ++ --- .../material_ui/test/goldens/goldens_test.dart | 2 +- script/flutter_goldens/lib/flutter_goldens.dart | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/material_ui/test/goldens/goldens_test.dart b/packages/material_ui/test/goldens/goldens_test.dart index be773a42a84..c4270a02678 100644 --- a/packages/material_ui/test/goldens/goldens_test.dart +++ b/packages/material_ui/test/goldens/goldens_test.dart @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'package:flutter/cupertino.dart'; import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:material_ui/material_ui.dart'; diff --git a/script/flutter_goldens/lib/flutter_goldens.dart b/script/flutter_goldens/lib/flutter_goldens.dart index 5a621e7557c..fbb02bfb735 100644 --- a/script/flutter_goldens/lib/flutter_goldens.dart +++ b/script/flutter_goldens/lib/flutter_goldens.dart @@ -11,6 +11,7 @@ import 'package:file/file.dart'; import 'package:file/local.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as path; import 'package:yaml/yaml.dart'; /// Main method that can be used in a `flutter_test_config.dart` file to set @@ -100,7 +101,9 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { }) { final Directory comparisonRoot = switch (suffix) { null => - fs.directory(defaultComparator.basedir).childDirectory('skia_goldens'), + fs + .directory(path.fromUri(defaultComparator.basedir)) + .childDirectory('skia_goldens'), _ => fs.systemTempDirectory.createTempSync(suffix), }; return comparisonRoot; @@ -109,7 +112,9 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { /// Returns the golden [File] identified by the given [Uri]. @protected File getGoldenFile(Uri uri) { - final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path); + final File goldenFile = fs + .directory(path.fromUri(basedir)) + .childFile(path.fromUri(uri)); return goldenFile; } @@ -122,8 +127,10 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { final File pubspec = current.childFile('pubspec.yaml'); if (pubspec.existsSync()) { try { - final yaml = loadYaml(pubspec.readAsStringSync()) as YamlMap; - return yaml['name'] as String?; + final Object? yaml = loadYaml(pubspec.readAsStringSync()); + if (yaml is YamlMap) { + return yaml['name'] as String?; + } } catch (e) { // Ignore parsing errors and keep looking }