diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java b/autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java index 46d3e8165..941e6f04b 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java +++ b/autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java @@ -3,7 +3,9 @@ import android.graphics.Bitmap; import android.graphics.Color; import android.media.Image; -import android.os.Build; +import android.util.Log; + +import androidx.annotation.NonNull; import com.stardust.autojs.core.opencv.Mat; import com.stardust.autojs.core.opencv.OpenCVHelper; @@ -14,31 +16,96 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; import java.nio.ByteBuffer; - -import androidx.annotation.NonNull; -import androidx.annotation.RequiresApi; +import java.util.HashSet; +import java.util.Set; /** * Created by Stardust on 2017/11/25. */ public class ImageWrapper { + private static final String TAG = "ImageWrapper"; + private static final int MAX_TRACKED_IMAGES = 10000; + private static final Set> sAllImages = new HashSet<>(); + private static final ReferenceQueue sStaleImages = new ReferenceQueue<>(); + private static final Object sLock = new Object(); + private static volatile boolean sTrackingEnabled = false; + private Mat mMat; private int mWidth; private int mHeight; private Bitmap mBitmap; + private boolean mRecycled; + + public static void enableTracking() { + synchronized (sLock) { + sTrackingEnabled = true; + drainStaleImages(); + sAllImages.clear(); + } + } + + public static void disableTracking() { + synchronized (sLock) { + sTrackingEnabled = false; + } + } + + /** + * 回收所有被追踪且仍存活的图像。仅由脚本显式调用（images.recycleAllImages()）； + * 追踪数量达到上限时不会自动调用，避免回收脚本仍在使用的图像。 + */ + public static void recycleAllTrackedImages() { + synchronized (sLock) { + drainStaleImages(); + for (WeakReference ref : sAllImages) { + ImageWrapper img = ref.get(); + if (img != null && img.isAlive()) { + try { + img.recycle(); + } catch (Exception e) { + Log.w(TAG, "Failed to recycle image", e); + } + } + } + sAllImages.clear(); + } + } + + public static int getTrackedImageCount() { + synchronized (sLock) { + drainStaleImages(); + return sAllImages.size(); + } + } + + /** + * 借助 ReferenceQueue 移除已被 GC 的弱引用，均摊 O(1)， + * 不再需要每次创建图像时遍历整个集合做 removeIf。 + */ + private static void drainStaleImages() { + Reference ref; + while ((ref = sStaleImages.poll()) != null) { + sAllImages.remove(ref); + } + } protected ImageWrapper(Mat mat) { mMat = mat; mWidth = mat.cols(); mHeight = mat.rows(); + trackThis(); } protected ImageWrapper(Bitmap bitmap) { mBitmap = bitmap; mWidth = bitmap.getWidth(); mHeight = bitmap.getHeight(); + trackThis(); } protected ImageWrapper(Bitmap bitmap, Mat mat) { @@ -46,12 +113,36 @@ protected ImageWrapper(Bitmap bitmap, Mat mat) { mMat = mat; mWidth = bitmap.getWidth(); mHeight = bitmap.getHeight(); + trackThis(); } public ImageWrapper(int width, int height) { this(Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)); } + private void trackThis() { + // 追踪默认关闭；锁外快速返回，避免每张图像都进入全局锁。 + if (!sTrackingEnabled) { + return; + } + synchronized (sLock) { + // 双重检查：等待锁期间可能已被 disableTracking()。 + if (!sTrackingEnabled) { + return; + } + drainStaleImages(); + if (sAllImages.size() == MAX_TRACKED_IMAGES) { + Log.w(TAG, "tracked images reached " + MAX_TRACKED_IMAGES + + "; images may not be released, call images.recycleAllImages()"); + } + // 达到上限只告警、继续登记，绝不在内部回收脚本仍在使用的图像。 + sAllImages.add(new WeakReference<>(this, sStaleImages)); + } + } + + public boolean isAlive() { + return (mRecycled || (mBitmap == null && mMat == null)) ? false : true; + } public static ImageWrapper ofImage(Image image) { if (image == null) { @@ -69,7 +160,6 @@ public static ImageWrapper ofMat(Mat mat) { return new ImageWrapper(mat); } - public static ImageWrapper ofBitmap(Bitmap bitmap) { if (bitmap == null) { return null; @@ -142,17 +232,21 @@ public Bitmap getBitmap() { } public void recycle() { + if (mRecycled) { + return; + } + mRecycled = true; mBitmap = null; if (mMat != null) { OpenCVHelper.release(mMat); mMat = null; } - } public void ensureNotRecycled() { - if (mBitmap == null && mMat == null) + if (mRecycled || (mBitmap == null && mMat == null)) { throw new IllegalStateException("image has been recycled"); + } } @Override @@ -167,4 +261,4 @@ public ImageWrapper clone() { } return new ImageWrapper(mBitmap.copy(mBitmap.getConfig(), true), mMat.clone()); } -} +} \ No newline at end of file diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java b/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java index c25e58060..99429e1bb 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java +++ b/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java @@ -3,8 +3,6 @@ import android.util.Log; import android.util.TimingLogger; -import androidx.annotation.NonNull; - import com.stardust.util.Nath; import org.opencv.core.Core; @@ -33,19 +31,24 @@ public static class Match { public final Point point; public final double similarity; public Integer templateIndex = 0; + public int width; + public int height; - public Match(Point point, double similarity) { + public Match(Point point, double similarity, int width, int height) { this.point = point; this.similarity = similarity; + this.width = width; + this.height = height; } - @NonNull @Override public String toString() { return "Match{" + "templateIndex=" + templateIndex + ", point=" + point + ", similarity=" + similarity + + ", width=" + width + + ", height=" + height + '}'; } } @@ -55,8 +58,11 @@ public String toString() { public static final int MAX_LEVEL_AUTO = -1; public static final int MATCHING_METHOD_DEFAULT = Imgproc.TM_CCOEFF_NORMED; - public static Point fastTemplateMatching(Mat img, Mat template, int matchMethod, float weakThreshold, float strictThreshold, int maxLevel, Boolean transparentMask) { - List result = fastTemplateMatching(img, template, matchMethod, weakThreshold, strictThreshold, maxLevel, 1, transparentMask); + public static Point fastTemplateMatching(Mat img, Mat template, int matchMethod, float weakThreshold, + float strictThreshold, int maxLevel, Boolean transparentMask, + boolean useGrayscale) { + List result = fastTemplateMatching(img, template, matchMethod, weakThreshold, + strictThreshold, maxLevel, 1, transparentMask, useGrayscale); if (result.isEmpty()) { return null; } @@ -72,12 +78,17 @@ public static Point fastTemplateMatching(Mat img, Mat template, int matchMethod, * @param weakThreshold 弱阈值。该值用于在每一轮模板匹配中检验是否继续匹配。如果相似度小于该值，则不再继续匹配。 * @param strictThreshold 强阈值。该值用于检验最终匹配结果，以及在每一轮匹配中如果相似度大于该值则直接返回匹配结果。 * @param maxLevel 图像金字塔的层数 + * @param limit 每层最多返回的候选点数量（控制计算量） + * @param transparentMask 是否启用透明蒙版 + * @param useGrayscale 是否先转为灰度图再匹配 * @return */ - public static List fastTemplateMatching(Mat img, Mat template, int matchMethod, float weakThreshold, float strictThreshold, int maxLevel, int limit, Boolean transparentMask) { + public static List fastTemplateMatching(Mat img, Mat template, int matchMethod, float weakThreshold, + float strictThreshold, int maxLevel, int limit, + Boolean transparentMask, boolean useGrayscale) { List templates = Collections.singletonList(template); return fastMultiTemplateMatching(img, templates, matchMethod, weakThreshold, - strictThreshold, maxLevel, limit, transparentMask); + strictThreshold, maxLevel, limit, transparentMask, useGrayscale); } /** @@ -92,27 +103,45 @@ public static List fastTemplateMatching(Mat img, Mat template, int matchM * @param maxLevel 金字塔层数（设为 MAX_LEVEL_AUTO 自动计算） * @param limit 每层最多返回的候选点数量（控制计算量） * @param transparentMask 是否启用透明蒙版 + * @param useGrayscale 是否先转为灰度图再匹配 * @return 按模板顺序返回匹配结果列表；若某模板无匹配，则对应列表为空 */ - public static List fastMultiTemplateMatching(Mat img, - List templates, - int matchMethod, - float weakThreshold, - float strictThreshold, - int maxLevel, - int limit, - Boolean transparentMask) { + public static List fastMultiTemplateMatching(Mat img, List templates, int matchMethod, + float weakThreshold, float strictThreshold, + int maxLevel, int limit, Boolean transparentMask, + boolean useGrayscale) { + // 灰度化会把 RGBA 转成单通道，alpha 不复存在，processingAlphaChannel() + // 之后只能拿到错误的蒙版。与其静默给出错误匹配结果，不如明确拒绝。 + if (useGrayscale && Boolean.TRUE.equals(transparentMask)) { + throw new IllegalArgumentException( + "useGrayscale and transparentMask cannot be used together: " + + "grayscale conversion drops the alpha channel"); + } TimingLogger logger = new TimingLogger(LOG_TAG, "fast_multi_tm"); List resourcesToRelease = new ArrayList<>(); + List allResults = new ArrayList<>(); try { + Mat workingImg = img; + List workingTemplates = templates; + + if (useGrayscale) { + workingImg = toGrayscaleIfNeeded(img, resourcesToRelease); + List grayTemplates = new ArrayList<>(); + for (Mat t : templates) { + grayTemplates.add(toGrayscaleIfNeeded(t, resourcesToRelease)); + } + workingTemplates = grayTemplates; + } // 确定全局最大金字塔层数（取所有模板允许层数的最大值） int globalMaxLevel; if (maxLevel == MAX_LEVEL_AUTO) { globalMaxLevel = 0; - for (Mat template : templates) { - int level = selectPyramidLevel(img, template); - if (level > globalMaxLevel) globalMaxLevel = level; + for (Mat template : workingTemplates) { + int level = selectPyramidLevel(workingImg, template); + if (level > globalMaxLevel) { + globalMaxLevel = level; + } } } else { globalMaxLevel = maxLevel; @@ -121,23 +150,17 @@ public static List fastMultiTemplateMatching(Mat img, // 构建主图金字塔（0 ~ globalMaxLevel） List imgPyramid = new ArrayList<>(); for (int level = 0; level <= globalMaxLevel; level++) { - Mat src = getPyramidDownAtLevel(img, level); + Mat src = getPyramidDownAtLevel(workingImg, level); imgPyramid.add(src); - if (src != img) { + if (src != workingImg) { resourcesToRelease.add(src); } } - List allResults = new ArrayList<>(); - for (int i = 0; i < templates.size(); i++) { - List result = matchTemplateWithPyramid(imgPyramid, - templates.get(i), - globalMaxLevel, - matchMethod, - weakThreshold, - strictThreshold, - limit, - transparentMask, + for (int i = 0; i < workingTemplates.size(); i++) { + Mat template = workingTemplates.get(i); + List result = matchTemplateWithPyramid(imgPyramid, template, globalMaxLevel, + matchMethod, weakThreshold, strictThreshold, limit, transparentMask, resourcesToRelease); for (Match r : result) { r.templateIndex = i; @@ -147,7 +170,6 @@ public static List fastMultiTemplateMatching(Mat img, logger.addSplit("multi_templates: " + templates.size()); logger.dumpToLog(); - return allResults; } finally { for (Mat mat : resourcesToRelease) { try { @@ -157,6 +179,8 @@ public static List fastMultiTemplateMatching(Mat img, } } } + + return allResults; } /** @@ -173,14 +197,10 @@ public static List fastMultiTemplateMatching(Mat img, * @param resourcesToRelease 资源回收列表 * @return 匹配结果（坐标已映射回原图） */ - private static List matchTemplateWithPyramid(List imgPyramid, - Mat template, - int globalMaxLevel, - int matchMethod, - float weakThreshold, - float strictThreshold, - int limit, - Boolean transparentMask, + private static List matchTemplateWithPyramid(List imgPyramid, Mat template, + int globalMaxLevel, int matchMethod, + float weakThreshold, float strictThreshold, + int limit, Boolean transparentMask, List resourcesToRelease) { // 计算模板自身的最大可行层数 int tMaxLevel = selectPyramidLevel(imgPyramid.get(0), template); @@ -204,13 +224,16 @@ private static List matchTemplateWithPyramid(List imgPyramid, List currentMatchResult = new ArrayList<>(); Mat src = imgPyramid.get(level); Mat currentTemplate = templatePyramid.get(level); - Mat currentMask = null; + Mat currentMask = null; if (transparentMask) { currentMask = TemplateMatchingKt.INSTANCE.processingAlphaChannel(currentTemplate); resourcesToRelease.add(currentMask); } + int actualWidth = currentTemplate.width() << level; + int actualHeight = currentTemplate.height() << level; + if (previousMatchResult.isEmpty()) { // 决定是否继续匹配（根据弱阈值跳过部分层） if (!isFirstMatching && !shouldContinueMatching(level, tMaxLevel)) { @@ -220,7 +243,7 @@ private static List matchTemplateWithPyramid(List imgPyramid, Mat matchResult = matchTemplate(src, currentTemplate, matchMethod, currentMask); resourcesToRelease.add(matchResult); getBestMatched(matchResult, currentTemplate, matchMethod, weakThreshold, - currentMatchResult, limit, null); + currentMatchResult, limit, null, actualWidth, actualHeight); } else { // 在上一层的匹配点附近进行局部匹配 for (Match match : previousMatchResult) { @@ -230,7 +253,7 @@ private static List matchTemplateWithPyramid(List imgPyramid, resourcesToRelease.add(m); resourcesToRelease.add(matchResult); getBestMatched(matchResult, currentTemplate, matchMethod, weakThreshold, - currentMatchResult, limit, r); + currentMatchResult, limit, r, actualWidth, actualHeight); } } @@ -257,6 +280,16 @@ private static List matchTemplateWithPyramid(List imgPyramid, return finalMatchResult; } + private static Mat toGrayscaleIfNeeded(Mat image, List resources) { + if (image.channels() == 1 && image.type() == CvType.CV_8UC1) { + return image; + } + Mat gray = new Mat(); + Imgproc.cvtColor(image, gray, Imgproc.COLOR_RGBA2GRAY); + resources.add(gray); + return gray; + } + private static Mat getPyramidDownAtLevel(Mat m, int level) { if (level == 0) { return m; @@ -297,10 +330,10 @@ private static Rect getROI(Point p, Mat src, Mat currentTemplate) { int w = (int) (currentTemplate.cols() * 1.5); int h = (int) (currentTemplate.rows() * 1.5); if (x + w >= src.cols()) { - w = src.cols() - x - 1; + w = src.cols() - x; } if (y + h >= src.rows()) { - h = src.rows() - y - 1; + h = src.rows() - y; } return new Rect(x, y, w, h); } @@ -316,7 +349,6 @@ private static int selectPyramidLevel(Mat img, Mat template) { return Math.min(6, maxLevel); } - private static Mat matchTemplate(Mat img, Mat temp, int match_method, Mat mask) { int result_cols = img.cols() - temp.cols() + 1; int result_rows = img.rows() - temp.rows() + 1; @@ -329,22 +361,30 @@ private static Mat matchTemplate(Mat img, Mat temp, int match_method, Mat mask) return result; } - private static void getBestMatched(Mat tmResult, Mat template, int matchMethod, float weakThreshold, List outResult, int limit, Rect rect) { + private static void getBestMatched(Mat tmResult, Mat template, int matchMethod, float weakThreshold, + List outResult, int limit, Rect rect, + int actualWidth, int actualHeight) { for (int i = 0; i < limit; i++) { - Match bestMatched = getBestMatched(tmResult, matchMethod, weakThreshold, rect); + Match bestMatched = getBestMatched(tmResult, matchMethod, weakThreshold, rect, + actualWidth, actualHeight); if (bestMatched == null) { break; } outResult.add(bestMatched); - Point start = new Point(Math.max(0, bestMatched.point.x - template.width() + 1), - Math.max(0, bestMatched.point.y - template.height() + 1)); - Point end = new Point(Math.min(tmResult.width(), bestMatched.point.x + template.width()), - Math.min(tmResult.height(), bestMatched.point.y + template.height())); + Point start = new Point( + Math.max(0, bestMatched.point.x - template.width() + 1), + Math.max(0, bestMatched.point.y - template.height() + 1) + ); + Point end = new Point( + Math.min(tmResult.width(), bestMatched.point.x + template.width()), + Math.min(tmResult.height(), bestMatched.point.y + template.height()) + ); Imgproc.rectangle(tmResult, start, end, new Scalar(0, 255, 0), -1); } } - private static Match getBestMatched(Mat tmResult, int matchMethod, float weakThreshold, Rect rect) { + private static Match getBestMatched(Mat tmResult, int matchMethod, float weakThreshold, + Rect rect, int actualWidth, int actualHeight) { TimingLogger logger = new TimingLogger(LOG_TAG, "best_matched_point"); Core.MinMaxLocResult mmr = Core.minMaxLoc(tmResult); logger.addSplit("minMaxLoc"); @@ -366,14 +406,16 @@ private static Match getBestMatched(Mat tmResult, int matchMethod, float weakThr } logger.addSplit("value:" + value); if (!Double.isFinite(value)) { - return getBestMatched(replaceNoFinite(tmResult), matchMethod, weakThreshold, rect); + return getBestMatched(replaceNoFinite(tmResult), matchMethod, weakThreshold, + rect, actualWidth, actualHeight); } - return new Match(pos, value); + return new Match(pos, value, actualWidth, actualHeight); } - private static Mat replaceNoFinite(Mat mat) { - if (mat.empty()) return null; + if (mat.empty()) { + return mat; + } if (mat.type() == CvType.CV_32FC1) { Core.patchNaNs(mat, 0.0); Mat infMask = new Mat(); diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/capture/ScreenCaptureManager.kt b/autojs/src/main/java/com/stardust/autojs/core/image/capture/ScreenCaptureManager.kt index a922b6d9f..ce73622ab 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/image/capture/ScreenCaptureManager.kt +++ b/autojs/src/main/java/com/stardust/autojs/core/image/capture/ScreenCaptureManager.kt @@ -19,8 +19,13 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeout import java.lang.ref.WeakReference import java.util.concurrent.CancellationException +import kotlin.time.Duration.Companion.milliseconds + +/** 绑定前台截图服务的最长等待时间；超时后失败返回，避免脚本线程永久挂起。 */ +private const val BIND_TIMEOUT_MILLIS = 10_000L class ScreenCaptureManager : ScreenCaptureRequester { @Volatile @@ -64,29 +69,45 @@ class ScreenCaptureManager : ScreenCaptureRequester { .getMediaProjection(Activity.RESULT_OK, result) CaptureForegroundService.setMediaProjection(context, mediaProjection!!) screenCapture = ScreenCapturer(mediaProjection!!, orientation) - } finally { serviceConnected.complete(Unit) - context.unbindService(this) + } catch (e: Exception) { + // 原来 finally 里无条件 complete(Unit)，getMediaProjection 抛异常时 + // 会伪装成成功，脚本随后拿到不可用的截屏器。 + serviceConnected.completeExceptionally(e) + } finally { + runCatching { context.unbindService(this) } + currentConnection = null } } override fun onServiceDisconnected(name: ComponentName?) { - // 服务意外断开时处理 + currentConnection = null serviceConnected.completeExceptionally(IllegalStateException("Service disconnected unexpectedly")) } } - // 先绑定服务再启动（避免延迟） val serviceIntent = Intent(context, CaptureForegroundService::class.java) - context.bindService( - serviceIntent, - connection, - Context.BIND_AUTO_CREATE - ) - - // 绑定后立即启动服务 - context.startForegroundService(serviceIntent) - serviceConnected.await() + val bound = context.bindService(serviceIntent, connection, Context.BIND_AUTO_CREATE) + if (!bound) { + serviceConnected.completeExceptionally( + IllegalStateException("bind CaptureForegroundService failed") + ) + } else { + currentConnection = connection + context.startForegroundService(serviceIntent) + } + + try { + // 服务被系统拒绝或迟迟不回调时不能无限挂起脚本线程。 + withTimeout(BIND_TIMEOUT_MILLIS) { serviceConnected.await() } + } catch (e: kotlinx.coroutines.TimeoutCancellationException) { + // 从未回调时 onServiceConnected 的 finally 不会解绑，这里补上。 + currentConnection?.let { + runCatching { context.unbindService(it) } + currentConnection = null + } + throw IllegalStateException("CaptureForegroundService bind timeout", e) + } } override fun requestScreenCaptureLegacy(context: Context, orientation: Int): ScriptPromiseAdapter { @@ -127,7 +148,7 @@ class ScreenCaptureManager : ScreenCaptureRequester { if (e.message?.contains("non-current") == true && retryCount < maxRetry) { retryCount++ Log.w("SCREEN_LEGACY", "MediaProjection expired, retrying ($retryCount/$maxRetry)") - delay(100) + delay(100.milliseconds) doRequest() } else { Log.e("SCREEN_LEGACY", "Failed after $maxRetry retries", e) @@ -163,20 +184,20 @@ class ScreenCaptureManager : ScreenCaptureRequester { } } - override fun recycle() { - // 取消协程 - currentCoroutineScope?.cancel() - currentCoroutineScope = null + override fun recycle() { + // 取消协程 + currentCoroutineScope?.cancel() + currentCoroutineScope = null - // 清理连接 - currentConnection = null + // 清理连接 + currentConnection = null - // 释放截图器 - screenCapture?.release() - screenCapture = null + // 释放截图器 + screenCapture?.release() + screenCapture = null - // 停止并释放 MediaProjection - mediaProjection?.stop() - mediaProjection = null - } + // 停止并释放 MediaProjection + mediaProjection?.stop() + mediaProjection = null + } } \ No newline at end of file diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/capture/ScreenCapturer.kt b/autojs/src/main/java/com/stardust/autojs/core/image/capture/ScreenCapturer.kt index f601144e1..193e855fa 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/image/capture/ScreenCapturer.kt +++ b/autojs/src/main/java/com/stardust/autojs/core/image/capture/ScreenCapturer.kt @@ -108,6 +108,7 @@ class ScreenCapturer( private fun refreshVirtualDisplay(orientation: Int) = synchronized(this) { latestImage.set(null) + cachedImageBitmap.set(null) mImageReader.close() val screenHeight = ScreenMetrics.getOrientationAwareScreenHeight(orientation) val screenWidth = ScreenMetrics.getOrientationAwareScreenWidth(orientation) diff --git a/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.kt b/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.kt index a01c458af..b40f4bdf2 100644 --- a/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.kt +++ b/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.kt @@ -15,6 +15,8 @@ import com.stardust.autojs.annotation.ScriptVariable import com.stardust.autojs.core.image.ColorFinder import com.stardust.autojs.core.image.ImageWrapper import com.stardust.autojs.core.image.TemplateMatching +import com.stardust.autojs.core.image.TemplateMatching.MATCHING_METHOD_DEFAULT +import com.stardust.autojs.core.image.capture.ScreenCapturer import com.stardust.autojs.core.image.capture.ScreenCaptureRequester import com.stardust.autojs.core.opencv.Mat import com.stardust.autojs.core.opencv.OpenCVHelper @@ -36,6 +38,13 @@ import java.io.FileOutputStream import java.io.IOException import java.net.HttpURLConnection import java.net.URL +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.math.sqrt + +/** legacy 截图权限请求的默认超时：给系统授权与内部重试留足时间。 */ +private const val DEFAULT_CAPTURE_TIMEOUT_MS = 15_000L /** * Created by Stardust on 2017/5/20. @@ -65,44 +74,8 @@ class Images( } } - fun requestScreenCaptureLegacy(orientation: Int): ScriptPromiseAdapter { - val outerPromise = ScriptPromiseAdapter() - val weakImages = java.lang.ref.WeakReference(this) - - try { - val images = weakImages.get() ?: run { - outerPromise.resolve(false) - return outerPromise - } - - val innerPromise = images.mScreenCaptureRequester.requestScreenCaptureLegacy( - images.mContext, - orientation - ) - - innerPromise.onResolve(object : ScriptPromiseAdapter.Callback { - override fun call(arg: Any?) { - outerPromise.resolve(arg) - } - }) - - innerPromise.onReject(object : ScriptPromiseAdapter.Callback { - override fun call(arg: Any?) { - val img = weakImages.get() - img?.mScriptRuntime?.toast(arg as String?) - Log.e(Images::class.java.name, "请求截图权限失败: $arg") - outerPromise.resolve(false) - } - }) - } catch (e: Exception) { - val images = weakImages.get() - images?.mScriptRuntime?.toast(e.message) - Log.e(Images::class.java.name, "请求截图权限失败", e) - outerPromise.resolve(false) - } - - return outerPromise - } + // legacy 截图权限统一走文件末尾的同步版本（Boolean），不再提供 + // 同名 1 参 Promise 重载，避免两种返回语义与 Rhino 重载歧义。 fun stopScreenCapturer() { mScreenCaptureRequester.recycle() @@ -241,6 +214,11 @@ class Images( fun releaseScreenCapturer() { disposables.forEach { it.dispose() } disposables.clear() + try { + mScreenCaptureRequester.recycle() + } catch (e: Exception) { + Log.e(Images::class.java.name, "Error recycling screen capture", e) + } } @JvmOverloads @@ -261,7 +239,9 @@ class Images( threshold: Float, rect: Rect?, maxLevel: Int, - transparentMask: Boolean = false + transparentMask: Boolean = false, + useGrayscale: Boolean = false, + matchMethod: Int = MATCHING_METHOD_DEFAULT ): Point? { initOpenCvIfNeeded() if (image == null) throw NullPointerException("image = null") @@ -271,8 +251,8 @@ class Images( src = Mat(src, rect) } val point = TemplateMatching.fastTemplateMatching( - src, template.mat, TemplateMatching.MATCHING_METHOD_DEFAULT, - weakThreshold, threshold, maxLevel, transparentMask + src, template.mat, matchMethod, + weakThreshold, threshold, maxLevel, transparentMask, useGrayscale ) if (point != null) { if (rect != null) { @@ -297,18 +277,21 @@ class Images( rect: Rect?, maxLevel: Int, limit: Int, - transparentMask: Boolean = false + transparentMask: Boolean = false, + useGrayscale: Boolean = false, + matchMethod: Int = MATCHING_METHOD_DEFAULT ): List { - val templates: List = listOf(template) return matchMultiTemplates( image, - templates, + listOf(template), weakThreshold, threshold, rect, maxLevel, limit, - transparentMask + transparentMask, + useGrayscale, + matchMethod ) } @@ -321,7 +304,9 @@ class Images( rect: Rect?, maxLevel: Int, limit: Int, - transparentMask: Boolean = false + transparentMask: Boolean = false, + useGrayscale: Boolean = false, + matchMethod: Int = MATCHING_METHOD_DEFAULT ): List { initOpenCvIfNeeded() if (image == null) throw NullPointerException("image = null") @@ -332,8 +317,8 @@ class Images( } val result = TemplateMatching.fastMultiTemplateMatching( - src, templates.map { it?.mat }, Imgproc.TM_CCOEFF_NORMED, - weakThreshold, threshold, maxLevel, limit, transparentMask + src, templates.map { it?.mat }, matchMethod, + weakThreshold, threshold, maxLevel, limit, transparentMask, useGrayscale ) for (match in result) { val point = match.point @@ -341,8 +326,10 @@ class Images( point.x += rect.x.toDouble() point.y += rect.y.toDouble() } + match.width *= mScreenMetrics.scaleX(1) + match.height *= mScreenMetrics.scaleY(1) point.x = mScreenMetrics.scaleX(point.x.toInt()).toDouble() - point.y = mScreenMetrics.scaleX(point.y.toInt()).toDouble() + point.y = mScreenMetrics.scaleY(point.y.toInt()).toDouble() } if (src !== image.mat) { OpenCVHelper.release(src) @@ -369,7 +356,6 @@ class Images( mScriptRuntime.console.info("opencv initialized") } - fun pixel(image: ImageWrapper?, x: Int, y: Int): Int { if (image == null) { throw NullPointerException("image = null") @@ -446,4 +432,173 @@ class Images( return Bitmap.createBitmap(origin, 0, 0, width, height, matrix, false) } -} + // ========== 新增方法 ========== + + /** + * 获取图像的亮度值 + * @param image 图像 + * @param sampleCount 采样数量 + * @param region 采样区域 + * @return 亮度值 + */ + @JvmOverloads + fun getBrightness(image: ImageWrapper?, sampleCount: Int = 50, region: Rect? = null): Float { + if (image == null) { + throw NullPointerException("image = null") + } + val img = image.mat + val w = img.cols() + val h = img.rows() + var startX = 0 + var startY = 0 + var endX = w + var endY = h + + if (region != null) { + startX = region.x.coerceAtLeast(0) + startY = region.y.coerceAtLeast(0) + endX = (region.x + region.width).coerceAtMost(w) + endY = (region.y + region.height).coerceAtMost(h) + } + + val regionW = endX - startX + val regionH = endY - startY + if (regionW <= 0 || regionH <= 0) return 0f + + // sampleCount 为 0/负数时原本会除零得到 Infinity，这里收敛到合法区间 + val samples = sampleCount.coerceIn(1, regionW * regionH) + val step = maxOf(1, sqrt((regionW * regionH).toDouble() / samples).toInt()) + var total = 0.0 + var count = 0 + val channels = img.channels() + + // OpenCV Mat.get(row, col) = (y, x) + for (y in startY until endY step step) { + for (x in startX until endX step step) { + val pixel = img.get(y, x) + val brightness = when (channels) { + 1, 2 -> pixel[0] + 3, 4 -> (pixel[0] + pixel[1] + pixel[2]) / 3.0 + else -> pixel[0] + } + total += brightness + count++ + } + } + + return if (count > 0) (total / count).toFloat() else 0f + } + + /** + * 请求截图权限（legacy 通道，不跳转应用），同步等待结果。 + * + * 提供 0 / 1 / 2 参三个重载，统一返回 Boolean： + * - 无参：自动方向 + 默认超时 + * - 1 参：Boolean 当作横屏开关，Number 当作方向常量，其它值回退自动方向 + * - 2 参：(方向或横屏开关, 超时毫秒) + * + * 参数一律用 Any? 接收：JS 省略实参时 Rhino 传的是 + * org.mozilla.javascript.Undefined，基本类型无法匹配会报「找不到方法」。 + * 三个重载 arity 不同，Rhino 按实参个数选择，不会歧义；不要再引入返回 + * ScriptPromiseAdapter 的同名重载，那会重演仅返回类型不同的歧义问题。 + * + * 用 CountDownLatch 而不是 runBlocking：脚本 forceStop 会 interrupt 线程， + * latch.await 能响应中断；结果回调由 ScreenCaptureManager 在主线程触发， + * 不依赖脚本线程的事件循环，因此不会死锁。 + */ + fun requestScreenCaptureLegacy(): Boolean = + requestLegacyWithTimeout(ScreenCapturer.ORIENTATION_AUTO, DEFAULT_CAPTURE_TIMEOUT_MS) + + fun requestScreenCaptureLegacy(arg: Any?): Boolean = + requestLegacyWithTimeout(normalizeOrientation(arg), DEFAULT_CAPTURE_TIMEOUT_MS) + + fun requestScreenCaptureLegacy(orientation: Any?, timeout: Any?): Boolean = + requestLegacyWithTimeout(normalizeOrientation(orientation), normalizeTimeout(timeout)) + + private fun normalizeOrientation(value: Any?): Int = when (value) { + is Boolean -> if (value) ScreenCapturer.ORIENTATION_LANDSCAPE + else ScreenCapturer.ORIENTATION_PORTRAIT + is Number -> value.toInt() + else -> ScreenCapturer.ORIENTATION_AUTO + } + + private fun normalizeTimeout(value: Any?): Long = + (value as? Number)?.toLong()?.takeIf { it > 0 } ?: DEFAULT_CAPTURE_TIMEOUT_MS + + private fun requestLegacyWithTimeout(orientation: Int, timeoutMs: Long): Boolean { + val latch = CountDownLatch(1) + val result = AtomicBoolean(false) + val delivered = AtomicBoolean(false) + + requestLegacyCapture(orientation) { ok -> + if (delivered.compareAndSet(false, true)) { + result.set(ok) + latch.countDown() + } + } + + return try { + if (latch.await(timeoutMs, TimeUnit.MILLISECONDS)) { + result.get() + } else { + Log.w(Images::class.java.name, "请求截图权限超时: ${timeoutMs}ms") + false + } + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + Log.w(Images::class.java.name, "请求截图权限等待被中断") + false + } + } + + /** 内部实现：把 ScreenCaptureRequester 的 Promise 转成一次性回调。 */ + private fun requestLegacyCapture(orientation: Int, callback: (Boolean) -> Unit) { + try { + val innerPromise = mScreenCaptureRequester.requestScreenCaptureLegacy(mContext, orientation) + innerPromise.onResolve(object : ScriptPromiseAdapter.Callback { + override fun call(arg: Any?) { + callback((arg as? Boolean) ?: true) + } + }) + innerPromise.onReject(object : ScriptPromiseAdapter.Callback { + override fun call(arg: Any?) { + mScriptRuntime.toast(arg?.toString()) + Log.e(Images::class.java.name, "请求截图权限失败: $arg") + callback(false) + } + }) + } catch (e: Exception) { + mScriptRuntime.toast(e.message) + Log.e(Images::class.java.name, "请求截图权限失败", e) + callback(false) + } + } + + /** + * 回收所有追踪的图像 + */ + fun recycleAllImages() { + ImageWrapper.recycleAllTrackedImages() + } + + /** + * 获取存活图像数量 + */ + fun getAliveImageCount(): Int { + return ImageWrapper.getTrackedImageCount() + } + + /** + * 启用图像追踪 + */ + fun enableImageTracking() { + ImageWrapper.enableTracking() + } + + /** + * 禁用图像追踪 + */ + fun disableImageTracking() { + ImageWrapper.disableTracking() + } +} \ No newline at end of file diff --git a/autojs/src/main/js/v6-api/src/images/index.ts b/autojs/src/main/js/v6-api/src/images/index.ts index f8288cae1..1ac1be47b 100644 --- a/autojs/src/main/js/v6-api/src/images/index.ts +++ b/autojs/src/main/js/v6-api/src/images/index.ts @@ -16,6 +16,8 @@ interface FindImageOptions { level?: number weakThreshold?: number transparentMask?: boolean + useGrayscale?: boolean + matchMethod?: number } function images() { @@ -56,7 +58,12 @@ images.requestScreenCapture = function (landscape: boolean) { return javaImages.requestScreenCapture(orientation); } -images.requestScreenCaptureLegacy = function (landscape: boolean) { +images.requestScreenCaptureLegacy = function (landscape?: boolean | number, timeout?: number) { + // 只给一个数字时视为超时（landscape 是 boolean，传数字只能是 timeout）。 + if (typeof landscape === 'number' && timeout === undefined) { + timeout = landscape; + landscape = undefined; + } const ScreenCapturer = com.stardust.autojs.core.image.capture.ScreenCapturer; var orientation = ScreenCapturer.ORIENTATION_AUTO; if (landscape === true) { @@ -65,7 +72,15 @@ images.requestScreenCaptureLegacy = function (landscape: boolean) { if (landscape === false) { orientation = ScreenCapturer.ORIENTATION_PORTRAIT; } - return javaImages.requestScreenCaptureLegacy(orientation); + // 按实参个数分发到 Java 的 0 / 1 / 2 参重载，不把 undefined 交给 + // Rhino 做 Java 方法匹配（基本类型无法匹配 Undefined）。 + if (landscape === undefined && timeout === undefined) { + return javaImages.requestScreenCaptureLegacy(); + } + if (timeout === undefined) { + return javaImages.requestScreenCaptureLegacy(orientation); + } + return javaImages.requestScreenCaptureLegacy(orientation, timeout); }; images.save = function (img: Image, path: string, format?: ImageFormat, quality?: number) { @@ -327,11 +342,14 @@ images.findImage = function (img: Image, template: Image, options?: FindImageOpt maxLevel = options.level; } var weakThreshold = options.weakThreshold || 0.6; - const transparentMask = !!options.transparentMask + const transparentMask = !!options.transparentMask; + const useGrayscale = !!options.useGrayscale; + // 0 是合法的 TM_SQDIFF，不能用 || 回退默认值。 + const matchMethod = options.matchMethod != null ? options.matchMethod : 5; if (options.region) { - return javaImages.findImage(img, template, weakThreshold, threshold, buildRegion(options.region, img), maxLevel, transparentMask); + return javaImages.findImage(img, template, weakThreshold, threshold, buildRegion(options.region, img), maxLevel, transparentMask, useGrayscale, matchMethod); } else { - return javaImages.findImage(img, template, weakThreshold, threshold, null, maxLevel, transparentMask); + return javaImages.findImage(img, template, weakThreshold, threshold, null, maxLevel, transparentMask, useGrayscale, matchMethod); } } @@ -352,11 +370,13 @@ images.matchMultiTemplates = function (img: Image, templates: Image[], var max = options.max || 5; var weakThreshold = options.weakThreshold || 0.6; var result; - const transparentMask = !!options.transparentMask + const transparentMask = !!options.transparentMask; + const useGrayscale = !!options.useGrayscale; + const matchMethod = options.matchMethod != null ? options.matchMethod : 5; if (options.region) { - result = javaImages.matchMultiTemplates(img, templates, weakThreshold, threshold, buildRegion(options.region, img), maxLevel, max, transparentMask); + result = javaImages.matchMultiTemplates(img, templates, weakThreshold, threshold, buildRegion(options.region, img), maxLevel, max, transparentMask, useGrayscale, matchMethod); } else { - result = javaImages.matchMultiTemplates(img, templates, weakThreshold, threshold, null, maxLevel, max, transparentMask); + result = javaImages.matchMultiTemplates(img, templates, weakThreshold, threshold, null, maxLevel, max, transparentMask, useGrayscale, matchMethod); } return new MatchingResult(result); } @@ -409,6 +429,37 @@ images.matToImage = function (img: Image) { return ImageWrapper.ofMat(img); } +images.getBrightness = function (img: Image, sampleCount?: number, region?: [number, number, number?, number?]) { + initIfNeeded(); + sampleCount = sampleCount || 50; + + var rect = null; + if (region && region.length >= 2) { + var x = region[0] || 0; + var y = region[1] || 0; + var w = region[2] !== undefined ? region[2] : img.width - x; + var h = region[3] !== undefined ? region[3] : img.height - y; + rect = new org.opencv.core.Rect(x, y, w, h); + } + + return javaImages.getBrightness(img, sampleCount, rect); +}; + +images.enableImageTracking = function () { + return javaImages.enableImageTracking(); +}; + +images.disableImageTracking = function () { + return javaImages.disableImageTracking(); +}; + +images.recycleAllImages = function () { + return javaImages.recycleAllImages(); +}; + +images.getAliveImageCount = function () { + return javaImages.getAliveImageCount(); +}; function toPointArray(points: unknown[]) { var arr = []; @@ -463,5 +514,4 @@ declare global { var findColorEquals: typeof images['findColorEquals'] } -export default images; - +export default images; \ No newline at end of file