diff --git a/image_based_actions/pom.xml b/image_based_actions/pom.xml
index ef5a616d..e456f4da 100644
--- a/image_based_actions/pom.xml
+++ b/image_based_actions/pom.xml
@@ -6,7 +6,7 @@
4.0.0
com.testsigma.addons
image_based_actions
- 1.0.16
+ 1.0.17
jar
@@ -67,6 +67,12 @@
3.17.0
+
+
+ org.apache.httpcomponents
+ httpclient
+ 4.5.14
+
diff --git a/image_based_actions/src/main/java/com/testsigma/addons/util/ImageMatchUtils.java b/image_based_actions/src/main/java/com/testsigma/addons/util/ImageMatchUtils.java
new file mode 100644
index 00000000..e9205021
--- /dev/null
+++ b/image_based_actions/src/main/java/com/testsigma/addons/util/ImageMatchUtils.java
@@ -0,0 +1,312 @@
+package com.testsigma.addons.util;
+
+import com.testsigma.sdk.Logger;
+
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.Iterator;
+
+public class ImageMatchUtils {
+
+ private static final int MAX_SCALES = 30;
+ private static final int MIN_TEMPLATE_DIM = 8;
+
+ public static class MatchResult {
+ public final boolean found;
+ public final int x1, y1, x2, y2;
+ public final double confidence;
+ public final String method;
+ public final String message;
+
+ public MatchResult(boolean found, int x1, int y1, int x2, int y2,
+ double confidence, String method, String message) {
+ this.found = found;
+ this.x1 = x1;
+ this.y1 = y1;
+ this.x2 = x2;
+ this.y2 = y2;
+ this.confidence = confidence;
+ this.method = method;
+ this.message = message;
+ }
+
+ public static MatchResult notFound(String method, String msg) {
+ return new MatchResult(false, 0, 0, 0, 0, 0, method, msg);
+ }
+ }
+
+ // ======================== Multi-Scale Search ========================
+
+ public static MatchResult searchMultiScale(double[][] baseData, int bw, int bh,
+ double[][] tmplGray, int tw, int th,
+ boolean edgeMode, double threshold,
+ String methodName, Logger logger) {
+ int tmplMaxDim = Math.max(tw, th);
+ double minScale = Math.max(0.05, (double) MIN_TEMPLATE_DIM / tmplMaxDim);
+ double maxScale = Math.min(4.0,
+ (double) Math.min(bw, bh) * 0.5 / Math.max(tw, th));
+
+ if (minScale >= maxScale) {
+ return MatchResult.notFound(methodName,
+ "Invalid scale range [" + minScale + ", " + maxScale + "]");
+ }
+
+ int numScales = Math.min(MAX_SCALES,
+ Math.max(10, (int) ((maxScale - minScale) / 0.03) + 1));
+ logger.info(String.format("%s: scale %.3f->%.3f, %d steps, edgeMode=%b",
+ methodName, minScale, maxScale, numScales, edgeMode));
+
+ double bestScore = -1;
+ int bestX = 0, bestY = 0, bestW = 0, bestH = 0;
+
+ for (int si = 0; si < numScales; si++) {
+ double scale = maxScale - (maxScale - minScale) * si / Math.max(1, numScales - 1);
+ int nw = (int) Math.round(tw * scale);
+ int nh = (int) Math.round(th * scale);
+
+ if (nw < MIN_TEMPLATE_DIM || nh < MIN_TEMPLATE_DIM || nw >= bw || nh >= bh) continue;
+
+ double[][] scaledTmpl = resize(tmplGray, tw, th, nw, nh);
+ double[][] searchTmpl = edgeMode
+ ? sobelEdgeDetection(scaledTmpl, nw, nh)
+ : scaledTmpl;
+
+ double tmplMean = mean(searchTmpl, nw, nh);
+ double tmplStd = std(searchTmpl, nw, nh, tmplMean);
+ if (tmplStd < 1e-6) continue;
+
+ int step = Math.max(1, Math.min(nw, nh) / 4);
+
+ double scaleBest = -1;
+ int sBestX = 0, sBestY = 0;
+
+ for (int y = 0; y <= bh - nh; y += step) {
+ for (int x = 0; x <= bw - nw; x += step) {
+ double ncc = computeNCC(baseData, bw, searchTmpl, nw, nh,
+ x, y, tmplMean, tmplStd);
+ if (ncc > scaleBest) {
+ scaleBest = ncc;
+ sBestX = x;
+ sBestY = y;
+ }
+ }
+ }
+
+ if (step > 1 && scaleBest > threshold * 0.6) {
+ int rx = sBestX, ry = sBestY;
+ for (int dy = -step; dy <= step; dy++) {
+ for (int dx = -step; dx <= step; dx++) {
+ int cx = rx + dx, cy = ry + dy;
+ if (cx < 0 || cy < 0 || cx + nw > bw || cy + nh > bh) continue;
+ double ncc = computeNCC(baseData, bw, searchTmpl, nw, nh,
+ cx, cy, tmplMean, tmplStd);
+ if (ncc > scaleBest) {
+ scaleBest = ncc;
+ sBestX = cx;
+ sBestY = cy;
+ }
+ }
+ }
+ }
+
+ if (scaleBest > bestScore) {
+ bestScore = scaleBest;
+ bestX = sBestX;
+ bestY = sBestY;
+ bestW = nw;
+ bestH = nh;
+ }
+ }
+
+ logger.info(String.format("%s: best NCC=%.4f at (%d,%d) size %dx%d",
+ methodName, bestScore, bestX, bestY, bestW, bestH));
+
+ if (bestScore >= threshold) {
+ return new MatchResult(true, bestX, bestY,
+ bestX + bestW, bestY + bestH, bestScore, methodName, "Matched");
+ }
+ return MatchResult.notFound(methodName,
+ String.format("Best NCC %.4f < threshold %.4f", bestScore, threshold));
+ }
+
+ // ======================== NCC Computation ========================
+
+ public static double computeNCC(double[][] base, int baseW,
+ double[][] tmpl, int tw, int th,
+ int sx, int sy,
+ double tmplMean, double tmplStd) {
+ int n = tw * th;
+ double baseMean = 0;
+ for (int y = 0; y < th; y++)
+ for (int x = 0; x < tw; x++)
+ baseMean += base[sy + y][sx + x];
+ baseMean /= n;
+
+ double crossCorr = 0;
+ double baseVar = 0;
+ for (int y = 0; y < th; y++) {
+ for (int x = 0; x < tw; x++) {
+ double bd = base[sy + y][sx + x] - baseMean;
+ double td = tmpl[y][x] - tmplMean;
+ crossCorr += bd * td;
+ baseVar += bd * bd;
+ }
+ }
+
+ double baseStd = Math.sqrt(baseVar / n);
+ if (baseStd < 1e-6) return 0;
+ return crossCorr / (n * baseStd * tmplStd);
+ }
+
+ // ======================== Image Processing ========================
+
+ public static double[][] toGrayscale(BufferedImage img) {
+ int w = img.getWidth(), h = img.getHeight();
+
+ BufferedImage opaque = new BufferedImage(w, h, BufferedImage.TYPE_INT_RGB);
+ Graphics2D g = opaque.createGraphics();
+ g.setColor(Color.WHITE);
+ g.fillRect(0, 0, w, h);
+ g.drawImage(img, 0, 0, null);
+ g.dispose();
+
+ double[][] gray = new double[h][w];
+ for (int y = 0; y < h; y++) {
+ for (int x = 0; x < w; x++) {
+ int rgb = opaque.getRGB(x, y);
+ int r = (rgb >> 16) & 0xFF;
+ int gv = (rgb >> 8) & 0xFF;
+ int b = rgb & 0xFF;
+ gray[y][x] = 0.299 * r + 0.587 * gv + 0.114 * b;
+ }
+ }
+ return gray;
+ }
+
+ public static double[][] resize(double[][] img, int ow, int oh, int nw, int nh) {
+ double[][] out = new double[nh][nw];
+ double xr = (double) ow / nw;
+ double yr = (double) oh / nh;
+ for (int y = 0; y < nh; y++) {
+ double srcY = y * yr;
+ int sy = Math.min((int) srcY, oh - 1);
+ int sy1 = Math.min(sy + 1, oh - 1);
+ double fy = srcY - sy;
+ for (int x = 0; x < nw; x++) {
+ double srcX = x * xr;
+ int sx = Math.min((int) srcX, ow - 1);
+ int sx1 = Math.min(sx + 1, ow - 1);
+ double fx = srcX - sx;
+ out[y][x] = (1 - fx) * (1 - fy) * img[sy][sx]
+ + fx * (1 - fy) * img[sy][sx1]
+ + (1 - fx) * fy * img[sy1][sx]
+ + fx * fy * img[sy1][sx1];
+ }
+ }
+ return out;
+ }
+
+ public static double[][] sobelEdgeDetection(double[][] gray, int w, int h) {
+ double[][] edges = new double[h][w];
+ double maxVal = 0;
+ for (int y = 1; y < h - 1; y++) {
+ for (int x = 1; x < w - 1; x++) {
+ double gx = -gray[y - 1][x - 1] + gray[y - 1][x + 1]
+ - 2 * gray[y][x - 1] + 2 * gray[y][x + 1]
+ - gray[y + 1][x - 1] + gray[y + 1][x + 1];
+ double gy = -gray[y - 1][x - 1] - 2 * gray[y - 1][x] - gray[y - 1][x + 1]
+ + gray[y + 1][x - 1] + 2 * gray[y + 1][x] + gray[y + 1][x + 1];
+ double mag = Math.sqrt(gx * gx + gy * gy);
+ edges[y][x] = mag;
+ if (mag > maxVal) maxVal = mag;
+ }
+ }
+ if (maxVal > 0) {
+ double inv = 255.0 / maxVal;
+ for (int y = 0; y < h; y++)
+ for (int x = 0; x < w; x++)
+ edges[y][x] *= inv;
+ }
+ return edges;
+ }
+
+ public static double mean(double[][] data, int w, int h) {
+ double sum = 0;
+ for (int y = 0; y < h; y++)
+ for (int x = 0; x < w; x++)
+ sum += data[y][x];
+ return sum / (w * h);
+ }
+
+ public static double std(double[][] data, int w, int h, double mean) {
+ double sumSq = 0;
+ for (int y = 0; y < h; y++)
+ for (int x = 0; x < w; x++) {
+ double d = data[y][x] - mean;
+ sumSq += d * d;
+ }
+ return Math.sqrt(sumSq / (w * h));
+ }
+
+ // ======================== File Utilities ========================
+
+ public static int[] getImageFileDimensions(File imageFile) throws Exception {
+ try (ImageInputStream iis = ImageIO.createImageInputStream(imageFile)) {
+ Iterator readers = ImageIO.getImageReaders(iis);
+ if (readers.hasNext()) {
+ ImageReader reader = readers.next();
+ try {
+ reader.setInput(iis);
+ return new int[]{reader.getWidth(0), reader.getHeight(0)};
+ } finally {
+ reader.dispose();
+ }
+ }
+ }
+ BufferedImage img = ImageIO.read(imageFile);
+ return new int[]{img.getWidth(), img.getHeight()};
+ }
+
+ public static File downloadImage(String fileName, String url, Logger logger) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Downloading: " + url);
+ URL urlObject = new URL(url);
+ String extension = ".png";
+ String urlPath = urlObject.getPath();
+ int dot = urlPath.lastIndexOf('.');
+ if (dot > 0) extension = urlPath.substring(dot);
+
+ File tempFile = File.createTempFile(fileName, extension);
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ logger.info("Downloaded to: " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Local path: " + url);
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Image access error: " + url);
+ throw new RuntimeException("Unable to access image: " + url, e);
+ }
+ }
+
+ public static void cleanupFile(File file) {
+ try {
+ if (file != null && file.exists() && file.isFile()) {
+ file.delete();
+ }
+ } catch (Exception ignored) {
+ }
+ }
+}
diff --git a/image_based_actions/src/main/java/com/testsigma/addons/util/ScreenshotUtils.java b/image_based_actions/src/main/java/com/testsigma/addons/util/ScreenshotUtils.java
new file mode 100644
index 00000000..7be6e64a
--- /dev/null
+++ b/image_based_actions/src/main/java/com/testsigma/addons/util/ScreenshotUtils.java
@@ -0,0 +1,131 @@
+package com.testsigma.addons.util;
+
+import com.testsigma.sdk.Logger;
+import com.testsigma.sdk.TestStepResult;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.FileEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+
+public class ScreenshotUtils {
+
+ private static final RequestConfig config = RequestConfig.custom()
+ .setConnectTimeout(30000)
+ .setConnectionRequestTimeout(30000)
+ .setSocketTimeout(30000)
+ .build();
+
+ /**
+ * Draws a red bounding box and green crosshair at the matched location on the image.
+ */
+ public static BufferedImage highlightClickLocation(BufferedImage image,
+ int x1, int y1, int x2, int y2) {
+ BufferedImage copy = new BufferedImage(image.getWidth(), image.getHeight(), BufferedImage.TYPE_INT_RGB);
+ Graphics2D g2d = copy.createGraphics();
+ g2d.drawImage(image, 0, 0, null);
+
+ g2d.setColor(Color.MAGENTA);
+ g2d.setStroke(new BasicStroke(3));
+ g2d.drawRect(x1, y1, x2 - x1, y2 - y1);
+
+ int centerX = (x1 + x2) / 2;
+ int centerY = (y1 + y2) / 2;
+ g2d.setColor(Color.GREEN);
+ g2d.setStroke(new BasicStroke(2));
+ g2d.drawRect(centerX-1, centerY-1, 1, 1); // small square at center
+// int crossSize = 15;
+// g2d.drawLine(centerX - crossSize, centerY, centerX + crossSize, centerY);
+// g2d.drawLine(centerX, centerY - crossSize, centerX, centerY + crossSize);
+// g2d.drawOval(centerX - crossSize, centerY - crossSize, crossSize * 2, crossSize * 2);
+
+ g2d.dispose();
+ return copy;
+ }
+
+ public static File saveScreenshotToFile(BufferedImage image, String fileName) throws Exception {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(image, "PNG", tempFile);
+ return tempFile;
+ }
+
+ public static boolean uploadScreenshotToS3(TestStepResult testStepResult, File screenshotFile, Logger logger) {
+ try {
+ String s3Url = testStepResult.getScreenshotUrl();
+ if (s3Url != null && !s3Url.isEmpty() && screenshotFile.exists()) {
+ boolean result = uploadFile(s3Url, screenshotFile.getAbsolutePath(), logger);
+ if (result) {
+ logger.info("Successfully uploaded screenshot to S3");
+ } else {
+ logger.info("Failed to upload screenshot to S3");
+ }
+ return result;
+ } else {
+ logger.info("S3 URL is null/empty or screenshot file doesn't exist, skipping upload");
+ return false;
+ }
+ } catch (Exception e) {
+ logger.info("Exception during screenshot upload: " + e.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * Highlights the matched region, saves to a temp file, and uploads to S3.
+ * Returns the temp file (caller should clean up).
+ */
+ public static File highlightAndUpload(BufferedImage baseImage, int x1, int y1, int x2, int y2,
+ String filePrefix, TestStepResult testStepResult, Logger logger) {
+ File highlightedFile = null;
+ try {
+ BufferedImage highlighted = highlightClickLocation(baseImage, x1, y1, x2, y2);
+ highlightedFile = saveScreenshotToFile(highlighted, filePrefix);
+ uploadScreenshotToS3(testStepResult, highlightedFile, logger);
+ } catch (Exception e) {
+ logger.info("Error during highlight and upload: " + e.getMessage());
+ }
+ return highlightedFile;
+ }
+
+ /**
+ * Uploads a plain (non-highlighted) screenshot — used on failure paths.
+ */
+ public static void uploadPlainScreenshot(File screenshotFile, TestStepResult testStepResult, Logger logger) {
+ try {
+ uploadScreenshotToS3(testStepResult, screenshotFile, logger);
+ } catch (Exception e) {
+ logger.info("Error uploading plain screenshot: " + e.getMessage());
+ }
+ }
+
+ private static boolean uploadFile(String s3SignedURL, String localPath, Logger logger) {
+ logger.info("Uploading to S3, presigned-URL: " + s3SignedURL);
+ File file = new File(localPath);
+ if (!file.exists()) {
+ logger.info("Local file does not exist: " + localPath);
+ return false;
+ }
+ try (CloseableHttpClient httpclient = HttpClients.custom().setDefaultRequestConfig(config).build()) {
+ HttpPut httpPut = new HttpPut(s3SignedURL);
+ httpPut.setEntity(new FileEntity(file));
+ HttpResponse response = httpclient.execute(httpPut);
+ int statusCode = response.getStatusLine().getStatusCode();
+ if (statusCode == 200) {
+ logger.info("Upload completed successfully");
+ return true;
+ } else {
+ logger.info("Upload failed with status code: " + statusCode);
+ return false;
+ }
+ } catch (Exception e) {
+ logger.info("Exception while uploading screenshot to S3: " + e.getMessage());
+ return false;
+ }
+ }
+}
diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnCoordinates.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnCoordinates.java
new file mode 100644
index 00000000..ec51fccc
--- /dev/null
+++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnCoordinates.java
@@ -0,0 +1,78 @@
+package com.testsigma.addons.windows;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.image.BufferedImage;
+import java.io.File;
+
+@Action(actionText = "Click on coordinates x-value: x-coordinate, y-value: y-coordinate",
+ description = "This action clicks on the screen at the specified coordinates.",
+ applicationType = ApplicationType.WINDOWS,
+ useCustomScreenshot = true)
+public class ClickOnCoordinates extends WindowsAction {
+
+ @TestData(reference = "x-coordinate")
+ private com.testsigma.sdk.TestData xCoordinate;
+
+ @TestData(reference = "y-coordinate")
+ private com.testsigma.sdk.TestData yCoordinate;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() {
+ File screenshotFile = null;
+ try {
+ String xCoordinateValue = xCoordinate.getValue().toString();
+ String yCoordinateValue = yCoordinate.getValue().toString();
+ int x = Integer.parseInt(xCoordinateValue);
+ int y = Integer.parseInt(yCoordinateValue);
+
+ BufferedImage screenCapture = new Robot().createScreenCapture(
+ new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()));
+
+ Graphics2D g2d = screenCapture.createGraphics();
+ g2d.setColor(Color.RED);
+ g2d.setStroke(new BasicStroke(2));
+ g2d.drawOval(x - 15, y - 15, 30, 30);
+ g2d.setColor(Color.GREEN);
+ g2d.setStroke(new BasicStroke(2));
+ g2d.drawLine(x - 15, y, x + 15, y);
+ g2d.drawLine(x, y - 15, x, y + 15);
+ g2d.dispose();
+
+ screenshotFile = File.createTempFile("click_coordinates_screenshot", ".png");
+ ImageIO.write(screenCapture, "PNG", screenshotFile);
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
+
+ logger.info("Clicking on coordinates: " + xCoordinateValue + ", " + yCoordinateValue);
+ Robot robot = new Robot();
+ robot.mouseMove(x, y);
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100);
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100);
+
+ setSuccessMessage("Successfully clicked on coordinates: " + xCoordinateValue + ", " + yCoordinateValue);
+ return Result.SUCCESS;
+ } catch (Exception e) {
+ logger.info("Failed to click on coordinates: " + e.getMessage());
+ setErrorMessage("Failed to click on coordinates: " + e.getMessage());
+ return Result.FAILED;
+ } finally {
+ if (screenshotFile != null && screenshotFile.exists()) {
+ screenshotFile.delete();
+ }
+ }
+ }
+}
diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageEdgeMatch.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageEdgeMatch.java
new file mode 100644
index 00000000..956e350a
--- /dev/null
+++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageEdgeMatch.java
@@ -0,0 +1,136 @@
+package com.testsigma.addons.windows;
+
+import com.testsigma.addons.util.ImageMatchUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.geom.AffineTransform;
+import java.awt.image.BufferedImage;
+import java.io.File;
+
+/**
+ * Edge-based structural matching using Sobel edge detection + multi-scale NCC on edge maps.
+ * Matches the structural contours of objects rather than raw pixel values,
+ * making it robust to color/contrast/background variations.
+ */
+@Data
+@Action(actionText = "Click on image image-url using edge-based matching",
+ description = "Clicks on image using Sobel edge detection and multi-scale NCC structural matching",
+ applicationType = ApplicationType.WINDOWS,
+ useCustomScreenshot = true)
+public class ClickOnImageEdgeMatch extends WindowsAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ private static final double EDGE_MATCH_THRESHOLD = 0.45;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== ClickOnImageEdgeMatch: Starting ===");
+
+ File screenshotFile = null;
+ File targetImageFile = null;
+ File highlightedFile = null;
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ logger.info("Target image URL: " + imageUrlValue);
+
+ targetImageFile = ImageMatchUtils.downloadImage("target_image", imageUrlValue, logger);
+ logger.info("Target image prepared: " + targetImageFile.getAbsolutePath());
+
+ Robot robot = new Robot();
+ Dimension logicalScreenSize = Toolkit.getDefaultToolkit().getScreenSize();
+ int logicalWidth = logicalScreenSize.width;
+ int logicalHeight = logicalScreenSize.height;
+ logger.info("Logical screen: " + logicalWidth + "x" + logicalHeight);
+
+ GraphicsDevice gd = GraphicsEnvironment.getLocalGraphicsEnvironment().getDefaultScreenDevice();
+ AffineTransform tx = gd.getDefaultConfiguration().getDefaultTransform();
+ double displayScaleX = tx.getScaleX();
+ double displayScaleY = tx.getScaleY();
+ logger.info("Display scale: " + displayScaleX + "x" + displayScaleY);
+
+ Rectangle screenRect = new Rectangle(logicalScreenSize);
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Capture dims: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ screenshotFile = File.createTempFile("edge_match_screenshot", ".png");
+ ImageIO.write(screenCapture, "PNG", screenshotFile);
+
+ int[] fileDims = ImageMatchUtils.getImageFileDimensions(screenshotFile);
+ int fileWidth = fileDims[0];
+ int fileHeight = fileDims[1];
+ logger.info("PNG file dims: " + fileWidth + "x" + fileHeight);
+
+ double scaleToLogicalX = (double) logicalWidth / fileWidth;
+ double scaleToLogicalY = (double) logicalHeight / fileHeight;
+
+ BufferedImage baseImage = ImageIO.read(screenshotFile);
+ BufferedImage templateImage = ImageIO.read(targetImageFile);
+ logger.info("Template dims: " + templateImage.getWidth() + "x" + templateImage.getHeight());
+
+ int bw = baseImage.getWidth(), bh = baseImage.getHeight();
+ int tw = templateImage.getWidth(), th = templateImage.getHeight();
+
+ double[][] baseGray = ImageMatchUtils.toGrayscale(baseImage);
+ double[][] tmplGray = ImageMatchUtils.toGrayscale(templateImage);
+ double[][] baseEdges = ImageMatchUtils.sobelEdgeDetection(baseGray, bw, bh);
+
+ long t0 = System.currentTimeMillis();
+ ImageMatchUtils.MatchResult result = ImageMatchUtils.searchMultiScale(
+ baseEdges, bw, bh, tmplGray, tw, th,
+ true, EDGE_MATCH_THRESHOLD, "EdgeStructural", logger);
+ logger.info("Edge matching took " + (System.currentTimeMillis() - t0) + "ms — "
+ + (result.found ? "FOUND conf=" + result.confidence : "NOT FOUND: " + result.message));
+
+ if (!result.found) {
+ setErrorMessage("Edge-based matching failed: " + result.message);
+ ScreenshotUtils.uploadPlainScreenshot(screenshotFile, testStepResult, logger);
+ return Result.FAILED;
+ }
+
+ highlightedFile = ScreenshotUtils.highlightAndUpload(
+ baseImage, result.x1, result.y1, result.x2, result.y2,
+ "edge_match_highlighted", testStepResult, logger);
+
+ int clickX = (int) (((result.x1 + result.x2) / 2.0) * scaleToLogicalX);
+ int clickY = (int) (((result.y1 + result.y2) / 2.0) * scaleToLogicalY);
+ logger.info("Clicking at (" + clickX + ", " + clickY + ") via " + result.method);
+
+ robot.mouseMove(clickX, clickY);
+ Thread.sleep(20);
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(50);
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(50);
+
+ setSuccessMessage(String.format(
+ "Successfully clicked on image at coordinates: %d, %d with confidence: %f",
+ clickX, clickY, result.confidence));
+ return Result.SUCCESS;
+ } catch (Exception e) {
+ logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
+ setErrorMessage("Failed to click on Image. Error: " + e.getMessage());
+ return Result.FAILED;
+ } finally {
+ ImageMatchUtils.cleanupFile(screenshotFile);
+ ImageMatchUtils.cleanupFile(targetImageFile);
+ ImageMatchUtils.cleanupFile(highlightedFile);
+ }
+ }
+}
diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageTemplateMatch.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageTemplateMatch.java
new file mode 100644
index 00000000..db39c2f7
--- /dev/null
+++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageTemplateMatch.java
@@ -0,0 +1,135 @@
+package com.testsigma.addons.windows;
+
+import com.testsigma.addons.util.ImageMatchUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.geom.AffineTransform;
+import java.awt.image.BufferedImage;
+import java.io.File;
+
+/**
+ * Multi-scale grayscale template matching using classical NCC (Normalized Cross-Correlation)
+ * across multiple scales with histogram-equalized preprocessing.
+ */
+@Data
+@Action(actionText = "Click on image image-url using template matching",
+ description = "Clicks on image using multi-scale grayscale NCC template matching",
+ applicationType = ApplicationType.WINDOWS,
+ useCustomScreenshot = true)
+public class ClickOnImageTemplateMatch extends WindowsAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ private static final double TEMPLATE_MATCH_THRESHOLD = 0.55;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== ClickOnImageTemplateMatch: Starting ===");
+
+ File screenshotFile = null;
+ File targetImageFile = null;
+ File highlightedFile = null;
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ logger.info("Target image URL: " + imageUrlValue);
+
+ targetImageFile = ImageMatchUtils.downloadImage("target_image", imageUrlValue, logger);
+ logger.info("Target image prepared: " + targetImageFile.getAbsolutePath());
+
+ Robot robot = new Robot();
+ Dimension logicalScreenSize = Toolkit.getDefaultToolkit().getScreenSize();
+ int logicalWidth = logicalScreenSize.width;
+ int logicalHeight = logicalScreenSize.height;
+ logger.info("Logical screen: " + logicalWidth + "x" + logicalHeight);
+
+ GraphicsDevice gd = GraphicsEnvironment.getLocalGraphicsEnvironment().getDefaultScreenDevice();
+ AffineTransform tx = gd.getDefaultConfiguration().getDefaultTransform();
+ double displayScaleX = tx.getScaleX();
+ double displayScaleY = tx.getScaleY();
+ logger.info("Display scale: " + displayScaleX + "x" + displayScaleY);
+
+ Rectangle screenRect = new Rectangle(logicalScreenSize);
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Capture dims: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ screenshotFile = File.createTempFile("template_match_screenshot", ".png");
+ ImageIO.write(screenCapture, "PNG", screenshotFile);
+
+ int[] fileDims = ImageMatchUtils.getImageFileDimensions(screenshotFile);
+ int fileWidth = fileDims[0];
+ int fileHeight = fileDims[1];
+ logger.info("PNG file dims: " + fileWidth + "x" + fileHeight);
+
+ double scaleToLogicalX = (double) logicalWidth / fileWidth;
+ double scaleToLogicalY = (double) logicalHeight / fileHeight;
+
+ BufferedImage baseImage = ImageIO.read(screenshotFile);
+ BufferedImage templateImage = ImageIO.read(targetImageFile);
+ logger.info("Template dims: " + templateImage.getWidth() + "x" + templateImage.getHeight());
+
+ int bw = baseImage.getWidth(), bh = baseImage.getHeight();
+ int tw = templateImage.getWidth(), th = templateImage.getHeight();
+
+ double[][] baseGray = ImageMatchUtils.toGrayscale(baseImage);
+ double[][] tmplGray = ImageMatchUtils.toGrayscale(templateImage);
+
+ long t0 = System.currentTimeMillis();
+ ImageMatchUtils.MatchResult result = ImageMatchUtils.searchMultiScale(
+ baseGray, bw, bh, tmplGray, tw, th,
+ false, TEMPLATE_MATCH_THRESHOLD, "TemplateMatch", logger);
+ logger.info("Template matching took " + (System.currentTimeMillis() - t0) + "ms — "
+ + (result.found ? "FOUND conf=" + result.confidence : "NOT FOUND: " + result.message));
+
+ if (!result.found) {
+ setErrorMessage("Template matching failed: " + result.message);
+ ScreenshotUtils.uploadPlainScreenshot(screenshotFile, testStepResult, logger);
+ return Result.FAILED;
+ }
+
+ highlightedFile = ScreenshotUtils.highlightAndUpload(
+ baseImage, result.x1, result.y1, result.x2, result.y2,
+ "template_match_highlighted", testStepResult, logger);
+
+ int clickX = (int) (((result.x1 + result.x2) / 2.0) * scaleToLogicalX);
+ int clickY = (int) (((result.y1 + result.y2) / 2.0) * scaleToLogicalY);
+ logger.info("Clicking at (" + clickX + ", " + clickY + ") via " + result.method);
+
+ robot.mouseMove(clickX, clickY);
+ Thread.sleep(50);
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(50);
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100);
+
+ setSuccessMessage(String.format(
+ "Successfully clicked on image at coordinates: %d, %d with confidence: %f",
+ clickX, clickY, result.confidence));
+ return Result.SUCCESS;
+
+ } catch (Exception e) {
+ logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
+ setErrorMessage("Failed to click on Image. Error: " + e.getMessage());
+ return Result.FAILED;
+ } finally {
+ ImageMatchUtils.cleanupFile(screenshotFile);
+ ImageMatchUtils.cleanupFile(targetImageFile);
+ ImageMatchUtils.cleanupFile(highlightedFile);
+ }
+ }
+}