Skip to content

Commit c253bb2

Browse files
ManojTestsigmaManojTestsigma
authored
[feat] Addons for windows advanced application type. (#211)
* [feat] Addons for windows advanced application type. * Added two new actions and addressed review comments. * Added missing classes. * [CUS-10875] added two new actions for desktop application type.
1 parent 2370bd6 commit c253bb2

16 files changed

Lines changed: 2251 additions & 0 deletions

‎windows_advanced_actions/pom.xml‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project
3+
xmlns="http://maven.apache.org/POM/4.0.0"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
6+
<modelVersion>4.0.0</modelVersion>
7+
<groupId>com.testsigma.addons</groupId>
8+
<artifactId>windows_advanced_actions</artifactId>
9+
<version>1.0.23</version>
10+
<packaging>jar</packaging>
11+
12+
<properties>
13+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
14+
<maven.compiler.source>11</maven.compiler.source>
15+
<maven.compiler.target>11</maven.compiler.target>
16+
<testsigma.sdk.version>1.2.24_cloud</testsigma.sdk.version>
17+
<junit.jupiter.version>5.8.0-M1</junit.jupiter.version>
18+
<testsigma.addon.maven.plugin>1.0.0</testsigma.addon.maven.plugin>
19+
<maven.source.plugin.version>3.2.1</maven.source.plugin.version>
20+
<lombok.version>1.18.30</lombok.version>
21+
22+
</properties>
23+
24+
<dependencies>
25+
<dependency>
26+
<groupId>com.testsigma</groupId>
27+
<artifactId>testsigma-java-sdk</artifactId>
28+
<version>${testsigma.sdk.version}</version>
29+
</dependency>
30+
<dependency>
31+
<groupId>org.projectlombok</groupId>
32+
<artifactId>lombok</artifactId>
33+
<version>${lombok.version}</version>
34+
<optional>true</optional>
35+
</dependency>
36+
<dependency>
37+
<groupId>org.junit.jupiter</groupId>
38+
<artifactId>junit-jupiter-api</artifactId>
39+
<version>${junit.jupiter.version}</version>
40+
<scope>test</scope>
41+
</dependency>
42+
<dependency>
43+
<groupId>org.testng</groupId>
44+
<artifactId>testng</artifactId>
45+
<version>6.14.3</version>
46+
</dependency>
47+
<!-- https://mvnrepository.com/artifact/org.seleniumhq.selenium/selenium-java -->
48+
<dependency>
49+
<groupId>org.seleniumhq.selenium</groupId>
50+
<artifactId>selenium-java</artifactId>
51+
<version>4.33.0</version>
52+
</dependency>
53+
<!-- https://mvnrepository.com/artifact/io.appium/java-client -->
54+
<dependency>
55+
<groupId>io.appium</groupId>
56+
<artifactId>java-client</artifactId>
57+
<version>9.4.0</version>
58+
</dependency>
59+
<dependency>
60+
<groupId>com.fasterxml.jackson.core</groupId>
61+
<artifactId>jackson-annotations</artifactId>
62+
<version>2.13.0</version>
63+
</dependency>
64+
65+
66+
67+
<!-- Apache HTTP Client for S3 uploads -->
68+
<dependency>
69+
<groupId>org.apache.httpcomponents</groupId>
70+
<artifactId>httpclient</artifactId>
71+
<version>4.5.14</version>
72+
</dependency>
73+
74+
<dependency>
75+
<groupId>com.fasterxml.jackson.core</groupId>
76+
<artifactId>jackson-databind</artifactId>
77+
<version>2.13.0</version>
78+
</dependency>
79+
<dependency>
80+
<groupId>org.apache.commons</groupId>
81+
<artifactId>commons-lang3</artifactId>
82+
<version>3.17.0</version>
83+
</dependency>
84+
<dependency>
85+
<groupId>commons-io</groupId>
86+
<artifactId>commons-io</artifactId>
87+
<version>2.17.0</version>
88+
</dependency>
89+
90+
91+
<dependency>
92+
<groupId>com.squareup.okhttp3</groupId>
93+
<artifactId>okhttp</artifactId>
94+
<version>4.9.1</version>
95+
</dependency>
96+
<dependency>
97+
<groupId>org.apache.poi</groupId>
98+
<artifactId>poi</artifactId>
99+
<version>5.2.4</version>
100+
</dependency>
101+
102+
<!-- DB2 JDBC Driver -->
103+
<dependency>
104+
<groupId>com.ibm.db2</groupId>
105+
<artifactId>jcc</artifactId>
106+
<version>11.5.8.0</version>
107+
</dependency>
108+
109+
</dependencies>
110+
<build>
111+
<finalName>windows_advanced_actions</finalName>
112+
<plugins>
113+
<plugin>
114+
<groupId>org.apache.maven.plugins</groupId>
115+
<artifactId>maven-shade-plugin</artifactId>
116+
<version>3.2.4</version>
117+
<executions>
118+
<execution>
119+
<phase>package</phase>
120+
<goals>
121+
<goal>shade</goal>
122+
</goals>
123+
</execution>
124+
</executions>
125+
</plugin>
126+
<plugin>
127+
<groupId>org.apache.maven.plugins</groupId>
128+
<artifactId>maven-source-plugin</artifactId>
129+
<version>${maven.source.plugin.version}</version>
130+
<executions>
131+
<execution>
132+
<id>attach-sources</id>
133+
<goals>
134+
<goal>jar</goal>
135+
</goals>
136+
</execution>
137+
</executions>
138+
</plugin>
139+
</plugins>
140+
</build>
141+
</project>

‎windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnImageWithTimeout.java‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
package com.testsigma.addons.windowsAdvanced;
2+
3+
import com.fasterxml.jackson.databind.ObjectMapper;
4+
import com.testsigma.addons.util.Constants;
5+
import com.testsigma.addons.util.ResponseObjectForFindImage;
6+
import com.testsigma.addons.util.ScreenshotUtils;
7+
import com.testsigma.sdk.Result;
8+
import com.testsigma.sdk.WindowsAdvancedAction;
9+
import com.testsigma.sdk.annotation.Action;
10+
import com.testsigma.sdk.annotation.TestData;
11+
import com.testsigma.sdk.annotation.TestStepResult;
12+
import okhttp3.*;
13+
import org.apache.commons.lang3.exception.ExceptionUtils;
14+
15+
import javax.imageio.ImageIO;
16+
import java.awt.*;
17+
import java.awt.image.BufferedImage;
18+
import java.io.File;
19+
import java.io.IOException;
20+
import java.io.InputStream;
21+
import java.net.URL;
22+
import java.nio.file.Files;
23+
import java.nio.file.StandardCopyOption;
24+
25+
@Action(actionText = "Click on the image image-url with timeout wait-time-in-seconds seconds with threshold threshold",
26+
description = "This action waits until the specified image appears on the screen (within the given timeout), "
27+
+ "then clicks on it. It takes an image URL (S3 URL or local file path), polls the screen every 1.5 seconds, "
28+
+ "finds that image on the current screen, and clicks on it. Threshold (0 to 1) controls match sensitivity. "
29+
+ "This works only for local executions.",
30+
applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
31+
displayName = "Click on the image with timeout",
32+
useCustomScreenshot = true)
33+
public class ClickOnImageWithTimeout extends WindowsAdvancedAction {
34+
35+
@TestData(reference = "image-url")
36+
private com.testsigma.sdk.TestData imageUrl;
37+
38+
@TestData(reference = "wait-time-in-seconds")
39+
private com.testsigma.sdk.TestData timeoutSeconds;
40+
41+
@TestData(reference = "threshold")
42+
private com.testsigma.sdk.TestData threshold;
43+
44+
@TestStepResult
45+
private com.testsigma.sdk.TestStepResult testStepResult;
46+
47+
private final ObjectMapper mapper = new ObjectMapper();
48+
private static final int POLLING_INTERVAL_MS = 1500;
49+
50+
@Override
51+
protected Result execute() {
52+
logger.info("=== Click On Image With Timeout: Starting Execution ===");
53+
54+
try {
55+
String imageUrlValue = imageUrl.getValue().toString();
56+
int timeoutMs = Integer.parseInt(timeoutSeconds.getValue().toString()) * 1000;
57+
String thresholdStr = threshold.getValue().toString().trim();
58+
double thresholdValue = Double.parseDouble(thresholdStr);
59+
if (thresholdValue < 0 || thresholdValue > 1) {
60+
setErrorMessage("Threshold must be between 0 and 1. Got: " + thresholdStr);
61+
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_failure_screenshot", logger);
62+
return Result.FAILED;
63+
}
64+
65+
logger.info("Looking for image from URL: " + imageUrlValue + " with timeout: "
66+
+ timeoutSeconds.getValue() + " seconds, threshold: " + thresholdStr);
67+
68+
File searchImageFile = urlToFileConverter("target_image", imageUrlValue);
69+
long startTime = System.currentTimeMillis();
70+
long endTime = startTime + timeoutMs;
71+
72+
while (System.currentTimeMillis() < endTime) {
73+
logger.info("Polling attempt - checking for image on screen");
74+
75+
Robot robot = new Robot();
76+
Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
77+
BufferedImage screenCapture = robot.createScreenCapture(screenRect);
78+
File baseImageFile = saveScreenshotToFile(screenCapture, "click_image_timeout_screenshot");
79+
80+
int[] center = findImageCoordinates(baseImageFile, searchImageFile, thresholdStr);
81+
82+
if (center != null) {
83+
int clickX = center[0];
84+
int clickY = center[1];
85+
logger.info("Image found at center (" + clickX + ", " + clickY + "). Performing click.");
86+
performClickWithRobot(clickX, clickY);
87+
setSuccessMessage("Image found and clicked at coordinates (" + clickX + ", " + clickY + ").");
88+
Thread.sleep(1000);
89+
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_success_screenshot", logger);
90+
return Result.SUCCESS;
91+
}
92+
93+
if (baseImageFile.exists()) {
94+
baseImageFile.delete();
95+
}
96+
97+
long remainingTime = endTime - System.currentTimeMillis();
98+
if (remainingTime > POLLING_INTERVAL_MS) {
99+
logger.info("Image not found yet. Waiting " + (POLLING_INTERVAL_MS / 1000)
100+
+ " second before next attempt. Remaining time: " + (remainingTime / 1000) + " seconds");
101+
Thread.sleep(POLLING_INTERVAL_MS);
102+
} else {
103+
break;
104+
}
105+
}
106+
107+
logger.debug("Timeout reached. Image was not found on the screen within "
108+
+ timeoutSeconds.getValue() + " seconds.");
109+
setErrorMessage("Image was not found on the screen within " + timeoutSeconds.getValue() + " seconds.");
110+
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_failure_screenshot", logger);
111+
return Result.FAILED;
112+
113+
} catch (NumberFormatException e) {
114+
logger.debug("Invalid number format: " + e.getMessage());
115+
setErrorMessage("Invalid input. Timeout must be a number (seconds). Threshold must be a number between 0 and 1.");
116+
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_failure_screenshot", logger);
117+
return Result.FAILED;
118+
} catch (Exception e) {
119+
logger.debug("Exception during click operation: " + e.getMessage());
120+
setErrorMessage("Error during click operation: " + e.getMessage());
121+
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_failure_screenshot", logger);
122+
return Result.FAILED;
123+
}
124+
}
125+
126+
/**
127+
* Finds the image on the screen and returns the center coordinates, or null if not found.
128+
* @param thresholdStr threshold for image match (0 to 1), from user input
129+
*/
130+
private int[] findImageCoordinates(File baseImageFile, File searchImageFile, String thresholdStr) {
131+
try {
132+
OkHttpClient client = new OkHttpClient();
133+
RequestBody requestBody = new MultipartBody.Builder()
134+
.setType(MultipartBody.FORM)
135+
.addFormDataPart("baseImageFile", baseImageFile.getName(),
136+
RequestBody.create(baseImageFile, MediaType.parse("image/png")))
137+
.addFormDataPart("searchImageFile", searchImageFile.getName(),
138+
RequestBody.create(searchImageFile, MediaType.parse("image/png")))
139+
.addFormDataPart("threshold", thresholdStr)
140+
.addFormDataPart("scale", "40")
141+
.addFormDataPart("occurance", "1")
142+
.build();
143+
144+
Request request = new Request.Builder()
145+
.url(Constants.VISUAL_SERVER_FIND_IMAGE_ENDPOINT)
146+
.post(requestBody)
147+
.addHeader("Authorization", "Bearer " + Constants.API_TOKEN)
148+
.build();
149+
150+
Response response = client.newCall(request).execute();
151+
if (!response.isSuccessful() || response.body() == null) {
152+
return null;
153+
}
154+
155+
String responseBody = response.body().string();
156+
ResponseObjectForFindImage responseObject = mapper.readValue(responseBody, ResponseObjectForFindImage.class);
157+
158+
if (Boolean.TRUE.equals(responseObject.getIsFound())) {
159+
int x1 = responseObject.getX1();
160+
int y1 = responseObject.getY1();
161+
int x2 = responseObject.getX2();
162+
int y2 = responseObject.getY2();
163+
int centerX = (x1 + x2) / 2;
164+
int centerY = (y1 + y2) / 2;
165+
return new int[]{centerX, centerY};
166+
}
167+
return null;
168+
} catch (IOException e) {
169+
logger.debug("Exception while finding image: " + ExceptionUtils.getStackTrace(e));
170+
return null;
171+
} catch (Exception e) {
172+
logger.debug("Exception: " + ExceptionUtils.getStackTrace(e));
173+
return null;
174+
}
175+
}
176+
177+
private void performClickWithRobot(int x, int y) throws Exception {
178+
Robot robot = new Robot();
179+
logger.info("Moving mouse to coordinates (" + x + ", " + y + ")");
180+
robot.mouseMove(x, y);
181+
Thread.sleep(200);
182+
logger.info("Pressing mouse button");
183+
robot.mousePress(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
184+
Thread.sleep(100);
185+
logger.info("Releasing mouse button");
186+
robot.mouseRelease(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
187+
Thread.sleep(200);
188+
logger.info("Click completed successfully");
189+
}
190+
191+
private static File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
192+
File tempFile = File.createTempFile(fileName, ".png");
193+
ImageIO.write(screenshot, "PNG", tempFile);
194+
return tempFile;
195+
}
196+
197+
private File urlToFileConverter(String fileName, String url) {
198+
try {
199+
if (url.startsWith("https://") || url.startsWith("http://")) {
200+
logger.info("Given is s3 url ...File name:" + fileName);
201+
URL urlObject = new URL(url);
202+
String baseName = fileName;
203+
String extension = "";
204+
int lastDotIndex = fileName.lastIndexOf('.');
205+
if (lastDotIndex > 0) {
206+
baseName = fileName.substring(0, lastDotIndex);
207+
extension = fileName.substring(lastDotIndex);
208+
} else {
209+
String urlPath = urlObject.getPath();
210+
int urlLastDotIndex = urlPath.lastIndexOf('.');
211+
if (urlLastDotIndex > 0) {
212+
extension = urlPath.substring(urlLastDotIndex);
213+
} else {
214+
extension = ".png";
215+
}
216+
}
217+
File tempFile = File.createTempFile(baseName, extension);
218+
try (InputStream in = urlObject.openStream()) {
219+
Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
220+
}
221+
logger.info("Temp file created: " + tempFile.getName() + " at " + tempFile.getAbsolutePath());
222+
return tempFile;
223+
} else {
224+
logger.info("Given is local file path..");
225+
return new File(url);
226+
}
227+
} catch (Exception e) {
228+
logger.info("Error while accessing: " + url);
229+
throw new RuntimeException("Unable to access the given file, please check the given inputs.");
230+
}
231+
}
232+
}

0 commit comments

Comments
 (0)