Sikulix select the picture of relative position (advanced version)

package com.welab.automation.projects.demo;

import io.appium.java_client.MobileBy;
import io.appium.java_client.MobileElement;
import io.appium.java_client.android.AndroidDriver;
import io.appium.java_client.service.local.AppiumDriverLocalService;
import io.appium.java_client.service.local.AppiumServiceBuilder;
import lombok.SneakyThrows;
import org.openqa.selenium.By;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.sikuli.script.Match;
import org.sikuli.script.Region;
import org.sikuli.script.Screen;

import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.util.Base64;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;

import static io.appium.java_client.service.local.flags.GeneralServerFlag.LOG_LEVEL;
import static io.appium.java_client.service.local.flags.GeneralServerFlag.SESSION_OVERRIDE;

public class TestSikulix {
    public static final String WIN_DRIVER_PATH = "D:/Program Files/nodejs/node.exe";
    public static final String WIN_APPIUM_PATH = "C:/Users/jeff.xie/AppData/Roaming/npm/node_modules/appium/build/lib/main.js";
    public static final String MAC_DRIVER_PATH = "/usr/local/bin/node";
    public static final String MAC_APPIUM_PATH = "/usr/local/lib/node_modules/appium/build/lib/main.js";
    int width;
    int height;
    AndroidDriver driver;


    public static void main(String[] args) throws URISyntaxException, IOException {
        TestSikulix testOpenCV = new TestSikulix();
        testOpenCV.startMain();
    }

    public DesiredCapabilities setAndroidDesiredCapabilities(){
        DesiredCapabilities devices  = new DesiredCapabilities();
        devices.setCapability("platformName", "Android");
        devices.setCapability("deviceName","emulator-5554");
        devices.setCapability("platformVersion", "10");
        devices.setCapability("appPackage","com.android.settings");
        devices.setCapability("appActivity","com.android.settings.Settings");
        devices.setCapability("automation","uiautomator2");
        devices.setCapability("unicodeKeyboard", true);
        devices.setCapability("resetKeyboard",true);
        return devices;
    }

    public void getWindowWidthAbdHeight(){
        width = driver.manage().window().getSize().width;
        height = driver.manage().window().getSize().height;
    }

    @SneakyThrows
    public void startMain()  throws URISyntaxException, IOException {
        // Need to open appium server, And start the simulator 
        //driver= new AndroidDriver(new URL("http://localhost:4723/wd/hub"),setAndroidDesiredCapabilities());

        // Don't need to open appium The desktop version 
        AppiumDriverLocalService service =startAppiumService();
        driver = new AndroidDriver(service.getUrl(), setAndroidDesiredCapabilities());

        Screen s = new Screen();
        //  Set element not found , Automatic wait time 10 second 
        s.setAutoWaitTimeout(10);
        String project_path=System.getProperty("user.dir");
        System.out.println(System.getProperty("user.dir"));
        String path = project_path+ "\\src\\main\\java\\com\\welab\\automation\\projects\\demo\\Notifications.png";

        String target ="D:\\images\\test\\test01.png";

        getWindowWidthAbdHeight();
        Region r = new Region(0,0,width,height);
        Match match=r.exists(target);
        //Match match = r.find(target);

        Region region=match.highlight();
        String locations=region.toStringShort();

        String matchString = match.toStringShort();
        System.out.println("matchString:"+matchString);
        System.out.println("match.x:"+match.x);
        System.out.println("match.y:"+match.y);
        System.out.println("match.h:"+match.h);
        System.out.println("match.w:"+match.w);
        System.out.println("match.getCenter:"+match.getCenter());
        System.out.println("match.getScore(): "+match.getScore());

        System.out.println("region.x:"+region.x);
        System.out.println("region.y:"+region.y);
        System.out.println("region.h:"+region.h);
        System.out.println("region.w:"+region.w);
        System.out.println("region.getCenter:"+region.getCenter());
        System.out.println("locations: "+locations);
        System.out.println("match.getScore(): "+match.getScore());

// [log] CLICK on L[120,648]@S(0) (546 msec) // [log] highlight M[46,621 148x54]On(0) S 10000 for -1.0 secs // region.x:46  The picture is on the left side of the interface 46 // region.y:621  The image in the interface transition position is 4621 // region.h:54  Height of picture  // region.w:148  Width of picture  // region.getCenter:L[120,648]@S(0) //  Click on the center point of the picture   Left position 46 +  Width  148/2 =120  Height position 621 +  Picture height  54/2 =648 // match.getScore(): 1.0  score 1.0  Description match is 100%
        System.out.println(match.h);
        System.out.println(match.w);
        match.highlight();

        Thread.sleep(5000);
        s.click(target);
        Thread.sleep(5000);
        s.click("D:\\images\\test\\test02.png");
        System.out.println("pass");
    }

    public  void clickMobileElementByPicture(String picPath) throws URISyntaxException, IOException {
        File file = new File(picPath);
        String base64String = Base64.getEncoder().encodeToString(Files.readAllBytes(file.toPath()));
        By by= MobileBy.image(base64String);
        findMobileElement(by).click();
    }

    public MobileElement findMobileElement(By by) {
        return (MobileElement) driver.findElement(by);
    }

    private static AppiumDriverLocalService startAppiumService() {
        AtomicInteger port = new AtomicInteger();
        AppiumDriverLocalService service = null;
        service = new AppiumServiceBuilder()
                .usingAnyFreePort()
                .withIPAddress("0.0.0.0")
                .withArgument(SESSION_OVERRIDE)
                .withArgument(LOG_LEVEL, "error")
                .usingDriverExecutable(new File("D:\\Program Files\\nodejs\\node.exe"))
                .withAppiumJS(new File("C:\\Users\\jeff.xie\\AppData\\Roaming\\npm\\node_modules\\appium\\build\\lib\\main.js"))
                .build();
        Optional.ofNullable(service).ifPresent(s -> {
            s.start();
            port.set(s.getUrl().getPort());
        });
        AppiumDriverLocalService appiumDriverLocalService = service;
        return service;
    }

}

Sikulix Select a picture or object in relative position
Generally speaking, there is such a demand , I already have a picture element , There will be a dynamically displayed object around this element , I'm going to do a click or hover Or simply picture the object save To local , Keep it as the object to be clicked on the next page , In this case, you can use sikuli To solve .
Here I still follow baidu Take the home page as an example , Maybe not very typical , But it's OK to be an example .
I first click the skin change button , Then when the skin changing picture is known , Click to take pictures from other directions , and save Go to the local location to retain the object to be clicked by the next script .

1. So let's set up a region object , This region It's a square area , I will default to the resolution size
Region r = new Region(0,0,1023,767);

2. Click the skin button
r.click("img/ChangeSK.png");

3. I have a skin picture in the center of all skin pictures
r.exists("img/SK1.png");
Under operation , The result is no problem ：
[debug] Region: exists: img/SK1.png has appeared

4. Find this picture and use it as a Match object ,highlight And print the similarity , At the same time print out width and height
Match SK1 = r.find("img/SK1.png");
System.out.println(baidu.getScore());
SK1.highlight();
System.out.println(SK1.h); 
System.out.println(SK1.w);
result 0.9999843239784241, Indicates that... Has been found

5. Create a new one Location object , To store this object in region The coordinates of
Location Loc1 = SK1.getTarget();
Note that describing a picture has four dimensions ,start x,start y,width,height.   there location It means x+height/2, y+width/2, That is, the central position
Here we need a tool to get the coordinates on the screen , In this way, the relative position of the picture can be accurately obtained during the manual test , There are many such tools on the Internet , Search and you will have

6. Get Location after , I just need to know in this SK1 An image of the same size above the object , How to do it here ：
Location Loc2 = Loc1.above(60);
Through the tool, we know that the position of the picture above is probably higher than 60pix, So build a new location object

7. Because we already know the picture above width and height, So now you need to create a new one Match Just the object
Match SK2 = new Match(SK1);
SK2.setLocation(Loc2);// It is wrong to do so , Because here setLocation Not the central axis , It is set It becomes the starting coordinate , It should be ：
SK2.y = SK2.y-60;

8, Print the similarity to see if it is found
SK2.highlight();
System.out.println(SK2.getScore());
The score is 1, That's all right.

9, Put this picture save To the local path
SK2.saveScreenCapture("img", "SK2");
SK2-1444961093063.png Save name should be followed by timestream, Should be to avoid coverage , Change sikuli jar Just recompile the source code in the package .

And so selenium Together, it can greatly enhance case Coverage of