spring-boot比spring更加好用的原因之一就是自动装配,可以引入官方提供的starter组件快速集成,免去各种复杂的配置。
现在我们来自己实现一个starter组件。
业务场景
现在有多个导出服务,都需要通过webdriver来请求、渲染html页面,然后获取渲染后的html。之前都是各个服务业务代码里面集成相关逻辑和配置,代码比较冗余,而且基础能力有调整的地方各个服务都需要同步调整源码。
解决办法
现在通过封装starter组件,让各个业务直接引入,业务服务无需关注webdriver实例的管理,以及获取html的内部实现逻辑。
代码实现
创建maven项目webdriver-spring-boot-starter
pom.xml添加依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.1.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.iflytek</groupId>
<artifactId>webdriver-spring-boot-starter</artifactId>
<version>1.0.0</version>
<name>webdriver-spring-boot-starter</name>
<description>webdriver-spring-boot-starter</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>3.141.59</version>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-chrome-driver</artifactId>
<version>3.141.59</version>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-server</artifactId>
<version>3.141.59</version>
</dependency>
<!-- https://mvnrepository.com/artifact/us.codecraft/webmagic-core -->
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId>
<version>0.7.3</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.16.16</version>
</dependency>
</dependencies>
<distributionManagement>
<repository>
<id>nexus-iflynote</id>
<url>https://xx.com/repository/iflynote-hosted/</url>
</repository>
<snapshotRepository>
<id>nexus-iflynote</id>
<url>https://xx.com/repository/iflynote-hosted/</url>
</snapshotRepository>
</distributionManagement>
<build>
<finalName>webdriver-spring-boot-starter</finalName>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>utf-8</encoding>
</configuration>
</plugin>
</plugins>
</build>
</project>
创建配置类WebdriverConfigProperties
package com.iflytek.webdriver.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* @Author: BillYu
* @Description:
* @Date: Created in 10:37 2022/2/25.
*/
@ConfigurationProperties(prefix = "webdriver")
@Component
@Data
public class WebdriverConfigProperties {
/**
* 实例最大个数
*/
private Integer instanceMaxNumber;
/**
* 最大等待时间
*/
private Integer maxWaitSeconds;
/**
* 唤醒间隔时间,尝试判断是否加载完成
*/
private Integer waitSleepMilliseconds;
}
创建WebDriverUtil,管理实例的初始化、更新、获取等
package com.iflytek.webdriver.config;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.NoSuchSessionException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
/**
* 1. webDriver.Close() //关闭当前焦点所在的窗口
* 2. webDriver.Quit() //调用dispose方法
* 3. webDriver.Dispose() //关闭所有窗口,并且安全关闭session
*
* @Author: BillYu
* @Description:
* @Date: Created in 08:53 2020-08-21.
*/
@Slf4j
public class WebDriverUtil {
private static List<WebDriverBean> list = new ArrayList<>();
private static List<WebDriverBean> webDriverList = Collections.synchronizedList(list);
public static Integer webdriverMaxNumber = 5;
private static Object lock = new Object();
/**
* 随机找一个可用的webdriver,如果不可用找一个可用的,如果全部不可用更新一个返回
* 10分钟更新一遍webdriver
*
* @return
*/
public static WebDriver getWebDriverInstance() {
List<Integer> validList = getValidWebDriverIndexList();
WebDriverBean webDriverBean;
if (validList.size() == 0) {
return getOneValidWebDriver();
} else if (validList.size() == 1) {
webDriverBean = webDriverList.get(validList.get(0));
} else {
int i = new Random().nextInt(validList.size());
webDriverBean = webDriverList.get(validList.get(i));
}
if (webDriverBean == null) {
return getOneValidWebDriver();
} else {
return webDriverBean.webDriver;
}
}
public static void quitAll() {
for (WebDriverBean webDriverBean : webDriverList) {
WebDriver webDriver = webDriverBean.webDriver;
if (webDriver != null) {
webDriver.close();
webDriver.quit();
webDriver = null;
}
}
}
public synchronized static void init() {
int needAddSize = webdriverMaxNumber - webDriverList.size();
for (int i = 0; i < needAddSize; i++) {
WebDriverBean webDriverBean = createWebDriverBean();
webDriverList.add(webDriverBean);
}
}
static List<Integer> getValidWebDriverIndexList() {
List<Integer> indexList = new ArrayList<>();
int i = 0;
for (WebDriverBean webdriver : webDriverList) {
if (webdriver != null && webdriver.webDriver != null && webdriver.valid && webdriver.sessionValid()) {
indexList.add(i);
}
i++;
}
return indexList;
}
static synchronized WebDriver getOneValidWebDriver() {
for (WebDriverBean webdriver : webDriverList) {
if (webdriver != null && webdriver.webDriver != null && webdriver.valid && webdriver.sessionValid()) {
return webdriver.webDriver;
}
}
WebDriverBean webDriverBean = createWebDriverBean();
webDriverList.add(webDriverBean);
return webDriverBean.webDriver;
}
public static void updateAllWebDriver() {
log.info("webdriver数量:{}", webDriverList.size());
synchronized (lock) {
int i = 0;
for (WebDriverBean webDriverBean : webDriverList) {
if (webDriverBean != null && webDriverBean.webDriver != null) {
webDriverBean.valid = false;
webDriverList.set(i, webDriverBean);
try {
//睡眠200s等待渲染任务执行完成
Thread.sleep(200000);
} catch (InterruptedException e) {
e.printStackTrace();
log.error("sleep exception{}", e);
//没有等到充足时间,恢复状态
webDriverBean.valid = true;
webDriverList.set(i, webDriverBean);
break;
}
try {
//更新session异常捕获
webDriverBean.webDriver.close();
webDriverBean.webDriver.quit();
}catch (NoSuchSessionException e){
log.info("更新时,webdriver失效", e);
}finally {
webDriverBean.webDriver = null;
}
if (i >= webdriverMaxNumber) {
//移除多余webdriver
webDriverList.remove(i);
} else {
//生成新的webdriver
webDriverBean = createWebDriverBean();
webDriverList.set(i, webDriverBean);
}
}
i++;
}
}
}
static class WebDriverBean {
private WebDriver webDriver;
private Boolean valid;
public WebDriverBean(WebDriver webDriver, Boolean valid) {
this.webDriver = webDriver;
this.valid = valid;
}
/**
* 添加webdriver session检测
* @return
*/
public boolean sessionValid(){
try {
//清除
webDriver.getWindowHandle();
}catch (NoSuchSessionException e){
log.info("检测webdriver session失效", e);
return false;
}
return true;
}
}
static WebDriverBean createWebDriverBean() {
log.info("开始创建webdriver");
ChromeOptions options = new ChromeOptions();
// 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
options.addArguments("--headless");
options.addArguments("--no-sandbox");
WebDriver webDriver = new ChromeDriver(options);
//打开一个空的tab页
// ((ChromeDriver) webDriver).executeScript("window.open('about:blank','_blank');");
log.info("初始化window:{}", String.valueOf(webDriver.getWindowHandles()));
WebDriverBean webDriverBean = new WebDriverBean(webDriver, true);
return webDriverBean;
}
}
创建WebdriverService类,用于封装渲染页面方法。
package com.iflytek.webdriver.service;
import com.iflytek.webdriver.config.WebDriverUtil;
import com.iflytek.webdriver.config.WebdriverConfigProperties;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
/**
* @Author: BillYu
* @Description:
* @Date: Created in 11:03 2022/2/25.
*/
@Slf4j
public class WebdriverService {
public WebdriverService(WebdriverConfigProperties webdriverConfigProperties) {
this.webdriverConfigProperties = webdriverConfigProperties;
}
private WebdriverConfigProperties webdriverConfigProperties;
public String getHtml(String url, String checkCompleteJS){
WebDriver webDriver = WebDriverUtil.getWebDriverInstance();
log.info("是否获取到webDriver:{} windows:{}", webDriver != null,webDriver.getWindowHandles().size());
try {
log.info("url{}", url);
//打开新的tab页
((ChromeDriver) webDriver).executeScript("window.open('" + url + "', '_blank');");
log.info("所有窗口:{}",String.valueOf(webDriver.getWindowHandles()));
WebDriverWait wait;
synchronized (webDriver) {
for (String tab : webDriver.getWindowHandles()) {
if (webDriver.switchTo().window(tab).getCurrentUrl().equals(url)) {
break;
}
}
wait = new WebDriverWait(webDriver, webdriverConfigProperties.getMaxWaitSeconds() , webdriverConfigProperties.getWaitSleepMilliseconds());
}
//等待前端加载完成
wait.until(ExpectedConditions.jsReturnsValue(checkCompleteJS));
String htmlText = null;
synchronized (webDriver) {
for (String tab : webDriver.getWindowHandles()) {
//切换到加载标签页获取html
if (webDriver.switchTo().window(tab).getCurrentUrl().equals(url)) {
WebElement webElement = webDriver.findElement(By.xpath("/html"));
htmlText = webElement.getAttribute("outerHTML");
//关闭当前窗口;
webDriver.close();
//关闭当前窗口后需要切换到默认
webDriver.switchTo().window((String) webDriver.getWindowHandles().toArray()[0]);
log.info("关闭后的窗口:{} 窗口数量:{}",String.valueOf(webDriver.getWindowHandles()), webDriver.getWindowHandles().size());
}
}
}
// 内容比较大(包含图片的base64),建议不要打印
// log.info("htmlText = " + htmlText);
return htmlText;
} catch (Exception e) {
log.error("获取html异常", e);
e.printStackTrace();
}
return null;
}
}
新建WebdriverAutoConfiguration配置类,这是starter自动配置的入口类。
package com.iflytek.webdriver.config;
import com.iflytek.webdriver.service.WebdriverService;
import org.openqa.selenium.WebDriver;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @Author: BillYu
* @Description:注册配置类
* 条件判断是否自动配置:@ConditionalOnClass @ConditionalOnProperty
* @Date: Created in 10:31 2022/2/25.
*/
@Configuration
@ConditionalOnProperty(prefix = "webdriver.autoconfig", name = "enable", havingValue = "true")
@EnableConfigurationProperties(WebdriverConfigProperties.class)
public class WebdriverAutoConfiguration {
@Autowired
private WebdriverConfigProperties webdriverConfigProperties;
@Bean
@ConditionalOnMissingBean
WebdriverService webdriverService(){
return new WebdriverService(webdriverConfigProperties);
}
}
新建@PreDestroy CommandLineRunner @Scheduled等实现类或方法。
在服务启动、销毁、运行中调用WebdriverUtil方法,以管理webdriver实例。
这里代码忽略
最后一步在resources下新建META-INF文件夹,在META-INF文件夹下新建spring.factories文件,文件内容:
org.springframework.boot.autoconfigure.EnableAutoConfiguration=com.iflytek.webdriver.config.WebdriverAutoConfiguration
然后mvn deploy,把starter jar包发布到maven私有仓库。
现在我们新建业务项目来使用webdriver-spring-boot-starter组件。
添加依赖
<dependency>
<groupId>com.iflytek</groupId>
<artifactId>webdriver-spring-boot-starter</artifactId>
<version>1.0.0</version>
</dependency>
application.properties添加配置
#启用自动配置
webdriver.autoconfig.enable=true
webdriver.instance-max-number=6
webdriver.wait-sleep-milliseconds=300
webdriver.max-wait-seconds=200
logging.level.root=debug
新建controller测试,引入WebdriverService,看看实例配置是否生效。
@Controller
@RequestMapping("/test")
public class TestController {
@Autowired
private WebdriverService webdriverService;
@GetMapping("/export")
@ResponseBody
public String export() {
String url = "http://localhost:7770/h/simple.html";
String html = webdriverService.getHtml(url,"return imageComplete()");
return html;
}
}