Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Adjustments to screenshot capture tool.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-04-16 08:55:32 +02:00
parent df1850bd45
commit fbbaf584ba
2 changed files with 32 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java
View File

@ -6,7 +6,9 @@ import nu.marginalia.service.module.DatabaseModule;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.openqa.selenium.OutputType; import org.openqa.selenium.OutputType;
import org.openqa.selenium.PageLoadStrategy; import org.openqa.selenium.PageLoadStrategy;
import org.openqa.selenium.TimeoutException;
import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeDriverService;
import org.openqa.selenium.chrome.ChromeOptions; import org.openqa.selenium.chrome.ChromeOptions;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -21,6 +23,10 @@ import java.sql.SQLException;
import java.time.Duration; import java.time.Duration;
import java.util.*; import java.util.*;
import org.openqa.selenium.support.ui.ExpectedCondition;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.openqa.selenium.JavascriptExecutor;
public class ScreenshotCaptureToolMain { public class ScreenshotCaptureToolMain {
private static final Logger logger = LoggerFactory.getLogger(ScreenshotCaptureToolMain.class); private static final Logger logger = LoggerFactory.getLogger(ScreenshotCaptureToolMain.class);
@ -29,6 +35,8 @@ public class ScreenshotCaptureToolMain {
DatabaseModule databaseModule = new DatabaseModule(); DatabaseModule databaseModule = new DatabaseModule();
var ds = databaseModule.provideConnection(); var ds = databaseModule.provideConnection();
System.setProperty(ChromeDriverService.CHROME_DRIVER_SILENT_OUTPUT_PROPERTY, "true");
ChromeDriver driver = initChromeDriver(); ChromeDriver driver = initChromeDriver();
List<EdgeDomain> crawlQueue = fetchCrawlQueue(ds, 100); List<EdgeDomain> crawlQueue = fetchCrawlQueue(ds, 100);
@ -53,7 +61,7 @@ public class ScreenshotCaptureToolMain {
System.setProperty("webdriver.chrome.driver", "./chromedriver"); System.setProperty("webdriver.chrome.driver", "./chromedriver");
ChromeOptions options = new ChromeOptions(); ChromeOptions options = new ChromeOptions();
options.setPageLoadStrategy(PageLoadStrategy.NORMAL); options.setPageLoadStrategy(PageLoadStrategy.NONE);
options.setPageLoadTimeout(Duration.ofSeconds(30)); options.setPageLoadTimeout(Duration.ofSeconds(30));
options.addArguments( options.addArguments(
@ -63,9 +71,11 @@ public class ScreenshotCaptureToolMain {
"window-size=1024,768", "window-size=1024,768",
"force-device-scale-factor=0.5", "force-device-scale-factor=0.5",
"high-dpi-support=0.5", "high-dpi-support=0.5",
"dns-prefetch-disable",
"disable-gpu", "disable-gpu",
"disable-dev-shm-usage", "disable-dev-shm-usage",
"disable-software-rasterizer" "disable-software-rasterizer",
"disable-extensions"
); );
return new ChromeDriver(options); return new ChromeDriver(options);
@ -104,10 +114,28 @@ public class ScreenshotCaptureToolMain {
try { try {
driver.get(domain.toRootUrl().toString()); driver.get(domain.toRootUrl().toString());
WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));
try {
wait.until((ExpectedCondition<Boolean>) wd -> {
if (wd instanceof JavascriptExecutor jse) {
return "complete".equals(jse.executeScript("return document.readyState"));
}
return true;
});
}
catch (TimeoutException ex) {
logger.info("Wait timed out, forcing window.stop()");
driver.executeScript("window.stop()");
}
final byte[] bytes = driver.getScreenshotAs(OutputType.BYTES); final byte[] bytes = driver.getScreenshotAs(OutputType.BYTES);
final var img = ImageIO.read(new ByteArrayInputStream(bytes)); final var img = ImageIO.read(new ByteArrayInputStream(bytes));
Path destPath = Files.createTempFile("website-screenshot-", ".webp"); Path destPath = Files.createTempFile("website-screenshot-", ".webp");
ImageIO.write(img, "webp", destPath.toFile()); ImageIO.write(img, "webp", destPath.toFile());

4
settings.gradle
View File

@ -164,8 +164,8 @@ dependencyResolutionManagement {
library('junit.jupiter.engine','org.junit.jupiter','junit-jupiter-engine').version('') library('junit.jupiter.engine','org.junit.jupiter','junit-jupiter-engine').version('')
library('mockito','org.mockito','mockito-junit-jupiter').version('4.5.1') library('mockito','org.mockito','mockito-junit-jupiter').version('4.5.1')
library('selenium.chrome','org.seleniumhq.selenium','selenium-chrome-driver').version('4.8.2') library('selenium.chrome','org.seleniumhq.selenium','selenium-chrome-driver').version('4.8.3')
library('selenium.java','org.seleniumhq.selenium','selenium-java').version('4.8.2') library('selenium.java','org.seleniumhq.selenium','selenium-java').version('4.8.3')
library('handlebars','com.github.jknack','handlebars').version('4.3.1') library('handlebars','com.github.jknack','handlebars').version('4.3.1')
library('handlebars.markdown','com.github.jknack','handlebars-markdown').version('4.2.1') library('handlebars.markdown','com.github.jknack','handlebars-markdown').version('4.2.1')