diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java b/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java index 334643b1..2ce1b4d1 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java +++ b/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java @@ -342,4 +342,44 @@ public class FileStorageService { stmt.executeUpdate(); } } + + public List getEachFileStorage() { + List ret = new ArrayList<>(); + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT PATH, TYPE, DESCRIPTION, ID, BASE_ID + FROM FILE_STORAGE_VIEW + """)) { + + long storageId; + long baseId; + String path; + String description; + FileStorageType type; + + try (var rs = stmt.executeQuery()) { + while (rs.next()) { + baseId = rs.getLong("BASE_ID"); + storageId = rs.getLong("ID"); + path = rs.getString("PATH"); + type = FileStorageType.valueOf(rs.getString("TYPE")); + description = rs.getString("DESCRIPTION"); + + var base = getStorageBase(new FileStorageBaseId(baseId)); + + ret.add(new FileStorage( + new FileStorageId(storageId), + base, + type, + path, + description + )); + } + } + } catch (SQLException e) { + e.printStackTrace(); + } + + return ret; + } } diff --git a/code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java b/code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java index cfd1df26..43d99d7d 100644 --- a/code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java +++ b/code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java @@ -131,7 +131,7 @@ public class FileStorageServiceTest { var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false, false); - var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldFail"); + var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldSucceed"); tempDirs.add(created.asPath()); var actual = storage.getStorage(created.id()); diff --git a/code/services-satellite/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java index dc6dd69d..663fa9d8 100644 --- a/code/services-satellite/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java +++ b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java @@ -15,6 +15,8 @@ import org.slf4j.LoggerFactory; import java.nio.file.Files; import java.nio.file.Path; +import java.sql.SQLException; +import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -27,6 +29,7 @@ public class FileStorageMonitorActor extends AbstractStateGraph { private static final String INITIAL = "INITIAL"; private static final String MONITOR = "MONITOR"; private static final String PURGE = "PURGE"; + private static final String REMOVE_STALE = "REMOVE-STALE"; private static final String END = "END"; private final FileStorageService fileStorageService; @@ -42,7 +45,10 @@ public class FileStorageMonitorActor extends AbstractStateGraph { public void init() { } - @GraphState(name = MONITOR, next = PURGE, resume = ResumeBehavior.RETRY, transitions = { PURGE }, + @GraphState(name = MONITOR, + next = PURGE, + resume = ResumeBehavior.RETRY, + transitions = { PURGE, REMOVE_STALE }, description = """ Monitor the file storage and trigger at transition to PURGE if any file storage area has been marked for deletion. @@ -52,12 +58,17 @@ public class FileStorageMonitorActor extends AbstractStateGraph { for (;;) { Optional toDeleteOpt = fileStorageService.findFileStorageToDelete(); - if (toDeleteOpt.isEmpty()) { - TimeUnit.SECONDS.sleep(10); - } - else { + if (toDeleteOpt.isPresent()) { transition(PURGE, toDeleteOpt.get().id()); } + + List allStorageItems = fileStorageService.getEachFileStorage(); + var missing = allStorageItems.stream().filter(storage -> !Files.exists(storage.asPath())).findAny(); + if (missing.isPresent()) { + transition(REMOVE_STALE, missing.get().id()); + } + + TimeUnit.SECONDS.sleep(10); } } @@ -79,4 +90,16 @@ public class FileStorageMonitorActor extends AbstractStateGraph { fileStorageService.removeFileStorage(storage.id()); } + + @GraphState( + name = REMOVE_STALE, + next = MONITOR, + resume = ResumeBehavior.RETRY, + description = """ + Remove file storage from the database if it doesn't exist on disk. + """ + ) + public void removeStale(FileStorageId id) throws SQLException { + fileStorageService.removeFileStorage(id); + } }