(converter) Automatically clean stale file storage records if they disappear on disk

code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java

@@ -342,4 +342,44 @@ public class FileStorageService {
             stmt.executeUpdate();
         }
     }
+
+    public List<FileStorage> getEachFileStorage() {
+        List<FileStorage> ret = new ArrayList<>();
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT PATH, TYPE, DESCRIPTION, ID, BASE_ID
+                     FROM FILE_STORAGE_VIEW
+                     """)) {
+
+            long storageId;
+            long baseId;
+            String path;
+            String description;
+            FileStorageType type;
+
+            try (var rs = stmt.executeQuery()) {
+                while (rs.next()) {
+                    baseId = rs.getLong("BASE_ID");
+                    storageId = rs.getLong("ID");
+                    path = rs.getString("PATH");
+                    type = FileStorageType.valueOf(rs.getString("TYPE"));
+                    description = rs.getString("DESCRIPTION");
+
+                    var base = getStorageBase(new FileStorageBaseId(baseId));
+
+                    ret.add(new FileStorage(
+                            new FileStorageId(storageId),
+                            base,
+                            type,
+                            path,
+                            description
+                    ));
+                }
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+
+        return ret;
+    }
 }

code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java

@@ -131,7 +131,7 @@ public class FileStorageServiceTest {
 
         var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false, false);
 
-        var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldFail");
+        var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldSucceed");
         tempDirs.add(created.asPath());
 
         var actual = storage.getStorage(created.id());

code/services-satellite/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java

@@ -15,6 +15,8 @@ import org.slf4j.LoggerFactory;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.sql.SQLException;
+import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
@@ -27,6 +29,7 @@ public class FileStorageMonitorActor extends AbstractStateGraph {
     private static final String INITIAL = "INITIAL";
     private static final String MONITOR = "MONITOR";
     private static final String PURGE = "PURGE";
+    private static final String REMOVE_STALE = "REMOVE-STALE";
     private static final String END = "END";
     private final FileStorageService fileStorageService;
 
@@ -42,7 +45,10 @@ public class FileStorageMonitorActor extends AbstractStateGraph {
     public void init() {
     }
 
-    @GraphState(name = MONITOR, next = PURGE, resume = ResumeBehavior.RETRY, transitions = { PURGE },
+    @GraphState(name = MONITOR,
+            next = PURGE,
+            resume = ResumeBehavior.RETRY,
+            transitions = { PURGE, REMOVE_STALE },
             description = """
                     Monitor the file storage and trigger at transition to PURGE if any file storage area
                     has been marked for deletion.
@@ -52,12 +58,17 @@ public class FileStorageMonitorActor extends AbstractStateGraph {
         for (;;) {
             Optional<FileStorage> toDeleteOpt = fileStorageService.findFileStorageToDelete();
 
-            if (toDeleteOpt.isEmpty()) {
+            if (toDeleteOpt.isPresent()) {
-                TimeUnit.SECONDS.sleep(10);
-            }
-            else {
                 transition(PURGE, toDeleteOpt.get().id());
             }
+
+            List<FileStorage> allStorageItems = fileStorageService.getEachFileStorage();
+            var missing = allStorageItems.stream().filter(storage -> !Files.exists(storage.asPath())).findAny();
+            if (missing.isPresent()) {
+                transition(REMOVE_STALE, missing.get().id());
+            }
+
+            TimeUnit.SECONDS.sleep(10);
         }
     }
 
@@ -79,4 +90,16 @@ public class FileStorageMonitorActor extends AbstractStateGraph {
 
         fileStorageService.removeFileStorage(storage.id());
     }
+
+    @GraphState(
+            name = REMOVE_STALE,
+            next = MONITOR,
+            resume = ResumeBehavior.RETRY,
+            description = """
+                        Remove file storage from the database if it doesn't exist on disk.
+                        """
+    )
+    public void removeStale(FileStorageId id) throws SQLException {
+        fileStorageService.removeFileStorage(id);
+    }
 }