Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(converter) Automatically clean stale file storage records if they disappear on disk

Browse Source
This commit is contained in:
Viktor Lofgren 2023-07-24 17:04:42 +02:00
parent 667b0ca0b0
commit 09fd0a1d0e
3 changed files with 69 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java
View File

@ -342,4 +342,44 @@ public class FileStorageService {
stmt.executeUpdate();
}
}
public List<FileStorage> getEachFileStorage() {
List<FileStorage> ret = new ArrayList<>();
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT PATH, TYPE, DESCRIPTION, ID, BASE_ID
FROM FILE_STORAGE_VIEW
""")) {
long storageId;
long baseId;
String path;
String description;
FileStorageType type;
try (var rs = stmt.executeQuery()) {
while (rs.next()) {
baseId = rs.getLong("BASE_ID");
storageId = rs.getLong("ID");
path = rs.getString("PATH");
type = FileStorageType.valueOf(rs.getString("TYPE"));
description = rs.getString("DESCRIPTION");
var base = getStorageBase(new FileStorageBaseId(baseId));
ret.add(new FileStorage(
new FileStorageId(storageId),
base,
type,
path,
description
));
}
}
} catch (SQLException e) {
e.printStackTrace();
}
return ret;
}
}

2
code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java
View File

@ -131,7 +131,7 @@ public class FileStorageServiceTest {
var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false, false);
var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldFail");
var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldSucceed");
tempDirs.add(created.asPath());
var actual = storage.getStorage(created.id());

33
code/services-satellite/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java
View File

@ -15,6 +15,8 @@ import org.slf4j.LoggerFactory;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@ -27,6 +29,7 @@ public class FileStorageMonitorActor extends AbstractStateGraph {
private static final String INITIAL = "INITIAL";
private static final String MONITOR = "MONITOR";
private static final String PURGE = "PURGE";
private static final String REMOVE_STALE = "REMOVE-STALE";
private static final String END = "END";
private final FileStorageService fileStorageService;
@ -42,7 +45,10 @@ public class FileStorageMonitorActor extends AbstractStateGraph {
public void init() {
}
@GraphState(name = MONITOR, next = PURGE, resume = ResumeBehavior.RETRY, transitions = { PURGE },
@GraphState(name = MONITOR,
next = PURGE,
resume = ResumeBehavior.RETRY,
transitions = { PURGE, REMOVE_STALE },
description = """
Monitor the file storage and trigger at transition to PURGE if any file storage area
has been marked for deletion.
@ -52,12 +58,17 @@ public class FileStorageMonitorActor extends AbstractStateGraph {
for (;;) {
Optional<FileStorage> toDeleteOpt = fileStorageService.findFileStorageToDelete();
if (toDeleteOpt.isEmpty()) {
TimeUnit.SECONDS.sleep(10);
}
else {
if (toDeleteOpt.isPresent()) {
transition(PURGE, toDeleteOpt.get().id());
}
List<FileStorage> allStorageItems = fileStorageService.getEachFileStorage();
var missing = allStorageItems.stream().filter(storage -> !Files.exists(storage.asPath())).findAny();
if (missing.isPresent()) {
transition(REMOVE_STALE, missing.get().id());
}
TimeUnit.SECONDS.sleep(10);
}
}
@ -79,4 +90,16 @@ public class FileStorageMonitorActor extends AbstractStateGraph {
fileStorageService.removeFileStorage(storage.id());
}
@GraphState(
name = REMOVE_STALE,
next = MONITOR,
resume = ResumeBehavior.RETRY,
description = """
Remove file storage from the database if it doesn't exist on disk.
"""
)
public void removeStale(FileStorageId id) throws SQLException {
fileStorageService.removeFileStorage(id);
}
}