From 867410c66b0e5a7760dee9cab340d7f7c42c2a2a Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 1 Aug 2023 18:05:43 +0200 Subject: [PATCH] (file-storage) Automatic file storage discovery via manifest file --- .../db/storage/FileStorageManifest.java | 51 +++++++++++++++ .../db/storage/FileStorageService.java | 65 +++++++++++++++++-- ..._07_0_005__file_storage_default_values.sql | 2 +- .../monitor/FileStorageMonitorActor.java | 3 + 4 files changed, 113 insertions(+), 8 deletions(-) create mode 100644 code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageManifest.java diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageManifest.java b/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageManifest.java new file mode 100644 index 00000000..f002a47d --- /dev/null +++ b/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageManifest.java @@ -0,0 +1,51 @@ +package nu.marginalia.db.storage; + +import com.google.gson.Gson; +import nu.marginalia.db.storage.model.FileStorage; +import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.model.gson.GsonFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Optional; + +record FileStorageManifest(FileStorageType type, String description) { + private static final Gson gson = GsonFactory.get(); + private static final String fileName = "marginalia-manifest.json"; + private static final Logger logger = LoggerFactory.getLogger(FileStorageManifest.class); + + public static Optional find(Path directory) { + Path expectedFileName = directory.resolve(fileName); + + if (!Files.isRegularFile(expectedFileName) || + !Files.isReadable(expectedFileName)) { + return Optional.empty(); + } + + try (var reader = Files.newBufferedReader(expectedFileName)) { + return Optional.of(gson.fromJson(reader, FileStorageManifest.class)); + } + catch (Exception e) { + logger.warn("Failed to read manifest " + expectedFileName, e); + return Optional.empty(); + } + } + + public void write(FileStorage dir) { + Path expectedFileName = dir.asPath().resolve(fileName); + + try (var writer = Files.newBufferedWriter(expectedFileName, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING)) + { + gson.toJson(this, writer); + } + catch (Exception e) { + logger.warn("Failed to write manifest " + expectedFileName, e); + } + } + +} diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java b/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java index 2ce1b4d1..e136dd0b 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java +++ b/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java @@ -2,25 +2,26 @@ package nu.marginalia.db.storage; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.db.storage.model.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Singleton; +import java.io.File; +import java.io.FileFilter; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; +import java.nio.file.*; import java.nio.file.attribute.PosixFilePermissions; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; +import java.util.*; /** Manages file storage for processes and services */ @Singleton public class FileStorageService { private final HikariDataSource dataSource; - + private final Logger logger = LoggerFactory.getLogger(FileStorageService.class); @Inject public FileStorageService(HikariDataSource dataSource) { this.dataSource = dataSource; @@ -65,6 +66,49 @@ public class FileStorageService { return null; } + public void synchronizeStorageManifests(FileStorageBase base) { + Set ignoredPaths = new HashSet<>(); + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT PATH FROM FILE_STORAGE WHERE BASE_ID = ? + """)) { + stmt.setLong(1, base.id().id()); + var rs = stmt.executeQuery(); + while (rs.next()) { + ignoredPaths.add(rs.getString(1)); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + + File basePathFile = Path.of(base.path()).toFile(); + File[] files = basePathFile.listFiles(pathname -> pathname.isDirectory() && !ignoredPaths.contains(pathname.getName())); + if (files == null) return; + for (File file : files) { + var maybeManifest = FileStorageManifest.find(file.toPath()); + if (maybeManifest.isEmpty()) continue; + var manifest = maybeManifest.get(); + + logger.info("Discovered new file storage: " + file.getName() + " (" + manifest.type() + ")"); + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + INSERT INTO FILE_STORAGE(BASE_ID, PATH, TYPE, DESCRIPTION) + VALUES (?, ?, ?, ?) + """)) { + stmt.setLong(1, base.id().id()); + stmt.setString(2, file.getName()); + stmt.setString(3, manifest.type().name()); + stmt.setString(4, manifest.description()); + stmt.execute(); + conn.commit(); + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + } public void relateFileStorages(FileStorageId source, FileStorageId target) { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" @@ -198,7 +242,14 @@ public class FileStorageService { var rs = query.executeQuery(); if (rs.next()) { - return getStorage(new FileStorageId(rs.getLong("ID"))); + var storage = getStorage(new FileStorageId(rs.getLong("ID"))); + + // Write a manifest file so we can pick this up later without needing to insert it into DB + // (e.g. when loading from outside the system) + var manifest = new FileStorageManifest(type, description); + manifest.write(storage); + + return storage; } } diff --git a/code/common/db/src/main/resources/db/migration/V23_07_0_005__file_storage_default_values.sql b/code/common/db/src/main/resources/db/migration/V23_07_0_005__file_storage_default_values.sql index 8d591965..3803911f 100644 --- a/code/common/db/src/main/resources/db/migration/V23_07_0_005__file_storage_default_values.sql +++ b/code/common/db/src/main/resources/db/migration/V23_07_0_005__file_storage_default_values.sql @@ -1,7 +1,7 @@ INSERT IGNORE INTO FILE_STORAGE_BASE(NAME, PATH, TYPE, PERMIT_TEMP) VALUES ('Index Storage', '/vol', 'SSD_INDEX', false), -('Data Storage', '/samples', 'SLOW', false); +('Data Storage', '/samples', 'SLOW', true); INSERT IGNORE INTO FILE_STORAGE(BASE_ID, PATH, DESCRIPTION, TYPE) SELECT ID, 'iw', "Index Staging Area", 'INDEX_STAGING' diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java index 663fa9d8..9f2ced26 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java @@ -4,6 +4,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.db.storage.FileStorageService; import nu.marginalia.db.storage.model.FileStorage; +import nu.marginalia.db.storage.model.FileStorageBaseType; import nu.marginalia.db.storage.model.FileStorageId; import nu.marginalia.mqsm.StateFactory; import nu.marginalia.mqsm.graph.AbstractStateGraph; @@ -68,6 +69,8 @@ public class FileStorageMonitorActor extends AbstractStateGraph { transition(REMOVE_STALE, missing.get().id()); } + fileStorageService.synchronizeStorageManifests(fileStorageService.getStorageBase(FileStorageBaseType.SLOW)); + TimeUnit.SECONDS.sleep(10); } }