Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(file-storage) Automatic file storage discovery via manifest file

Browse Source
This commit is contained in:
Viktor Lofgren 2023-08-01 18:05:43 +02:00
parent 483c2dbb44
commit 867410c66b
4 changed files with 113 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageManifest.java Normal file
View File

@ -0,0 +1,51 @@
package nu.marginalia.db.storage;
import com.google.gson.Gson;
import nu.marginalia.db.storage.model.FileStorage;
import nu.marginalia.db.storage.model.FileStorageType;
import nu.marginalia.model.gson.GsonFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Optional;
record FileStorageManifest(FileStorageType type, String description) {
private static final Gson gson = GsonFactory.get();
private static final String fileName = "marginalia-manifest.json";
private static final Logger logger = LoggerFactory.getLogger(FileStorageManifest.class);
public static Optional<FileStorageManifest> find(Path directory) {
Path expectedFileName = directory.resolve(fileName);
if (!Files.isRegularFile(expectedFileName) ||
!Files.isReadable(expectedFileName)) {
return Optional.empty();
}
try (var reader = Files.newBufferedReader(expectedFileName)) {
return Optional.of(gson.fromJson(reader, FileStorageManifest.class));
}
catch (Exception e) {
logger.warn("Failed to read manifest " + expectedFileName, e);
return Optional.empty();
}
}
public void write(FileStorage dir) {
Path expectedFileName = dir.asPath().resolve(fileName);
try (var writer = Files.newBufferedWriter(expectedFileName,
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING))
{
gson.toJson(this, writer);
}
catch (Exception e) {
logger.warn("Failed to write manifest " + expectedFileName, e);
}
}
}

65
code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java
View File

@ -2,25 +2,26 @@ package nu.marginalia.db.storage;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.db.storage.model.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.*;
import java.nio.file.attribute.PosixFilePermissions;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.*;
/** Manages file storage for processes and services
*/
@Singleton
public class FileStorageService {
private final HikariDataSource dataSource;
private final Logger logger = LoggerFactory.getLogger(FileStorageService.class);
@Inject
public FileStorageService(HikariDataSource dataSource) {
this.dataSource = dataSource;
@ -65,6 +66,49 @@ public class FileStorageService {
return null;
}
public void synchronizeStorageManifests(FileStorageBase base) {
Set<String> ignoredPaths = new HashSet<>();
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT PATH FROM FILE_STORAGE WHERE BASE_ID = ?
""")) {
stmt.setLong(1, base.id().id());
var rs = stmt.executeQuery();
while (rs.next()) {
ignoredPaths.add(rs.getString(1));
}
} catch (SQLException e) {
throw new RuntimeException(e);
}
File basePathFile = Path.of(base.path()).toFile();
File[] files = basePathFile.listFiles(pathname -> pathname.isDirectory() && !ignoredPaths.contains(pathname.getName()));
if (files == null) return;
for (File file : files) {
var maybeManifest = FileStorageManifest.find(file.toPath());
if (maybeManifest.isEmpty()) continue;
var manifest = maybeManifest.get();
logger.info("Discovered new file storage: " + file.getName() + " (" + manifest.type() + ")");
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
INSERT INTO FILE_STORAGE(BASE_ID, PATH, TYPE, DESCRIPTION)
VALUES (?, ?, ?, ?)
""")) {
stmt.setLong(1, base.id().id());
stmt.setString(2, file.getName());
stmt.setString(3, manifest.type().name());
stmt.setString(4, manifest.description());
stmt.execute();
conn.commit();
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
}
public void relateFileStorages(FileStorageId source, FileStorageId target) {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
@ -198,7 +242,14 @@ public class FileStorageService {
var rs = query.executeQuery();
if (rs.next()) {
return getStorage(new FileStorageId(rs.getLong("ID")));
var storage = getStorage(new FileStorageId(rs.getLong("ID")));
// Write a manifest file so we can pick this up later without needing to insert it into DB
// (e.g. when loading from outside the system)
var manifest = new FileStorageManifest(type, description);
manifest.write(storage);
return storage;
}
}

2
code/common/db/src/main/resources/db/migration/V23_07_0_005__file_storage_default_values.sql
View File

@ -1,7 +1,7 @@
INSERT IGNORE INTO FILE_STORAGE_BASE(NAME, PATH, TYPE, PERMIT_TEMP)
VALUES
('Index Storage', '/vol', 'SSD_INDEX', false),
('Data Storage', '/samples', 'SLOW', false);
('Data Storage', '/samples', 'SLOW', true);
INSERT IGNORE INTO FILE_STORAGE(BASE_ID, PATH, DESCRIPTION, TYPE)
SELECT ID, 'iw', "Index Staging Area", 'INDEX_STAGING'

3
code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.db.storage.FileStorageService;
import nu.marginalia.db.storage.model.FileStorage;
import nu.marginalia.db.storage.model.FileStorageBaseType;
import nu.marginalia.db.storage.model.FileStorageId;
import nu.marginalia.mqsm.StateFactory;
import nu.marginalia.mqsm.graph.AbstractStateGraph;
@ -68,6 +69,8 @@ public class FileStorageMonitorActor extends AbstractStateGraph {
transition(REMOVE_STALE, missing.get().id());
}
fileStorageService.synchronizeStorageManifests(fileStorageService.getStorageBase(FileStorageBaseType.SLOW));
TimeUnit.SECONDS.sleep(10);
}
}