Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(file-storage) Deprecate mustClean flag

Browse Source
This commit is contained in:
Viktor Lofgren 2023-08-01 22:32:30 +02:00
parent 867410c66b
commit 659d2134ba
4 changed files with 11 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java
View File

@ -8,7 +8,6 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject; import javax.inject.Inject;
import javax.inject.Singleton; import javax.inject.Singleton;
import java.io.File; import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.nio.file.*; import java.nio.file.*;
@ -46,7 +45,7 @@ public class FileStorageService {
public FileStorageBase getStorageBase(FileStorageBaseId type) throws SQLException { public FileStorageBase getStorageBase(FileStorageBaseId type) throws SQLException {
try (var conn = dataSource.getConnection(); try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement(""" var stmt = conn.prepareStatement("""
SELECT ID, NAME, PATH, TYPE, MUST_CLEAN, PERMIT_TEMP SELECT ID, NAME, PATH, TYPE, PERMIT_TEMP
FROM FILE_STORAGE_BASE WHERE ID = ? FROM FILE_STORAGE_BASE WHERE ID = ?
""")) { """)) {
stmt.setLong(1, type.id()); stmt.setLong(1, type.id());
@ -57,8 +56,7 @@ public class FileStorageService {
FileStorageBaseType.valueOf(rs.getString(4)), FileStorageBaseType.valueOf(rs.getString(4)),
rs.getString(2), rs.getString(2),
rs.getString(3), rs.getString(3),
rs.getBoolean(5), rs.getBoolean(5)
rs.getBoolean(6)
); );
} }
} }
@ -156,7 +154,7 @@ public class FileStorageService {
public FileStorageBase getStorageBase(FileStorageBaseType type) throws SQLException { public FileStorageBase getStorageBase(FileStorageBaseType type) throws SQLException {
try (var conn = dataSource.getConnection(); try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement(""" var stmt = conn.prepareStatement("""
SELECT ID, NAME, PATH, TYPE, MUST_CLEAN, PERMIT_TEMP SELECT ID, NAME, PATH, TYPE, PERMIT_TEMP
FROM FILE_STORAGE_BASE WHERE TYPE = ? FROM FILE_STORAGE_BASE WHERE TYPE = ?
""")) { """)) {
stmt.setString(1, type.name()); stmt.setString(1, type.name());
@ -167,8 +165,7 @@ public class FileStorageService {
FileStorageBaseType.valueOf(rs.getString(4)), FileStorageBaseType.valueOf(rs.getString(4)),
rs.getString(2), rs.getString(2),
rs.getString(3), rs.getString(3),
rs.getBoolean(5), rs.getBoolean(5)
rs.getBoolean(6)
); );
} }
} }
@ -176,7 +173,7 @@ public class FileStorageService {
return null; return null;
} }
public FileStorageBase createStorageBase(String name, Path path, FileStorageBaseType type, boolean mustClean, boolean permitTemp) throws SQLException, FileNotFoundException { public FileStorageBase createStorageBase(String name, Path path, FileStorageBaseType type, boolean permitTemp) throws SQLException, FileNotFoundException {
if (!Files.exists(path)) { if (!Files.exists(path)) {
throw new FileNotFoundException("Storage base path does not exist: " + path); throw new FileNotFoundException("Storage base path does not exist: " + path);
@ -184,14 +181,13 @@ public class FileStorageService {
try (var conn = dataSource.getConnection(); try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement(""" var stmt = conn.prepareStatement("""
INSERT INTO FILE_STORAGE_BASE(NAME, PATH, TYPE, MUST_CLEAN, PERMIT_TEMP) INSERT INTO FILE_STORAGE_BASE(NAME, PATH, TYPE, PERMIT_TEMP)
VALUES (?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?)
""")) { """)) {
stmt.setString(1, name); stmt.setString(1, name);
stmt.setString(2, path.toString()); stmt.setString(2, path.toString());
stmt.setString(3, type.name()); stmt.setString(3, type.name());
stmt.setBoolean(4, mustClean); stmt.setBoolean(4, permitTemp);
stmt.setBoolean(5, permitTemp);
int update = stmt.executeUpdate(); int update = stmt.executeUpdate();
if (update < 0) { if (update < 0) {

2
code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBase.java
View File

@ -9,14 +9,12 @@ import java.nio.file.Path;
* @param type the type of the storage base * @param type the type of the storage base
* @param name the name of the storage base * @param name the name of the storage base
* @param path the path of the storage base * @param path the path of the storage base
* @param mustClean if true, the storage is small and *must* be cleaned after use
* @param permitTemp if true, the storage may be used for temporary files * @param permitTemp if true, the storage may be used for temporary files
*/ */
public record FileStorageBase(FileStorageBaseId id, public record FileStorageBase(FileStorageBaseId id,
FileStorageBaseType type, FileStorageBaseType type,
String name, String name,
String path, String path,
boolean mustClean,
boolean permitTemp boolean permitTemp
) { ) {
public Path asPath() { public Path asPath() {

1
code/common/db/src/main/resources/db/migration/V23_07_0_004__file_storage.sql
View File

@ -3,7 +3,6 @@ CREATE TABLE IF NOT EXISTS FILE_STORAGE_BASE (
NAME VARCHAR(255) NOT NULL UNIQUE, NAME VARCHAR(255) NOT NULL UNIQUE,
PATH VARCHAR(255) NOT NULL UNIQUE COMMENT 'The path to the storage base', PATH VARCHAR(255) NOT NULL UNIQUE COMMENT 'The path to the storage base',
TYPE ENUM ('SSD_INDEX', 'SSD_WORK', 'SLOW', 'BACKUP') NOT NULL, TYPE ENUM ('SSD_INDEX', 'SSD_WORK', 'SLOW', 'BACKUP') NOT NULL,
MUST_CLEAN BOOLEAN NOT NULL DEFAULT FALSE COMMENT 'If true, the storage must be cleaned after use',
PERMIT_TEMP BOOLEAN NOT NULL DEFAULT FALSE COMMENT 'If true, the storage can be used for temporary files' PERMIT_TEMP BOOLEAN NOT NULL DEFAULT FALSE COMMENT 'If true, the storage can be used for temporary files'
) )
CHARACTER SET utf8mb4 CHARACTER SET utf8mb4

9
code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java
View File

@ -97,11 +97,10 @@ public class FileStorageServiceTest {
String name = "test-" + UUID.randomUUID(); String name = "test-" + UUID.randomUUID();
var storage = new FileStorageService(dataSource); var storage = new FileStorageService(dataSource);
var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false, false); var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false);
Assertions.assertEquals(name, base.name()); Assertions.assertEquals(name, base.name());
Assertions.assertEquals(FileStorageBaseType.SLOW, base.type()); Assertions.assertEquals(FileStorageBaseType.SLOW, base.type());
Assertions.assertFalse(base.mustClean());
Assertions.assertFalse(base.permitTemp()); Assertions.assertFalse(base.permitTemp());
} }
@Test @Test
@ -110,7 +109,7 @@ public class FileStorageServiceTest {
var storage = new FileStorageService(dataSource); var storage = new FileStorageService(dataSource);
var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false, false); var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false);
try { try {
storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldFail"); storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldFail");
@ -129,7 +128,7 @@ public class FileStorageServiceTest {
var storage = new FileStorageService(dataSource); var storage = new FileStorageService(dataSource);
var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false, false); var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false);
var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldSucceed"); var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldSucceed");
tempDirs.add(created.asPath()); tempDirs.add(created.asPath());
@ -144,7 +143,7 @@ public class FileStorageServiceTest {
var storage = new FileStorageService(dataSource); var storage = new FileStorageService(dataSource);
var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false, true); var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, true);
var fileStorage = storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldSucceed"); var fileStorage = storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldSucceed");
Assertions.assertTrue(Files.exists(fileStorage.asPath())); Assertions.assertTrue(Files.exists(fileStorage.asPath()));