Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(control-service) Add timestamp to file storages.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-09-02 14:01:04 +02:00
parent 676e7c7947
commit 9e185e80ce
6 changed files with 85 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java
View File

@ -13,6 +13,7 @@ import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.PosixFilePermissions;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.*;
/** Manages file storage for processes and services
@ -292,6 +293,7 @@ public class FileStorageService {
new FileStorageId(rs.getLong("ID")),
base,
type,
LocalDateTime.now(),
newDir.toString(),
description
);
@ -305,7 +307,7 @@ public class FileStorageService {
public FileStorage getStorageByType(FileStorageType type) throws SQLException {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT PATH, DESCRIPTION, ID, BASE_ID
SELECT PATH, DESCRIPTION, ID, BASE_ID, CREATE_DATE
FROM FILE_STORAGE_VIEW WHERE TYPE = ?
""")) {
stmt.setString(1, type.name());
@ -314,11 +316,13 @@ public class FileStorageService {
long baseId;
String path;
String description;
LocalDateTime createDateTime;
try (var rs = stmt.executeQuery()) {
if (rs.next()) {
baseId = rs.getLong("BASE_ID");
storageId = rs.getLong("ID");
createDateTime = rs.getTimestamp("CREATE_DATE").toLocalDateTime();
path = rs.getString("PATH");
description = rs.getString("DESCRIPTION");
}
@ -332,6 +336,7 @@ public class FileStorageService {
new FileStorageId(storageId),
base,
type,
createDateTime,
path,
description
);
@ -344,7 +349,7 @@ public class FileStorageService {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT PATH, TYPE, DESCRIPTION, ID, BASE_ID
SELECT PATH, TYPE, DESCRIPTION, CREATE_DATE, ID, BASE_ID
FROM FILE_STORAGE_VIEW WHERE ID = ?
""")) {
stmt.setLong(1, id.id());
@ -354,6 +359,7 @@ public class FileStorageService {
String path;
String description;
FileStorageType type;
LocalDateTime createDateTime;
try (var rs = stmt.executeQuery()) {
if (rs.next()) {
@ -362,6 +368,7 @@ public class FileStorageService {
type = FileStorageType.valueOf(rs.getString("TYPE"));
path = rs.getString("PATH");
description = rs.getString("DESCRIPTION");
createDateTime = rs.getTimestamp("CREATE_DATE").toLocalDateTime();
}
else {
return null;
@ -373,6 +380,7 @@ public class FileStorageService {
new FileStorageId(storageId),
base,
type,
createDateTime,
path,
description
);
@ -394,7 +402,7 @@ public class FileStorageService {
List<FileStorage> ret = new ArrayList<>();
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT PATH, TYPE, DESCRIPTION, ID, BASE_ID
SELECT PATH, TYPE, DESCRIPTION, CREATE_DATE, ID, BASE_ID
FROM FILE_STORAGE_VIEW
""")) {
@ -402,6 +410,7 @@ public class FileStorageService {
long baseId;
String path;
String description;
LocalDateTime createDateTime;
FileStorageType type;
try (var rs = stmt.executeQuery()) {
@ -411,13 +420,14 @@ public class FileStorageService {
path = rs.getString("PATH");
type = FileStorageType.valueOf(rs.getString("TYPE"));
description = rs.getString("DESCRIPTION");
createDateTime = rs.getTimestamp("CREATE_DATE").toLocalDateTime();
var base = getStorageBase(new FileStorageBaseId(baseId));
ret.add(new FileStorage(
new FileStorageId(storageId),
base,
type,
createDateTime,
path,
description
));

30
code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorage.java
View File

@ -1,6 +1,8 @@
package nu.marginalia.db.storage.model;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Objects;
/**
* Represents a file storage area
@ -15,10 +17,38 @@ public record FileStorage(
FileStorageId id,
FileStorageBase base,
FileStorageType type,
LocalDateTime createDateTime,
String path,
String description)
{
public Path asPath() {
return Path.of(path);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FileStorage that = (FileStorage) o;
// Exclude timestamp as it may different due to how the objects
// are constructed
if (!Objects.equals(id, that.id)) return false;
if (!Objects.equals(base, that.base)) return false;
if (type != that.type) return false;
if (!Objects.equals(path, that.path)) return false;
return Objects.equals(description, that.description);
}
@Override
public int hashCode() {
int result = id != null ? id.hashCode() : 0;
result = 31 * result + (base != null ? base.hashCode() : 0);
result = 31 * result + (type != null ? type.hashCode() : 0);
result = 31 * result + (path != null ? path.hashCode() : 0);
result = 31 * result + (description != null ? description.hashCode() : 0);
return result;
}
}

2
code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java
View File

@ -31,7 +31,7 @@ class LoaderIndexJournalWriterTest {
tempDir = Files.createTempDirectory(getClass().getSimpleName());
FileStorageService storageService = Mockito.mock(FileStorageService.class);
Mockito.when(storageService.getStorageByType(FileStorageType.INDEX_STAGING)).
thenReturn(new FileStorage(null, null, null, tempDir.toString(),
thenReturn(new FileStorage(null, null, null, null, tempDir.toString(),
"test"));
writer = new LoaderIndexJournalWriter(storageService);
}

27
code/services-core/control-service/src/main/java/nu/marginalia/control/model/FileStorageWithActions.java
View File

@ -4,6 +4,10 @@ import nu.marginalia.db.storage.model.FileStorage;
import nu.marginalia.db.storage.model.FileStorageBaseType;
import nu.marginalia.db.storage.model.FileStorageType;
import java.nio.file.Path;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
public record FileStorageWithActions(FileStorage storage) {
public boolean isCrawlable() {
return storage.type() == FileStorageType.CRAWL_SPEC;
@ -27,4 +31,27 @@ public record FileStorageWithActions(FileStorage storage) {
return baseType == FileStorageBaseType.SLOW
|| baseType == FileStorageBaseType.BACKUP;
}
public String getRelPath() {
Path basePath = storage.base().asPath();
Path storagePath = storage.asPath();
return basePath.relativize(storagePath)
.toString();
}
public String getTimestampFull() {
return storage.createDateTime().format(DateTimeFormatter.ISO_DATE_TIME);
}
public String getTimestamp() {
var ctime = storage.createDateTime();
if (ctime.isAfter(LocalDate.now().atStartOfDay())) {
return storage.createDateTime().format(DateTimeFormatter.ISO_TIME);
}
else {
return storage.createDateTime().format(DateTimeFormatter.ISO_DATE);
}
}
}

15
code/services-core/control-service/src/main/resources/templates/control/partials/storage-table.hdb
View File

@ -1,22 +1,22 @@
<table>
<table class="table-rh-2">
{{#each storage}}
<tr>
<th>Type</th>
<th>Name</th>
<th>Path</th>
<th>Permit Temp</th>
</tr>
<tr>
<td>{{base.type}}</td>
<td>{{base.name}}</td>
<td>{{base.path}}</td>
<td>{{base.permitTemp}}</td>
</tr>
<tr>
<th></th>
<th>Type</th>
<th>Path</th>
<th>Description</th>
<tr>
<th>Created</th>
<th colspan="2">Description</th>
</tr>
{{#each storage}}
<tr>
@ -24,8 +24,11 @@
<a href="/storage/{{storage.id}}">Info</a>
</td>
<td>{{storage.type}}</td>
<td>{{storage.path}}</td>
<td>{{storage.description}}</td>
<td>{{relPath}}</td>
</tr>
<tr>
<td title="{{timestampFull}}">{{timestamp}}</td>
<td colspan="2">{{storage.description}}</td>
</tr>
{{/each}}
{{/each}}

7
code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.Random;
import java.util.UUID;
@ -52,9 +53,9 @@ public class IndexQueryServiceIntegrationTestModule extends AbstractModule {
try {
var fileStorageServiceMock = Mockito.mock(FileStorageService.class);
when(fileStorageServiceMock.getStorageByType(FileStorageType.SEARCH_SETS)).thenReturn(new FileStorage(null, null, null, fastDir.toString(), null));
when(fileStorageServiceMock.getStorageByType(FileStorageType.INDEX_LIVE)).thenReturn(new FileStorage(null, null, null, fastDir.toString(), null));
when(fileStorageServiceMock.getStorageByType(FileStorageType.INDEX_STAGING)).thenReturn(new FileStorage(null, null, null, slowDir.toString(), null));
when(fileStorageServiceMock.getStorageByType(FileStorageType.SEARCH_SETS)).thenReturn(new FileStorage(null, null, null, LocalDateTime.now(), fastDir.toString(), null));
when(fileStorageServiceMock.getStorageByType(FileStorageType.INDEX_LIVE)).thenReturn(new FileStorage(null, null, null, LocalDateTime.now(), fastDir.toString(), null));
when(fileStorageServiceMock.getStorageByType(FileStorageType.INDEX_STAGING)).thenReturn(new FileStorage(null, null, null, LocalDateTime.now(), slowDir.toString(), null));
bind(FileStorageService.class).toInstance(fileStorageServiceMock);