Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(*) Fix merge gore

Browse Source
This commit is contained in:
Viktor Lofgren 2023-08-09 13:33:28 +02:00
parent 47f3855a4b
commit 251fc63b42
15 changed files with 79 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java
View File

@ -182,7 +182,7 @@ public class FileStorageService {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
INSERT INTO FILE_STORAGE_BASE(NAME, PATH, TYPE, PERMIT_TEMP)
VALUES (?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?)
""")) {
stmt.setString(1, name);
stmt.setString(2, path.toString());

2
code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeat.java
View File

@ -44,7 +44,7 @@ public class ServiceHeartbeat {
Runtime.getRuntime().addShutdownHook(new Thread(this::shutDown));
}
public <T extends Enum<T>> ServiceTaskHeartbeat<T> createServiceProcessHeartbeat(Class<T> steps, String processName) {
public <T extends Enum<T>> ServiceTaskHeartbeat<T> createServiceTaskHeartbeat(Class<T> steps, String processName) {
return new ServiceTaskHeartbeat<>(steps, configuration, processName, dataSource);
}

2
code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java
View File

@ -64,7 +64,7 @@ public class ForwardIndexConverter {
logger.info("Domain Rankings size = {}", domainRankings.size());
try (var progress = heartbeat.createServiceProcessHeartbeat(TaskSteps.class, "forwardIndexConverter")) {
try (var progress = heartbeat.createServiceTaskHeartbeat(TaskSteps.class, "forwardIndexConverter")) {
progress.progress(TaskSteps.GET_DOC_IDS);
LongArray docsFileId = getDocIds(outputFileDocsId, journalReader);

12
code/features-index/index-forward/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java
View File

@ -1,7 +1,6 @@
package nu.marginalia.index.forward;
import lombok.SneakyThrows;
import nu.marginalia.dict.OffHeapDictionaryHashMap;
import nu.marginalia.index.journal.model.IndexJournalEntry;
import nu.marginalia.index.journal.writer.IndexJournalWriterImpl;
import nu.marginalia.index.journal.writer.IndexJournalWriter;
@ -9,10 +8,13 @@ import nu.marginalia.lexicon.journal.KeywordLexiconJournalMode;
import nu.marginalia.ranking.DomainRankings;
import nu.marginalia.lexicon.KeywordLexicon;
import nu.marginalia.lexicon.journal.KeywordLexiconJournal;
import nu.marginalia.service.control.ServiceHeartbeat;
import nu.marginalia.service.control.ServiceTaskHeartbeat;
import nu.marginalia.test.TestUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -22,6 +24,7 @@ import java.nio.file.Path;
import java.util.stream.IntStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.when;
class ForwardIndexConverterTest {
@ -98,7 +101,12 @@ class ForwardIndexConverterTest {
@Test
void testForwardIndex() throws IOException {
new ForwardIndexConverter(indexFile.toFile(), docsFileId, docsFileData, new DomainRankings()).convert();
// RIP fairies
var serviceHeartbeat = Mockito.mock(ServiceHeartbeat.class);
when(serviceHeartbeat.createServiceTaskHeartbeat(Mockito.any(), Mockito.any()))
.thenReturn(Mockito.mock(ServiceTaskHeartbeat.class));
new ForwardIndexConverter(serviceHeartbeat, indexFile.toFile(), docsFileId, docsFileData, new DomainRankings()).convert();
var forwardReader = new ForwardIndexReader(docsFileId, docsFileData);

2
code/features-index/index-reverse/src/main/java/nu/marginalia/index/full/ReverseIndexFullConverter.java
View File

@ -76,7 +76,7 @@ public class ReverseIndexFullConverter {
final Path intermediateUrlsFile = Files.createTempFile(tmpFileDir, "urls-sorted", ".dat");
try (var progress = heartbeat.createServiceProcessHeartbeat(TaskSteps.class, "reverseIndexFullConverter")) {
try (var progress = heartbeat.createServiceTaskHeartbeat(TaskSteps.class, "reverseIndexFullConverter")) {
progress.progress(TaskSteps.ACCUMULATE_STATISTICS);
final IndexJournalStatistics statistics = journalReader.getStatistics();

2
code/features-index/index-reverse/src/main/java/nu/marginalia/index/priority/ReverseIndexPriorityConverter.java
View File

@ -75,7 +75,7 @@ public class ReverseIndexPriorityConverter {
final Path intermediateUrlsFile = Files.createTempFile(tmpFileDir, "urls-sorted", ".dat");
try (var progress = heartbeat.createServiceProcessHeartbeat(TaskSteps.class, "reverseIndexPriorityConverter")) {
try (var progress = heartbeat.createServiceTaskHeartbeat(TaskSteps.class, "reverseIndexPriorityConverter")) {
progress.progress(TaskSteps.ACCUMULATE_STATISTICS);
final IndexJournalStatistics statistics = journalReader.getStatistics();

9
code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest.java
View File

@ -14,6 +14,7 @@ import nu.marginalia.lexicon.KeywordLexicon;
import nu.marginalia.lexicon.journal.KeywordLexiconJournal;
import nu.marginalia.model.idx.DocumentMetadata;
import nu.marginalia.service.control.ServiceHeartbeat;
import nu.marginalia.service.control.ServiceTaskHeartbeat;
import nu.marginalia.test.TestUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -28,6 +29,7 @@ import java.util.stream.IntStream;
import java.util.stream.LongStream;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.mockito.Mockito.when;
class ReverseIndexFullConverterTest {
KeywordLexicon keywordLexicon;
@ -86,8 +88,13 @@ class ReverseIndexFullConverterTest {
var docsFile = dataDir.resolve("docs.dat");
var journalReader = new IndexJournalReaderSingleCompressedFile(indexFile);
// RIP fairies
var serviceHeartbeat = Mockito.mock(ServiceHeartbeat.class);
when(serviceHeartbeat.createServiceTaskHeartbeat(Mockito.any(), Mockito.any()))
.thenReturn(Mockito.mock(ServiceTaskHeartbeat.class));
new ReverseIndexFullConverter(
Mockito.mock(ServiceHeartbeat.class),
serviceHeartbeat,
tmpDir, journalReader, new DomainRankings(), wordsFile, docsFile)
.convert();

17
code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest2.java
View File

@ -15,6 +15,7 @@ import nu.marginalia.ranking.DomainRankings;
import nu.marginalia.lexicon.KeywordLexicon;
import nu.marginalia.lexicon.journal.KeywordLexiconJournal;
import nu.marginalia.service.control.ServiceHeartbeat;
import nu.marginalia.service.control.ServiceTaskHeartbeat;
import nu.marginalia.test.TestUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@ -30,6 +31,8 @@ import java.util.Arrays;
import java.util.stream.IntStream;
import java.util.stream.LongStream;
import static org.mockito.Mockito.when;
class ReverseIndexFullConverterTest2 {
KeywordLexicon keywordLexicon;
@ -119,7 +122,12 @@ class ReverseIndexFullConverterTest2 {
Path tmpDir = Path.of("/tmp");
new ReverseIndexFullConverter(Mockito.mock(ServiceHeartbeat.class), tmpDir, new IndexJournalReaderSingleCompressedFile(indexFile), new DomainRankings(), wordsFile, docsFile).convert();
// RIP fairies
var serviceHeartbeat = Mockito.mock(ServiceHeartbeat.class);
when(serviceHeartbeat.createServiceTaskHeartbeat(Mockito.any(), Mockito.any()))
.thenReturn(Mockito.mock(ServiceTaskHeartbeat.class));
new ReverseIndexFullConverter(serviceHeartbeat, tmpDir, new IndexJournalReaderSingleCompressedFile(indexFile), new DomainRankings(), wordsFile, docsFile).convert();
var reverseReader = new ReverseIndexFullReader(wordsFile, docsFile);
@ -144,7 +152,12 @@ class ReverseIndexFullConverterTest2 {
Path tmpDir = Path.of("/tmp");
new ReverseIndexFullConverter(Mockito.mock(ServiceHeartbeat.class), tmpDir, new IndexJournalReaderSingleCompressedFile(indexFile, null, ReverseIndexPriorityParameters::filterPriorityRecord), new DomainRankings(), wordsFile, docsFile).convert();
// RIP fairies
var serviceHeartbeat = Mockito.mock(ServiceHeartbeat.class);
when(serviceHeartbeat.createServiceTaskHeartbeat(Mockito.any(), Mockito.any()))
.thenReturn(Mockito.mock(ServiceTaskHeartbeat.class));
new ReverseIndexFullConverter(serviceHeartbeat, tmpDir, new IndexJournalReaderSingleCompressedFile(indexFile, null, ReverseIndexPriorityParameters::filterPriorityRecord), new DomainRankings(), wordsFile, docsFile).convert();
var reverseReader = new ReverseIndexFullReader(wordsFile, docsFile);

17
code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexPriorityConverterTest2.java
View File

@ -15,6 +15,7 @@ import nu.marginalia.lexicon.journal.KeywordLexiconJournal;
import nu.marginalia.lexicon.journal.KeywordLexiconJournalMode;
import nu.marginalia.ranking.DomainRankings;
import nu.marginalia.service.control.ServiceHeartbeat;
import nu.marginalia.service.control.ServiceTaskHeartbeat;
import nu.marginalia.test.TestUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@ -30,6 +31,8 @@ import java.util.Arrays;
import java.util.stream.IntStream;
import java.util.stream.LongStream;
import static org.mockito.Mockito.when;
class ReverseIndexPriorityConverterTest2 {
KeywordLexicon keywordLexicon;
@ -119,7 +122,12 @@ class ReverseIndexPriorityConverterTest2 {
Path tmpDir = Path.of("/tmp");
new ReverseIndexPriorityConverter(Mockito.mock(ServiceHeartbeat.class), tmpDir, new IndexJournalReaderSingleCompressedFile(indexFile), new DomainRankings(), wordsFile, docsFile).convert();
// RIP fairies
var serviceHeartbeat = Mockito.mock(ServiceHeartbeat.class);
when(serviceHeartbeat.createServiceTaskHeartbeat(Mockito.any(), Mockito.any()))
.thenReturn(Mockito.mock(ServiceTaskHeartbeat.class));
new ReverseIndexPriorityConverter(serviceHeartbeat, tmpDir, new IndexJournalReaderSingleCompressedFile(indexFile), new DomainRankings(), wordsFile, docsFile).convert();
var reverseReader = new ReverseIndexPriorityReader(wordsFile, docsFile);
@ -144,7 +152,12 @@ class ReverseIndexPriorityConverterTest2 {
Path tmpDir = Path.of("/tmp");
new ReverseIndexPriorityConverter(Mockito.mock(ServiceHeartbeat.class), tmpDir, new IndexJournalReaderSingleCompressedFile(indexFile, null, ReverseIndexPriorityParameters::filterPriorityRecord), new DomainRankings(), wordsFile, docsFile).convert();
// RIP fairies
var serviceHeartbeat = Mockito.mock(ServiceHeartbeat.class);
when(serviceHeartbeat.createServiceTaskHeartbeat(Mockito.any(), Mockito.any()))
.thenReturn(Mockito.mock(ServiceTaskHeartbeat.class));
new ReverseIndexPriorityConverter(serviceHeartbeat, tmpDir, new IndexJournalReaderSingleCompressedFile(indexFile, null, ReverseIndexPriorityParameters::filterPriorityRecord), new DomainRankings(), wordsFile, docsFile).convert();
var reverseReader = new ReverseIndexPriorityReader(wordsFile, docsFile);

9
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java
View File

@ -166,15 +166,6 @@ public class CrawlerMain {
finally {
heartbeat.shutDown();
}
pool.execute(() -> {
try {
fetchDomain(crawlingSpecification);
}
finally {
taskSem.release();
}
});
}
class CrawlTask implements DumbThreadPool.Task {

15
code/services-core/control-service/src/test/java/nu/marginalia/control/svc/ApiKeyServiceTest.java
View File

@ -3,15 +3,13 @@ package nu.marginalia.control.svc;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.control.model.ApiKeyModel;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.parallel.Execution;
import org.testcontainers.containers.MariaDBContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import java.sql.SQLException;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
@ -46,6 +44,15 @@ public class ApiKeyServiceTest {
mariaDBContainer.close();
}
@AfterEach
public void cleanDb() {
try (var conn = dataSource.getConnection(); var stmt = conn.createStatement()) {
stmt.executeUpdate("TRUNCATE TABLE EC_API_KEY");
} catch (SQLException e) {
e.printStackTrace();
}
}
@Test
void getKeys() {
var apiKeyService = new ApiKeyService(dataSource);

5
code/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java
View File

@ -2,14 +2,11 @@ package nu.marginalia.index;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import nu.marginalia.db.storage.FileStorageService;
import nu.marginalia.db.storage.model.FileStorageType;
import nu.marginalia.index.forward.ForwardIndexConverter;
import nu.marginalia.index.forward.ForwardIndexReader;
import nu.marginalia.index.journal.reader.IndexJournalReaderSingleCompressedFile;
import nu.marginalia.index.journal.writer.IndexJournalWriter;
import nu.marginalia.index.journal.writer.IndexJournalWriterImpl;
import nu.marginalia.index.priority.ReverseIndexPriorityConverter;
import nu.marginalia.index.full.ReverseIndexFullConverter;
import nu.marginalia.index.priority.ReverseIndexPriorityReader;
@ -110,7 +107,7 @@ public class IndexServicesFactory {
FINISHED
}
public void convertIndex(DomainRankings domainRankings) throws IOException {
try (var hb = heartbeat.createServiceProcessHeartbeat(ConvertSteps.class, "index-conversion")) {
try (var hb = heartbeat.createServiceTaskHeartbeat(ConvertSteps.class, "index-conversion")) {
hb.progress(ConvertSteps.FORWARD_INDEX);
convertForwardIndex(domainRankings);

2
code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexSearchSetsService.java
View File

@ -103,7 +103,7 @@ public class IndexSearchSetsService {
FINISHED
}
public void recalculateAll() {
try (var processHeartbeat = heartbeat.createServiceProcessHeartbeat(RepartitionSteps.class, "repartitionAll")) {
try (var processHeartbeat = heartbeat.createServiceTaskHeartbeat(RepartitionSteps.class, "repartitionAll")) {
processHeartbeat.progress(RepartitionSteps.UPDATE_ACADEMIA);
updateAcademiaDomainsSet();

4
code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTest.java
View File

@ -18,6 +18,7 @@ import nu.marginalia.lexicon.KeywordLexicon;
import nu.marginalia.model.idx.WordFlags;
import nu.marginalia.model.idx.DocumentMetadata;
import nu.marginalia.model.idx.WordMetadata;
import nu.marginalia.service.control.ServiceHeartbeat;
import nu.marginalia.service.server.Initialization;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@ -49,6 +50,9 @@ public class IndexQueryServiceIntegrationTest {
@Inject
KeywordLexicon keywordLexicon;
@Inject
ServiceHeartbeat heartbeat;
@Inject
IndexJournalWriter indexJournalWriter;

11
code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java
View File

@ -17,6 +17,7 @@ import nu.marginalia.index.util.TestUtil;
import nu.marginalia.index.client.model.query.SearchSetIdentifier;
import nu.marginalia.service.control.ServiceEventLog;
import nu.marginalia.service.control.ServiceHeartbeat;
import nu.marginalia.service.control.ServiceTaskHeartbeat;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.service.module.ServiceConfiguration;
import org.mockito.Mockito;
@ -62,8 +63,14 @@ public class IndexQueryServiceIntegrationTestModule extends AbstractModule {
when(fileStorageServiceMock.getStorageByType(FileStorageType.INDEX_LIVE)).thenReturn(new FileStorage(null, null, null, fastDir.toString(), null));
when(fileStorageServiceMock.getStorageByType(FileStorageType.INDEX_STAGING)).thenReturn(new FileStorage(null, null, null, slowDir.toString(), null));
var serviceHeartbeat = Mockito.mock(ServiceHeartbeat.class);
// RIP fairies
when(serviceHeartbeat.createServiceTaskHeartbeat(Mockito.any(), Mockito.any()))
.thenReturn(Mockito.mock(ServiceTaskHeartbeat.class));
bind(ServiceHeartbeat.class).toInstance(serviceHeartbeat);
var servicesFactory = new IndexServicesFactory(
Mockito.mock(ServiceHeartbeat.class),
serviceHeartbeat,
fileStorageServiceMock
);
bind(IndexServicesFactory.class).toInstance(servicesFactory);
@ -81,7 +88,7 @@ public class IndexQueryServiceIntegrationTestModule extends AbstractModule {
bind(KeywordLexiconReadOnlyView.class).toInstance(new KeywordLexiconReadOnlyView(keywordLexicon));
bind(ServiceEventLog.class).toInstance(Mockito.mock(ServiceEventLog.class));
bind(ServiceHeartbeat.class).toInstance(Mockito.mock(ServiceHeartbeat.class));
bind(IndexJournalWriter.class).toInstance(new IndexJournalWriterImpl(keywordLexicon,
slowDir.resolve("page-index.dat")));