From 3889c4bdd9f97f9cc9f80fc4c812ccf9a83920bc Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Mon, 9 Oct 2023 15:12:30 +0200 Subject: [PATCH] (refactor) Remove features-search and update documentation --- code/api/readme.md | 4 ++-- code/common/linkdb/readme.md | 2 +- code/common/model/readme.md | 1 - code/features-index/domain-ranking/readme.md | 2 +- code/features-index/readme.md | 4 ++-- .../result-ranking/build.gradle | 0 .../result-ranking/readme.md | 2 +- .../main/java/nu/marginalia/ranking/ResultKeywordSet.java | 0 .../main/java/nu/marginalia/ranking/ResultValuator.java | 0 .../java/nu/marginalia/ranking/factors/Bm25Factor.java | 0 .../nu/marginalia/ranking/factors/PriorityTermBonus.java | 0 .../nu/marginalia/ranking/factors/TermCoherenceFactor.java | 0 .../java/nu/marginalia/ranking/ResultValuatorTest.java | 0 .../ranking/factors/TermCoherenceFactorTest.java | 0 .../query-parser/build.gradle | 0 .../query-parser/readme.md | 2 +- .../java/nu/marginalia/language/EnglishDictionary.java | 0 .../src/main/java/nu/marginalia/ngrams/DenseBitMap.java | 0 .../main/java/nu/marginalia/ngrams/NGramBloomFilter.java | 0 .../main/java/nu/marginalia/query_parser/QueryParser.java | 0 .../java/nu/marginalia/query_parser/QueryPermutation.java | 0 .../java/nu/marginalia/query_parser/QueryTokenizer.java | 0 .../java/nu/marginalia/query_parser/QueryVariants.java | 0 .../main/java/nu/marginalia/query_parser/token/Token.java | 0 .../java/nu/marginalia/query_parser/token/TokenType.java | 0 .../nu/marginalia/query_parser/token/TokenVisitor.java | 0 .../java/nu/marginalia/transform_list/TransformList.java | 0 .../test/java/nu/marginalia/ngrams/DenseBitMapTest.java | 0 .../nu/marginalia/query_parser/BodyQueryParserTest.java | 0 .../java/nu/marginalia/query_parser/QueryParserTest.java | 0 .../java/nu/marginalia/query_parser/QueryVariantsTest.java | 0 .../nu/marginalia/transform_list/TransformListTest.java | 0 .../test/java/nu/marginalia/util/TestLanguageModels.java | 0 code/features-qs/readme.md | 7 +++++++ code/features-search/readme.md | 4 +--- code/libraries/language-processing/readme.md | 2 +- code/process-models/crawling-model/readme.md | 1 - code/processes/crawling-process/readme.md | 2 +- code/processes/loading-process/readme.md | 3 +-- code/processes/readme.md | 3 +-- code/readme.md | 2 +- code/services-application/search-service/build.gradle | 4 ++-- code/services-application/search-service/readme.md | 2 -- code/services-core/index-service/build.gradle | 2 +- code/services-core/query-service/build.gradle | 2 +- code/services-core/query-service/readme.md | 2 +- settings.gradle | 4 ++-- 47 files changed, 28 insertions(+), 29 deletions(-) rename code/{features-search => features-index}/result-ranking/build.gradle (100%) rename code/{features-search => features-index}/result-ranking/readme.md (83%) rename code/{features-search => features-index}/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java (100%) rename code/{features-search => features-index}/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java (100%) rename code/{features-search => features-index}/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java (100%) rename code/{features-search => features-index}/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java (100%) rename code/{features-search => features-index}/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java (100%) rename code/{features-search => features-index}/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java (100%) rename code/{features-search => features-index}/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java (100%) rename code/{features-search => features-qs}/query-parser/build.gradle (100%) rename code/{features-search => features-qs}/query-parser/readme.md (79%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenVisitor.java (100%) rename code/{features-search => features-qs}/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java (100%) rename code/{features-search => features-qs}/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java (100%) rename code/{features-search => features-qs}/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java (100%) rename code/{features-search => features-qs}/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java (100%) rename code/{features-search => features-qs}/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java (100%) rename code/{features-search => features-qs}/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java (100%) rename code/{features-search => features-qs}/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java (100%) create mode 100644 code/features-qs/readme.md diff --git a/code/api/readme.md b/code/api/readme.md index 9bee706e..31000815 100644 --- a/code/api/readme.md +++ b/code/api/readme.md @@ -3,7 +3,7 @@ ## Core Services * [assistant-api](assistant-api/) -* [search-api](search-api/) +* [query-api](query-api/) * [index-api](index-api/) These are clients for the [core services](../services-core/), along with what models @@ -20,4 +20,4 @@ requests can be sent. [process-mqapi](process-mqapi/) defines requests and inboxes for the message queue based API used for interacting with processes. -See [libraries/message-queue](../libraries/message-queue) and [services-application/control-service](../services-application/control-service). \ No newline at end of file +See [libraries/message-queue](../libraries/message-queue) and [services-application/control-service](../services-core/control-service). \ No newline at end of file diff --git a/code/common/linkdb/readme.md b/code/common/linkdb/readme.md index a87166bc..567ec746 100644 --- a/code/common/linkdb/readme.md +++ b/code/common/linkdb/readme.md @@ -8,4 +8,4 @@ this information take effect in production immediately, even before the information was searchable. It is constructed by the [loading-process](../../processes/loading-process), and consumed -by the [search-service](../../services-core/search-service). \ No newline at end of file +by the [index-service](../../services-core/index-service). \ No newline at end of file diff --git a/code/common/model/readme.md b/code/common/model/readme.md index e58b9a3e..84337753 100644 --- a/code/common/model/readme.md +++ b/code/common/model/readme.md @@ -6,7 +6,6 @@ This package contains common models to the search engine * [EdgeDomain](src/main/java/nu/marginalia/model/EdgeDomain.java) * [EdgeUrl](src/main/java/nu/marginalia/model/EdgeUrl.java) -* [EdgeId](src/main/java/nu/marginalia/model/id/EdgeId.java) * [DocumentMetadata](src/main/java/nu/marginalia/model/idx/DocumentMetadata.java) * [DocumentFlags](src/main/java/nu/marginalia/model/idx/DocumentFlags.java) * [WordMetadata](src/main/java/nu/marginalia/model/idx/WordMetadata.java) diff --git a/code/features-index/domain-ranking/readme.md b/code/features-index/domain-ranking/readme.md index 2db92499..4274c417 100644 --- a/code/features-index/domain-ranking/readme.md +++ b/code/features-index/domain-ranking/readme.md @@ -17,7 +17,7 @@ Contains domain ranking algorithms. ## See Also -* [features-search/result-ranking](../../features-search/result-ranking) - Ranks search results +* [result-ranking](../result-ranking) - Ranks search results ## Useful Resources diff --git a/code/features-index/readme.md b/code/features-index/readme.md index 3ae01618..28fe56b2 100644 --- a/code/features-index/readme.md +++ b/code/features-index/readme.md @@ -14,11 +14,11 @@ These indices rely heavily on the [libraries/btree](../libraries/btree) and [lib ## Algorithms -* [domain-ranking](domain-ranking/) contains ranking algorithms. +* [domain-ranking](domain-ranking/) contains domain ranking algorithms. +* [result-ranking](result-ranking/) contains logic for ranking search results by relevance. # Libraries * [index-query](index-query/) contains structures for evaluating search queries. * [index-journal](index-journal/) contains tools for writing and reading index data. -* [lexicon](lexicon/) contains a mapping between words' string representation and an unique integer identifier. diff --git a/code/features-search/result-ranking/build.gradle b/code/features-index/result-ranking/build.gradle similarity index 100% rename from code/features-search/result-ranking/build.gradle rename to code/features-index/result-ranking/build.gradle diff --git a/code/features-search/result-ranking/readme.md b/code/features-index/result-ranking/readme.md similarity index 83% rename from code/features-search/result-ranking/readme.md rename to code/features-index/result-ranking/readme.md index fb892a38..99ba8647 100644 --- a/code/features-search/result-ranking/readme.md +++ b/code/features-index/result-ranking/readme.md @@ -11,4 +11,4 @@ results higher. ## See Also -* [features-index/domain-ranking](../../features-index/domain-ranking) - Ranks domains \ No newline at end of file +* [features-index/domain-ranking](../domain-ranking) - Ranks domains \ No newline at end of file diff --git a/code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java similarity index 100% rename from code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java rename to code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java diff --git a/code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java similarity index 100% rename from code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java rename to code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java diff --git a/code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java similarity index 100% rename from code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java rename to code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java diff --git a/code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java similarity index 100% rename from code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java rename to code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java diff --git a/code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java similarity index 100% rename from code/features-search/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java rename to code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java diff --git a/code/features-search/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java b/code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java similarity index 100% rename from code/features-search/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java rename to code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java diff --git a/code/features-search/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java b/code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java similarity index 100% rename from code/features-search/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java rename to code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java diff --git a/code/features-search/query-parser/build.gradle b/code/features-qs/query-parser/build.gradle similarity index 100% rename from code/features-search/query-parser/build.gradle rename to code/features-qs/query-parser/build.gradle diff --git a/code/features-search/query-parser/readme.md b/code/features-qs/query-parser/readme.md similarity index 79% rename from code/features-search/query-parser/readme.md rename to code/features-qs/query-parser/readme.md index 08ba9dd1..1296be13 100644 --- a/code/features-search/query-parser/readme.md +++ b/code/features-qs/query-parser/readme.md @@ -1,6 +1,6 @@ # Query Parser -End-user search query parsing tools used by the [search-service](../../services-core/search-service). +End-user search query parsing tools used by the [query-service](../../services-core/query-service). ## Central Classes diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenVisitor.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenVisitor.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenVisitor.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenVisitor.java diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java b/code/features-qs/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java similarity index 100% rename from code/features-search/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java rename to code/features-qs/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java diff --git a/code/features-search/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java similarity index 100% rename from code/features-search/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java rename to code/features-qs/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java diff --git a/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java similarity index 100% rename from code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java rename to code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java diff --git a/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java similarity index 100% rename from code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java rename to code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java diff --git a/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java similarity index 100% rename from code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java rename to code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java diff --git a/code/features-search/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java similarity index 100% rename from code/features-search/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java rename to code/features-qs/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java diff --git a/code/features-search/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java similarity index 100% rename from code/features-search/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java rename to code/features-qs/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java diff --git a/code/features-qs/readme.md b/code/features-qs/readme.md new file mode 100644 index 00000000..c0d77dec --- /dev/null +++ b/code/features-qs/readme.md @@ -0,0 +1,7 @@ +**Note**: This package is called `features-qs` rather than `features-query` because the latter, +though more consistent with other packages like features-index, would be very confusing +as there are other packages elsewhere with the 'query' name (e.g. features-index/index-query). + +## Contents + +* [query-parser](query-parser/) contains code for parsing the user-facing query grammar. diff --git a/code/features-search/readme.md b/code/features-search/readme.md index f7811607..93f0b256 100644 --- a/code/features-search/readme.md +++ b/code/features-search/readme.md @@ -3,8 +3,6 @@ These are bits of search-engine related code that are relatively isolated pieces of business logic, that benefit from the clarity of being kept separate from the rest of the search engine code. - -* [query-parser](query-parser/) contains code for parsing the user-facing query grammar. -* [result-ranking](result-ranking/) contains logic for ranking search results by relevance. +* * [screenshots](screenshots/) and [random-websites](random-websites/) contains SQL queries random exploration mode. diff --git a/code/libraries/language-processing/readme.md b/code/libraries/language-processing/readme.md index 20821b13..08965755 100644 --- a/code/libraries/language-processing/readme.md +++ b/code/libraries/language-processing/readme.md @@ -13,4 +13,4 @@ its words, how they stem, POS tags, and so on. [features-convert/keyword-extraction](../../features-convert/keyword-extraction) uses this code to identify which keywords are important. -[features-search/query-parser](../../features-search/query-parser) also does some language processing. \ No newline at end of file +[features-qs/query-parser](../../features-qs/query-parser) also does some language processing. \ No newline at end of file diff --git a/code/process-models/crawling-model/readme.md b/code/process-models/crawling-model/readme.md index 3a9b78dc..d360e80f 100644 --- a/code/process-models/crawling-model/readme.md +++ b/code/process-models/crawling-model/readme.md @@ -7,7 +7,6 @@ Contains models shared by the [crawling-process](../../processes/crawling-proces * [CrawledDocument](src/main/java/nu/marginalia/crawling/model/CrawledDocument.java) * [CrawledDomain](src/main/java/nu/marginalia/crawling/model/CrawledDomain.java) -* [CrawlingSpecification](src/main/java/nu/marginalia/crawling/model/spec/CrawlingSpecification.java) ### Serialization * [CrawledDomainReader](src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java) diff --git a/code/processes/crawling-process/readme.md b/code/processes/crawling-process/readme.md index 1133baae..f76da78a 100644 --- a/code/processes/crawling-process/readme.md +++ b/code/processes/crawling-process/readme.md @@ -8,7 +8,7 @@ into per-domain snapshots. * [CrawlerMain](src/main/java/nu/marginalia/crawl/CrawlerMain.java) orchestrates the crawling. * [CrawlerRetreiver](src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java) visits known addresses from a domain and downloads each document. -* [HttpFetcher](src/main/java/nu/marginalia/crawl/retreival/HttpFetcher.java) +* [HttpFetcher](src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java) fetches a URL. ## See Also diff --git a/code/processes/loading-process/readme.md b/code/processes/loading-process/readme.md index c3cdf325..ec0c12fd 100644 --- a/code/processes/loading-process/readme.md +++ b/code/processes/loading-process/readme.md @@ -6,5 +6,4 @@ the index-service. ## Central Classes -* [LoaderMain](src/main/java/nu/marginalia/loading/LoaderMain.java) main class. -* [Loader](src/main/java/nu/marginalia/loading/loader/Loader.java) evaluates loading instructions. \ No newline at end of file +* [LoaderMain](src/main/java/nu/marginalia/loading/LoaderMain.java) main class. \ No newline at end of file diff --git a/code/processes/readme.md b/code/processes/readme.md index b2839a09..0722502a 100644 --- a/code/processes/readme.md +++ b/code/processes/readme.md @@ -5,8 +5,7 @@ The [crawling-process](crawling-process/) fetches website contents and saves them as compressed JSON models described in [crawling-model](../process-models/crawling-model/). -The operation is specified by a crawl job specification. This is generated by [tools/crawl-job-extractor](../tools/crawl-job-extractor/) -based on the content in the database. +The operation is specified by a [crawl specification](../process-models/crawl-spec), which can be created in the control GUI. ## 2. Converting Process diff --git a/code/readme.md b/code/readme.md index f67f0aee..dd6403a1 100644 --- a/code/readme.md +++ b/code/readme.md @@ -20,7 +20,7 @@ A map of the most important components and how they relate can be found below. * * [assistant](services-core/assistant-service) * [application services](services-application/) "microservices", stateless providing additional functionality and making an application out of the search engine. * * [api](services-application/api-service) - public API -* * [search](services-core/search-service) - marginalia search application +* * [search](services-application/search-service) - marginalia search application * * [dating](services-application/dating-service) - [https://explore.marginalia.nu/](https://explore.marginalia.nu/) * * [explorer](services-application/explorer-service) - [https://explore2.marginalia.nu/](https://explore2.marginalia.nu/) * an [internal API](api/) diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index 249e6219..47c83d63 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -41,8 +41,8 @@ dependencies { implementation project(':code:features-search:screenshots') implementation project(':code:features-search:random-websites') - implementation project(':code:features-search:query-parser') - implementation project(':code:features-search:result-ranking') + implementation project(':code:features-qs:query-parser') + implementation project(':code:features-index:result-ranking') implementation libs.bundles.slf4j diff --git a/code/services-application/search-service/readme.md b/code/services-application/search-service/readme.md index 7f32c34a..02362a55 100644 --- a/code/services-application/search-service/readme.md +++ b/code/services-application/search-service/readme.md @@ -20,8 +20,6 @@ appropriate services. * [CommandEvaluator](src/main/java/nu/marginalia/search/command/CommandEvaluator.java) interprets a user query and acts upon it, dealing with special operations like `browse:` or `site:`. -* [QueryFactory](src/main/java/nu/marginalia/search/query/QueryFactory.java) parses a search query into a machine readable query specification. - * [SearchQueryIndexService](src/main/java/nu/marginalia/search/svc/SearchQueryIndexService.java) passes a parsed search query to the index service, and then decorates the search results so that they can be rendered. diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 85159cdf..2b9f9aff 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -37,7 +37,7 @@ dependencies { implementation project(':code:features-index:index-forward') implementation project(':code:features-index:index-reverse') implementation project(':code:features-index:domain-ranking') - implementation project(':code:features-search:result-ranking') + implementation project(':code:features-index:result-ranking') implementation project(':third-party:commons-codec') testImplementation project(path: ':code:services-core:control-service') testImplementation project(':code:common:process') diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index 46353c22..5477d634 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -30,7 +30,7 @@ dependencies { implementation project(':code:api:index-api') implementation project(':code:api:query-api') implementation project(':code:common:service-discovery') - implementation project(':code:features-search:query-parser') + implementation project(':code:features-qs:query-parser') implementation project(':code:features-index:index-query') implementation project(':code:libraries:language-processing') implementation project(':code:libraries:term-frequency-dict') diff --git a/code/services-core/query-service/readme.md b/code/services-core/query-service/readme.md index aa335ce4..a609f4f7 100644 --- a/code/services-core/query-service/readme.md +++ b/code/services-core/query-service/readme.md @@ -11,5 +11,5 @@ to that format. ## See Also * [api/query-api](../../api/query-api) -* [features-search/query-parser](../../features-search/query-parser) +* [features-qs/query-parser](../../features-qs/query-parser) * [features-index/index-query](../../features-index/index-query) \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 1e9087af..2b9226f5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -26,8 +26,8 @@ include 'code:libraries:message-queue' include 'code:features-search:screenshots' include 'code:features-search:random-websites' -include 'code:features-search:query-parser' -include 'code:features-search:result-ranking' +include 'code:features-qs:query-parser' +include 'code:features-index:result-ranking' include 'code:features-convert:adblock' include 'code:features-convert:stackexchange-xml'