diff --git a/marginalia_nu/build.gradle b/marginalia_nu/build.gradle index b2115fb0..eb553649 100644 --- a/marginalia_nu/build.gradle +++ b/marginalia_nu/build.gradle @@ -59,12 +59,12 @@ dependencies { implementation "com.sparkjava:spark-core:2.9.3" implementation 'com.opencsv:opencsv:5.6' - implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.17.1' + implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.2' + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.2' + implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.17.2' + implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.2' + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.2' + implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.17.2' implementation 'org.slf4j:slf4j-api:1.7.36' @@ -76,7 +76,6 @@ dependencies { implementation 'com.github.ThatJavaNerd:JRAW:1.1.0' implementation group: 'com.h2database', name: 'h2', version: '2.1.210' - testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.3.1' implementation 'org.jsoup:jsoup:1.14.3' implementation group: 'com.github.crawler-commons', name: 'crawler-commons', version: '1.2' @@ -86,7 +85,7 @@ dependencies { implementation 'com.zaxxer:HikariCP:5.0.1' - implementation 'org.apache.opennlp:opennlp-tools:1.9.3' + implementation 'org.apache.opennlp:opennlp-tools:1.9.4' implementation 'io.prometheus:simpleclient:0.15.0' implementation 'io.prometheus:simpleclient_servlet:0.15.0' implementation 'io.prometheus:simpleclient_httpserver:0.15.0' @@ -123,15 +122,19 @@ dependencies { testImplementation 'org.projectlombok:lombok:1.18.24' testAnnotationProcessor 'org.projectlombok:lombok:1.18.24' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.5.1' + + testImplementation platform('org.testcontainers:testcontainers-bom:1.17.2') + testImplementation 'org.testcontainers:mariadb:1.17.2' + testImplementation "org.testcontainers:junit-jupiter:1.17.2" + e2eTestImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' e2eTestRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' e2eTestImplementation 'org.projectlombok:lombok:1.18.24' - e2eTestAnnotationProcessor 'org.projectlombok:lombok:1.18.22' - e2eTestImplementation 'org.testcontainers:mariadb:1.17.1' - e2eTestImplementation 'org.testcontainers:nginx:1.17.1' - e2eTestImplementation 'org.testcontainers:testcontainers:1.17.1' - e2eTestImplementation "org.testcontainers:junit-jupiter:1.17.1" - e2eTestImplementation "org.testcontainers:selenium:1.17.1" + e2eTestAnnotationProcessor 'org.projectlombok:lombok:1.18.24' + e2eTestImplementation 'org.testcontainers:nginx:1.17.2' + e2eTestImplementation "org.testcontainers:junit-jupiter:1.17.2" + e2eTestImplementation "org.testcontainers:selenium:1.17.2" e2eTestImplementation 'org.seleniumhq.selenium:selenium-remote-driver:4.1.4' e2eTestImplementation 'org.seleniumhq.selenium:selenium-chrome-driver:4.1.4' } diff --git a/marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/EdgeSearchE2ETest.java b/marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/EdgeSearchE2ETest.java index af43e462..08408de2 100644 --- a/marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/EdgeSearchE2ETest.java +++ b/marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/EdgeSearchE2ETest.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.List; import static nu.marginalia.wmsa.configuration.ServiceDescriptor.*; +import static org.junit.jupiter.api.Assertions.assertEquals; @Tag("e2e") @Testcontainers @@ -156,6 +157,16 @@ public class EdgeSearchE2ETest extends E2ETestBase { return wikipediaFiles.toString(); } + private List getTitlesFromSearchResults(String html) { + List ret = new ArrayList<>(); + + for (var title : Jsoup.parse(html).select(".card.search-result > h2")) { + ret.add(title.text()); + } + + return ret; + } + @Test public void testFrontPage() throws IOException { var driver = chrome.getWebDriver(); @@ -173,8 +184,9 @@ public class EdgeSearchE2ETest extends E2ETestBase { driver.get("http://proxyNginx/search?query=bird&profile=corpo"); System.out.println(driver.getTitle()); - System.out.println(driver.findElement(new By.ByXPath("//*")).getAttribute("outerHTML")); + var html = driver.findElement(new By.ByXPath("//*")).getAttribute("outerHTML"); + assertEquals(List.of("Bird"), getTitlesFromSearchResults(html)); Files.move(driver.getScreenshotAs(OutputType.FILE).toPath(), screenshotFilename("query")); } @@ -187,20 +199,23 @@ public class EdgeSearchE2ETest extends E2ETestBase { System.out.println(driver.getTitle()); System.out.println(driver.findElement(new By.ByXPath("//*")).getAttribute("outerHTML")); - Files.move(driver.getScreenshotAs(OutputType.FILE).toPath(), screenshotFilename("site-info")); } + @Test public void testSiteSearch() throws IOException { var driver = chrome.getWebDriver(); driver.get("http://proxyNginx/search?query=site:wikipedia.local%20frog"); System.out.println(driver.getTitle()); - System.out.println(driver.findElement(new By.ByXPath("//*")).getAttribute("outerHTML")); + var html = driver.findElement(new By.ByXPath("//*")).getAttribute("outerHTML"); + + assertEquals(List.of("Frog", "Binomial nomenclature", "Amphibian", "Mantis"), getTitlesFromSearchResults(html)); Files.move(driver.getScreenshotAs(OutputType.FILE).toPath(), screenshotFilename("site-search")); } + @Test public void testBrowse() throws IOException { var driver = chrome.getWebDriver(); @@ -209,7 +224,6 @@ public class EdgeSearchE2ETest extends E2ETestBase { System.out.println(driver.getTitle()); System.out.println(driver.findElement(new By.ByXPath("//*")).getAttribute("outerHTML")); - Files.move(driver.getScreenshotAs(OutputType.FILE).toPath(), screenshotFilename("browse")); } @Test @@ -220,7 +234,6 @@ public class EdgeSearchE2ETest extends E2ETestBase { System.out.println(driver.getTitle()); System.out.println(driver.findElement(new By.ByXPath("//*")).getAttribute("outerHTML")); - Files.move(driver.getScreenshotAs(OutputType.FILE).toPath(), screenshotFilename("define")); } @Test diff --git a/marginalia_nu/src/e2e/resources/init.sh b/marginalia_nu/src/e2e/resources/init.sh index 5409f787..50dbd406 100644 --- a/marginalia_nu/src/e2e/resources/init.sh +++ b/marginalia_nu/src/e2e/resources/init.sh @@ -69,4 +69,4 @@ memex memex dating dating EOF -WMSA_HOME=${HOME} java -Dsmall-ram=TRUE -Dservice-host=0.0.0.0 -jar /WMSA.jar start $1 \ No newline at end of file +WMSA_HOME=${HOME} java -server -Xmx2G -Dsmall-ram=TRUE -Dservice-host=0.0.0.0 -jar /WMSA.jar start $1 \ No newline at end of file diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/btree/BTreeWriter.java b/marginalia_nu/src/main/java/nu/marginalia/util/btree/BTreeWriter.java index 28ac4914..b43faca7 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/btree/BTreeWriter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/btree/BTreeWriter.java @@ -3,6 +3,7 @@ package nu.marginalia.util.btree; import nu.marginalia.util.btree.model.BTreeContext; import nu.marginalia.util.btree.model.BTreeHeader; import nu.marginalia.util.multimap.MultimapFileLong; +import nu.marginalia.util.multimap.MultimapFileLongSlice; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,9 +13,9 @@ import java.io.IOException; public class BTreeWriter { private final Logger logger = LoggerFactory.getLogger(BTreeWriter.class); private final BTreeContext ctx; - private final MultimapFileLong map; + private final MultimapFileLongSlice map; - public BTreeWriter(MultimapFileLong map, BTreeContext ctx) { + public BTreeWriter(MultimapFileLongSlice map, BTreeContext ctx) { this.map = map; this.ctx = ctx; } @@ -31,13 +32,18 @@ public class BTreeWriter { return size; } - public long write(long offset, int numEntries, WriteCallback writeIndex) + /** Construct a BTree with numEntries entries at offset in the associated map + * + * @return The size of the written data + */ + public long write(long offset, int numEntries, WriteCallback writeIndexCallback) throws IOException { - var header = makeHeader(offset, numEntries); + BTreeHeader header = makeHeader(offset, numEntries); header.write(map, offset); - writeIndex.write(header.dataOffsetLongs()); + + writeIndexCallback.write(map.atOffset(header.dataOffsetLongs())); if (header.layers() < 1) { return ctx.calculateSize(numEntries); diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/btree/WriteCallback.java b/marginalia_nu/src/main/java/nu/marginalia/util/btree/WriteCallback.java index 70bd8132..a6225db1 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/btree/WriteCallback.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/btree/WriteCallback.java @@ -1,7 +1,9 @@ package nu.marginalia.util.btree; +import nu.marginalia.util.multimap.MultimapFileLongSlice; + import java.io.IOException; public interface WriteCallback { - void write(long offset) throws IOException; + void write(MultimapFileLongSlice slice) throws IOException; } diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/btree/model/BTreeHeader.java b/marginalia_nu/src/main/java/nu/marginalia/util/btree/model/BTreeHeader.java index 4951f5b8..8d68b424 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/btree/model/BTreeHeader.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/btree/model/BTreeHeader.java @@ -1,6 +1,7 @@ package nu.marginalia.util.btree.model; import nu.marginalia.util.multimap.MultimapFileLong; +import nu.marginalia.util.multimap.MultimapFileLongSlice; public record BTreeHeader(int layers, int numEntries, long indexOffsetLongs, long dataOffsetLongs) { public BTreeHeader { @@ -28,7 +29,7 @@ public record BTreeHeader(int layers, int numEntries, long indexOffsetLongs, lon return padding; } - public void write(MultimapFileLong dest, long offset) { + public void write(MultimapFileLongSlice dest, long offset) { dest.put(offset, ((long) layers << 32L) | ((long)numEntries & 0xFFFF_FFFFL)); dest.put(offset+1, indexOffsetLongs); dest.put(offset+2, dataOffsetLongs); diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/hash/LongPairHashMap.java b/marginalia_nu/src/main/java/nu/marginalia/util/hash/LongPairHashMap.java index 6f8912a9..d1e056b9 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/hash/LongPairHashMap.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/hash/LongPairHashMap.java @@ -1,9 +1,7 @@ package nu.marginalia.util.hash; -import io.prometheus.client.Gauge; import lombok.EqualsAndHashCode; import lombok.Getter; -import nu.marginalia.wmsa.edge.index.service.index.wordstable.IndexWordsTable; import nu.marginalia.util.multimap.MultimapFileLong; import nu.marginalia.util.PrimeUtil; import org.slf4j.Logger; @@ -17,9 +15,7 @@ import static java.lang.Math.round; */ public class LongPairHashMap { private static final Logger logger = LoggerFactory.getLogger(LongPairHashMap.class); - private static final Gauge probe_count_metrics - = Gauge.build("wmsa_wordfile_hash_map_probe_count", "Probing Count") - .register(); + private static final long MAGIC_WORD = 0xE00E00E00E0E0E0EL; // it's the data police private final long hashTableSize; private final MultimapFileLong data; @@ -27,26 +23,37 @@ public class LongPairHashMap { private int sz = 0; private static final int HEADER_SIZE = 2; - public LongPairHashMap(MultimapFileLong data, long size) { + private LongPairHashMap(MultimapFileLong data, long hashTableSize, long maxProbeLength) { this.data = data; - // Actually use a prime size for Donald Knuth reasons - hashTableSize = PrimeUtil.nextPrime(size, 1); - maxProbeLength = hashTableSize / 2; + this.hashTableSize = hashTableSize; + this.maxProbeLength = maxProbeLength; + } - logger.debug("Table size = " + hashTableSize); + public static LongPairHashMap createNew(MultimapFileLong data, long size) { + var tableSize = PrimeUtil.nextPrime(size, 1); + var ret = new LongPairHashMap(data, tableSize, tableSize/2); - data.put(0, IndexWordsTable.Strategy.HASH.ordinal()); - data.put(1, hashTableSize); - for (int i = 2; i < hashTableSize; i++) { + data.put(0, MAGIC_WORD); + data.put(1, tableSize); + + for (int i = 2; i < tableSize; i++) { data.put(HEADER_SIZE + 2L*i, 0); } - } - public LongPairHashMap(MultimapFileLong data) { - this.data = data; - hashTableSize = data.get(1); - maxProbeLength = hashTableSize / 10; - logger.debug("Table size = " + hashTableSize); + return ret; + } + + public static LongPairHashMap loadExisting(MultimapFileLong data) { + long key = data.get(0); + + if (key != MAGIC_WORD) { + logger.warn("LongPairHashMap lacks magic word, could this be garbage data?"); + } + + var hashTableSize = data.get(1); + var maxProbeLength = hashTableSize / 10; + + return new LongPairHashMap(data, hashTableSize, maxProbeLength); } public int size() { @@ -91,8 +98,6 @@ public class LongPairHashMap { final var val = getCell(idx); if (!val.isSet()) { - probe_count_metrics.set(j); - return setValue(data, idx); } else if (val.getKey() == data.getKey()) { diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLong.java b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLong.java index dca8248e..f381a977 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLong.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLong.java @@ -21,7 +21,7 @@ import static java.nio.channels.FileChannel.MapMode.READ_WRITE; import static nu.marginalia.util.FileSizeUtil.readableSize; -public class MultimapFileLong implements AutoCloseable { +public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice { private final ArrayList buffers = new ArrayList<>(); private final ArrayList mappedByteBuffers = new ArrayList<>(); @@ -196,10 +196,12 @@ public class MultimapFileLong implements AutoCloseable { } } + @Override public long size() { return fileLength; } + @Override public void put(long idx, long val) { if (idx >= mappedSize) grow(idx); @@ -214,6 +216,7 @@ public class MultimapFileLong implements AutoCloseable { } } + @Override public long get(long idx) { if (idx >= mappedSize) grow(idx); @@ -229,10 +232,12 @@ public class MultimapFileLong implements AutoCloseable { } + @Override public void read(long[] vals, long idx) { read(vals, vals.length, idx); } + @Override public void read(long[] vals, int n, long idx) { if (idx+n >= mappedSize) { grow(idx+n); @@ -257,10 +262,12 @@ public class MultimapFileLong implements AutoCloseable { } + @Override public void write(long[] vals, long idx) { write(vals, vals.length, idx); } + @Override public void write(long[] vals, int n, long idx) { if (idx+n >= mappedSize) { grow(idx+n); @@ -285,6 +292,7 @@ public class MultimapFileLong implements AutoCloseable { } + @Override public void write(LongBuffer vals, long idx) { int n = vals.limit() - vals.position(); if (idx+n >= mappedSize) { @@ -310,6 +318,7 @@ public class MultimapFileLong implements AutoCloseable { } + @Override public void transferFromFileChannel(FileChannel sourceChannel, long destOffset, long sourceStart, long sourceEnd) throws IOException { int length = (int)(sourceEnd - sourceStart); diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLongOffsetSlice.java b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLongOffsetSlice.java new file mode 100644 index 00000000..c2630ddc --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLongOffsetSlice.java @@ -0,0 +1,70 @@ +package nu.marginalia.util.multimap; + +import java.io.IOException; +import java.nio.LongBuffer; +import java.nio.channels.FileChannel; + +public class MultimapFileLongOffsetSlice implements MultimapFileLongSlice { + private final long off; + private final MultimapFileLongSlice map; + + public MultimapFileLongOffsetSlice(MultimapFileLongSlice map, long off) { + this.off = off; + this.map = map; + } + + @Override + public long size() { + return map.size() - off; + } + + @Override + public void put(long idx, long val) { + map.put(off+idx, val); + } + + @Override + public long get(long idx) { + return map.get(off+idx); + } + + @Override + public void read(long[] vals, long idx) { + map.read(vals, idx+off); + } + + @Override + public void read(long[] vals, int n, long idx) { + map.read(vals, n, idx+off); + } + + @Override + public void write(long[] vals, long idx) { + map.write(vals, idx+off); + } + + @Override + public void write(long[] vals, int n, long idx) { + map.write(vals, n, idx+off); + } + + @Override + public void write(LongBuffer vals, long idx) { + map.write(vals, idx+off); + } + + @Override + public void transferFromFileChannel(FileChannel sourceChannel, long destOffset, long sourceStart, long sourceEnd) + throws IOException { + map.transferFromFileChannel(sourceChannel, destOffset + off, sourceStart, sourceEnd); + } + + @Override + public MultimapFileLongSlice atOffset(long off) { + // If we don't override this, the default implementation would build a pyramid of + // MultimapFileLongSlice(MultimapFileLongSlice(MultimapFileLongSlice(MultimapFileLongSlice(MultimapFileLongSlice(...))) + // if this is called iteratively (e.g. to walk over a file) + + return new MultimapFileLongOffsetSlice(map, this.off + off); + } +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLongSlice.java b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLongSlice.java new file mode 100644 index 00000000..abf29f51 --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLongSlice.java @@ -0,0 +1,29 @@ +package nu.marginalia.util.multimap; + +import java.io.IOException; +import java.nio.LongBuffer; +import java.nio.channels.FileChannel; + +public interface MultimapFileLongSlice { + long size(); + + void put(long idx, long val); + + long get(long idx); + + void read(long[] vals, long idx); + + void read(long[] vals, int n, long idx); + + void write(long[] vals, long idx); + + void write(long[] vals, int n, long idx); + + void write(LongBuffer vals, long idx); + + void transferFromFileChannel(FileChannel sourceChannel, long destOffset, long sourceStart, long sourceEnd) throws IOException; + + default MultimapFileLongSlice atOffset(long off) { + return new MultimapFileLongOffsetSlice(this, off); + } +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSearcher.java b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSearcher.java index c961ac0e..005888d8 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSearcher.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSearcher.java @@ -4,9 +4,9 @@ import lombok.experimental.Delegate; public class MultimapSearcher { @Delegate - private final MultimapFileLong mmf; + private final MultimapFileLongSlice mmf; - public MultimapSearcher(MultimapFileLong mmf) { + public MultimapSearcher(MultimapFileLongSlice mmf) { this.mmf = mmf; } diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSorter.java b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSorter.java index 6ca4f64f..61dd04c4 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSorter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSorter.java @@ -13,10 +13,10 @@ import static nu.marginalia.util.multimap.MultimapFileLong.WORD_SIZE; public class MultimapSorter { private final Path tmpFileDir; private final int internalSortLimit; - private final MultimapFileLong multimapFileLong; + private final MultimapFileLongSlice multimapFileLong; private final long[] buffer; - public MultimapSorter(MultimapFileLong multimapFileLong, Path tmpFileDir, int internalSortLimit) { + public MultimapSorter(MultimapFileLongSlice multimapFileLong, Path tmpFileDir, int internalSortLimit) { this.multimapFileLong = multimapFileLong; this.tmpFileDir = tmpFileDir; this.internalSortLimit = internalSortLimit; diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/AcademiaRank.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/AcademiaRank.java deleted file mode 100644 index 272a1798..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/AcademiaRank.java +++ /dev/null @@ -1,49 +0,0 @@ -package nu.marginalia.util.ranking; - -import com.zaxxer.hikari.HikariDataSource; -import gnu.trove.list.TIntList; -import gnu.trove.list.array.TIntArrayList; -import gnu.trove.map.hash.TIntIntHashMap; -import it.unimi.dsi.fastutil.ints.IntArrays; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.sql.SQLException; - -public class AcademiaRank { - private final TIntArrayList result; - private static final Logger logger = LoggerFactory.getLogger(AcademiaRank.class); - - public AcademiaRank(HikariDataSource ds, String... origins) throws IOException { - - TIntList rankingResults = new BetterStandardPageRank(ds, origins).pageRank(100_000); - TIntIntHashMap idToRanking = new TIntIntHashMap(100_000, 0.5f, -1, 1_000_000_000); - - for (int i = 0; i < rankingResults.size(); i++) { - idToRanking.put(rankingResults.get(i), i); - } - - result = new TIntArrayList(10000); - try (var conn = ds.getConnection(); - var stmt = conn.prepareStatement("select EC_DOMAIN.ID,COUNT(SOURCE_DOMAIN_ID) AS CNT from EC_DOMAIN INNER JOIN DOMAIN_METADATA ON DOMAIN_METADATA.ID=EC_DOMAIN.ID INNER JOIN EC_DOMAIN_LINK ON EC_DOMAIN_LINK.DEST_DOMAIN_ID=EC_DOMAIN.ID WHERE INDEXED>0 AND STATE>=0 AND STATE<2 AND ((VISITED_URLS>1000+1500*RANK AND RANK<1) OR (GOOD_URLS>1000 AND URL_PART LIKE '%edu')) GROUP BY EC_DOMAIN.ID HAVING CNT<1500 ORDER BY RANK ASC")) { - - stmt.setFetchSize(1000); - var rsp = stmt.executeQuery(); - while (rsp.next()) { - result.add(rsp.getInt(1)); - } - } - catch (SQLException ex) { - logger.error("SQL error", ex); - } - - int[] internalArray = result.toArray(); - IntArrays.quickSort(internalArray, (a,b) -> idToRanking.get(a) - idToRanking.get(b)); - result.set(0, internalArray); - } - - public TIntArrayList getResult() { - return result; - } -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BetterReversePageRank.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BetterReversePageRank.java index f2889ad6..7d3b17c4 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BetterReversePageRank.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BetterReversePageRank.java @@ -1,15 +1,11 @@ package nu.marginalia.util.ranking; -import com.zaxxer.hikari.HikariDataSource; - -import java.io.IOException; - public class BetterReversePageRank extends RankingAlgorithm { - public BetterReversePageRank(HikariDataSource dataSource, String... origins) { - super(dataSource, origins); + public BetterReversePageRank(RankingDomainFetcher domains, String... origins) { + super(domains, origins); } @Override diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BetterStandardPageRank.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BetterStandardPageRank.java index 5b64fa73..f1f9b0b1 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BetterStandardPageRank.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BetterStandardPageRank.java @@ -1,14 +1,10 @@ package nu.marginalia.util.ranking; -import com.zaxxer.hikari.HikariDataSource; - -import java.io.IOException; - public class BetterStandardPageRank extends RankingAlgorithm { - public BetterStandardPageRank(HikariDataSource dataSource, String... origins) { - super(dataSource, origins); + public BetterStandardPageRank(RankingDomainFetcher domains, String... origins) { + super(domains, origins); } @Override diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BuggyReversePageRank.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BuggyReversePageRank.java index 1e87776c..485ba353 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BuggyReversePageRank.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BuggyReversePageRank.java @@ -1,15 +1,11 @@ package nu.marginalia.util.ranking; -import com.zaxxer.hikari.HikariDataSource; - -import java.io.IOException; - public class BuggyReversePageRank extends RankingAlgorithm { - public BuggyReversePageRank(HikariDataSource dataSource, String... origins) { - super(dataSource, origins); + public BuggyReversePageRank(RankingDomainFetcher domains, String... origins) { + super(domains, origins); } @Override diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BuggyStandardPageRank.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BuggyStandardPageRank.java index a3d7b87e..836bcdfe 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BuggyStandardPageRank.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/BuggyStandardPageRank.java @@ -1,14 +1,10 @@ package nu.marginalia.util.ranking; -import com.zaxxer.hikari.HikariDataSource; - -import java.io.IOException; - public class BuggyStandardPageRank extends RankingAlgorithm { - public BuggyStandardPageRank(HikariDataSource dataSource, String... origins) { - super(dataSource, origins); + public BuggyStandardPageRank(RankingDomainFetcher domains, String... origins) { + super(domains, origins); } @Override diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingAlgorithm.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingAlgorithm.java index fd76989c..4d255087 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingAlgorithm.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingAlgorithm.java @@ -1,224 +1,129 @@ package nu.marginalia.util.ranking; -import com.zaxxer.hikari.HikariDataSource; import gnu.trove.list.TIntList; import gnu.trove.list.array.TIntArrayList; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.map.hash.TIntObjectHashMap; -import gnu.trove.set.hash.TIntHashSet; import it.unimi.dsi.fastutil.ints.IntComparator; -import lombok.AllArgsConstructor; -import lombok.Data; import nu.marginalia.wmsa.configuration.module.DatabaseModule; import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklistImpl; -import nu.marginalia.wmsa.edge.model.crawl.EdgeDomainIndexingState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.sql.SQLException; import java.util.*; import java.util.function.IntToDoubleFunction; import java.util.stream.IntStream; import it.unimi.dsi.fastutil.ints.IntArrays; public abstract class RankingAlgorithm { - final TIntObjectHashMap domainsById = new TIntObjectHashMap<>(); - final TIntIntHashMap domainIndexToId = new TIntIntHashMap(); - final TIntIntHashMap domainIdToIndex = new TIntIntHashMap(); + protected final TIntObjectHashMap domainsById = new TIntObjectHashMap<>(); + protected final TIntIntHashMap domainIndexToId = new TIntIntHashMap(); + protected final TIntIntHashMap domainIdToIndex = new TIntIntHashMap(); - private final TIntHashSet spamDomains; - private final HikariDataSource dataSource; - - TIntArrayList[] linkDataSrc2Dest; - TIntArrayList[] linkDataDest2Src; + protected TIntArrayList[] linkDataSrc2Dest; + protected TIntArrayList[] linkDataDest2Src; public final Set originDomains = new HashSet<>(); public final Set originDomainIds = new HashSet<>(); private int maxKnownUrls = Integer.MAX_VALUE; - private static final boolean getNames = true; - private final Logger logger = LoggerFactory.getLogger(getClass()); - public static void main(String... args) throws IOException { - var rpr = new BuggyReversePageRank(new DatabaseModule().provideConnection(), "wiki.xxiivv.com"); - var spr = new BuggyStandardPageRank(new DatabaseModule().provideConnection(), "memex.marginalia.nu"); + private final RankingDomainFetcher domains; - var rankVector = spr.pageRankVector(); - var norm = rankVector.norm(); - rpr.pageRank(i -> rankVector.get(i) / norm, 25).forEach(i -> { - System.out.println(spr.domainNameFromId(i)); - return true; - }); - } + public RankingAlgorithm(RankingDomainFetcher domains, String... origins) { + this.domains = domains; - public String domainNameFromId(int id) { - return domainsById.get(id).name; - } - public boolean isPeripheral(int id) { - return domainsById.get(id).peripheral; - } - - public RankingAlgorithm(HikariDataSource dataSource, String... origins) { - this.dataSource = dataSource; - var blacklist = new EdgeDomainBlacklistImpl(dataSource); - - spamDomains = blacklist.getSpamDomains(); originDomains.addAll(Arrays.asList(origins)); - try (var conn = dataSource.getConnection()) { + domains.getDomains(domainData -> { + int id = domainData.id; - String s; - if (getNames) { - s = "SELECT EC_DOMAIN.ID,URL_PART,DOMAIN_ALIAS,STATE,KNOWN_URLS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID INNER JOIN EC_DOMAIN_LINK ON SOURCE_DOMAIN_ID=EC_DOMAIN.ID WHERE ((INDEXED>1 AND STATE >= 0) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND QUALITY_RAW>=-20 AND SOURCE_DOMAIN_ID!=DEST_DOMAIN_ID GROUP BY EC_DOMAIN.ID"; + domainsById.put(id, domainData); + + domainIndexToId.put(domainIndexToId.size(), id); + domainIdToIndex.put(id, domainIdToIndex.size()); + }); + + linkDataSrc2Dest = new TIntArrayList[domainIndexToId.size()]; + linkDataDest2Src = new TIntArrayList[domainIndexToId.size()]; + + domains.eachDomainLink((src, dst) -> { + if (src == dst) return; + + if (domainsById.contains(src) && domainsById.contains(dst)) { + + int srcIdx = domainIdToIndex.get(src); + int dstIdx = domainIdToIndex.get(domainsById.get(dst).resolveAlias()); + + if (linkDataSrc2Dest[srcIdx] == null) { + linkDataSrc2Dest[srcIdx] = new TIntArrayList(); + } + linkDataSrc2Dest[srcIdx].add(dstIdx); + + if (linkDataDest2Src[dstIdx] == null) { + linkDataDest2Src[dstIdx] = new TIntArrayList(); + } + linkDataDest2Src[dstIdx].add(srcIdx); + } + }); + + for (var namePattern : this.originDomains) { + domains.domainsByPattern(namePattern, i -> { + int ival = domainIdToIndex.get(i); + if (ival != domainIdToIndex.getNoEntryValue() || domainIndexToId.get(0) == i) { + originDomainIds.add(ival); } else { - s = "SELECT EC_DOMAIN.ID,\"\",DOMAIN_ALIAS,STATE,KNOWN_URLS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID INNER JOIN EC_DOMAIN_LINK ON SOURCE_DOMAIN_ID=EC_DOMAIN.ID WHERE ((INDEXED>1 AND STATE >= 0) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND QUALITY_RAW>=-20 AND SOURCE_DOMAIN_ID!=DEST_DOMAIN_ID GROUP BY EC_DOMAIN.ID"; + logger.debug("No value for {}", i); } - try (var stmt = conn.prepareStatement(s)) { - stmt.setFetchSize(10000); - var rsp = stmt.executeQuery(); - while (rsp.next()) { - int id = rsp.getInt(1); - if (!spamDomains.contains(id)) { - - domainsById.put(id, new DomainData(id, rsp.getString(2), rsp.getInt(3), rsp.getInt(4), rsp.getInt(5), false)); - - domainIndexToId.put(domainIndexToId.size(), id); - domainIdToIndex.put(id, domainIdToIndex.size()); - } - } - } - - - linkDataSrc2Dest = new TIntArrayList[domainIndexToId.size()]; - linkDataDest2Src = new TIntArrayList[domainIndexToId.size()]; - - try (var stmt = conn.prepareStatement("SELECT SOURCE_DOMAIN_ID, DEST_DOMAIN_ID FROM EC_DOMAIN_LINK")) { - stmt.setFetchSize(10000); - - var rsp = stmt.executeQuery(); - - while (rsp.next()) { - int src = rsp.getInt(1); - int dst = rsp.getInt(2); - - if (src == dst) continue; - - if (domainsById.contains(src) && domainsById.contains(dst)) { - - int srcIdx = domainIdToIndex.get(src); - int dstIdx = domainIdToIndex.get(domainsById.get(dst).resolveAlias()); - - if (linkDataSrc2Dest[srcIdx] == null) { - linkDataSrc2Dest[srcIdx] = new TIntArrayList(); - } - linkDataSrc2Dest[srcIdx].add(dstIdx); - - if (linkDataDest2Src[dstIdx] == null) { - linkDataDest2Src[dstIdx] = new TIntArrayList(); - } - linkDataDest2Src[dstIdx].add(srcIdx); - } - } - } - - try (var stmt = conn.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE URL_PART LIKE ?")) { - for (var seed : this.originDomains) { - stmt.setString(1, seed); - var rsp = stmt.executeQuery(); - while (rsp.next()) { - int i = rsp.getInt(1); - int ival = domainIdToIndex.get(i); - if (ival != domainIdToIndex.getNoEntryValue() || domainIndexToId.get(0) == i) { - originDomainIds.add(ival); - } - else { - logger.debug("No value for {}", i); - } - } - logger.debug("{} -> {}", seed, originDomainIds.size()); - } - } - - logger.info("Origin Domains: {}", originDomainIds.size()); - - } catch (SQLException throwables) { - logger.error("SQL error", throwables); + }); } + logger.info("Origin Domains: {}", originDomainIds.size()); } - public void addPeripheralNodes(boolean includeErrorStates) { + public void addPeripheralNodes() { int newNodesIdxCutoff = domainIdToIndex.size(); logger.info("Inserting peripheral nodes"); - try (var conn = dataSource.getConnection()) { - String s; - if (getNames) { - s = "SELECT EC_DOMAIN.ID,URL_PART,DOMAIN_ALIAS,STATE,KNOWN_URLS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID LEFT JOIN EC_DOMAIN_LINK ON SOURCE_DOMAIN_ID=EC_DOMAIN.ID WHERE ((INDEXED>1 AND STATE >= 0) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND QUALITY_RAW>=-20 AND EC_DOMAIN_LINK.ID IS NULL GROUP BY EC_DOMAIN.ID"; + domains.getPeripheralDomains(domainData -> { + int id = domainData.id; + + if (domainsById.put(id, domainData) == null) { // true if id was not already present + domainIndexToId.put(domainIndexToId.size(), id); + domainIdToIndex.put(id, domainIdToIndex.size()); } - else { - s = "SELECT EC_DOMAIN.ID,\"\",DOMAIN_ALIAS,STATE,KNOWN_URLS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID LEFT JOIN EC_DOMAIN_LINK ON SOURCE_DOMAIN_ID=EC_DOMAIN.ID WHERE ((INDEXED>1 AND STATE >= 0) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND QUALITY_RAW>=-20 AND EC_DOMAIN_LINK.ID IS NULL GROUP BY EC_DOMAIN.ID"; - } - try (var stmt = conn.prepareStatement(s)) { - stmt.setFetchSize(10000); - var rsp = stmt.executeQuery(); + }); - while (rsp.next()) { - int id = rsp.getInt(1); + linkDataSrc2Dest = Arrays.copyOf(linkDataSrc2Dest, domainIndexToId.size()); + linkDataDest2Src = Arrays.copyOf(linkDataDest2Src, domainIndexToId.size()); - if (!spamDomains.contains(id)) { - domainsById.put(id, new DomainData(id, rsp.getString(2), rsp.getInt(3), rsp.getInt(4), rsp.getInt(5), true)); + domains.eachDomainLink((src, dst) -> { + if (src == dst) return; - domainIndexToId.put(domainIndexToId.size(), id); - domainIdToIndex.put(id, domainIdToIndex.size()); - } + if (domainsById.contains(src) && domainsById.contains(dst)) { + int srcIdx = domainIdToIndex.get(src); + int dstIdx = domainIdToIndex.get(domainsById.get(dst).resolveAlias()); + + // This looks like a bug, but it improves the results + if (srcIdx < newNodesIdxCutoff || dstIdx < newNodesIdxCutoff) + return; + + if (linkDataSrc2Dest[srcIdx] == null) { + linkDataSrc2Dest[srcIdx] = new TIntArrayList(); } + linkDataSrc2Dest[srcIdx].add(dstIdx); - } - - linkDataSrc2Dest = Arrays.copyOf(linkDataSrc2Dest, domainIndexToId.size()); - linkDataDest2Src = Arrays.copyOf(linkDataDest2Src, domainIndexToId.size()); - - try (var stmt = conn.prepareStatement("SELECT SOURCE_DOMAIN_ID, DEST_DOMAIN_ID FROM EC_DOMAIN_LINK")) { - stmt.setFetchSize(10000); - - var rsp = stmt.executeQuery(); - - while (rsp.next()) { - int src = rsp.getInt(1); - int dst = rsp.getInt(2); - - if (src == dst) continue; - - if (domainsById.contains(src) && domainsById.contains(dst)) { - - int srcIdx = domainIdToIndex.get(src); - int dstIdx = domainIdToIndex.get(domainsById.get(dst).resolveAlias()); - - // This looks like a bug, but it improves the results - if (srcIdx < newNodesIdxCutoff || dstIdx < newNodesIdxCutoff) - continue; - - if (linkDataSrc2Dest[srcIdx] == null) { - linkDataSrc2Dest[srcIdx] = new TIntArrayList(); - } - linkDataSrc2Dest[srcIdx].add(dstIdx); - - if (linkDataDest2Src[dstIdx] == null) { - linkDataDest2Src[dstIdx] = new TIntArrayList(); - } - linkDataDest2Src[dstIdx].add(srcIdx); - } + if (linkDataDest2Src[dstIdx] == null) { + linkDataDest2Src[dstIdx] = new TIntArrayList(); } + linkDataDest2Src[dstIdx].add(srcIdx); } - } catch (SQLException throwables) { - logger.error("SQL error", throwables); - } + }); logger.info("Peripheral nodes inserted {} -> {}", newNodesIdxCutoff, domainIdToIndex.size()); } @@ -271,14 +176,14 @@ public abstract class RankingAlgorithm { return rank.getRanking(resultCount); } - public TIntList pageRankWithPeripheralNodes(int resultCount, boolean includeErrorStates) { + public TIntList pageRankWithPeripheralNodes(int resultCount) { RankVector rank = new RankVector(1.d / domainsById.size()); int iter_max = 100; for (int i = 0; i < iter_max; i++) { if (i == iter_max-1) { - addPeripheralNodes(includeErrorStates); + addPeripheralNodes(); } RankVector newRank = createNewRankVector(rank); @@ -323,7 +228,7 @@ public abstract class RankingAlgorithm { abstract RankVector createNewRankVector(RankVector rank); - public boolean includeInRanking(DomainData data) { + public boolean includeInRanking(RankingDomainData data) { if (data.isAlias()) return false; if (data.isSpecial()) @@ -445,32 +350,4 @@ public abstract class RankingAlgorithm { } } - @Data - @AllArgsConstructor - static class DomainData { - public final int id; - public final String name; - private int alias; - private int state; - public final int knownUrls; - public boolean peripheral; - - public int resolveAlias() { - if (alias == 0) return id; - return alias; - } - - public boolean isAlias() { - return alias != 0; - } - - public boolean isSpecial() { - return EdgeDomainIndexingState.SPECIAL.code == state; - } - - public boolean isSocialMedia() { - return EdgeDomainIndexingState.SOCIAL_MEDIA.code == state; - } - } - } diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingDomainData.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingDomainData.java new file mode 100644 index 00000000..c29ed704 --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingDomainData.java @@ -0,0 +1,33 @@ +package nu.marginalia.util.ranking; + +import lombok.AllArgsConstructor; +import lombok.Data; +import nu.marginalia.wmsa.edge.model.crawl.EdgeDomainIndexingState; + +@Data +@AllArgsConstructor +class RankingDomainData { + public final int id; + public final String name; + private int alias; + private EdgeDomainIndexingState state; + public final int knownUrls; + public boolean peripheral; + + public int resolveAlias() { + if (alias == 0) return id; + return alias; + } + + public boolean isAlias() { + return alias != 0; + } + + public boolean isSpecial() { + return EdgeDomainIndexingState.SPECIAL == state; + } + + public boolean isSocialMedia() { + return EdgeDomainIndexingState.SOCIAL_MEDIA == state; + } +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingDomainFetcher.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingDomainFetcher.java new file mode 100644 index 00000000..79285a83 --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/RankingDomainFetcher.java @@ -0,0 +1,105 @@ +package nu.marginalia.util.ranking; + +import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklistImpl; +import nu.marginalia.wmsa.edge.model.crawl.EdgeDomainIndexingState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.function.Consumer; +import java.util.function.IntConsumer; + +public class RankingDomainFetcher { + private final HikariDataSource dataSource; + private final EdgeDomainBlacklistImpl blacklist; + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private final boolean getNames = false; + + @Inject + public RankingDomainFetcher(HikariDataSource dataSource, EdgeDomainBlacklistImpl blacklist) { + this.dataSource = dataSource; + this.blacklist = blacklist; + } + + public void getDomains(Consumer consumer) { + String query; + if (getNames) { + query = "SELECT EC_DOMAIN.ID,DOMAIN_NAME,DOMAIN_ALIAS,STATE,KNOWN_URLS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID INNER JOIN EC_DOMAIN_LINK ON SOURCE_DOMAIN_ID=EC_DOMAIN.ID WHERE ((INDEXED>1 AND IS_ALIVE) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND SOURCE_DOMAIN_ID!=DEST_DOMAIN_ID GROUP BY EC_DOMAIN.ID"; + } + else { + query = "SELECT EC_DOMAIN.ID,\"\",DOMAIN_ALIAS,STATE,KNOWN_URLS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID INNER JOIN EC_DOMAIN_LINK ON SOURCE_DOMAIN_ID=EC_DOMAIN.ID WHERE ((INDEXED>1 AND IS_ALIVE) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND SOURCE_DOMAIN_ID!=DEST_DOMAIN_ID GROUP BY EC_DOMAIN.ID"; + } + + getDomains(query, consumer); + } + + + public void getPeripheralDomains(Consumer consumer) { + String query; + if (getNames) { + query = "SELECT EC_DOMAIN.ID,DOMAIN_NAME,DOMAIN_ALIAS,STATE,KNOWN_URLS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID LEFT JOIN EC_DOMAIN_LINK ON SOURCE_DOMAIN_ID=EC_DOMAIN.ID WHERE ((INDEXED>1 AND IS_ALIVE) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND EC_DOMAIN_LINK.ID IS NULL GROUP BY EC_DOMAIN.ID"; + } + else { + query = "SELECT EC_DOMAIN.ID,\"\",DOMAIN_ALIAS,STATE,KNOWN_URLS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID LEFT JOIN EC_DOMAIN_LINK ON SOURCE_DOMAIN_ID=EC_DOMAIN.ID WHERE ((INDEXED>1 AND IS_ALIVE) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND EC_DOMAIN_LINK.ID IS NULL GROUP BY EC_DOMAIN.ID"; + } + + getDomains(query, consumer); + } + + private void getDomains(String query, Consumer consumer) { + try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(query)) { + stmt.setFetchSize(10000); + var rsp = stmt.executeQuery(); + while (rsp.next()) { + int id = rsp.getInt(1); + if (!blacklist.isBlacklisted(id)) { + consumer.accept(new RankingDomainData(id, rsp.getString(2), rsp.getInt(3), EdgeDomainIndexingState.valueOf(rsp.getString(4)), rsp.getInt(5), false)); + } + } + } + catch (SQLException ex) { + logger.error("Failed to fetch domains", ex); + } + } + + public void eachDomainLink(DomainLinkConsumer consumer) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement("SELECT SOURCE_DOMAIN_ID, DEST_DOMAIN_ID FROM EC_DOMAIN_LINK")) + { + stmt.setFetchSize(10000); + + var rsp = stmt.executeQuery(); + + while (rsp.next()) { + int src = rsp.getInt(1); + int dst = rsp.getInt(2); + + consumer.accept(src, dst); + } + } + catch (SQLException ex) { + logger.error("Failed to fetch domain links", ex); + } + } + + public void domainsByPattern(String pattern, IntConsumer idConsumer) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME LIKE ?")) { + stmt.setString(1, pattern); + var rsp = stmt.executeQuery(); + while (rsp.next()) { + idConsumer.accept(rsp.getInt(1)); + } + } + catch (SQLException ex) { + logger.error("Failed to fetch domains by pattern", ex); + } + } + + public interface DomainLinkConsumer { + void accept(int from, int to); + } +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/old/OldReversePageRankV2.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/old/OldReversePageRankV2.java index 6a214278..02823563 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/old/OldReversePageRankV2.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/old/OldReversePageRankV2.java @@ -66,7 +66,7 @@ public class OldReversePageRankV2 { originDomains.add("memex.marginalia.nu"); try (var conn = dataSource.getConnection()) { - try (var stmt = conn.prepareStatement("SELECT ID,INDEXED,STATE FROM EC_DOMAIN WHERE INDEXED>1 AND STATE>=0 AND QUALITY_RAW>=-10")) { + try (var stmt = conn.prepareStatement("SELECT ID,INDEXED,STATE FROM EC_DOMAIN WHERE INDEXED>1 AND IS_ALIVE")) { stmt.setFetchSize(10000); var rsp = stmt.executeQuery(); while (rsp.next()) { @@ -90,7 +90,7 @@ public class OldReversePageRankV2 { } } - try (var stmt = conn.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE URL_PART=?")) { + try (var stmt = conn.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=?")) { stmt.setFetchSize(10000); for (var seed : this.originDomains) { diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/old/StandardPageRank.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/old/StandardPageRank.java index c42b28dd..74bef70a 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/old/StandardPageRank.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/old/StandardPageRank.java @@ -48,7 +48,7 @@ public class StandardPageRank { originDomains.addAll(Arrays.asList(origins)); try (var conn = dataSource.getConnection()) { - try (var stmt = conn.prepareStatement("SELECT ID,INDEXED,STATE,URL_PART FROM EC_DOMAIN WHERE INDEXED>1 AND STATE>=0 AND QUALITY>=-10")) { + try (var stmt = conn.prepareStatement("SELECT ID,INDEXED,STATE,DOMAIN_NAME FROM EC_DOMAIN WHERE INDEXED>1 AND IS_ALIVE AND QUALITY>=-10")) { stmt.setFetchSize(10000); var rsp = stmt.executeQuery(); while (rsp.next()) { @@ -78,7 +78,7 @@ public class StandardPageRank { } } - try (var stmt = conn.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE URL_PART=?")) { + try (var stmt = conn.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=?")) { for (var seed : this.originDomains) { stmt.setString(1, seed); var rsp = stmt.executeQuery(); diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/DedupTool.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/DedupTool.java index a5ea8b06..d6f95f51 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/DedupTool.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/DedupTool.java @@ -50,7 +50,7 @@ public class DedupTool { Map>> domainToHashToUrl = new HashMap<>(); try (var conn = ds.getConnection(); - var fetchStmt = conn.prepareStatement("SELECT URL_TOP_DOMAIN_ID,DATA_HASH,URL,EC_URL.ID,EC_DOMAIN.URL_PART FROM EC_URL INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID WHERE DATA_HASH IS NOT NULL"); + var fetchStmt = conn.prepareStatement("SELECT URL_TOP_DOMAIN_ID,DATA_HASH,URL,EC_URL.ID,EC_DOMAIN.DOMAIN_NAME FROM EC_URL INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID WHERE DATA_HASH IS NOT NULL"); var updateStmt = conn.prepareStatement("UPDATE EC_URL SET STATE='redirect' WHERE ID=?") ) { diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/PerusePageRankV2.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/PerusePageRankV2.java index 85a691c2..3f3ce6a5 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/PerusePageRankV2.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/PerusePageRankV2.java @@ -112,10 +112,10 @@ public class PerusePageRankV2 { try (var conn = dataSource.getConnection()) { String s; if (getNames) { - s = "SELECT EC_DOMAIN.ID,URL_PART,DOMAIN_ALIAS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID WHERE ((INDEXED>1 AND STATE >= 0) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND QUALITY_RAW>=-20 GROUP BY EC_DOMAIN.ID"; + s = "SELECT EC_DOMAIN.ID,DOMAIN_NAME,DOMAIN_ALIAS FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID WHERE ((INDEXED>1 AND IS_ALIVE) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) GROUP BY EC_DOMAIN.ID"; } else { - s = "SELECT EC_DOMAIN.ID,\"\",DOMAIN_ALIAS FROM EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_METADATA.ID WHERE ((INDEXED>1 AND STATE >= 0) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) AND QUALITY_RAW>=-20 GROUP BY EC_DOMAIN.ID"; + s = "SELECT EC_DOMAIN.ID,\"\",DOMAIN_ALIAS FROM EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_METADATA.ID WHERE ((INDEXED>1 AND IS_ALIVE) OR (INDEXED=1 AND VISITED_URLS=KNOWN_URLS AND GOOD_URLS>0)) GROUP BY EC_DOMAIN.ID"; } try (var stmt = conn.prepareStatement(s)) { stmt.setFetchSize(10000); diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/TestAcademiaRankTool.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/TestAcademiaRankTool.java deleted file mode 100644 index 38192b35..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/TestAcademiaRankTool.java +++ /dev/null @@ -1,30 +0,0 @@ -package nu.marginalia.util.ranking.tool; - -import lombok.SneakyThrows; -import nu.marginalia.util.ranking.AcademiaRank; -import nu.marginalia.wmsa.configuration.module.DatabaseModule; -import org.mariadb.jdbc.Driver; - -import java.io.IOException; - -public class TestAcademiaRankTool { - - @SneakyThrows - public static void main(String... args) { - Driver driver = new Driver(); - var conn = new DatabaseModule().provideConnection(); - - var rank = new AcademiaRank(new DatabaseModule().provideConnection(), "www.perseus.tufts.edu", "xroads.virginia.edu"); - var res = rank.getResult(); - - try (var c = conn.getConnection(); var stmt = c.prepareStatement("SELECT URL_PART FROM EC_DOMAIN WHERE ID=?")) { - for (int i = 0; i < Math.min(res.size(), 100); i++) { - stmt.setInt(1, res.getQuick(i)); - var rsp = stmt.executeQuery(); - while (rsp.next()) - System.out.println(rsp.getString(1)); - } - } - } - -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/UpdateDomainRanksTool.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/UpdateDomainRanksTool.java index 71ec72a6..f80d307f 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/UpdateDomainRanksTool.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/UpdateDomainRanksTool.java @@ -3,12 +3,13 @@ package nu.marginalia.util.ranking.tool; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; import nu.marginalia.util.ranking.BuggyStandardPageRank; +import nu.marginalia.util.ranking.RankingDomainFetcher; import nu.marginalia.wmsa.configuration.module.DatabaseModule; +import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklistImpl; import org.mariadb.jdbc.Driver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.sql.SQLException; import java.util.HashSet; import java.util.Set; @@ -43,12 +44,14 @@ public class UpdateDomainRanksTool { var uploader = new Thread(() -> uploadThread(conn), "Uploader"); logger.info("Ranking"); - var spr = new BuggyStandardPageRank(new DatabaseModule().provideConnection(),"memex.marginalia.nu"); + var ds = new DatabaseModule().provideConnection(); + var domains = new RankingDomainFetcher(ds, new EdgeDomainBlacklistImpl(ds)); + var spr = new BuggyStandardPageRank(domains, "memex.marginalia.nu"); rankMax = spr.size()*2; uploader.start(); - spr.pageRankWithPeripheralNodes(rankMax, false).forEach(i -> { + spr.pageRankWithPeripheralNodes(rankMax).forEach(i -> { try { uploadQueue.put(i); } catch (InterruptedException e) { @@ -83,11 +86,6 @@ public class UpdateDomainRanksTool { } } - logger.info("Recalculating quality"); - try (var stmt = conn.prepareStatement("UPDATE EC_DOMAIN SET QUALITY=-5*RANK+IF(RANK=1,RANK*GREATEST(QUALITY_RAW,QUALITY_ORIGINAL)/2, 0)")) { - stmt.executeUpdate(); - } - } catch (SQLException | InterruptedException throwables) { throwables.printStackTrace(); } diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/UpdateDomainRanksTool2.java b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/UpdateDomainRanksTool2.java index 336b35fd..f46fb390 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/UpdateDomainRanksTool2.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/ranking/tool/UpdateDomainRanksTool2.java @@ -3,12 +3,13 @@ package nu.marginalia.util.ranking.tool; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; import nu.marginalia.util.ranking.BetterReversePageRank; +import nu.marginalia.util.ranking.RankingDomainFetcher; import nu.marginalia.wmsa.configuration.module.DatabaseModule; +import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklistImpl; import org.mariadb.jdbc.Driver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.sql.SQLException; import java.util.HashSet; import java.util.Set; @@ -45,7 +46,9 @@ public class UpdateDomainRanksTool2 { logger.info("Ranking"); // "memex.marginalia.nu", "wiki.xxiivv.com", "bikobatanari.art", "sadgrl.online", "lileks.com", // "www.rep.routledge.com", "www.personal.kent.edu", "xroads.virginia.edu", "classics.mit.edu", "faculty.washington.edu", "monadnock.net" - var rpr = new BetterReversePageRank(new DatabaseModule().provideConnection(), "memex.marginalia.nu", "bikobatanari.art", "sadgrl.online", "wiki.xxiivv.com", "%neocities.org"); + var ds = new DatabaseModule().provideConnection(); + var domains = new RankingDomainFetcher(ds, new EdgeDomainBlacklistImpl(ds)); + var rpr = new BetterReversePageRank(domains, "memex.marginalia.nu", "bikobatanari.art", "sadgrl.online", "wiki.xxiivv.com", "%neocities.org"); // var rpr = new BetterStandardPageRank(new DatabaseModule().provideConnection(), "%edu"); // var spr = new BetterStandardPageRank(new DatabaseModule().provideConnection(), "memex.marginalia.nu"); @@ -58,7 +61,7 @@ public class UpdateDomainRanksTool2 { rankMax = rpr.size(); - rpr.pageRankWithPeripheralNodes(rankMax, false).forEach(i -> { + rpr.pageRankWithPeripheralNodes(rankMax).forEach(i -> { try { uploadQueue.put(i); } catch (InterruptedException e) { @@ -94,9 +97,6 @@ public class UpdateDomainRanksTool2 { } logger.info("Recalculating quality"); - try (var stmt = conn.prepareStatement("UPDATE EC_DOMAIN SET QUALITY=-5*RANK+IF(RANK=1,RANK*GREATEST(QUALITY_RAW,QUALITY_ORIGINAL)/2, 0)")) { - stmt.executeUpdate(); - } } catch (SQLException | InterruptedException throwables) { throwables.printStackTrace(); diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ReindexTriggerMain.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ReindexTriggerMain.java index 050152bc..55648dfd 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ReindexTriggerMain.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ReindexTriggerMain.java @@ -29,7 +29,7 @@ public class ReindexTriggerMain { .build(); try (var ds = db.provideConnection(); var conn = ds.getConnection(); var stmt = conn.createStatement()) { - var rs = stmt.executeQuery("SELECT ID, URL_PART, STATE, INDEXED FROM EC_DOMAIN LIMIT 100"); + var rs = stmt.executeQuery("SELECT ID, DOMAIN_NAME, STATE, INDEXED FROM EC_DOMAIN LIMIT 100"); while (rs.next()) { System.out.printf("%d %s %s %d\n", rs.getInt(1), @@ -38,7 +38,7 @@ public class ReindexTriggerMain { rs.getInt(4)); } - rs = stmt.executeQuery("SELECT ID, DOMAIN_ID, URL, VISITED, STATE FROM EC_URL LIMIT 100"); + rs = stmt.executeQuery("SELECT ID, DOMAIN_ID, PATH, VISITED, STATE FROM EC_URL LIMIT 100"); while (rs.next()) { System.out.printf("%d %d %s %d %s\n", rs.getInt(1), diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/Interpreter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/Interpreter.java index 8755716c..c0698dde 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/Interpreter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/Interpreter.java @@ -14,7 +14,7 @@ public interface Interpreter { void loadRssFeed(EdgeUrl[] rssFeed); void loadDomainLink(DomainLink[] links); - void loadProcessedDomain(EdgeDomain domain, EdgeDomainIndexingState state, double quality); + void loadProcessedDomain(EdgeDomain domain, EdgeDomainIndexingState state, String ip); void loadProcessedDocument(LoadProcessedDocument loadProcessedDocument); void loadProcessedDocumentWithError(LoadProcessedDocumentWithError loadProcessedDocumentWithError); diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/instruction/LoadProcessedDomain.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/instruction/LoadProcessedDomain.java index 065d6211..2b1fd631 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/instruction/LoadProcessedDomain.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/instruction/LoadProcessedDomain.java @@ -6,11 +6,11 @@ import nu.marginalia.wmsa.edge.converting.interpreter.Interpreter; import nu.marginalia.wmsa.edge.model.EdgeDomain; import nu.marginalia.wmsa.edge.model.crawl.EdgeDomainIndexingState; -public record LoadProcessedDomain(EdgeDomain domain, EdgeDomainIndexingState state, double quality) implements Instruction { +public record LoadProcessedDomain(EdgeDomain domain, EdgeDomainIndexingState state, String ip) implements Instruction { @Override public void apply(Interpreter interpreter) { - interpreter.loadProcessedDomain(domain, state, quality); + interpreter.loadProcessedDomain(domain, state, ip); } @Override diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/Loader.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/Loader.java index 140a762a..49a39457 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/Loader.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/Loader.java @@ -76,9 +76,9 @@ public class Loader implements Interpreter { } @Override - public void loadProcessedDomain(EdgeDomain domain, EdgeDomainIndexingState state, double quality) { - logger.debug("loadProcessedDomain({}, {}, {})", domain, state, quality); - sqlLoadProcessedDomain.load(data, domain, state, quality); + public void loadProcessedDomain(EdgeDomain domain, EdgeDomainIndexingState state, String ip) { + logger.debug("loadProcessedDomain({}, {}, {})", domain, state, ip); + sqlLoadProcessedDomain.load(data, domain, state, ip); } @Override diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainLinks.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainLinks.java index e0978828..6750bd33 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainLinks.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainLinks.java @@ -30,7 +30,7 @@ public class SqlLoadDomainLinks { INSERT IGNORE INTO EC_DOMAIN_LINK (SOURCE_DOMAIN_ID, DEST_DOMAIN_ID) SELECT SOURCE.ID,DEST.ID FROM EC_DOMAIN SOURCE INNER JOIN EC_DOMAIN DEST - ON SOURCE.URL_PART=FROM_DOMAIN AND DEST.URL_PART=TO_DOMAIN; + ON SOURCE.DOMAIN_NAME=FROM_DOMAIN AND DEST.DOMAIN_NAME=TO_DOMAIN; END """); } @@ -61,8 +61,8 @@ public class SqlLoadDomainLinks { } } } - catch (SQLException sql) { - sql.printStackTrace(); + catch (SQLException ex) { + logger.warn("SQL error inserting domain links", ex); } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomains.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomains.java index 18cc40bd..76a839c9 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomains.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomains.java @@ -25,15 +25,9 @@ public class SqlLoadDomains { stmt.execute(""" CREATE PROCEDURE INSERT_DOMAIN ( IN DOMAIN_NAME VARCHAR(255), - IN SUB_DOMAIN VARCHAR(255), IN TOP_DOMAIN VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci) BEGIN - INSERT IGNORE INTO EC_TOP_DOMAIN (URL_PART) VALUES (TOP_DOMAIN); - - INSERT IGNORE INTO EC_DOMAIN(URL_PART, URL_SUBDOMAIN, URL_TOP_DOMAIN_ID) - SELECT DOMAIN_NAME,SUB_DOMAIN,ID - FROM EC_TOP_DOMAIN - WHERE EC_TOP_DOMAIN.URL_PART=TOP_DOMAIN; + INSERT IGNORE INTO EC_DOMAIN(DOMAIN_NAME, DOMAIN_TOP) VALUES (DOMAIN_NAME, TOP_DOMAIN); END """); } @@ -46,10 +40,9 @@ public class SqlLoadDomains { public void load(LoaderData data, EdgeDomain domain) { try (var connection = dataSource.getConnection()) { - try (var insertCall = connection.prepareCall("CALL INSERT_DOMAIN(?,?,?)")) { + try (var insertCall = connection.prepareCall("CALL INSERT_DOMAIN(?,?)")) { insertCall.setString(1, domain.toString()); - insertCall.setString(2, domain.subDomain); - insertCall.setString(3, domain.domain); + insertCall.setString(2, domain.domain); insertCall.addBatch(); var ret = insertCall.executeUpdate(); @@ -57,12 +50,11 @@ public class SqlLoadDomains { logger.warn("load({}) -- bad row count {}", domain, ret); } - connection.commit(); findIdForTargetDomain(connection, data); } } catch (SQLException ex) { - ex.printStackTrace(); + logger.warn("SQL error inserting domain", ex); } @@ -73,12 +65,11 @@ public class SqlLoadDomains { try (var connection = dataSource.getConnection()) { connection.setAutoCommit(false); - try (var insertCall = connection.prepareCall("CALL INSERT_DOMAIN(?,?,?)")) { + try (var insertCall = connection.prepareCall("CALL INSERT_DOMAIN(?,?)")) { for (var domain : domains) { insertCall.setString(1, domain.toString()); - insertCall.setString(2, domain.subDomain); - insertCall.setString(3, domain.domain); + insertCall.setString(2, domain.domain); insertCall.addBatch(); } var ret = insertCall.executeBatch(); @@ -95,7 +86,7 @@ public class SqlLoadDomains { findIdForTargetDomain(connection, data); } catch (SQLException ex) { - ex.printStackTrace(); + logger.warn("SQL error inserting domains", ex); } } @@ -104,7 +95,7 @@ public class SqlLoadDomains { return; } - try (var query = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE URL_PART=?")) + try (var query = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=?")) { var targetDomain = data.getTargetDomain(); @@ -118,7 +109,7 @@ public class SqlLoadDomains { } } catch (SQLException ex) { - ex.printStackTrace(); + logger.warn("SQL error finding id for domain", ex); } } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocument.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocument.java index b25a657b..e2e25fff 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocument.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocument.java @@ -31,14 +31,14 @@ public class SqlLoadProcessedDocument { IN TITLE VARCHAR(255), IN DESCRIPTION VARCHAR(255), IN LENGTH INT, - IN QUALITY_MEASURE DOUBLE, IN FEATURES INT, IN STANDARD VARCHAR(32), + IN QUALITY DOUBLE, IN HASH INT) BEGIN SET FOREIGN_KEY_CHECKS=0; - REPLACE INTO EC_PAGE_DATA(ID, TITLE, DESCRIPTION, WORDS_TOTAL, FORMAT, FEATURES) VALUES (URL_ID, TITLE, DESCRIPTION, LENGTH, STANDARD, FEATURES); - UPDATE EC_URL SET VISITED=1, STATE=STATE, QUALITY_MEASURE=QUALITY_MEASURE, DATA_HASH=HASH WHERE ID=URL_ID; + REPLACE INTO EC_PAGE_DATA(ID, TITLE, DESCRIPTION, WORDS_TOTAL, FORMAT, FEATURES, DATA_HASH, QUALITY) VALUES (URL_ID, TITLE, DESCRIPTION, LENGTH, STANDARD, FEATURES, HASH, QUALITY); + UPDATE EC_URL SET VISITED=1, STATE=STATE WHERE ID=URL_ID; SET FOREIGN_KEY_CHECKS=1; END """); @@ -47,7 +47,8 @@ public class SqlLoadProcessedDocument { IN URL_ID INT, IN STATE VARCHAR(32)) BEGIN - UPDATE EC_URL SET VISITED=1, STATE=STATE, QUALITY_MEASURE=-100, DATA_HASH=NULL WHERE ID=URL_ID; + UPDATE EC_URL SET VISITED=1, STATE=STATE WHERE ID=URL_ID; + DELETE FROM EC_PAGE_DATA WHERE ID=URL_ID; END """); @@ -61,6 +62,7 @@ public class SqlLoadProcessedDocument { public void load(LoaderData data, List documents) { try (var conn = dataSource.getConnection(); var stmt = conn.prepareCall("CALL INSERT_PAGE_VISIT(?, ?, ?, ?, ?, ?, ?, ?, ?)")) { + conn.setAutoCommit(false); for (var doc : documents) { int urlId = data.getUrlId(doc.url()); @@ -74,9 +76,9 @@ public class SqlLoadProcessedDocument { stmt.setString(3, doc.title()); stmt.setString(4, doc.description()); stmt.setInt(5, doc.length()); - stmt.setDouble(6, doc.quality()); - stmt.setInt(7, doc.htmlFeatures()); - stmt.setString(8, doc.standard().name()); + stmt.setInt(6, doc.htmlFeatures()); + stmt.setString(7, doc.standard().name()); + stmt.setDouble(8, doc.quality()); stmt.setInt(9, (int) doc.hash()); stmt.addBatch(); } @@ -89,8 +91,8 @@ public class SqlLoadProcessedDocument { } conn.commit(); - } catch (SQLException e) { - e.printStackTrace(); + } catch (SQLException ex) { + logger.warn("SQL error inserting document", ex); } @@ -117,8 +119,8 @@ public class SqlLoadProcessedDocument { logger.warn("load({}) -- bad row count {}", documents.get(rv), ret[rv]); } } - } catch (SQLException e) { - e.printStackTrace(); + } catch (SQLException ex) { + logger.warn("SQL error inserting failed document", ex); } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDomain.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDomain.java index 64607b3a..018d76c9 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDomain.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDomain.java @@ -25,12 +25,12 @@ public class SqlLoadProcessedDomain { stmt.execute("DROP PROCEDURE IF EXISTS INITIALIZE_DOMAIN"); stmt.execute(""" CREATE PROCEDURE INITIALIZE_DOMAIN ( - IN ST INT, + IN ST ENUM('ACTIVE', 'EXHAUSTED', 'SPECIAL', 'SOCIAL_MEDIA', 'BLOCKED', 'REDIR', 'ERROR', 'UNKNOWN'), IN IDX INT, - IN QUAL DOUBLE, - IN DID INT) + IN DID INT, + IN IP VARCHAR(32)) BEGIN - UPDATE EC_DOMAIN SET INDEX_DATE=NOW(), STATE=ST, DOMAIN_ALIAS=NULL, INDEXED=GREATEST(INDEXED,IDX), QUALITY=QUAL, QUALITY_RAW=QUAL, QUALITY_ORIGINAL=QUAL WHERE ID=DID; + UPDATE EC_DOMAIN SET INDEX_DATE=NOW(), STATE=ST, DOMAIN_ALIAS=NULL, INDEXED=GREATEST(INDEXED,IDX), IP=IP WHERE ID=DID; DELETE FROM EC_DOMAIN_LINK WHERE SOURCE_DOMAIN_ID=DID; END """); @@ -41,7 +41,7 @@ public class SqlLoadProcessedDomain { } } - public void load(LoaderData data, EdgeDomain domain, EdgeDomainIndexingState state, double quality) { + public void load(LoaderData data, EdgeDomain domain, EdgeDomainIndexingState state, String ip) { data.setTargetDomain(domain); loadDomains.load(data, domain); @@ -49,18 +49,17 @@ public class SqlLoadProcessedDomain { try (var conn = dataSource.getConnection(); var initCall = conn.prepareCall("CALL INITIALIZE_DOMAIN(?,?,?,?)")) { - initCall.setInt(1, state.code); + initCall.setString(1, state.name()); initCall.setInt(2, 1 + data.sizeHint / 100); - initCall.setDouble(3, quality); - initCall.setInt(4, data.getDomainId(domain)); + initCall.setInt(3, data.getDomainId(domain)); + initCall.setString(4, ip); int rc = initCall.executeUpdate(); if (rc < 1) { - logger.warn("load({},{},{}) -- bad rowcount {}", domain, state, quality, rc); + logger.warn("load({},{}) -- bad rowcount {}", domain, state, rc); } - conn.commit(); } catch (SQLException ex) { - ex.printStackTrace(); + logger.warn("SQL error initializing domain", ex); } } @@ -69,9 +68,9 @@ public class SqlLoadProcessedDomain { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" UPDATE EC_DOMAIN TARGET - INNER JOIN EC_DOMAIN ALIAS ON ALIAS.URL_PART=? + INNER JOIN EC_DOMAIN ALIAS ON ALIAS.DOMAIN_NAME=? SET TARGET.DOMAIN_ALIAS=ALIAS.ID - WHERE TARGET.URL_PART=? + WHERE TARGET.DOMAIN_NAME=? """)) { stmt.setString(1, link.to().toString()); stmt.setString(2, link.from().toString()); @@ -81,7 +80,7 @@ public class SqlLoadProcessedDomain { } } catch (SQLException ex) { - ex.printStackTrace(); + logger.warn("SQL error inserting domain alias", ex); } } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadUrls.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadUrls.java index 7d8851ca..ba9ae43a 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadUrls.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadUrls.java @@ -1,11 +1,13 @@ package nu.marginalia.wmsa.edge.converting.loader; +import com.google.common.hash.Hashing; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.wmsa.edge.model.EdgeUrl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.sql.Types; @@ -25,12 +27,13 @@ public class SqlLoadUrls { stmt.execute(""" CREATE PROCEDURE INSERT_URL ( IN PROTO VARCHAR(255), - IN DOMAIN_NAME VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci, + IN DOMAIN VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci, IN PORT INT, - IN URL VARCHAR(255) + IN PATH VARCHAR(255), + IN PATH_HASH BIGINT ) BEGIN - INSERT IGNORE INTO EC_URL (PROTO,DOMAIN_ID,PORT,URL) SELECT PROTO,ID,PORT,URL FROM EC_DOMAIN WHERE URL_PART=DOMAIN_NAME; + INSERT IGNORE INTO EC_URL (PROTO,DOMAIN_ID,PORT,PATH,PATH_HASH) SELECT PROTO,ID,PORT,PATH,PATH_HASH FROM EC_DOMAIN WHERE DOMAIN_NAME=DOMAIN; END """); } @@ -42,8 +45,8 @@ public class SqlLoadUrls { public void load(LoaderData data, EdgeUrl[] urls) { try (var conn = dataSource.getConnection(); - var insertCall = conn.prepareCall("CALL INSERT_URL(?,?,?,?)"); - var queryCall = conn.prepareStatement("SELECT ID, PROTO, URL FROM EC_URL WHERE DOMAIN_ID=?") + var insertCall = conn.prepareCall("CALL INSERT_URL(?,?,?,?, ?)"); + var queryCall = conn.prepareStatement("SELECT ID, PROTO, PATH FROM EC_URL WHERE DOMAIN_ID=?") ) { conn.setAutoCommit(false); @@ -58,6 +61,7 @@ public class SqlLoadUrls { insertCall.setNull(3, Types.INTEGER); } insertCall.setString(4, url.path); + insertCall.setLong(5, hashPath(url.path)); insertCall.addBatch(); } var ret = insertCall.executeBatch(); @@ -86,7 +90,11 @@ public class SqlLoadUrls { } catch (SQLException ex) { - ex.printStackTrace(); + logger.warn("SQL error inserting URLs", ex); } } + + private long hashPath(String path) { + return Hashing.murmur3_128().hashString(path, StandardCharsets.UTF_8).asLong(); + } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/InstructionsCompiler.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/InstructionsCompiler.java index d36cb830..b75de436 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/InstructionsCompiler.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/InstructionsCompiler.java @@ -15,7 +15,7 @@ public class InstructionsCompiler { public List compile(ProcessedDomain domain) { List ret = new ArrayList<>(domain.size()*4); - ret.add(new LoadProcessedDomain(domain.domain, domain.state, domain.averageQuality().orElse(-5.))); + ret.add(new LoadProcessedDomain(domain.domain, domain.state, domain.ip)); if (domain.documents != null) { compileUrls(ret, domain.documents); diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/CrawlJobExtractorMain.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/CrawlJobExtractorMain.java index 2f25d6d7..52fe338a 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/CrawlJobExtractorMain.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/CrawlJobExtractorMain.java @@ -34,11 +34,10 @@ public class CrawlJobExtractorMain { private static final String domainsSql = """ - SELECT ID, LOWER(EC_DOMAIN.URL_PART) + SELECT ID, LOWER(EC_DOMAIN.DOMAIN_NAME) FROM EC_DOMAIN - WHERE QUALITY_RAW>-100 - AND INDEXED>0 - AND STATE<2 + WHERE INDEXED>0 + AND STATE='ACTIVE' OR STATE='EXHAUSTED' ORDER BY INDEX_DATE ASC, DISCOVER_DATE ASC, @@ -49,8 +48,8 @@ public class CrawlJobExtractorMain { private static final String urlsSql = """ - SELECT CONCAT(PROTO, "://", ?, URL) - FROM EC_URL + SELECT URL + FROM EC_URL_VIEW WHERE DOMAIN_ID=? ORDER BY VISITED DESC, diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/CrawlJobExtractorPageRankMain.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/CrawlJobExtractorPageRankMain.java index 21935fd0..ea1946fc 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/CrawlJobExtractorPageRankMain.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/CrawlJobExtractorPageRankMain.java @@ -6,6 +6,7 @@ import com.google.common.hash.Hashing; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.util.ranking.RankingDomainFetcher; import nu.marginalia.wmsa.configuration.module.DatabaseModule; import nu.marginalia.wmsa.edge.crawling.model.CrawlingSpecification; import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklistImpl; @@ -30,19 +31,19 @@ public class CrawlJobExtractorPageRankMain { """ SELECT ID FROM EC_DOMAIN - WHERE URL_PART=? + WHERE DOMAIN_NAME=? """; private static final String specificDomainSqlFromId = """ - SELECT LOWER(URL_PART) + SELECT LOWER(DOMAIN_NAME) FROM EC_DOMAIN WHERE ID=? """; private static final String urlsSql = """ - SELECT CONCAT(PROTO, "://", ?, URL) - FROM EC_URL + SELECT URL + FROM EC_URL_VIEW WHERE DOMAIN_ID=? ORDER BY VISITED DESC, @@ -73,10 +74,12 @@ public class CrawlJobExtractorPageRankMain { Gson gson = new GsonBuilder().create(); - var rpr = new BetterReversePageRank(new DatabaseModule().provideConnection(), "bikobatanari.art", "sadgrl.online", "wiki.xxiivv.com", "%neocities.org"); + var ds = new DatabaseModule().provideConnection(); + var domains = new RankingDomainFetcher(ds, new EdgeDomainBlacklistImpl(ds)); + var rpr = new BetterReversePageRank(domains, "bikobatanari.art", "sadgrl.online", "wiki.xxiivv.com", "%neocities.org"); rpr.setMaxKnownUrls(750); - var targetDomainIds = rpr.pageRankWithPeripheralNodes(rpr.size(), false); + var targetDomainIds = rpr.pageRankWithPeripheralNodes(rpr.size()); try (var out = new PrintWriter(new ZstdOutputStream(new BufferedOutputStream(new FileOutputStream(outFile.toFile()))))) { final var extractor = new CrawlJobExtractorPageRankMain(new DatabaseModule().provideConnection()); diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/EdgeDataStoreDao.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/EdgeDataStoreDao.java index 81e8dd58..2f309b07 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/EdgeDataStoreDao.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/EdgeDataStoreDao.java @@ -13,44 +13,14 @@ import java.util.Optional; @ImplementedBy(EdgeDataStoreDaoImpl.class) public interface EdgeDataStoreDao { - boolean isBlacklisted(EdgeDomain domain); - EdgeId getDomainId(EdgeDomain domain); - EdgeId getUrlId(EdgeUrl domain); - EdgeUrl getUrl(EdgeId id); - EdgeUrlDetails getUrlDetails(EdgeId id); - List getDomainNeighbors(EdgeId domainId, EdgeDomainBlacklist backlist, int count); List getDomainNeighborsAdjacent(EdgeId domainId, EdgeDomainBlacklist backlist, int count); + List getRandomDomains(int count, EdgeDomainBlacklist backlist); List getUrlDetailsMulti(List> ids); - List> getDomainIdsFromUrlIds(Collection> urlIds); - EdgeDomain getDomain(EdgeId id); - List> inboudUrls(EdgeId id, int limit); - List> outboundUrls(EdgeId id, int limit); - Optional> resolveAmbiguousDomain(String name); - - - int getPagesKnown(EdgeId domainId); - int getPagesVisited(EdgeId domainId); - int getPagesIndexed(EdgeId domainId); - - int getIncomingLinks(EdgeId domainId); - int getOutboundLinks(EdgeId domainId); - - double getDomainQuality(EdgeId domainId); - - EdgeDomainIndexingState getDomainState(EdgeId domainId); - - List getLinkingDomains(EdgeId domainId); - - List getNewUrls(EdgeId domainId, Collection links); - - double getRank(EdgeId domainId); - - void updateDomainIndexTimestamp(EdgeDomain domain, EdgeDomainIndexingState state, EdgeDomain alias, int minIndexed); } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/EdgeDataStoreDaoImpl.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/EdgeDataStoreDaoImpl.java index a214bb15..30ea2256 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/EdgeDataStoreDaoImpl.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/EdgeDataStoreDaoImpl.java @@ -33,7 +33,6 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { private final Cache> urlIdCache = CacheBuilder.newBuilder().maximumSize(100_000).build(); private final Cache> domainIdCache = CacheBuilder.newBuilder().maximumSize(10_000).build(); - private static final String DEFAULT_PROTOCOL = "http"; public static double QUALITY_LOWER_BOUND_CUTOFF = -15.; @Inject public EdgeDataStoreDaoImpl(HikariDataSource dataSource) @@ -48,30 +47,13 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { domainIdCache.invalidateAll(); } - @SneakyThrows - @Override - public boolean isBlacklisted(EdgeDomain domain) { - - try (var connection = dataSource.getConnection()) { - try (var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?")) { - stmt.setString(1, domain.domain); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return true; - } else { - return false; - } - } - } - } - @SneakyThrows @Override public EdgeId getDomainId(EdgeDomain domain) { try (var connection = dataSource.getConnection()) { return domainIdCache.get(domain, () -> { - try (var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE URL_PART=?")) { + try (var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=?")) { stmt.setString(1, domain.toString()); var rsp = stmt.executeQuery(); if (rsp.next()) { @@ -86,104 +68,14 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { } } - @Override - @SneakyThrows - public EdgeId getUrlId(EdgeUrl url) { - try (var connection = dataSource.getConnection()) { - - return urlIdCache.get(url, () -> { - try (var stmt = connection.prepareStatement("SELECT ID FROM EC_URL_VIEW WHERE URL_PATH=? AND URL_DOMAIN=? AND URL_PROTO=?")) { - stmt.setString(1, url.path); - stmt.setString(2, url.domain.toString()); - stmt.setString(3, url.proto); - - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return new EdgeId<>(rsp.getInt(1)); - } - } - // Lenient mode for http->https upgrades etc - try (var stmt = connection.prepareStatement("SELECT ID FROM EC_URL_VIEW WHERE URL_PATH=? AND URL_DOMAIN=?")) { - stmt.setString(1, url.path); - stmt.setString(2, url.domain.toString()); - - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return new EdgeId<>(rsp.getInt(1)); - } - } - throw new NoSuchElementException(url.toString()); - }); - } - catch (UncheckedExecutionException ex) { - throw ex.getCause(); + private String idList(List> ids) { + StringJoiner j = new StringJoiner(",", "(", ")"); + for (var id : ids) { + j.add(Integer.toString(id.getId())); } + return j.toString(); } - - @SneakyThrows - @Override - public List> getDomainIdsFromUrlIds(Collection> urlIds) { - List> results = new ArrayList<>(urlIds.size()); - - if (urlIds.isEmpty()) - return results; - - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.prepareStatement("SELECT DOMAIN_ID FROM EC_URL WHERE ID IN " + urlIds - .stream() - .map(EdgeId::getId) - .map(Object::toString) - .collect(Collectors.joining(",", "(", ")")))) - { - var rsp = stmt.executeQuery(); - while (rsp.next()) { - results.add(new EdgeId<>(rsp.getInt(1))); - } - - } - } - - return results; - } - - static final Pattern badChars = Pattern.compile("[';\\\\]"); - private String saneString(String s) { - return "\'"+badChars.matcher(s).replaceAll("?")+"\'"; - } - @SneakyThrows - @Override - public EdgeUrl getUrl(EdgeId id) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.createStatement()) { - var rsp = stmt.executeQuery("SELECT URL_PROTO, URL_DOMAIN,URL_PORT,URL_PATH FROM EC_URL_VIEW WHERE ID=" + id.getId()); - if (rsp.next()) { - return new EdgeUrl(rsp.getString(1), new EdgeDomain(rsp.getString(2)), rsp.getInt(3), rsp.getString(4)); - } - throw new NoSuchElementException(); - } - } - } - - @SneakyThrows - @Override - public EdgeUrlDetails getUrlDetails(EdgeId id) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.createStatement()) { - var rsp = stmt.executeQuery("SELECT ID,URL_PROTO,URL_DOMAIN,URL_PORT,URL_PATH,TITLE,DESCRIPTION,URL_QUALITY_MEASURE,DOMAIN_QUALITY_MEASURE,IFNULL(EC_DOMAIN_LINK_AGGREGATE.LINKS,1),WORDS_TOTAL,FORMAT,FEATURES,\"\",QUALITY_RAW,DOMAIN_STATE,DATA_HASH FROM EC_URL_VIEW LEFT JOIN EC_DOMAIN_LINK_AGGREGATE ON EC_DOMAIN_LINK_AGGREGATE.DOMAIN_ID=EC_URL_VIEW.DOMAIN_ID WHERE ID=" + id.getId()); - if (rsp.next()) { - EdgeUrl url = new EdgeUrl(rsp.getString(2), new EdgeDomain(rsp.getString(3)), rsp.getInt(4), rsp.getString(5)); - return new EdgeUrlDetails(rsp.getInt(1), url, rsp.getString(6), rsp.getString(7), rsp.getDouble(8), rsp.getDouble(15), rsp.getDouble(9), rsp.getInt(10), rsp.getInt(11), rsp.getString(12), rsp.getInt(13), EdgePageScoreAdjustment.zero(), Integer.MAX_VALUE, Double.MAX_VALUE, rsp.getString(14), rsp.getInt(16), 0, rsp.getInt(17)); - } - throw new NoSuchElementException(); - } - } - } - - @SneakyThrows @Override public List getUrlDetailsMulti(List> ids) { @@ -193,16 +85,39 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { List result = new ArrayList<>(ids.size()); try (var connection = dataSource.getConnection()) { - // This is SQL-injection safe, the IDs are of type int - String idString = ids.stream().map(EdgeId::getId).map(Objects::toString).collect(Collectors.joining(",", "(", ")")); - try (var stmt = connection.prepareStatement("SELECT ID,URL_PROTO,URL_DOMAIN,URL_PORT,URL_PATH,TITLE,DESCRIPTION,URL_QUALITY_MEASURE,DOMAIN_QUALITY_MEASURE,IFNULL(EC_DOMAIN_LINK_AGGREGATE.LINKS,1),WORDS_TOTAL,FORMAT,FEATURES,\"\",QUALITY_RAW,DOMAIN_STATE,DATA_HASH FROM EC_URL_VIEW LEFT JOIN EC_DOMAIN_LINK_AGGREGATE ON EC_DOMAIN_LINK_AGGREGATE.DOMAIN_ID=EC_URL_VIEW.DOMAIN_ID WHERE ID IN " + idString)) { + String idString = idList(ids); + + try (var stmt = connection.prepareStatement( + """ + SELECT ID, URL, + TITLE, DESCRIPTION, + QUALITY, + WORDS_TOTAL, FORMAT, FEATURES, + IP, DOMAIN_STATE, + DATA_HASH + FROM EC_URL_VIEW WHERE ID IN + """ + idString)) { stmt.setFetchSize(ids.size()); var rsp = stmt.executeQuery(); while (rsp.next()) { - EdgeUrl url = new EdgeUrl(rsp.getString(2), new EdgeDomain(rsp.getString(3)), rsp.getInt(4), rsp.getString(5)); - var val = new EdgeUrlDetails(rsp.getInt(1), url, rsp.getString(6), rsp.getString(7), rsp.getDouble(8), rsp.getDouble(15), rsp.getDouble(9), rsp.getInt(10), rsp.getInt(11), rsp.getString(12), rsp.getInt(13), EdgePageScoreAdjustment.zero(), Integer.MAX_VALUE, Double.MAX_VALUE, rsp.getString(14), rsp.getInt(16), 0, rsp.getInt(17)); + EdgeUrl url = new EdgeUrl(rsp.getString(2)); + var val = new EdgeUrlDetails(rsp.getInt(1), url, + rsp.getString(3), // title + rsp.getString(4), // description + rsp.getDouble(5), // quality + rsp.getInt(6), // wordsTotal + rsp.getString(7), // format + rsp.getInt(8), // features + rsp.getString(9), // ip + EdgeDomainIndexingState.valueOf(rsp.getString(10)), // domainState + rsp.getInt(11), // dataHash + EdgePageScoreAdjustment.zero(), // urlQualityAdjustment + Integer.MAX_VALUE, // rankingId + Double.MAX_VALUE, // termScore + 0 // queryLength + ); if (val.urlQuality >= QUALITY_LOWER_BOUND_CUTOFF) { result.add(val); } @@ -214,82 +129,13 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { return result; } - @Override - public List getDomainNeighbors(EdgeId domainId, EdgeDomainBlacklist blacklist, int count) { - final Set domains = new HashSet<>(count*3); - - final String q = "SELECT EC_DOMAIN.ID AS NEIGHBOR_ID, URL_PART from EC_DOMAIN_NEIGHBORS INNER JOIN EC_DOMAIN ON NEIGHBOR_ID=EC_DOMAIN.ID WHERE STATE<2 AND DOMAIN_ALIAS IS NULL AND EC_DOMAIN_NEIGHBORS.DOMAIN_ID = ? ORDER BY ADJ_IDX LIMIT ?"; - - try (var connection = dataSource.getConnection()) { - try (var stmt = connection.prepareStatement(q)) { - stmt.setFetchSize(count); - stmt.setInt(1, domainId.getId()); - stmt.setInt(2, count); - var rsp = stmt.executeQuery(); - while (rsp.next()) { - int id = rsp.getInt(1); - String domain = rsp.getString(2); - - if (!blacklist.isBlacklisted(id)) { - var url = new EdgeUrl(DEFAULT_PROTOCOL, new EdgeDomain(domain), null, "/"); - - domains.add(new BrowseResult(url, id)); - } - } - } - - final String q2 = "SELECT EC_DOMAIN.ID, URL_PART FROM EC_DOMAIN_LINK INNER JOIN EC_DOMAIN ON DEST_DOMAIN_ID=EC_DOMAIN.ID WHERE SOURCE_DOMAIN_ID=? AND STATE<2 AND DOMAIN_ALIAS IS NULL GROUP BY EC_DOMAIN.ID ORDER BY RANK ASC LIMIT ?"; - try (var stmt = connection.prepareStatement(q2)) { - - stmt.setFetchSize(count); - stmt.setInt(1, domainId.getId()); - stmt.setInt(2, count); - var rsp = stmt.executeQuery(); - while (rsp.next()) { - int id = rsp.getInt(1); - String domain = rsp.getString(2); - - if (!blacklist.isBlacklisted(id)) { - var url = new EdgeUrl(DEFAULT_PROTOCOL, new EdgeDomain(domain), null, "/"); - - domains.add(new BrowseResult(url, id)); - } - } - } - - final String q3 = "SELECT EC_DOMAIN.ID, URL_PART FROM EC_DOMAIN_LINK INNER JOIN EC_DOMAIN ON DEST_DOMAIN_ID=EC_DOMAIN.ID WHERE DEST_DOMAIN_ID=? AND STATE<2 AND DOMAIN_ALIAS IS NULL GROUP BY EC_DOMAIN.ID ORDER BY RANK ASC LIMIT ?"; - try (var stmt = connection.prepareStatement(q3)) { - stmt.setFetchSize(count); - stmt.setInt(1, domainId.getId()); - stmt.setInt(2, count); - - var rsp = stmt.executeQuery(); - while (rsp.next()) { - int id = rsp.getInt(1); - String domain = rsp.getString(2); - - if (!blacklist.isBlacklisted(id)) { - var url = new EdgeUrl(DEFAULT_PROTOCOL, new EdgeDomain(domain), null, "/"); - - domains.add(new BrowseResult(url, id)); - } - } - } - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - - - return new ArrayList<>(domains); - } - @Override public List getDomainNeighborsAdjacent(EdgeId domainId, EdgeDomainBlacklist blacklist, int count) { final Set domains = new HashSet<>(count*3); final String q = """ - SELECT EC_DOMAIN.ID AS NEIGHBOR_ID, URL_PART, COUNT(*) AS CNT + SELECT EC_DOMAIN.ID AS NEIGHBOR_ID, DOMAIN_NAME, COUNT(*) AS CNT FROM EC_DOMAIN_NEIGHBORS INNER JOIN EC_DOMAIN ON NEIGHBOR_ID=EC_DOMAIN.ID INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID @@ -316,16 +162,14 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { String domain = rsp.getString(2); if (!blacklist.isBlacklisted(id)) { - var url = new EdgeUrl(DEFAULT_PROTOCOL, new EdgeDomain(domain), null, "/"); - - domains.add(new BrowseResult(url, id)); + domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id)); } } } if (domains.size() < count/2) { final String q2 = """ - SELECT EC_DOMAIN.ID, URL_PART + SELECT EC_DOMAIN.ID, DOMAIN_NAME FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID INNER JOIN EC_DOMAIN_LINK B ON DEST_DOMAIN_ID=EC_DOMAIN.ID @@ -347,9 +191,7 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { String domain = rsp.getString(2); if (!blacklist.isBlacklisted(id)) { - var url = new EdgeUrl(DEFAULT_PROTOCOL, new EdgeDomain(domain), null, "/"); - - domains.add(new BrowseResult(url, id)); + domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id)); } } } @@ -357,11 +199,11 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { if (domains.size() < count/2) { final String q3 = """ - SELECT EC_DOMAIN.ID, URL_PART - FROM EC_DOMAIN - INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID + SELECT EC_DOMAIN.ID, DOMAIN_NAME + FROM EC_DOMAIN + INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID INNER JOIN EC_DOMAIN_LINK B ON B.SOURCE_DOMAIN_ID=EC_DOMAIN.ID - INNER JOIN EC_DOMAIN_LINK O ON O.DEST_DOMAIN_ID=EC_DOMAIN.ID + INNER JOIN EC_DOMAIN_LINK O ON O.DEST_DOMAIN_ID=EC_DOMAIN.ID WHERE B.DEST_DOMAIN_ID=? AND STATE<2 AND KNOWN_URLS<1000 @@ -381,9 +223,7 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { String domain = rsp.getString(2); if (!blacklist.isBlacklisted(id)) { - var url = new EdgeUrl(DEFAULT_PROTOCOL, new EdgeDomain(domain), null, "/"); - - domains.add(new BrowseResult(url, id)); + domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id)); } } } @@ -399,7 +239,15 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { @Override public List getRandomDomains(int count, EdgeDomainBlacklist blacklist) { - final String q = "SELECT DOMAIN_ID,URL_PART FROM EC_RANDOM_DOMAINS INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID WHERE STATE<2 AND DOMAIN_ALIAS IS NULL ORDER BY RAND() LIMIT ?"; + final String q = """ + SELECT DOMAIN_ID, DOMAIN_NAME + FROM EC_RANDOM_DOMAINS + INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID + WHERE STATE<2 + AND DOMAIN_ALIAS IS NULL + ORDER BY RAND() + LIMIT ? + """; List domains = new ArrayList<>(count); try (var conn = dataSource.getConnection()) { try (var stmt = conn.prepareStatement(q)) { @@ -410,9 +258,7 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { String domain = rsp.getString(2); if (!blacklist.isBlacklisted(id)) { - var url = new EdgeUrl(DEFAULT_PROTOCOL, new EdgeDomain(domain), null, "/"); - - domains.add(new BrowseResult(url, id)); + domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id)); } } } @@ -428,7 +274,7 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { public EdgeDomain getDomain(EdgeId id) { try (var connection = dataSource.getConnection()) { - try (var stmt = connection.prepareStatement("SELECT URL_PART FROM EC_DOMAIN WHERE ID=?")) { + try (var stmt = connection.prepareStatement("SELECT DOMAIN_NAME FROM EC_DOMAIN WHERE ID=?")) { stmt.setInt(1, id.getId()); var rsp = stmt.executeQuery(); if (rsp.next()) { @@ -439,330 +285,4 @@ public class EdgeDataStoreDaoImpl implements EdgeDataStoreDao { } } - @Override @SneakyThrows - public List> inboudUrls(EdgeId id, int limit) { - - List> ret = new ArrayList<>(); - try (var connection = dataSource.getConnection()) { - - try (var stmt = - connection.prepareStatement("SELECT SRC_URL_ID FROM EC_RELATED_LINKS_IN WHERE DEST_URL_ID=? ORDER BY SRC_URL_QUALITY DESC LIMIT ?")) { - stmt.setFetchSize(limit); - stmt.setInt(1, id.getId()); - stmt.setInt(2, limit); - var rsp = stmt.executeQuery(); - while (rsp.next()) { - ret.add(new EdgeId<>(rsp.getInt(1))); - } - } - - } - - return ret; - } - - - @Override @SneakyThrows - public List> outboundUrls(EdgeId id, int limit) { - - List> ret = new ArrayList<>(); - try (var connection = dataSource.getConnection()) { - - try (var stmt = - connection.prepareStatement("SELECT DEST_URL_ID FROM EC_RELATED_LINKS_IN WHERE SRC_URL_ID=? ORDER BY SRC_URL_QUALITY DESC LIMIT ?")) { - stmt.setFetchSize(limit); - stmt.setInt(1, id.getId()); - stmt.setInt(2, limit); - var rsp = stmt.executeQuery(); - while (rsp.next()) { - ret.add(new EdgeId<>(rsp.getInt(1))); - } - } - - } - - return ret; - } - - @Override - public Optional> resolveAmbiguousDomain(String name) { - try (var connection = dataSource.getConnection()) { - try (var stmt = connection.prepareStatement("SELECT IFNULL(DOMAIN_ALIAS,ID) FROM EC_DOMAIN WHERE URL_PART=?")) { - stmt.setString(1, name); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return Optional.of(new EdgeId<>(rsp.getInt(1))); - } - } - - try (var stmt = connection.prepareStatement("SELECT IFNULL(DOMAIN_ALIAS,ID) FROM EC_DOMAIN WHERE URL_PART=?")) { - stmt.setString(1, "https://"+name); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return Optional.of(new EdgeId<>(rsp.getInt(1))); - } - } - - try (var stmt = connection.prepareStatement("SELECT IFNULL(DOMAIN_ALIAS,ID) FROM EC_DOMAIN WHERE URL_PART=?")) { - stmt.setString(1, "http://"+name); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return Optional.of(new EdgeId<>(rsp.getInt(1))); - } - } - - try (var stmt = connection.prepareStatement("SELECT IFNULL(DOMAIN_ALIAS,ID) FROM EC_DOMAIN WHERE URL_PART=?")) { - stmt.setString(1, "https://www."+name); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return Optional.of(new EdgeId<>(rsp.getInt(1))); - } - } - - try (var stmt = connection.prepareStatement("SELECT IFNULL(DOMAIN_ALIAS,ID) FROM EC_DOMAIN WHERE URL_PART=?")) { - stmt.setString(1, "http://www."+name); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return Optional.of(new EdgeId<>(rsp.getInt(1))); - } - } - - } catch (SQLException throwables) { - logger.info("Could not resolve domain id for {}", name); - } - - return Optional.empty(); - } - - @SneakyThrows - @Override - public int getPagesKnown(EdgeId domainId) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.prepareStatement("SELECT KNOWN_URLS FROM DOMAIN_METADATA WHERE ID=?")) { - stmt.setInt(1, domainId.getId()); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return rsp.getInt(1); - } - } catch (Exception ex) { - logger.error("DB error", ex); - } - return 0; - } - } - - @SneakyThrows - @Override - public int getPagesVisited(EdgeId domainId) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.prepareStatement("SELECT VISITED_URLS FROM DOMAIN_METADATA WHERE ID=?")) { - stmt.setInt(1, domainId.getId()); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return rsp.getInt(1); - } - } catch (Exception ex) { - logger.error("DB error", ex); - } - return 0; - } - } - - - @SneakyThrows - @Override - public int getPagesIndexed(EdgeId domainId) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.prepareStatement("SELECT GOOD_URLS FROM DOMAIN_METADATA WHERE ID=?")) { - stmt.setInt(1, domainId.getId()); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return rsp.getInt(1); - } - } catch (Exception ex) { - logger.error("DB error", ex); - } - return 0; - } - } - - @SneakyThrows - @Override - public int getIncomingLinks(EdgeId domainId) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.prepareStatement("SELECT COUNT(ID) FROM EC_DOMAIN_LINK WHERE DEST_DOMAIN_ID=?")) { - stmt.setInt(1, domainId.getId()); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return rsp.getInt(1); - } - } catch (Exception ex) { - logger.error("DB error", ex); - } - return 0; - } - } - @SneakyThrows - @Override - public int getOutboundLinks(EdgeId domainId) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.prepareStatement("SELECT COUNT(ID) FROM EC_DOMAIN_LINK WHERE SOURCE_DOMAIN_ID=?")) { - stmt.setInt(1, domainId.getId()); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return rsp.getInt(1); - } - } catch (Exception ex) { - logger.error("DB error", ex); - } - return 0; - } - } - - @SneakyThrows - @Override - public double getDomainQuality(EdgeId domainId) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.prepareStatement("SELECT QUALITY FROM EC_DOMAIN WHERE ID=?")) { - stmt.setInt(1, domainId.getId()); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return rsp.getDouble(1); - } - } catch (Exception ex) { - logger.error("DB error", ex); - } - return -5; - } - } - - @Override - public EdgeDomainIndexingState getDomainState(EdgeId domainId) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.prepareStatement("SELECT STATE FROM EC_DOMAIN WHERE ID=?")) { - stmt.setInt(1, domainId.getId()); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return EdgeDomainIndexingState.fromCode(rsp.getInt(1)); - } - } catch (Exception ex) { - logger.error("DB error", ex); - } - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - return EdgeDomainIndexingState.ERROR; - } - - @Override - public List getLinkingDomains(EdgeId domainId) { - try (var connection = dataSource.getConnection()) { - List results = new ArrayList<>(25); - try (var stmt = connection.prepareStatement("SELECT SOURCE_URL FROM EC_RELATED_LINKS_VIEW WHERE DEST_DOMAIN_ID=? ORDER BY SOURCE_DOMAIN_ID LIMIT 25")) { - stmt.setInt(1, domainId.getId()); - var rsp = stmt.executeQuery(); - while (rsp.next()) { - results.add(new EdgeDomain(rsp.getString(1))); - } - return results; - } catch (Exception ex) { - logger.error("DB error", ex); - } - - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - return Collections.emptyList(); - } - - @Override - public List getNewUrls(EdgeId domainId, Collection links) { - Map edgeUrlByPath = links.stream().collect(Collectors.toMap(EdgeUrl::getPath, Function.identity(), (a,b)->a)); - - try (var connection = dataSource.getConnection()) { - try (var stmt = connection.prepareStatement("SELECT URL FROM EC_URL WHERE DOMAIN_ID=?")) { - stmt.setFetchSize(500); - stmt.setInt(1, domainId.getId()); - var rs = stmt.executeQuery(); - while (rs.next()) { - edgeUrlByPath.remove(rs.getString(1)); - } - } - } - catch (Exception ex) { - return Collections.emptyList(); - } - return new ArrayList<>(edgeUrlByPath.values()); - - } - - @Override - public double getRank(EdgeId domainId) { - try (var connection = dataSource.getConnection()) { - - try (var stmt = connection.prepareStatement("SELECT IFNULL(RANK, 1) FROM EC_DOMAIN WHERE ID=?")) { - stmt.setInt(1, domainId.getId()); - var rsp = stmt.executeQuery(); - if (rsp.next()) { - return rsp.getDouble(1); - } - } catch (Exception ex) { - logger.error("DB error", ex); - } - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - return 1; - } - - @Override - public void updateDomainIndexTimestamp(EdgeDomain domain, EdgeDomainIndexingState state, EdgeDomain alias, int minIndexed) { - try (var connection = dataSource.getConnection(); - var stmt = connection.prepareStatement("UPDATE EC_DOMAIN SET INDEX_DATE=NOW(), STATE=?, DOMAIN_ALIAS=?, INDEXED=GREATEST(INDEXED,?) WHERE ID=?")) { - stmt.setInt(1, state.code); - if (null == alias) { - stmt.setNull(2, Types.INTEGER); - } - else { - stmt.setInt(2, getDomainId(alias).getId()); - } - - stmt.setInt(3, minIndexed); - stmt.setInt(4, getDomainId(domain).getId()); - stmt.executeUpdate(); - connection.commit(); - } - catch (SQLException throwables) { - logger.error("SQL error", throwables); - } - } - - @SneakyThrows - private double getDomainQuality(Connection connection, EdgeDomain src) { - try (var stmt = connection.prepareStatement("SELECT QUALITY_RAW FROM EC_DOMAIN WHERE URL_PART=?")) { - stmt.setString(1, src.toString()); - var res = stmt.executeQuery(); - - if (res.next()) { - var q = res.getDouble(1); - if (q > 0.5) { - logger.warn("gDQ({}) -> 1", src); - } - return 0; - } - } - catch (SQLException ex) { - logger.error("DB error", ex); - } - - return -5; - } - } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/task/EdgeDomainBlacklistImpl.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/task/EdgeDomainBlacklistImpl.java index f4cbb8d0..334ec5a9 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/task/EdgeDomainBlacklistImpl.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/data/dao/task/EdgeDomainBlacklistImpl.java @@ -50,7 +50,7 @@ public class EdgeDomainBlacklistImpl implements EdgeDomainBlacklist { final TIntHashSet result = new TIntHashSet(1_000_000); try (var connection = dataSource.getConnection()) { - try (var stmt = connection.prepareStatement("SELECT EC_DOMAIN.ID FROM EC_DOMAIN INNER JOIN EC_TOP_DOMAIN ON EC_DOMAIN.URL_TOP_DOMAIN_ID = EC_TOP_DOMAIN.ID INNER JOIN EC_DOMAIN_BLACKLIST ON EC_DOMAIN_BLACKLIST.URL_DOMAIN = EC_TOP_DOMAIN.URL_PART")) { + try (var stmt = connection.prepareStatement("SELECT EC_DOMAIN.ID FROM EC_DOMAIN INNER JOIN EC_DOMAIN_BLACKLIST ON EC_DOMAIN_BLACKLIST.URL_DOMAIN = EC_DOMAIN.DOMAIN_TOP")) { stmt.setFetchSize(1000); var rsp = stmt.executeQuery(); while (rsp.next()) { diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/radix/EdgeIndexBucket.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexBucket.java similarity index 93% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/radix/EdgeIndexBucket.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexBucket.java index 2e8fdcd2..05bcfe75 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/radix/EdgeIndexBucket.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexBucket.java @@ -1,13 +1,13 @@ -package nu.marginalia.wmsa.edge.index.radix; +package nu.marginalia.wmsa.edge.index; import nu.marginalia.wmsa.edge.index.EdgeIndexControl; import nu.marginalia.wmsa.edge.index.IndexServicesFactory; import nu.marginalia.wmsa.edge.index.model.EdgeIndexSearchTerms; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndexReader; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndexWriter; -import nu.marginalia.wmsa.edge.index.service.query.IndexSearchBudget; -import nu.marginalia.wmsa.edge.index.service.query.Query; +import nu.marginalia.wmsa.edge.index.reader.SearchIndexReader; +import nu.marginalia.wmsa.edge.index.journal.SearchIndexWriter; +import nu.marginalia.wmsa.edge.index.reader.query.IndexSearchBudget; +import nu.marginalia.wmsa.edge.index.reader.query.Query; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexControl.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexControl.java index b590af55..ab7c73fe 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexControl.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexControl.java @@ -3,7 +3,9 @@ package nu.marginalia.wmsa.edge.index; import com.google.inject.Inject; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.index.ConversionUnnecessaryException; +import nu.marginalia.wmsa.edge.index.conversion.ConversionUnnecessaryException; + +import java.io.IOException; public class EdgeIndexControl { @@ -27,7 +29,10 @@ public class EdgeIndexControl { System.gc(); } catch (ConversionUnnecessaryException unnecessary) { - + // swallow quietly + } + catch (IOException e) { + e.printStackTrace(); } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexService.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexService.java index a04a4c83..de6276a8 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexService.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexService.java @@ -15,9 +15,9 @@ import nu.marginalia.wmsa.configuration.server.Initialization; import nu.marginalia.wmsa.configuration.server.MetricsServer; import nu.marginalia.wmsa.configuration.server.Service; import nu.marginalia.wmsa.edge.index.model.*; -import nu.marginalia.wmsa.edge.index.service.SearchIndexes; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndexWriterImpl; -import nu.marginalia.wmsa.edge.index.service.query.IndexSearchBudget; +import nu.marginalia.wmsa.edge.index.reader.SearchIndexes; +import nu.marginalia.wmsa.edge.index.journal.SearchIndexWriterImpl; +import nu.marginalia.wmsa.edge.index.reader.query.IndexSearchBudget; import nu.marginalia.util.dict.DictionaryHashMap; import nu.marginalia.wmsa.edge.model.*; import nu.marginalia.wmsa.edge.model.crawl.EdgePageWordSet; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java index fb58ac0e..61e64b41 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java @@ -5,12 +5,16 @@ import com.google.inject.Singleton; import com.google.inject.name.Named; import lombok.SneakyThrows; import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklist; +import nu.marginalia.wmsa.edge.index.conversion.ConversionUnnecessaryException; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexConverter; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexPreconverter; +import nu.marginalia.wmsa.edge.index.journal.SearchIndexWriterImpl; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.radix.EdgeIndexBucket; -import nu.marginalia.wmsa.edge.index.service.dictionary.DictionaryReader; -import nu.marginalia.wmsa.edge.index.service.dictionary.DictionaryWriter; -import nu.marginalia.wmsa.edge.index.service.index.*; -import nu.marginalia.wmsa.edge.index.service.query.SearchIndexPartitioner; +import nu.marginalia.wmsa.edge.index.dictionary.DictionaryReader; +import nu.marginalia.wmsa.edge.index.dictionary.DictionaryWriter; +import nu.marginalia.wmsa.edge.index.reader.SearchIndex; +import nu.marginalia.wmsa.edge.index.reader.SearchIndexReader; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexPartitioner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,7 +93,7 @@ public class IndexServicesFactory { } - public SearchIndexConverter getIndexConverter(int id, IndexBlock block) throws ConversionUnnecessaryException { + public SearchIndexConverter getIndexConverter(int id, IndexBlock block) throws ConversionUnnecessaryException, IOException { return new SearchIndexConverter(block, id, tmpFileDir, preconverterOutputFile.get(id), indexWriteWordsFile.get(id, block.id), diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/ConversionUnnecessaryException.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/ConversionUnnecessaryException.java similarity index 80% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/ConversionUnnecessaryException.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/ConversionUnnecessaryException.java index fd7f529f..2242f476 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/ConversionUnnecessaryException.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/ConversionUnnecessaryException.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service.index; +package nu.marginalia.wmsa.edge.index.conversion; public class ConversionUnnecessaryException extends Exception { public ConversionUnnecessaryException() { diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchEngineRanking.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchEngineRanking.java similarity index 97% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchEngineRanking.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchEngineRanking.java index abaced82..220a9708 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchEngineRanking.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchEngineRanking.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service; +package nu.marginalia.wmsa.edge.index.conversion; import gnu.trove.list.TIntList; import gnu.trove.map.hash.TIntIntHashMap; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexConverter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexConverter.java similarity index 75% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexConverter.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexConverter.java index c9b69386..0827b4e7 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexConverter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexConverter.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service.index; +package nu.marginalia.wmsa.edge.index.conversion; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -6,9 +6,10 @@ import gnu.trove.set.hash.TIntHashSet; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklist; +import nu.marginalia.wmsa.edge.index.conversion.words.WordIndexOffsetsTable; +import nu.marginalia.wmsa.edge.index.journal.SearchIndexWriterImpl; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.index.wordstable.WordsTableWriter; -import nu.marginalia.wmsa.edge.index.service.query.SearchIndexPartitioner; +import nu.marginalia.wmsa.edge.index.conversion.words.WordsTableWriter; import nu.marginalia.util.btree.BTreeWriter; import nu.marginalia.util.btree.model.BTreeContext; import nu.marginalia.util.multimap.MultimapFileLong; @@ -32,18 +33,24 @@ public class SearchIndexConverter { private final long fileLength; private final long urlsFileSize; + private final Path tmpFileDir; + private final FileChannel urlsTmpFileChannel; private final int wordCount; private final MultimapFileLong urlsTmpFileMap; private final Logger logger = LoggerFactory.getLogger(getClass()); private final IndexBlock block; private final int bucketId; - @org.jetbrains.annotations.NotNull + + private final File urlsFile; private final SearchIndexPartitioner partitioner; private final TIntHashSet spamDomains; private final MultimapSorter urlTmpFileSorter; + private final static int internalSortLimit = + Boolean.getBoolean("small-ram") ? 1024*1024 : 1024*1024*256; + @SneakyThrows public static long wordCount(File inputFile) { try (RandomAccessFile raf = new RandomAccessFile(inputFile, "r")) { @@ -52,7 +59,6 @@ public class SearchIndexConverter { } } - @SneakyThrows @Inject public SearchIndexConverter(IndexBlock block, int bucketId, @Named("tmp-file-dir") Path tmpFileDir, @@ -61,13 +67,15 @@ public class SearchIndexConverter { @Named("edge-index-write-urls-file") File outputFileUrls, SearchIndexPartitioner partitioner, EdgeDomainBlacklist blacklist) - throws ConversionUnnecessaryException + throws ConversionUnnecessaryException, IOException { this.block = block; this.bucketId = bucketId; - urlsFile = outputFileUrls; + this.tmpFileDir = tmpFileDir; + this.urlsFile = outputFileUrls; this.partitioner = partitioner; this.spamDomains = blacklist.getSpamDomains(); + logger.info("Converting {} ({}) {}", block.id, block, inputFile); Files.deleteIfExists(outputFileWords.toPath()); @@ -89,18 +97,16 @@ public class SearchIndexConverter { urlsFileSize = getUrlsSize(buffer, inputChannel); var tmpUrlsFile = Files.createTempFile(tmpFileDir, "urls-sorted", ".dat"); - - var urlsTmpFileRaf = new RandomAccessFile(tmpUrlsFile.toFile(), "rw"); urlsTmpFileChannel = urlsTmpFileRaf.getChannel(); urlsTmpFileMap = new MultimapFileLong(urlsTmpFileRaf, FileChannel.MapMode.READ_WRITE, urlsFileSize, 8*1024*1024, false); - urlTmpFileSorter = urlsTmpFileMap.createSorter(tmpFileDir, 1024*1024*256); + urlTmpFileSorter = urlsTmpFileMap.createSorter(tmpFileDir, internalSortLimit); logger.info("Creating word index table {} for block {} ({})", outputFileWords, block.id, block); - long[] wordIndexTable = createWordIndexTable(outputFileWords, inputChannel); + WordIndexOffsetsTable wordIndexTable = createWordIndexTable(outputFileWords, inputChannel); logger.info("Creating word urls table {} for block {} ({})", outputFileUrls, block.id, block); - createUrlTable(tmpFileDir, buffer, raf, wordIndexTable); + createUrlTable(buffer, raf, wordIndexTable); Files.delete(tmpUrlsFile); raf.close(); @@ -140,99 +146,69 @@ public class SearchIndexConverter { return reader.size; } - private void createUrlTable(Path tmpFileDir, ByteBuffer buffer, RandomAccessFile raf, long[] wordIndexTable) throws IOException { - logger.debug("Table size = {}", wordIndexTable.length); - int[] wordIndex = new int[wordIndexTable.length]; + private void createUrlTable(ByteBuffer buffer, RandomAccessFile raf, WordIndexOffsetsTable wordOffsetsTable) throws IOException { + logger.info("Table size = {}", wordOffsetsTable.length()); + raf.seek(FILE_HEADER_SIZE); var channel = raf.getChannel(); try (RandomWriteFunnel rwf = new RandomWriteFunnel(tmpFileDir, urlsFileSize, 10_000_000)) { - var reader = new IndexReader(buffer, channel) { + int[] wordWriteOffset = new int[wordOffsetsTable.length()]; + + new IndexReader(buffer, channel) { @Override public void eachWord(long urlId, int wordId) throws IOException { - if (wordId >= wordIndex.length) + if (wordId >= wordWriteOffset.length) return; - if (wordId != 0) { - if (!(wordIndexTable[wordId - 1] + wordIndex[wordId] <= wordIndexTable[wordId])) { - logger.error("Crazy state: wordId={}, index={}, lower={}, upper={}", - wordId, - wordIndex[wordId], - wordIndexTable[wordId - 1], - wordIndexTable[wordId]); - throw new IllegalStateException(); - } - } if (wordId > 0) { - rwf.put(wordIndexTable[wordId - 1] + wordIndex[wordId]++, translateUrl(urlId)); + rwf.put(wordOffsetsTable.get(wordId - 1) + wordWriteOffset[wordId]++, translateUrl(urlId)); } else { - rwf.put(wordIndex[wordId]++, translateUrl(urlId)); + rwf.put(wordWriteOffset[wordId]++, translateUrl(urlId)); } } - }; - - reader.read(); + }.read(); rwf.write(urlsTmpFileChannel); } urlsTmpFileChannel.force(false); + logger.info("URL TMP Table: {} Mb", channel.position()/(1024*1024)); - logger.debug("URL TMP Table: {} Mb", channel.position()/(1024*1024)); + if (wordOffsetsTable.length() > 0) { + logger.info("Sorting urls table"); + + wordOffsetsTable.forEach(urlTmpFileSorter::sort); - if (wordIndexTable.length > 0) { - logger.debug("Sorting urls table"); - sortUrls(wordIndexTable); urlsTmpFileMap.force(); } else { logger.warn("urls table empty -- nothing to sort"); } - - long idx = 0; - + logger.info("Writing BTree"); try (var urlsFileMap = MultimapFileLong.forOutput(urlsFile.toPath(), 1024)) { var writer = new BTreeWriter(urlsFileMap, urlsBTreeContext); - if (wordIndexTable[0] != 0) { - int start = 0; - int end = (int) wordIndexTable[0]; + wordOffsetsTable.fold((accumulatorIdx, start, length) -> { + // Note: The return value is accumulated into accumulatorIdx! - idx += writer.write(idx, (int) wordIndexTable[0], - offset -> urlsFileMap.transferFromFileChannel(urlsTmpFileChannel, offset, start, end)); - } + return writer.write(accumulatorIdx, length, + slice -> slice.transferFromFileChannel(urlsTmpFileChannel, 0, start, start + length)); + }); - for (int i = 1; i < wordIndexTable.length; i++) { - if (wordIndexTable[i] != wordIndexTable[i - 1]) { - long start = wordIndexTable[i-1]; - long end = wordIndexTable[i]; - - idx += writer.write(idx, (int) (end-start), - offset -> urlsFileMap.transferFromFileChannel(urlsTmpFileChannel, offset, start, end)); - } - } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while writing BTree", e); } } - @SneakyThrows - private void sortUrls(long[] wordIndices) { - urlTmpFileSorter.sort( 0, (int) wordIndices[0]); - - for (int i = 1; i < wordIndices.length; i++) { - urlTmpFileSorter.sort(wordIndices[i-1], (int) (wordIndices[i] - wordIndices[i-1])); - } - } - - private long[] createWordIndexTable(File outputFileWords, FileChannel inputChannel) throws Exception { + private WordIndexOffsetsTable createWordIndexTable(File outputFileWords, FileChannel inputChannel) throws IOException { inputChannel.position(FILE_HEADER_SIZE); logger.debug("Table size = {}", wordCount); WordsTableWriter wordsTableWriter = new WordsTableWriter(wordCount); - ByteBuffer buffer = ByteBuffer.allocateDirect(8*SearchIndexWriterImpl.MAX_BLOCK_SIZE); + ByteBuffer buffer = ByteBuffer.allocateDirect(8* SearchIndexWriterImpl.MAX_BLOCK_SIZE); logger.debug("Reading words"); diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchIndexDao.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexDao.java similarity index 64% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchIndexDao.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexDao.java index 615fbc34..fcf6d175 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchIndexDao.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexDao.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service; +package nu.marginalia.wmsa.edge.index.conversion; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -10,7 +10,7 @@ import lombok.SneakyThrows; import nu.marginalia.util.ranking.BetterReversePageRank; import nu.marginalia.util.ranking.BetterStandardPageRank; import nu.marginalia.util.ranking.BuggyStandardPageRank; -import nu.marginalia.wmsa.configuration.module.DatabaseModule; +import nu.marginalia.util.ranking.RankingDomainFetcher; import nu.marginalia.wmsa.edge.index.model.RankingSettings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,41 +18,28 @@ import org.slf4j.LoggerFactory; @Singleton public class SearchIndexDao { private final HikariDataSource dataSource; + private RankingDomainFetcher rankingDomains; private final RankingSettings rankingSettings; private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject public SearchIndexDao(HikariDataSource dataSource, + RankingDomainFetcher rankingDomains, RankingSettings rankingSettings) { this.dataSource = dataSource; + this.rankingDomains = rankingDomains; this.rankingSettings = rankingSettings; logger.info("SearchIndexDao ranking settings = {}", rankingSettings); } - @SneakyThrows - public TIntHashSet getSpamDomains() { - final TIntHashSet result = new TIntHashSet(1_000_000); - - try (var connection = dataSource.getConnection()) { - try (var stmt = connection.prepareStatement("SELECT EC_DOMAIN.ID FROM EC_DOMAIN INNER JOIN EC_TOP_DOMAIN ON EC_DOMAIN.URL_TOP_DOMAIN_ID = EC_TOP_DOMAIN.ID INNER JOIN EC_DOMAIN_BLACKLIST ON EC_DOMAIN_BLACKLIST.URL_DOMAIN = EC_TOP_DOMAIN.URL_PART")) { - var rsp = stmt.executeQuery(); - while (rsp.next()) { - result.add(rsp.getInt(1)); - } - } - } - - return result; - } - @SneakyThrows public TIntHashSet goodUrls() { TIntHashSet domains = new TIntHashSet(10_000_000, 0.5f, -1); TIntHashSet urls = new TIntHashSet(100_000_000, 0.5f, -1); try (var connection = dataSource.getConnection()) { - try (var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_ALIAS IS NULL AND STATE>=0")) { + try (var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_ALIAS IS NULL AND IS_ALIVE")) { stmt.setFetchSize(10_000); var rsp = stmt.executeQuery(); while (rsp.next()) { @@ -79,36 +66,36 @@ public class SearchIndexDao { @SneakyThrows public TIntList getRetroDomains() { - var spr = new BetterStandardPageRank(dataSource,rankingSettings.retro.toArray(String[]::new)); - return spr.pageRankWithPeripheralNodes(spr.size()/2, false); + var spr = new BetterStandardPageRank(rankingDomains,rankingSettings.retro.toArray(String[]::new)); + return spr.pageRankWithPeripheralNodes(spr.size()/2); } @SneakyThrows public TIntList getSmallWebDomains() { - var rpr = new BetterReversePageRank(new DatabaseModule().provideConnection(), rankingSettings.small.toArray(String[]::new)); + var rpr = new BetterReversePageRank(rankingDomains, rankingSettings.small.toArray(String[]::new)); rpr.setMaxKnownUrls(750); - return rpr.pageRankWithPeripheralNodes(rpr.size(), false); + return rpr.pageRankWithPeripheralNodes(rpr.size()); } @SneakyThrows public TIntList getAcademiaDomains() { - var spr = new BetterStandardPageRank(new DatabaseModule().provideConnection(), rankingSettings.academia.toArray(String[]::new)); - return spr.pageRankWithPeripheralNodes(spr.size()/2, false); + var spr = new BetterStandardPageRank(rankingDomains, rankingSettings.academia.toArray(String[]::new)); + return spr.pageRankWithPeripheralNodes(spr.size()/2); } @SneakyThrows public TIntList getStandardDomains() { - var spr = new BuggyStandardPageRank(dataSource,rankingSettings.standard.toArray(String[]::new)); - return spr.pageRankWithPeripheralNodes(spr.size()/2, false); + var spr = new BuggyStandardPageRank(rankingDomains,rankingSettings.standard.toArray(String[]::new)); + return spr.pageRankWithPeripheralNodes(spr.size()/2); } @SneakyThrows public TIntList getSpecialDomains() { TIntArrayList results = new TIntArrayList(); try (var connection = dataSource.getConnection(); - var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE STATE=2") + var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE STATE='SPECIAL'") ) { var rs = stmt.executeQuery(); while (rs.next()) { diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/SearchIndexPartitioner.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexPartitioner.java similarity index 96% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/SearchIndexPartitioner.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexPartitioner.java index cf281116..bf5a1d74 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/SearchIndexPartitioner.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexPartitioner.java @@ -1,11 +1,9 @@ -package nu.marginalia.wmsa.edge.index.service.query; +package nu.marginalia.wmsa.edge.index.conversion; import com.google.inject.Inject; import com.google.inject.Singleton; import gnu.trove.set.hash.TIntHashSet; import lombok.SneakyThrows; -import nu.marginalia.wmsa.edge.index.service.SearchEngineRanking; -import nu.marginalia.wmsa.edge.index.service.SearchIndexDao; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexPreconverter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexPreconverter.java similarity index 97% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexPreconverter.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexPreconverter.java index 5149b546..9e851025 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexPreconverter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexPreconverter.java @@ -1,10 +1,9 @@ -package nu.marginalia.wmsa.edge.index.service.index; +package nu.marginalia.wmsa.edge.index.conversion; import com.google.inject.Inject; import gnu.trove.set.hash.TIntHashSet; import lombok.SneakyThrows; import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklist; -import nu.marginalia.wmsa.edge.index.service.query.SearchIndexPartitioner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordIndexLengthsTable.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordIndexLengthsTable.java new file mode 100644 index 00000000..464e9388 --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordIndexLengthsTable.java @@ -0,0 +1,10 @@ +package nu.marginalia.wmsa.edge.index.conversion.words; + +public class WordIndexLengthsTable { + final long[] table; + + public WordIndexLengthsTable(int size) { + this.table = new long[size]; + } + public void increment(int idx) { table[idx]++; } +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordIndexOffsetsTable.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordIndexOffsetsTable.java new file mode 100644 index 00000000..29b88509 --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordIndexOffsetsTable.java @@ -0,0 +1,67 @@ +package nu.marginalia.wmsa.edge.index.conversion.words; + +import java.io.IOException; + +public class WordIndexOffsetsTable { + final long[] table; + public final int numberOfUsedWords; + + public WordIndexOffsetsTable(long[] table, int numberOfUsedWords) { + + this.table = table; + this.numberOfUsedWords = numberOfUsedWords; + } + + public int length() { + return table.length; + } + + public void forEach(OffsetTableEntryConsumer o) throws IOException { + if (table[0] > 0) { + o.accept(0, (int) table[0]); + } + + for (int i = 1; i < table.length; i++) { + long start = table[i-1]; + int length = (int) (table[i] - start); + + if (length != 0) { + o.accept(start, length); + } + } + } + + /** + * Fold over each span in the file, left to right + */ + public long fold(OffsetTableEntryFoldConsumer o) throws IOException { + long total = 0; + + if (table[0] > 0) { + total = o.accept(total,0, (int) table[0]); + } + + for (int i = 1; i < table.length; i++) { + long start = table[i-1]; + int length = (int) (table[i] - start); + + if (length != 0) { + total += o.accept(total, start, length); + } + } + + return total; + } + + public long get(int i) { + return table[i]; + } + + public interface OffsetTableEntryConsumer { + void accept(long start, int length) throws IOException; + } + + public interface OffsetTableEntryFoldConsumer { + long accept(long accumulator, long start, int length) throws IOException; + } +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordIndexTables.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordIndexTables.java new file mode 100644 index 00000000..2056948b --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordIndexTables.java @@ -0,0 +1,56 @@ +package nu.marginalia.wmsa.edge.index.conversion.words; + +/** Contains a stateful table of word index offsets, initially in lengths mode + * where the table contains how many postings exist for each word; then in offsets + * mode, where the lengths are converted into the necessary offsets for each block + * of document data. + * + * Caveat! This uses the same underlying array to conserve space. + * + */ +public class WordIndexTables { + private WordIndexLengthsTable lengthsTable; + private WordIndexOffsetsTable offsetsTable; + + private boolean converted = false; + + public WordIndexTables(int size) { + lengthsTable = new WordIndexLengthsTable(size); + } + + public WordIndexLengthsTable lengths() { + if (converted) throw new IllegalStateException("Table has been converted"); + + return lengthsTable; + } + + public WordIndexOffsetsTable offsets() { + if (!converted) throw new IllegalStateException("Table has not been converted"); + + return offsetsTable; + } + + public void convert() { + if (converted) throw new IllegalStateException("Table has been converted"); + + // Go from lengths to offsets, i.e. + // BEFORE: 1, 2, 1, 3, 0, 2 + // AFTER: 1, 3, 4, 7, 7, 9 + + long[] table = lengthsTable.table; + int numberOfUsedWords = 0; + + if (table[0] != 0) numberOfUsedWords = 1; + + for (int i = 1; i < table.length; i++) { + if (table[i] != 0) { + numberOfUsedWords++; + } + table[i] += table[i-1]; + } + + lengthsTable = null; + offsetsTable = new WordIndexOffsetsTable(table, numberOfUsedWords); + converted = true; + } +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordsTableWriter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordsTableWriter.java new file mode 100644 index 00000000..7f762ff3 --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/words/WordsTableWriter.java @@ -0,0 +1,75 @@ +package nu.marginalia.wmsa.edge.index.conversion.words; + +import nu.marginalia.util.btree.BTreeWriter; +import nu.marginalia.util.btree.model.BTreeContext; +import nu.marginalia.util.multimap.MultimapFileLong; +import nu.marginalia.util.multimap.MultimapFileLongSlice; +import nu.marginalia.wmsa.edge.index.reader.IndexWordsTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; + +import static nu.marginalia.wmsa.edge.index.conversion.SearchIndexConverter.urlsBTreeContext; + +public class WordsTableWriter { + private final WordIndexTables table; + private final Logger logger = LoggerFactory.getLogger(getClass()); + + public static final BTreeContext wordsBTreeContext = new BTreeContext(7, 2, 0x0000_0000_FFFF_FFFFL, 8); + + public WordsTableWriter(int length) { + table = new WordIndexTables(length); + } + + public void acceptWord(int wordId) { + table.lengths().increment(wordId); + } + + public WordIndexOffsetsTable getTable() { + return table.offsets(); + } + + public void write(File file) throws IOException { + table.convert(); + + logger.info("Writing table - {} max", table.offsets().numberOfUsedWords); + + final int tableSize = table.offsets().numberOfUsedWords; + + try (var mmf = MultimapFileLong.forOutput(file.toPath(), tableSize/8L)) { + mmf.put(0, IndexWordsTable.Strategy.BTREE.ordinal()); + long offset = 1; + + var writer = new BTreeWriter(mmf, wordsBTreeContext); + + writer.write(offset, tableSize, this::writeBTreeBlock); + } + } + + private void writeBTreeBlock(MultimapFileLongSlice mapSlice) { + long urlFileOffset = 0; + int idx = 0; + + var offsetTable = table.offsets().table; + + if (offsetTable[0] != 0) { + int length = (int) offsetTable[0]; + mapSlice.put(idx++, (long)length<<32); + mapSlice.put(idx++, 0); + + urlFileOffset += (urlsBTreeContext.calculateSize(length)); + } + + for (int i = 1; i < offsetTable.length; i++) { + final int length = (int)(offsetTable[i] - offsetTable[i-1]); + + if (length > 0) { + mapSlice.put(idx++, (long)length << 32 | i); + mapSlice.put(idx++, urlFileOffset); + + urlFileOffset += (urlsBTreeContext.calculateSize(length)); + } + } + } +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/dictionary/DictionaryReader.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/dictionary/DictionaryReader.java similarity index 92% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/dictionary/DictionaryReader.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/dictionary/DictionaryReader.java index 90d270d2..ca10c000 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/dictionary/DictionaryReader.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/dictionary/DictionaryReader.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service.dictionary; +package nu.marginalia.wmsa.edge.index.dictionary; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/dictionary/DictionaryWriter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/dictionary/DictionaryWriter.java similarity index 99% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/dictionary/DictionaryWriter.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/dictionary/DictionaryWriter.java index 9ce1b149..906231be 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/dictionary/DictionaryWriter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/dictionary/DictionaryWriter.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service.dictionary; +package nu.marginalia.wmsa.edge.index.dictionary; import com.google.inject.Inject; import com.google.inject.Singleton; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/dictionary/TokenCompressor.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/dictionary/TokenCompressor.java similarity index 97% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/dictionary/TokenCompressor.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/dictionary/TokenCompressor.java index 9f26fffd..5a3d73ab 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/dictionary/TokenCompressor.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/dictionary/TokenCompressor.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service.dictionary; +package nu.marginalia.wmsa.edge.index.dictionary; import nu.marginalia.util.ByteFolder; import nu.marginalia.util.dict.DictionaryHashMap; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexWriter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/journal/SearchIndexWriter.java similarity index 88% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexWriter.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/journal/SearchIndexWriter.java index ca5d70b3..11fc186a 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexWriter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/journal/SearchIndexWriter.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service.index; +package nu.marginalia.wmsa.edge.index.journal; import nu.marginalia.wmsa.edge.index.model.IndexBlock; import nu.marginalia.wmsa.edge.model.EdgeDomain; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexWriterImpl.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/journal/SearchIndexWriterImpl.java similarity index 96% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexWriterImpl.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/journal/SearchIndexWriterImpl.java index 2f482815..cf76ada2 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexWriterImpl.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/journal/SearchIndexWriterImpl.java @@ -1,10 +1,10 @@ -package nu.marginalia.wmsa.edge.index.service.index; +package nu.marginalia.wmsa.edge.index.journal; import io.reactivex.rxjava3.disposables.Disposable; import io.reactivex.rxjava3.schedulers.Schedulers; import lombok.SneakyThrows; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.dictionary.DictionaryWriter; +import nu.marginalia.wmsa.edge.index.dictionary.DictionaryWriter; import nu.marginalia.wmsa.edge.model.EdgeDomain; import nu.marginalia.wmsa.edge.model.EdgeId; import nu.marginalia.wmsa.edge.model.EdgeUrl; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/wordstable/BtreeWordsTable.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/IndexWordsTable.java similarity index 58% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/wordstable/BtreeWordsTable.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/IndexWordsTable.java index 0a6a70c0..2bde1aa7 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/wordstable/BtreeWordsTable.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/IndexWordsTable.java @@ -1,36 +1,80 @@ -package nu.marginalia.wmsa.edge.index.service.index.wordstable; +package nu.marginalia.wmsa.edge.index.reader; import com.upserve.uppend.blobs.NativeIO; import nu.marginalia.util.btree.BTreeReader; import nu.marginalia.util.btree.model.BTreeHeader; import nu.marginalia.util.multimap.MultimapFileLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; import java.util.function.LongConsumer; -import static nu.marginalia.wmsa.edge.index.service.index.wordstable.WordsTableWriter.wordsBTreeContext; +import static nu.marginalia.wmsa.edge.index.conversion.words.WordsTableWriter.wordsBTreeContext; -public class BtreeWordsTable extends IndexWordsTable{ - private final MultimapFileLong words; - private final BTreeReader reader; - private final BTreeHeader header; - private final int HEADER_OFFSET = 1; +public class IndexWordsTable implements AutoCloseable { + protected final MultimapFileLong words; + protected final BTreeReader reader; + protected final BTreeHeader header; + protected final int HEADER_OFFSET = 1; + final Logger logger = LoggerFactory.getLogger(getClass()); - public BtreeWordsTable(MultimapFileLong words) { + private static final int BUFFER_SIZE = 1024*1024*64; + + public IndexWordsTable(MultimapFileLong words) { this.words = words; - reader = new BTreeReader(words, wordsBTreeContext); header = reader.getHeader(HEADER_OFFSET); madvise(); } - private void madvise() { + public static IndexWordsTable ofFile(RandomAccessFile file) throws IOException { + var wordsFile = openWordsFile(file); + long signature = wordsFile.get(0); + + if (signature == Strategy.BTREE.ordinal()) { + return new IndexWordsTable(wordsFile); + } + + throw new IllegalArgumentException("Unknown signature " + signature); + } + + private static MultimapFileLong openWordsFile(RandomAccessFile wordsFile) throws IOException { + return new MultimapFileLong(wordsFile, + FileChannel.MapMode.READ_ONLY, wordsFile.length(), BUFFER_SIZE, false); + } + + public long positionForWord(int wordId) { + + long offset = reader.offsetForEntry(header, wordId); + if (offset < 0) { + return -1L; + } + + return words.get(offset+1); + } + + public int wordLength(int wordId) { + + long offset = reader.offsetForEntry(header, wordId); + if (offset < 0) { + return -1; + } + + return (int)(words.get(offset) >> 32); + } + + protected void madvise() { words.advice(NativeIO.Advice.Random); words.advice0(NativeIO.Advice.WillNeed); var h = reader.getHeader(HEADER_OFFSET); int length = (int)(h.dataOffsetLongs() - h.indexOffsetLongs()); + words.adviceRange(NativeIO.Advice.WillNeed, h.indexOffsetLongs(), length); words.pokeRange(h.indexOffsetLongs(), length); } @@ -58,31 +102,13 @@ public class BtreeWordsTable extends IndexWordsTable{ } } - @Override - public long positionForWord(int wordId) { - - long offset = reader.offsetForEntry(header, wordId); - if (offset < 0) { - return -1L; - } - - return words.get(offset+1); - } - - @Override - public int wordLength(int wordId) { - - long offset = reader.offsetForEntry(header, wordId); - if (offset < 0) { - return -1; - } - - return (int)(words.get(offset) >> 32); - } - @Override public void close() throws Exception { words.close(); } + public enum Strategy { + BTREE + } + } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndex.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndex.java similarity index 93% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndex.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndex.java index 17e62437..042f8f54 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndex.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndex.java @@ -1,20 +1,18 @@ -package nu.marginalia.wmsa.edge.index.service.index; +package nu.marginalia.wmsa.edge.index.reader; import com.google.inject.Inject; import com.google.inject.name.Named; import com.upserve.uppend.blobs.NativeIO; import io.reactivex.rxjava3.schedulers.Schedulers; -import nu.marginalia.wmsa.edge.index.service.index.wordstable.IndexWordsTable; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexConverter; import nu.marginalia.util.btree.BTreeReader; import nu.marginalia.util.multimap.MultimapFileLong; -import org.eclipse.jetty.util.thread.ThreadPool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.util.concurrent.ForkJoinPool; import java.util.stream.LongStream; public class SearchIndex implements AutoCloseable { diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexReader.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndexReader.java similarity index 96% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexReader.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndexReader.java index 7baeb8ae..8e7fea81 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndexReader.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndexReader.java @@ -1,13 +1,13 @@ -package nu.marginalia.wmsa.edge.index.service.index; +package nu.marginalia.wmsa.edge.index.reader; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.query.IndexQueryBuilder; -import nu.marginalia.wmsa.edge.index.service.query.IndexSearchBudget; -import nu.marginalia.wmsa.edge.index.service.query.Query; +import nu.marginalia.wmsa.edge.index.reader.query.IndexQueryBuilder; +import nu.marginalia.wmsa.edge.index.reader.query.IndexSearchBudget; +import nu.marginalia.wmsa.edge.index.reader.query.Query; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,10 +105,8 @@ public class SearchIndexReader implements AutoCloseable { .mapToLong(idx -> idx.numUrls(word)) .sum() ); - } - public IndexBlock getBlockForResult(int searchTerm, long urlId) { for (var block : indicesBySearchOrder) { var index = indices.get(block); diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchIndexes.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndexes.java similarity index 91% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchIndexes.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndexes.java index 91065101..863c0c65 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchIndexes.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndexes.java @@ -1,13 +1,13 @@ -package nu.marginalia.wmsa.edge.index.service; +package nu.marginalia.wmsa.edge.index.reader; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.wmsa.configuration.server.Initialization; import nu.marginalia.wmsa.edge.index.IndexServicesFactory; -import nu.marginalia.wmsa.edge.index.radix.EdgeIndexBucket; -import nu.marginalia.wmsa.edge.index.service.dictionary.DictionaryReader; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndexWriterImpl; -import nu.marginalia.wmsa.edge.index.service.query.SearchIndexPartitioner; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexPartitioner; +import nu.marginalia.wmsa.edge.index.EdgeIndexBucket; +import nu.marginalia.wmsa.edge.index.dictionary.DictionaryReader; +import nu.marginalia.wmsa.edge.index.journal.SearchIndexWriterImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/IndexQueryBuilder.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/query/IndexQueryBuilder.java similarity index 97% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/IndexQueryBuilder.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/query/IndexQueryBuilder.java index be217057..6f54dd2d 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/IndexQueryBuilder.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/query/IndexQueryBuilder.java @@ -1,7 +1,7 @@ -package nu.marginalia.wmsa.edge.index.service.query; +package nu.marginalia.wmsa.edge.index.reader.query; import com.google.common.collect.Streams; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndex; +import nu.marginalia.wmsa.edge.index.reader.SearchIndex; import java.util.Collection; import java.util.List; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/IndexSearchBudget.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/query/IndexSearchBudget.java similarity index 87% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/IndexSearchBudget.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/query/IndexSearchBudget.java index 2ec30e65..3608f70a 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/IndexSearchBudget.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/query/IndexSearchBudget.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service.query; +package nu.marginalia.wmsa.edge.index.reader.query; public class IndexSearchBudget { diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/Query.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/query/Query.java similarity index 73% rename from marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/Query.java rename to marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/query/Query.java index 09f7701b..5f343d54 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/query/Query.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/query/Query.java @@ -1,4 +1,4 @@ -package nu.marginalia.wmsa.edge.index.service.query; +package nu.marginalia.wmsa.edge.index.reader.query; import java.util.stream.LongStream; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchOrder.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchOrder.java deleted file mode 100644 index d1c9f10a..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/SearchOrder.java +++ /dev/null @@ -1,6 +0,0 @@ -package nu.marginalia.wmsa.edge.index.service; - -public enum SearchOrder { - ASCENDING, - REVERSED -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/wordstable/IndexWordsTable.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/wordstable/IndexWordsTable.java deleted file mode 100644 index 5b557db1..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/wordstable/IndexWordsTable.java +++ /dev/null @@ -1,48 +0,0 @@ -package nu.marginalia.wmsa.edge.index.service.index.wordstable; - -import nu.marginalia.util.multimap.MultimapFileLong; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; -import java.util.function.LongConsumer; - -public abstract class IndexWordsTable implements AutoCloseable { - final Logger logger = LoggerFactory.getLogger(getClass()); - - private static final int BUFFER_SIZE = 1024*1024*64; - - public static IndexWordsTable ofFile(RandomAccessFile file) throws IOException { - var wordsFile = openWordsFile(file); - long signature = wordsFile.get(0); - - if (signature == Strategy.BTREE.ordinal()) { - return new BtreeWordsTable(wordsFile); - } - throw new IllegalArgumentException("Unknown signature " + signature); - } - - private static MultimapFileLong openWordsFile(RandomAccessFile wordsFile) throws IOException { - return new MultimapFileLong(wordsFile, - FileChannel.MapMode.READ_ONLY, wordsFile.length(), BUFFER_SIZE, false); - } - - public abstract long positionForWord(int wordId); - - public abstract int wordLength(int wordId); - public abstract void forEachWordsOffset(LongConsumer offsetConsumer); - - @Override - public void close() throws Exception { - - } - - public record TableWordRange(long start, long end) {} - - public enum Strategy { - FLAT, HASH, BTREE_OLD, BTREE - } - -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/wordstable/WordsTableWriter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/wordstable/WordsTableWriter.java deleted file mode 100644 index 3097dd47..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/wordstable/WordsTableWriter.java +++ /dev/null @@ -1,85 +0,0 @@ -package nu.marginalia.wmsa.edge.index.service.index.wordstable; - -import nu.marginalia.util.btree.BTreeWriter; -import nu.marginalia.util.btree.model.BTreeContext; -import nu.marginalia.util.multimap.MultimapFileLong; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; - -import static nu.marginalia.wmsa.edge.index.service.index.SearchIndexConverter.urlsBTreeContext; - -public class WordsTableWriter { - private final long[] table; - private final Logger logger = LoggerFactory.getLogger(getClass()); - - public static final BTreeContext wordsBTreeContext = new BTreeContext(7, 2, 0x0000_0000_FFFF_FFFFL, 8); - - public WordsTableWriter(int length) { - table = new long[length]; - } - - public void acceptWord(int wordId) { - if (wordId >= table.length) { - logger.warn("Invalid word-id {}", wordId); - } - else { - table[wordId]++; - } - } - - public long[] getTable() { - return table; - } - public void write(File file) throws Exception { - - int tableSize = 0; - - if (table[0] != 0) tableSize = 1; - - for (int i = 1; i < table.length; i++) { - if (table[i] != 0) { - tableSize++; - } - table[i] += table[i-1]; - } - - logger.info("Writing table {} words {} max", tableSize, table.length); - - writeBtreeWordsFile(file, table, tableSize); - - } - - private void writeBtreeWordsFile(File outputFileWords, long[] table, int tableSize) throws Exception { - try (var mmf = MultimapFileLong.forOutput(outputFileWords.toPath(), tableSize/8L)) { - mmf.put(0, IndexWordsTable.Strategy.BTREE.ordinal()); - long offset = 1; - - var writer = new BTreeWriter(mmf, wordsBTreeContext); - - writer.write(offset, tableSize, (idx) -> { - long urlFileOffset = 0; - - if (table[0] != 0) { - int length = (int) table[0]; - mmf.put(idx++, (long)length<<32); - mmf.put(idx++, 0); - - urlFileOffset += (urlsBTreeContext.calculateSize(length)); - } - - for (int i = 1; i < table.length; i++) { - if (table[i] != table[i - 1]) { - int length = (int)(table[i] - table[i-1]); - mmf.put(idx++, (long)length << 32 | i); - mmf.put(idx++, urlFileOffset); - - urlFileOffset += (urlsBTreeContext.calculateSize(length)); - } - } - }); - } - } - -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/EdgeDomain.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/EdgeDomain.java index 53740c95..d1945c9e 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/EdgeDomain.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/EdgeDomain.java @@ -55,8 +55,11 @@ public class EdgeDomain implements WideHashable { } } } + } - + public EdgeUrl toRootUrl() { + // Set default protocol to http, as most https websites redirect http->https, but few http websites redirect https->http + return new EdgeUrl("http", this, null, "/"); } public String toString() { diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/crawl/EdgeDomainIndexingState.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/crawl/EdgeDomainIndexingState.java index 119da59d..b10d0e88 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/crawl/EdgeDomainIndexingState.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/crawl/EdgeDomainIndexingState.java @@ -1,27 +1,12 @@ package nu.marginalia.wmsa.edge.model.crawl; public enum EdgeDomainIndexingState { - ACTIVE(0), - EXHAUSTED(1), - SPECIAL(2), - SOCIAL_MEDIA(3), - BLOCKED(-1), - REDIR(-2), - ERROR(-3), - UNKNOWN(-100); - - public final int code; - - EdgeDomainIndexingState(int code) { - this.code = code; - } - - public static EdgeDomainIndexingState fromCode(int code) { - for (var state : values()) { - if (state.code == code) { - return state; - } - } - return UNKNOWN; - } + ACTIVE, + EXHAUSTED, + SPECIAL, + SOCIAL_MEDIA, + BLOCKED, + REDIR, + ERROR, + UNKNOWN } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/search/EdgeSearchSpecification.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/search/EdgeSearchSpecification.java index 0063efd9..02c7197a 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/search/EdgeSearchSpecification.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/search/EdgeSearchSpecification.java @@ -2,7 +2,6 @@ package nu.marginalia.wmsa.edge.model.search; import lombok.*; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.SearchOrder; import java.util.Arrays; import java.util.Collections; @@ -21,14 +20,13 @@ public class EdgeSearchSpecification { public final int limitTotal; public final String humanQuery; - public final SearchOrder searchOrder; public boolean stagger; public boolean experimental; public static EdgeSearchSpecification justIncludes(String... words) { return new EdgeSearchSpecification( IntStream.range(0, DYNAMIC_BUCKET_LENGTH+1).boxed().toList(), - Collections.singletonList(new EdgeSearchSubquery(Arrays.asList(words), Collections.emptyList(), IndexBlock.Title)), 10, 10, 10, "", SearchOrder.ASCENDING, false, false); + Collections.singletonList(new EdgeSearchSubquery(Arrays.asList(words), Collections.emptyList(), IndexBlock.Title)), 10, 10, 10, "", false, false); } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/search/EdgeUrlDetails.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/search/EdgeUrlDetails.java index ed5fd013..d46aa79e 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/search/EdgeUrlDetails.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/search/EdgeUrlDetails.java @@ -16,25 +16,24 @@ public class EdgeUrlDetails { public String description; public double urlQuality; - public double urlQualityRaw; - public double domainQuality; - public int links; // DEAD public int words; public String format; public int features; - public EdgePageScoreAdjustment urlQualityAdjustment; - public long rankingId; - public double termScore; public String ip; // BROKEN - public int domainState; - public int queryLength; + public EdgeDomainIndexingState domainState; + public int dataHash; + public EdgePageScoreAdjustment urlQualityAdjustment; + public long rankingId; + public double termScore; + public int queryLength; + public long rankingIdAdjustment() { int penalty = 0; @@ -136,7 +135,7 @@ public class EdgeUrlDetails { return HtmlFeature.hasFeature(features, HtmlFeature.COOKIES); } public boolean isSpecialDomain() { - return domainState == EdgeDomainIndexingState.SPECIAL.code; + return domainState == EdgeDomainIndexingState.SPECIAL; } public int getLogRank() { return (int) Math.round(Math.min(Math.log(1+rankingId),10)); } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchOperator.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchOperator.java index 10675cc5..66004dee 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchOperator.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchOperator.java @@ -11,7 +11,6 @@ import nu.marginalia.wmsa.edge.assistant.dict.WikiArticles; import nu.marginalia.wmsa.edge.data.dao.EdgeDataStoreDao; import nu.marginalia.wmsa.edge.index.client.EdgeIndexClient; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.SearchOrder; import nu.marginalia.wmsa.edge.model.*; import nu.marginalia.wmsa.edge.model.search.*; import nu.marginalia.wmsa.edge.search.model.DecoratedSearchResultSet; @@ -136,7 +135,7 @@ public class EdgeSearchOperator { sqs.add(new EdgeSearchSubquery(Arrays.asList(termsInclude), Collections.emptyList(), block)); - EdgeSearchSpecification specs = new EdgeSearchSpecification(profile.buckets, sqs, 100, limitPerDomain, limitTotal, "", SearchOrder.ASCENDING, EdgeSearchProfile.YOLO.equals(profile), false); + EdgeSearchSpecification specs = new EdgeSearchSpecification(profile.buckets, sqs, 100, limitPerDomain, limitTotal, "", EdgeSearchProfile.YOLO.equals(profile), false); return performQuery(ctx, new EdgeSearchQuery(specs), true); } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchProfile.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchProfile.java index 05fcaa04..212d09ab 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchProfile.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchProfile.java @@ -1,7 +1,6 @@ package nu.marginalia.wmsa.edge.search; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.SearchOrder; import java.util.Arrays; import java.util.Collections; @@ -9,27 +8,27 @@ import java.util.List; import java.util.stream.Collectors; public enum EdgeSearchProfile { - DEFAULT("default", SearchOrder.ASCENDING, + DEFAULT("default", Collections.emptyList(), List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords), 0, 1), - MODERN("modern", SearchOrder.ASCENDING, + MODERN("modern", Collections.emptyList(), List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords), 2), - CORPO("corpo", SearchOrder.ASCENDING, + CORPO("corpo", Collections.emptyList(), List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords), 4, 5, 6, 7), - YOLO("yolo", SearchOrder.ASCENDING, + YOLO("yolo", Collections.emptyList(), List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords), 0, 2, 1, 3, 4, 6), - CORPO_CLEAN("corpo-clean", SearchOrder.ASCENDING, + CORPO_CLEAN("corpo-clean", Collections.emptyList(), List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords), 4, 5), - ACADEMIA("academia", SearchOrder.ASCENDING, + ACADEMIA("academia", Collections.emptyList(), List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords), 3), @@ -37,17 +36,15 @@ public enum EdgeSearchProfile { public final String name; - public final SearchOrder order; public final List additionalSearchTerm; public final List buckets; public final List indexBlocks; - EdgeSearchProfile(String name, SearchOrder order, + EdgeSearchProfile(String name, List additionalSearchTerm, List indexBlocks, int... buckets) { this.name = name; - this.order = order; this.additionalSearchTerm = additionalSearchTerm; this.indexBlocks = indexBlocks; this.buckets = Arrays.stream(buckets).boxed().collect(Collectors.toList()); diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/command/commands/SiteSearchCommand.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/command/commands/SiteSearchCommand.java index 60520aa9..6e341721 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/command/commands/SiteSearchCommand.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/command/commands/SiteSearchCommand.java @@ -32,7 +32,7 @@ import java.util.regex.Pattern; public class SiteSearchCommand implements SearchCommandInterface { private final EdgeDataStoreDao dataStoreDao; private final EdgeSearchOperator searchOperator; - private DomainInformationService domainInformationService; + private final DomainInformationService domainInformationService; private final Logger logger = LoggerFactory.getLogger(getClass()); private final MustacheRenderer siteInfoRenderer; @@ -91,7 +91,7 @@ public class SiteSearchCommand implements SearchCommandInterface { logger.info("Fetching Site Info: {}", word); var results = domainInformationService.domainInfo(word) - .orElseGet(() -> new DomainInformation(null, false, 0, 0, 0, 0, 0, 0, 0, EdgeDomainIndexingState.UNKNOWN, Collections.emptyList())); + .orElseGet(() -> new DomainInformation(null, false, 0, 0, 0, 0, 0, 0, EdgeDomainIndexingState.UNKNOWN, Collections.emptyList())); logger.debug("Results = {}", results); diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/model/DomainInformation.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/model/DomainInformation.java index c5c19187..d94ae487 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/model/DomainInformation.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/model/DomainInformation.java @@ -18,7 +18,6 @@ public class DomainInformation { int pagesIndexed; int incomingLinks; int outboundLinks; - double nominalQuality; double ranking; EdgeDomainIndexingState state; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryFactory.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryFactory.java index d3775dd9..1d77a9d0 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryFactory.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryFactory.java @@ -138,7 +138,6 @@ public class QueryFactory { .subqueries(subqueries) .limitByBucket(50) .limitTotal(100) - .searchOrder(profile.order) .humanQuery(query) .buckets(profile.buckets); diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/results/SearchResultDecorator.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/results/SearchResultDecorator.java index 487e1556..22b24aca 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/results/SearchResultDecorator.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/results/SearchResultDecorator.java @@ -107,7 +107,7 @@ public class SearchResultDecorator { private double calculateTermScore(IndexBlock block, EdgeSearchResultItem resultItem, EdgeUrlDetails details) { return valuator.evaluateTerms(resultItem.scores, block, details.words) / Math.sqrt(1 + resultItem.queryLength) - + ((details.domainState == EdgeDomainIndexingState.SPECIAL.code) ? 1.25 : 0); + + ((details.domainState == EdgeDomainIndexingState.SPECIAL) ? 1.25 : 0); } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/siteinfo/DomainInformationService.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/siteinfo/DomainInformationService.java index 54179d64..2f79a9ea 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/siteinfo/DomainInformationService.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/siteinfo/DomainInformationService.java @@ -1,24 +1,43 @@ package nu.marginalia.wmsa.edge.search.siteinfo; +import com.zaxxer.hikari.HikariDataSource; +import lombok.SneakyThrows; import nu.marginalia.wmsa.edge.data.dao.EdgeDataStoreDao; +import nu.marginalia.wmsa.edge.data.dao.EdgeDataStoreDaoImpl; import nu.marginalia.wmsa.edge.model.EdgeDomain; import nu.marginalia.wmsa.edge.model.EdgeId; import nu.marginalia.wmsa.edge.model.crawl.EdgeDomainIndexingState; import nu.marginalia.wmsa.edge.search.model.DomainInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Singleton; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; +/* + TODO: This class needs to be refactored, a lot of + these SQL queries are redundant and can be + collapsed into one single query that fetches + all the information + */ @Singleton public class DomainInformationService { - private EdgeDataStoreDao dataStore; + private EdgeDataStoreDaoImpl dataStoreDao; + private HikariDataSource dataSource; + private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject - public DomainInformationService(EdgeDataStoreDao dataStore) { - this.dataStore = dataStore; + public DomainInformationService( + EdgeDataStoreDaoImpl dataStoreDao, + HikariDataSource dataSource) { + this.dataStoreDao = dataStoreDao; + this.dataSource = dataSource; } @@ -28,29 +47,28 @@ public class DomainInformationService { if (domainId == null) { return Optional.empty(); } - EdgeDomain domain = dataStore.getDomain(domainId); + EdgeDomain domain = dataStoreDao.getDomain(domainId); - boolean blacklisted = dataStore.isBlacklisted(domain); - int pagesKnown = dataStore.getPagesKnown(domainId); - int pagesVisited = dataStore.getPagesVisited(domainId); - int pagesIndexed = dataStore.getPagesIndexed(domainId); - int incomingLinks = dataStore.getIncomingLinks(domainId); - int outboundLinks = dataStore.getOutboundLinks(domainId); - double rank = Math.round(10000.0*(1.0-dataStore.getRank(domainId)))/100; - EdgeDomainIndexingState state = dataStore.getDomainState(domainId); - double nominalQuality = Math.round(100*100*Math.exp(dataStore.getDomainQuality(domainId)))/100.; - List linkingDomains = dataStore.getLinkingDomains(domainId); + boolean blacklisted = isBlacklisted(domain); + int pagesKnown = getPagesKnown(domainId); + int pagesVisited = getPagesVisited(domainId); + int pagesIndexed = getPagesIndexed(domainId); + int incomingLinks = getIncomingLinks(domainId); + int outboundLinks = getOutboundLinks(domainId); + double rank = Math.round(10000.0*(1.0-getRank(domainId)))/100; + EdgeDomainIndexingState state = getDomainState(domainId); + List linkingDomains = getLinkingDomains(domainId); - return Optional.of(new DomainInformation(domain, blacklisted, pagesKnown, pagesVisited, pagesIndexed, incomingLinks, outboundLinks, nominalQuality, rank, state, linkingDomains)); + return Optional.of(new DomainInformation(domain, blacklisted, pagesKnown, pagesVisited, pagesIndexed, incomingLinks, outboundLinks, rank, state, linkingDomains)); } private EdgeId getDomainFromPartial(String site) { try { - return dataStore.getDomainId(new EdgeDomain(site)); + return dataStoreDao.getDomainId(new EdgeDomain(site)); } catch (Exception ex) { try { - return dataStore.getDomainId(new EdgeDomain(site)); + return dataStoreDao.getDomainId(new EdgeDomain(site)); } catch (Exception ex2) { return null; @@ -58,4 +76,178 @@ public class DomainInformationService { } } + + @SneakyThrows + public boolean isBlacklisted(EdgeDomain domain) { + + try (var connection = dataSource.getConnection()) { + try (var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?")) { + stmt.setString(1, domain.domain); + var rsp = stmt.executeQuery(); + if (rsp.next()) { + return true; + } else { + return false; + } + } + } + } + + @SneakyThrows + public int getPagesKnown(EdgeId domainId) { + try (var connection = dataSource.getConnection()) { + + try (var stmt = connection.prepareStatement("SELECT KNOWN_URLS FROM DOMAIN_METADATA WHERE ID=?")) { + stmt.setInt(1, domainId.getId()); + var rsp = stmt.executeQuery(); + if (rsp.next()) { + return rsp.getInt(1); + } + } catch (Exception ex) { + logger.error("DB error", ex); + } + return 0; + } + } + + @SneakyThrows + public int getPagesVisited(EdgeId domainId) { + try (var connection = dataSource.getConnection()) { + + try (var stmt = connection.prepareStatement("SELECT VISITED_URLS FROM DOMAIN_METADATA WHERE ID=?")) { + stmt.setInt(1, domainId.getId()); + var rsp = stmt.executeQuery(); + if (rsp.next()) { + return rsp.getInt(1); + } + } catch (Exception ex) { + logger.error("DB error", ex); + } + return 0; + } + } + + + @SneakyThrows + public int getPagesIndexed(EdgeId domainId) { + try (var connection = dataSource.getConnection()) { + + try (var stmt = connection.prepareStatement("SELECT GOOD_URLS FROM DOMAIN_METADATA WHERE ID=?")) { + stmt.setInt(1, domainId.getId()); + var rsp = stmt.executeQuery(); + if (rsp.next()) { + return rsp.getInt(1); + } + } catch (Exception ex) { + logger.error("DB error", ex); + } + return 0; + } + } + + @SneakyThrows + public int getIncomingLinks(EdgeId domainId) { + try (var connection = dataSource.getConnection()) { + + try (var stmt = connection.prepareStatement("SELECT COUNT(ID) FROM EC_DOMAIN_LINK WHERE DEST_DOMAIN_ID=?")) { + stmt.setInt(1, domainId.getId()); + var rsp = stmt.executeQuery(); + if (rsp.next()) { + return rsp.getInt(1); + } + } catch (Exception ex) { + logger.error("DB error", ex); + } + return 0; + } + } + @SneakyThrows + public int getOutboundLinks(EdgeId domainId) { + try (var connection = dataSource.getConnection()) { + + try (var stmt = connection.prepareStatement("SELECT COUNT(ID) FROM EC_DOMAIN_LINK WHERE SOURCE_DOMAIN_ID=?")) { + stmt.setInt(1, domainId.getId()); + var rsp = stmt.executeQuery(); + if (rsp.next()) { + return rsp.getInt(1); + } + } catch (Exception ex) { + logger.error("DB error", ex); + } + return 0; + } + } + + @SneakyThrows + public double getDomainQuality(EdgeId domainId) { + try (var connection = dataSource.getConnection()) { + + try (var stmt = connection.prepareStatement("SELECT QUALITY FROM EC_DOMAIN WHERE ID=?")) { + stmt.setInt(1, domainId.getId()); + var rsp = stmt.executeQuery(); + if (rsp.next()) { + return rsp.getDouble(1); + } + } catch (Exception ex) { + logger.error("DB error", ex); + } + return -5; + } + } + + public EdgeDomainIndexingState getDomainState(EdgeId domainId) { + try (var connection = dataSource.getConnection()) { + + try (var stmt = connection.prepareStatement("SELECT STATE FROM EC_DOMAIN WHERE ID=?")) { + stmt.setInt(1, domainId.getId()); + var rsp = stmt.executeQuery(); + if (rsp.next()) { + return EdgeDomainIndexingState.valueOf(rsp.getString(1)); + } + } catch (Exception ex) { + logger.error("DB error", ex); + } + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + return EdgeDomainIndexingState.ERROR; + } + + public List getLinkingDomains(EdgeId domainId) { + try (var connection = dataSource.getConnection()) { + List results = new ArrayList<>(25); + try (var stmt = connection.prepareStatement("SELECT SOURCE_URL FROM EC_RELATED_LINKS_VIEW WHERE DEST_DOMAIN_ID=? ORDER BY SOURCE_DOMAIN_ID LIMIT 25")) { + stmt.setInt(1, domainId.getId()); + var rsp = stmt.executeQuery(); + while (rsp.next()) { + results.add(new EdgeDomain(rsp.getString(1))); + } + return results; + } catch (Exception ex) { + logger.error("DB error", ex); + } + + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + return Collections.emptyList(); + } + + public double getRank(EdgeId domainId) { + try (var connection = dataSource.getConnection()) { + + try (var stmt = connection.prepareStatement("SELECT IFNULL(RANK, 1) FROM EC_DOMAIN WHERE ID=?")) { + stmt.setInt(1, domainId.getId()); + var rsp = stmt.executeQuery(); + if (rsp.next()) { + return rsp.getDouble(1); + } + } catch (Exception ex) { + logger.error("DB error", ex); + } + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + return 1; + } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/tools/IndexMergerMain.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/tools/IndexMergerMain.java index bb946238..05c67481 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/tools/IndexMergerMain.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/tools/IndexMergerMain.java @@ -3,12 +3,13 @@ package nu.marginalia.wmsa.edge.tools; import com.google.inject.Inject; import gnu.trove.set.hash.TIntHashSet; import lombok.SneakyThrows; +import nu.marginalia.util.ranking.RankingDomainFetcher; import nu.marginalia.wmsa.configuration.module.DatabaseModule; import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklist; import nu.marginalia.wmsa.edge.data.dao.task.EdgeDomainBlacklistImpl; import nu.marginalia.wmsa.edge.index.model.RankingSettings; -import nu.marginalia.wmsa.edge.index.service.SearchIndexDao; -import nu.marginalia.wmsa.edge.index.service.query.SearchIndexPartitioner; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexDao; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexPartitioner; import org.mariadb.jdbc.Driver; import org.roaringbitmap.longlong.Roaring64Bitmap; import org.slf4j.Logger; @@ -59,7 +60,9 @@ public class IndexMergerMain { } var hikari = new DatabaseModule().provideConnection(); - var partitioner = new SearchIndexPartitioner(new SearchIndexDao(hikari, new RankingSettings())); + var ds = new DatabaseModule().provideConnection(); + var domains = new RankingDomainFetcher(ds, new EdgeDomainBlacklistImpl(ds)); + var partitioner = new SearchIndexPartitioner(new SearchIndexDao(hikari, domains, new RankingSettings())); var blacklist = new EdgeDomainBlacklistImpl(hikari); new IndexMergerMain(file1, file2, outputFile, partitioner, blacklist); diff --git a/marginalia_nu/src/main/resources/sql/edge-crawler-cache.sql b/marginalia_nu/src/main/resources/sql/edge-crawler-cache.sql index fc9e515d..36ab040a 100644 --- a/marginalia_nu/src/main/resources/sql/edge-crawler-cache.sql +++ b/marginalia_nu/src/main/resources/sql/edge-crawler-cache.sql @@ -1,24 +1,11 @@ -DROP TABLE IF EXISTS EC_URL_LINK; -DROP VIEW IF EXISTS EC_PAGE_VIEW; - -DROP TABLE IF EXISTS DISC_DOMAIN_TAG; -DROP TABLE IF EXISTS DISC_TAG; -DROP TABLE IF EXISTS DISC_USER; - -DROP TABLE IF EXISTS EC_DOMAIN_NEIGHBORS; +DROP TABLE IF EXISTS DOMAIN_METADATA; DROP TABLE IF EXISTS EC_FEED_URL; DROP TABLE IF EXISTS EC_DOMAIN_LINK; DROP TABLE IF EXISTS EC_PAGE_DATA; DROP TABLE IF EXISTS EC_URL; +DROP TABLE IF EXISTS EC_DOMAIN_NEIGHBORS; DROP TABLE IF EXISTS EC_DOMAIN; -DROP TABLE IF EXISTS EC_TOP_DOMAIN; -DROP TABLE IF EXISTS EC_URL_DETAILS; -DROP VIEW IF EXISTS EC_URL_VIEW; -DROP VIEW IF EXISTS EC_URL_PART_HASH; -DROP TABLE IF EXISTS EC_URL_WORD; -DROP TABLE IF EXISTS EC_DICTIONARY; -DROP TABLE IF EXISTS DOMAIN_METADATA; CREATE TABLE IF NOT EXISTS DOMAIN_METADATA ( ID INT PRIMARY KEY, @@ -27,52 +14,31 @@ CREATE TABLE IF NOT EXISTS DOMAIN_METADATA ( GOOD_URLS INT DEFAULT 0 ); -CREATE TABLE IF NOT EXISTS EC_TOP_DOMAIN ( - ID INT PRIMARY KEY AUTO_INCREMENT, - URL_PART VARCHAR(255) UNIQUE NOT NULL, - ALIVE BOOLEAN DEFAULT TRUE NOT NULL -) -CHARACTER SET utf8mb4 -COLLATE utf8mb4_unicode_ci; CREATE TABLE IF NOT EXISTS EC_DOMAIN ( ID INT PRIMARY KEY AUTO_INCREMENT, - URL_PART VARCHAR(255) UNIQUE NOT NULL, - INDEXED INT DEFAULT 0 NOT NULL, - QUALITY DOUBLE DEFAULT -5 NOT NULL, - QUALITY_RAW DOUBLE DEFAULT -5 NOT NULL, - QUALITY_ORIGINAL DOUBLE DEFAULT -5 NOT NULL, - URL_TOP_DOMAIN_ID INT NOT NULL, - URL_SUBDOMAIN VARCHAR(255) NOT NULL, - STATE INT DEFAULT 0 NOT NULL, + DOMAIN_NAME VARCHAR(255) UNIQUE NOT NULL, + DOMAIN_TOP VARCHAR(255) NOT NULL, + + INDEXED INT DEFAULT 0 NOT NULL COMMENT "~number of documents visited / 100", + STATE ENUM('ACTIVE', 'EXHAUSTED', 'SPECIAL', 'SOCIAL_MEDIA', 'BLOCKED', 'REDIR', 'ERROR', 'UNKNOWN') NOT NULL DEFAULT 'active' COMMENT "@see EdgeDomainIndexingState", RANK DOUBLE, - DOMAIN_ALIAS INTEGER, + IP VARCHAR(32), INDEX_DATE TIMESTAMP DEFAULT NOW(), DISCOVER_DATE TIMESTAMP DEFAULT NOW(), - FOREIGN KEY (URL_TOP_DOMAIN_ID) REFERENCES EC_TOP_DOMAIN(ID) ON DELETE CASCADE -) -CHARACTER SET utf8mb4 -COLLATE utf8mb4_unicode_ci; - -CREATE TABLE IF NOT EXISTS EC_DOMAIN_HISTORY ( - ID INT PRIMARY KEY AUTO_INCREMENT, - URL_PART VARCHAR(255) UNIQUE NOT NULL, - QUALITY_MEASURE DOUBLE DEFAULT -5 NOT NULL, - INBOUND_LINKS INT DEFAULT 1, - LINK_ADJUSTED_QUALITY DOUBLE GENERATED ALWAYS AS (0.3*QUALITY_MEASURE + 0.7*QUALITY_MEASURE / GREATEST(1, INBOUND_LINKS)), - RANK DOUBLE + IS_ALIVE BOOLEAN AS (STATE='ACTIVE' OR STATE='EXHAUSTED' OR STATE='SPECIAL' OR STATE='SOCIAL_MEDIA') VIRTUAL ) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE IF NOT EXISTS EC_DOMAIN_BLACKLIST ( ID INT PRIMARY KEY AUTO_INCREMENT, - URL_DOMAIN VARCHAR(255) UNIQUE NOT NULL + URL_DOMAIN VARCHAR(255) UNIQUE NOT NULL ) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; @@ -81,18 +47,15 @@ CREATE TABLE IF NOT EXISTS EC_URL ( ID INT PRIMARY KEY AUTO_INCREMENT, DOMAIN_ID INT NOT NULL, PROTO ENUM('http','https','gemini') NOT NULL, - URL VARCHAR(255) NOT NULL, + PATH VARCHAR(255) NOT NULL COLLATE utf8mb4_bin, PORT INT, + PATH_HASH BIGINT NOT NULL COMMENT "Hash of PATH for uniqueness check by domain", VISITED BOOLEAN NOT NULL DEFAULT FALSE, - DATA_HASH INTEGER, - QUALITY_MEASURE DOUBLE, STATE ENUM('ok', 'redirect', 'dead', 'archived', 'disqualified') NOT NULL DEFAULT 'ok', - IP VARCHAR(32), - - CONSTRAINT CONS UNIQUE (DOMAIN_ID, URL), + CONSTRAINT CONS UNIQUE (DOMAIN_ID, PATH_HASH), FOREIGN KEY (DOMAIN_ID) REFERENCES EC_DOMAIN(ID) ON DELETE CASCADE ) CHARACTER SET utf8mb4 @@ -101,13 +64,15 @@ COLLATE utf8mb4_unicode_ci; CREATE TABLE IF NOT EXISTS EC_PAGE_DATA ( ID INT PRIMARY KEY AUTO_INCREMENT, - TITLE VARCHAR(255), - DESCRIPTION VARCHAR(255), + TITLE VARCHAR(255) NOT NULL, + DESCRIPTION VARCHAR(255) NOT NULL, - WORDS_DISTINCT INTEGER, - WORDS_TOTAL INTEGER, - FORMAT VARCHAR(8), - FEATURES INT, + WORDS_TOTAL INTEGER NOT NULL, + FORMAT ENUM('PLAIN', 'UNKNOWN', 'HTML123', 'HTML4', 'XHTML', 'HTML5', 'MARKDOWN') NOT NULL, + FEATURES INT COMMENT "Bit-encoded feature set of document, @see HtmlFeature" NOT NULL, + + DATA_HASH INTEGER NOT NULL, + QUALITY DOUBLE NOT NULL, FOREIGN KEY (ID) REFERENCES EC_URL(ID) ON DELETE CASCADE ) @@ -115,13 +80,9 @@ CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE EC_FEED_URL ( - ID INT PRIMARY KEY AUTO_INCREMENT, - DOMAIN_ID INT NOT NULL, - PROTO VARCHAR(8) NOT NULL, - URL VARCHAR(255) NOT NULL, - PORT INT, + URL VARCHAR(255) PRIMARY KEY, + DOMAIN_ID INT, - CONSTRAINT CONS UNIQUE (DOMAIN_ID, URL), FOREIGN KEY (DOMAIN_ID) REFERENCES EC_DOMAIN(ID) ON DELETE CASCADE ) CHARACTER SET utf8mb4 @@ -150,92 +111,63 @@ CREATE TABLE IF NOT EXISTS EC_DOMAIN_LINK ( FOREIGN KEY (DEST_DOMAIN_ID) REFERENCES EC_DOMAIN(ID) ON DELETE CASCADE ); -CREATE TABLE IF NOT EXISTS EC_DOMAIN_LINK_AGGREGATE ( - DOMAIN_ID INT PRIMARY KEY NOT NULL, - LINKS INT -); - CREATE OR REPLACE VIEW EC_URL_VIEW AS SELECT - EC_DOMAIN.URL_PART AS URL_DOMAIN, - EC_URL.URL AS URL_PATH, - EC_TOP_DOMAIN.URL_PART AS URL_TOP, + IF(PORT IS NULL, + CONCAT(EC_URL.PROTO, "://", EC_DOMAIN.DOMAIN_NAME, EC_URL.PATH), + CONCAT(EC_URL.PROTO, "://", EC_DOMAIN.DOMAIN_NAME, ":", EC_URL.PORT, EC_URL.PATH)) + AS URL, + EC_URL.PATH_HASH AS PATH_HASH, + EC_URL.PATH AS PATH, + EC_DOMAIN.DOMAIN_NAME AS DOMAIN_NAME, + EC_DOMAIN.DOMAIN_TOP AS DOMAIN_TOP, EC_URL.ID AS ID, EC_DOMAIN.ID AS DOMAIN_ID, - EC_TOP_DOMAIN.ID AS TOP_DOMAIN_ID, - EC_URL.PROTO AS URL_PROTO, - EC_URL.PORT AS URL_PORT, + EC_URL.VISITED AS VISITED, - EC_URL.DATA_HASH AS DATA_HASH, - EC_URL.QUALITY_MEASURE AS URL_QUALITY_MEASURE, - EC_DOMAIN.QUALITY AS DOMAIN_QUALITY_MEASURE, - EC_DOMAIN.QUALITY_RAW AS QUALITY_RAW, + + EC_PAGE_DATA.QUALITY AS QUALITY, + EC_PAGE_DATA.DATA_HASH AS DATA_HASH, EC_PAGE_DATA.TITLE AS TITLE, EC_PAGE_DATA.DESCRIPTION AS DESCRIPTION, - EC_URL.IP AS IP, - EC_DOMAIN.STATE AS STATE, EC_PAGE_DATA.WORDS_TOTAL AS WORDS_TOTAL, EC_PAGE_DATA.FORMAT AS FORMAT, EC_PAGE_DATA.FEATURES AS FEATURES, + + EC_DOMAIN.IP AS IP, + EC_DOMAIN.STATE AS STATE, EC_DOMAIN.RANK AS RANK, EC_DOMAIN.STATE AS DOMAIN_STATE FROM EC_URL LEFT JOIN EC_PAGE_DATA ON EC_PAGE_DATA.ID = EC_URL.ID INNER JOIN EC_DOMAIN - ON EC_URL.DOMAIN_ID = EC_DOMAIN.ID - INNER JOIN EC_TOP_DOMAIN - ON EC_DOMAIN.URL_TOP_DOMAIN_ID=EC_TOP_DOMAIN.ID; - -CREATE OR REPLACE VIEW EC_DISCOVER_TASKS_VIEW AS - SELECT - ID, - URL_PART - FROM EC_DOMAIN - WHERE - DOMAIN_ALIAS IS NULL - AND INDEXED = 0 - ORDER BY QUALITY DESC, ID ASC; + ON EC_URL.DOMAIN_ID = EC_DOMAIN.ID; CREATE OR REPLACE VIEW EC_RELATED_LINKS_VIEW AS SELECT SOURCE_DOMAIN_ID, - SOURCE_DOMAIN.URL_PART AS SOURCE_URL, - SOURCE_TOP_DOMAIN.URL_PART AS SOURCE_TOP_URL, + SOURCE_DOMAIN.DOMAIN_NAME AS SOURCE_DOMAIN, + SOURCE_DOMAIN.DOMAIN_TOP AS SOURCE_TOP_DOMAIN, DEST_DOMAIN_ID, - DEST_DOMAIN.URL_PART AS DEST_URL, - DEST_TOP_DOMAIN.URL_PART AS DEST_TOP_URL + DEST_DOMAIN.DOMAIN_NAME AS DEST_DOMAIN, + DEST_DOMAIN.DOMAIN_TOP AS DEST_TOP_DOMAIN FROM EC_DOMAIN_LINK INNER JOIN EC_DOMAIN AS SOURCE_DOMAIN ON SOURCE_DOMAIN.ID=SOURCE_DOMAIN_ID - INNER JOIN EC_TOP_DOMAIN AS SOURCE_TOP_DOMAIN - ON SOURCE_TOP_DOMAIN.ID=SOURCE_DOMAIN.URL_TOP_DOMAIN_ID INNER JOIN EC_DOMAIN AS DEST_DOMAIN ON DEST_DOMAIN.ID=DEST_DOMAIN_ID - INNER JOIN EC_TOP_DOMAIN AS DEST_TOP_DOMAIN - ON DEST_TOP_DOMAIN.ID=DEST_DOMAIN.URL_TOP_DOMAIN_ID ; CREATE OR REPLACE VIEW EC_RELATED_LINKS_IN AS SELECT IN_URL.ID AS SRC_URL_ID, - IN_URL.QUALITY_MEASURE AS SRC_URL_QUALITY, - OUT_URL.ID AS DEST_URL_ID, - OUT_URL.QUALITY_MEASURE AS DEST_URL_QUALITY - FROM EC_URL AS IN_URL - INNER JOIN EC_DOMAIN_LINK - ON IN_URL.DOMAIN_ID=EC_DOMAIN_LINK.SOURCE_DOMAIN_ID - INNER JOIN EC_URL AS OUT_URL - ON OUT_URL.DOMAIN_ID=EC_DOMAIN_LINK.DEST_DOMAIN_ID - WHERE IN_URL.VISITED=TRUE - AND IN_URL.DATA_HASH IS NOT NULL - AND OUT_URL.VISITED=TRUE - AND OUT_URL.DATA_HASH IS NOT NULL; - -CREATE TABLE IF NOT EXISTS EC_DOMAIN_BACKLINKS ( - ID INT PRIMARY KEY, - LINKEDNESS INT -); + OUT_URL.ID AS DEST_URL_ID + FROM EC_DOMAIN_LINK + INNER JOIN EC_URL AS IN_URL ON IN_URL.DOMAIN_ID=EC_DOMAIN_LINK.SOURCE_DOMAIN_ID + INNER JOIN EC_URL AS OUT_URL ON OUT_URL.DOMAIN_ID=EC_DOMAIN_LINK.DEST_DOMAIN_ID + WHERE IN_URL.VISITED AND IN_URL.STATE = 'ok' + AND OUT_URL.VISITED AND OUT_URL.STATE = 'ok'; CREATE TABLE IF NOT EXISTS EC_API_KEY ( LICENSE_KEY VARCHAR(255) UNIQUE, @@ -245,16 +177,8 @@ CREATE TABLE IF NOT EXISTS EC_API_KEY ( RATE INT DEFAULT 10 ); -CREATE INDEX IF NOT EXISTS EC_DOMAIN_RANK_INDEX ON EC_DOMAIN (RANK); -CREATE INDEX IF NOT EXISTS EC_DOMAIN_QUALITY_INDEX ON EC_DOMAIN (QUALITY,STATE); - CREATE INDEX IF NOT EXISTS EC_DOMAIN_INDEXED_INDEX ON EC_DOMAIN (INDEXED); -CREATE INDEX IF NOT EXISTS EC_DOMAIN_ID_INDEXED_INDEX ON EC_DOMAIN (ID, INDEXED); -CREATE INDEX IF NOT EXISTS EC_DOMAIN_TRIO ON EC_DOMAIN (STATE, DOMAIN_ALIAS, INDEXED, QUALITY); - -CREATE INDEX IF NOT EXISTS EC_URL_VISITED ON EC_URL (VISITED); -CREATE INDEX IF NOT EXISTS EC_URL_VISITED_STATE ON EC_URL (VISITED, STATE); -CREATE INDEX IF NOT EXISTS EC_URL_IP ON EC_URL (IP); +CREATE INDEX IF NOT EXISTS EC_DOMAIN_TOP_DOMAIN ON EC_DOMAIN (DOMAIN_TOP); ---; diff --git a/marginalia_nu/src/main/resources/templates/edge/site-info-gmi.hdb b/marginalia_nu/src/main/resources/templates/edge/site-info-gmi.hdb index 5696b251..cd8abf67 100644 --- a/marginalia_nu/src/main/resources/templates/edge/site-info-gmi.hdb +++ b/marginalia_nu/src/main/resources/templates/edge/site-info-gmi.hdb @@ -10,5 +10,4 @@ Pages Known: {{pagesKnown}} Pages Indexed: {{pagesKnown}} Inbound Links: {{inboundLinks}} Outbound Links: {{outboundLinks}} -Nominal Quality: {{nominalQuality}}% Crawl Ranking: {{ranking}}% \ No newline at end of file diff --git a/marginalia_nu/src/main/resources/templates/edge/site-info.hdb b/marginalia_nu/src/main/resources/templates/edge/site-info.hdb index 19b585b8..837f320d 100644 --- a/marginalia_nu/src/main/resources/templates/edge/site-info.hdb +++ b/marginalia_nu/src/main/resources/templates/edge/site-info.hdb @@ -37,7 +37,6 @@

Links

- Nominal Quality: {{nominalQuality}}%
Crawl Ranking: {{ranking}}%
Incoming Links: {{incomingLinks}}
Outbound Links: {{outboundLinks}}
diff --git a/marginalia_nu/src/test/java/nu/marginalia/util/TestUtil.java b/marginalia_nu/src/test/java/nu/marginalia/util/TestUtil.java index 84b9f165..26d397a8 100644 --- a/marginalia_nu/src/test/java/nu/marginalia/util/TestUtil.java +++ b/marginalia_nu/src/test/java/nu/marginalia/util/TestUtil.java @@ -43,7 +43,7 @@ public class TestUtil { logger.info("Running script {}", scriptFile); try (var scriptStream = ClassLoader.getSystemResourceAsStream(scriptFile); var stmt = conn.createStatement()) { - for (String s : new String(scriptStream.readAllBytes()).split(";")) { + for (String s : new String(scriptStream.readAllBytes()).split("(;|---)")) { if (!s.isBlank()) { try { Assertions.assertTrue(stmt.executeUpdate(s) >= 0); diff --git a/marginalia_nu/src/test/java/nu/marginalia/util/btree/BTreeWriterTest.java b/marginalia_nu/src/test/java/nu/marginalia/util/btree/BTreeWriterTest.java index 1915d989..875cda37 100644 --- a/marginalia_nu/src/test/java/nu/marginalia/util/btree/BTreeWriterTest.java +++ b/marginalia_nu/src/test/java/nu/marginalia/util/btree/BTreeWriterTest.java @@ -90,10 +90,10 @@ class BTreeWriterTest { { var writer = new BTreeWriter(mmf, ctx); - writer.write(0, toPut.size(), (offset) -> { + writer.write(0, toPut.size(), (slice) -> { for (int i = 0; i < data.length; i++) { - mmf.put(offset + 2L*i, data[i]); - mmf.put(offset + 2L*i + 1, i); + slice.put(2L*i, data[i]); + slice.put( 2L*i + 1, i); } }); mmf.force(); @@ -133,10 +133,10 @@ class BTreeWriterTest { { var writer = new BTreeWriter(mmf, ctx); - writer.write( 0, toPut.size(), (offset) -> { + writer.write( 0, toPut.size(), (slice) -> { for (int i = 0; i < data.length; i++) { - mmf.put(offset + 2L*i, data[i]); - mmf.put(offset + 2L*i + 1, i); + slice.put(2L*i, data[i]); + slice.put(2L*i + 1, i); } }); mmf.force(); @@ -182,9 +182,9 @@ class BTreeWriterTest { try (MultimapFileLong mmf = MultimapFileLong.forOutput(tempFile, 1000)) { { var writer = new BTreeWriter(mmf, ctx); - writer.write(0, toPut.size(), (offset) -> { + writer.write(0, toPut.size(), (slice) -> { for (int i = 0; i < data.length; i++) { - mmf.put(offset + i, data[i]); + slice.put(i, data[i]); } }); mmf.force(); @@ -235,9 +235,9 @@ class BTreeWriterTest { try (MultimapFileLong mmf = MultimapFileLong.forOutput(tempFile, 1000)) { { var writer = new BTreeWriter(mmf, ctx); - writer.write(0, toPut.size(), (offset) -> { + writer.write(0, toPut.size(), (slice) -> { for (int i = 0; i < data.length; i++) { - mmf.put(offset + i, data[i]); + slice.put(i, data[i]); } }); mmf.force(); @@ -288,10 +288,10 @@ class BTreeWriterTest { try (MultimapFileLong mmf = MultimapFileLong.forOutput(tempFile, 1000)) { { var writer = new BTreeWriter(mmf, ctx); - writer.write(0, toPut.size(), (offset) -> { + writer.write(0, toPut.size(), (slice) -> { for (int i = 0; i < data.length; i++) { - mmf.put(offset + i*2L, data[i]); - mmf.put(offset + i*2L+1, i); + slice.put(i*2L, data[i]); + slice.put(i*2L+1, i); } }); mmf.force(); diff --git a/marginalia_nu/src/test/java/nu/marginalia/util/hash/LongPairHashMapTest.java b/marginalia_nu/src/test/java/nu/marginalia/util/hash/LongPairHashMapTest.java index 326c9b15..9331a998 100644 --- a/marginalia_nu/src/test/java/nu/marginalia/util/hash/LongPairHashMapTest.java +++ b/marginalia_nu/src/test/java/nu/marginalia/util/hash/LongPairHashMapTest.java @@ -27,7 +27,7 @@ class LongPairHashMapTest { try { RandomAccessFile raf = new RandomAccessFile(tempFile.toFile(), "rw"); MultimapFileLong mmf = new MultimapFileLong(raf, FileChannel.MapMode.READ_WRITE, 10000, 1000, true); - var lphm = new LongPairHashMap(mmf, 1024); + var lphm = LongPairHashMap.createNew(mmf, 1024); toPut.forEach(i -> { lphm.put(new LongPairHashMap.CellData(i, i)); }); @@ -36,7 +36,7 @@ class LongPairHashMapTest { RandomAccessFile raf2 = new RandomAccessFile(tempFile.toFile(), "rw"); MultimapFileLong mmf2 = new MultimapFileLong(raf2, FileChannel.MapMode.READ_WRITE, 10000, 1000, true); - var lphm2 = new LongPairHashMap(mmf2); + var lphm2 = LongPairHashMap.loadExisting(mmf2); toPut.forEach(i -> { Assertions.assertTrue(lphm2.get(i).isSet()); Assertions.assertEquals(i, (int) lphm2.get(i).getKey()); diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainLinksTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainLinksTest.java new file mode 100644 index 00000000..d839bbb2 --- /dev/null +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainLinksTest.java @@ -0,0 +1,48 @@ +package nu.marginalia.wmsa.edge.converting.loader; + +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.util.TestUtil; +import nu.marginalia.wmsa.edge.converting.interpreter.instruction.DomainLink; +import nu.marginalia.wmsa.edge.model.EdgeDomain; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +class SqlLoadDomainLinksTest { + @Container + static MariaDBContainer mariaDBContainer = new MariaDBContainer<>("mariadb") + .withDatabaseName("WMSA_prod") + .withUsername("wmsa") + .withPassword("wmsa") + .withInitScript("sql/edge-crawler-cache.sql") + .withNetworkAliases("mariadb"); + + HikariDataSource dataSource; + LoaderData loaderData; + @BeforeEach + public void setUp() { + dataSource = TestUtil.getConnection(mariaDBContainer.getJdbcUrl()); + + var loadDomains = new SqlLoadDomains(dataSource); + loaderData = new LoaderData(10); + + loaderData.setTargetDomain(new EdgeDomain("www.marginalia.nu")); + loadDomains.load(loaderData, new EdgeDomain[] { new EdgeDomain("www.marginalia.nu"), new EdgeDomain("memex.marginalia.nu") }); + } + + @AfterEach + public void tearDown() { + dataSource.close(); + } + + @Test + public void loadDomainLinks() { + var loader = new SqlLoadDomainLinks(dataSource); + loader.load(new DomainLink[] { new DomainLink(new EdgeDomain("www.marginalia.nu"), new EdgeDomain("memex.marginalia.nu")) }); + } + +} \ No newline at end of file diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainsTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainsTest.java new file mode 100644 index 00000000..25dd18b4 --- /dev/null +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadDomainsTest.java @@ -0,0 +1,52 @@ +package nu.marginalia.wmsa.edge.converting.loader; + +import nu.marginalia.util.TestUtil; +import nu.marginalia.wmsa.edge.model.EdgeDomain; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +class SqlLoadDomainsTest { + @Container + static MariaDBContainer mariaDBContainer = new MariaDBContainer<>("mariadb") + .withDatabaseName("WMSA_prod") + .withUsername("wmsa") + .withPassword("wmsa") + .withInitScript("sql/edge-crawler-cache.sql") + .withNetworkAliases("mariadb"); + + @Test + public void loadDomain() { + + try (var dataSource = TestUtil.getConnection(mariaDBContainer.getJdbcUrl());) { + var loadDomains = new SqlLoadDomains(dataSource); + var loaderData = new LoaderData(10); + + loaderData.setTargetDomain(new EdgeDomain("www.marginalia.nu")); + loadDomains.load(loaderData, new EdgeDomain("www.marginalia.nu")); + + assertTrue(loaderData.getDomainId(new EdgeDomain("www.marginalia.nu")) >= 0); + } + + } + + @Test + public void loadDomains() { + + try (var dataSource = TestUtil.getConnection(mariaDBContainer.getJdbcUrl());) { + var loadDomains = new SqlLoadDomains(dataSource); + var loaderData = new LoaderData(10); + + loaderData.setTargetDomain(new EdgeDomain("www.marginalia.nu")); + loadDomains.load(loaderData, new EdgeDomain[] { new EdgeDomain("www.marginalia.nu"), new EdgeDomain("memex.marginalia.nu") }); + + assertTrue(loaderData.getDomainId(new EdgeDomain("www.marginalia.nu")) >= 0); + assertTrue(loaderData.getDomainId(new EdgeDomain("memex.marginalia.nu")) >= 0); + } + + } +} \ No newline at end of file diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocumentTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocumentTest.java new file mode 100644 index 00000000..ecb0e88a --- /dev/null +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocumentTest.java @@ -0,0 +1,94 @@ +package nu.marginalia.wmsa.edge.converting.loader; + +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.util.TestUtil; +import nu.marginalia.wmsa.edge.converting.interpreter.instruction.LoadProcessedDocument; +import nu.marginalia.wmsa.edge.converting.processor.logic.HtmlFeature; +import nu.marginalia.wmsa.edge.data.dao.EdgeDataStoreDaoImpl; +import nu.marginalia.wmsa.edge.model.EdgeDomain; +import nu.marginalia.wmsa.edge.model.EdgeId; +import nu.marginalia.wmsa.edge.model.EdgeUrl; +import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard; +import nu.marginalia.wmsa.edge.model.crawl.EdgeUrlState; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.net.URISyntaxException; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Testcontainers +class SqlLoadProcessedDocumentTest { + @Container + static MariaDBContainer mariaDBContainer = new MariaDBContainer<>("mariadb") + .withDatabaseName("WMSA_prod") + .withUsername("wmsa") + .withPassword("wmsa") + .withInitScript("sql/edge-crawler-cache.sql") + .withNetworkAliases("mariadb"); + + HikariDataSource dataSource; + LoaderData loaderData; + EdgeDataStoreDaoImpl dataStoreDao; + + @BeforeEach + public void setUp() throws URISyntaxException { + dataSource = TestUtil.getConnection(mariaDBContainer.getJdbcUrl()); + dataStoreDao = new EdgeDataStoreDaoImpl(dataSource); + + var loadDomains = new SqlLoadDomains(dataSource); + var loadUrls = new SqlLoadUrls(dataSource); + + loaderData = new LoaderData(10); + + loaderData.setTargetDomain(new EdgeDomain("www.marginalia.nu")); + loadDomains.load(loaderData, new EdgeDomain("www.marginalia.nu")); + + loadUrls.load(loaderData, new EdgeUrl[]{new EdgeUrl("https://www.marginalia.nu/")}); + + } + + @AfterEach + public void tearDown() { + dataStoreDao.clearCaches(); + dataSource.close(); + } + + @Test + public void loadProcessedDocument() throws URISyntaxException { + var loader = new SqlLoadProcessedDocument(dataSource); + var url = new EdgeUrl("https://www.marginalia.nu/"); + + loader.load(loaderData, List.of(new LoadProcessedDocument( + url, + EdgeUrlState.OK, + "TITLE", + "DESCR", + HtmlFeature.encode(Set.of(HtmlFeature.AFFILIATE_LINK)), + EdgeHtmlStandard.HTML5, + 100, + 12345, + -3.14 + ))); + + var details = dataStoreDao.getUrlDetailsMulti(List.of(new EdgeId<>(loaderData.getUrlId(new EdgeUrl("https://www.marginalia.nu/"))))); + assertEquals(1, details.size()); + + var urlDetails = details.get(0); + + assertEquals("TITLE", urlDetails.getTitle()); + assertEquals("DESCR", urlDetails.getDescription()); + assertTrue(urlDetails.isAffiliate()); + assertEquals(100, urlDetails.words); + assertEquals(12345, urlDetails.dataHash); + assertEquals(-3.14, urlDetails.getUrlQuality()); + } + +} \ No newline at end of file diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDomainTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDomainTest.java new file mode 100644 index 00000000..eb66da92 --- /dev/null +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDomainTest.java @@ -0,0 +1,54 @@ +package nu.marginalia.wmsa.edge.converting.loader; + +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.util.TestUtil; +import nu.marginalia.wmsa.edge.converting.interpreter.instruction.DomainLink; +import nu.marginalia.wmsa.edge.model.EdgeDomain; +import nu.marginalia.wmsa.edge.model.crawl.EdgeDomainIndexingState; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +class SqlLoadProcessedDomainTest { + @Container + static MariaDBContainer mariaDBContainer = new MariaDBContainer<>("mariadb") + .withDatabaseName("WMSA_prod") + .withUsername("wmsa") + .withPassword("wmsa") + .withInitScript("sql/edge-crawler-cache.sql") + .withNetworkAliases("mariadb"); + + HikariDataSource dataSource; + LoaderData loaderData; + @BeforeEach + public void setUp() { + + dataSource = TestUtil.getConnection(mariaDBContainer.getJdbcUrl()); + + var loadDomains = new SqlLoadDomains(dataSource); + loaderData = new LoaderData(10); + + loaderData.setTargetDomain(new EdgeDomain("www.marginalia.nu")); + loadDomains.load(loaderData, new EdgeDomain[]{ new EdgeDomain("www.marginalia.nu"), new EdgeDomain("memex.marginalia.nu") }); + } + + @AfterEach + public void tearDown() { + dataSource.close(); + } + + @Test + public void loadProcessedDomain() { + var loader = new SqlLoadProcessedDomain(dataSource, new SqlLoadDomains(dataSource)); + loader.load(loaderData, new EdgeDomain("www.marginalia.nu"), EdgeDomainIndexingState.BLOCKED, "127.0.0.1"); + } + @Test + public void loadDomainAlias() { + var loader = new SqlLoadProcessedDomain(dataSource, new SqlLoadDomains(dataSource)); + loader.loadAlias(loaderData, new DomainLink(new EdgeDomain("memex.marginalia.nu"), new EdgeDomain("www.marginalia.nu"))); + } +} \ No newline at end of file diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadUrlsTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadUrlsTest.java new file mode 100644 index 00000000..5afac733 --- /dev/null +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadUrlsTest.java @@ -0,0 +1,50 @@ +package nu.marginalia.wmsa.edge.converting.loader; + +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.util.TestUtil; +import nu.marginalia.wmsa.edge.model.EdgeDomain; +import nu.marginalia.wmsa.edge.model.EdgeUrl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.net.URISyntaxException; + +@Testcontainers +class SqlLoadUrlsTest { + @Container + static MariaDBContainer mariaDBContainer = new MariaDBContainer<>("mariadb") + .withDatabaseName("WMSA_prod") + .withUsername("wmsa") + .withPassword("wmsa") + .withInitScript("sql/edge-crawler-cache.sql") + .withNetworkAliases("mariadb"); + + HikariDataSource dataSource; + LoaderData loaderData; + @BeforeEach + public void setUp() { + dataSource = TestUtil.getConnection(mariaDBContainer.getJdbcUrl()); + + var loadDomains = new SqlLoadDomains(dataSource); + loaderData = new LoaderData(10); + + loaderData.setTargetDomain(new EdgeDomain("www.marginalia.nu")); + loadDomains.load(loaderData, new EdgeDomain("www.marginalia.nu")); + } + + @AfterEach + public void tearDown() { + dataSource.close(); + } + + @Test + public void loadUrl() throws URISyntaxException { + var loadUrls = new SqlLoadUrls(dataSource); + loadUrls.load(loaderData, new EdgeUrl[] { new EdgeUrl("https://www.marginalia.nu/") }); + } + +} \ No newline at end of file diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/DictionaryWriterTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/DictionaryWriterTest.java index 180576fc..961d8304 100644 --- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/DictionaryWriterTest.java +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/DictionaryWriterTest.java @@ -1,11 +1,11 @@ package nu.marginalia.wmsa.edge.index.service; import lombok.SneakyThrows; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexPartitioner; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.dictionary.DictionaryReader; -import nu.marginalia.wmsa.edge.index.service.dictionary.DictionaryWriter; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndexConverter; -import nu.marginalia.wmsa.edge.index.service.query.SearchIndexPartitioner; +import nu.marginalia.wmsa.edge.index.dictionary.DictionaryReader; +import nu.marginalia.wmsa.edge.index.dictionary.DictionaryWriter; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexConverter; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/EdgeIndexClientTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/EdgeIndexClientTest.java index 6b029da9..2b2da0fd 100644 --- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/EdgeIndexClientTest.java +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/EdgeIndexClientTest.java @@ -3,14 +3,14 @@ package nu.marginalia.wmsa.edge.index.service; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; import nu.marginalia.util.TestUtil; -import nu.marginalia.wmsa.client.exception.RemoteException; import nu.marginalia.wmsa.configuration.server.Context; import nu.marginalia.wmsa.configuration.server.Initialization; import nu.marginalia.wmsa.edge.index.EdgeIndexService; import nu.marginalia.wmsa.edge.index.IndexServicesFactory; import nu.marginalia.wmsa.edge.index.client.EdgeIndexClient; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexPartitioner; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.query.SearchIndexPartitioner; +import nu.marginalia.wmsa.edge.index.reader.SearchIndexes; import nu.marginalia.wmsa.edge.model.crawl.EdgePageWordSet; import nu.marginalia.wmsa.edge.model.search.EdgeSearchSpecification; import nu.marginalia.wmsa.edge.model.EdgeId; @@ -23,7 +23,6 @@ import org.junit.jupiter.api.parallel.ResourceAccessMode; import org.junit.jupiter.api.parallel.ResourceLock; import spark.Spark; -import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -31,7 +30,6 @@ import java.util.List; import java.util.stream.Collectors; import static nu.marginalia.util.TestUtil.getConnection; -import static nu.marginalia.wmsa.edge.index.EdgeIndexService.DYNAMIC_BUCKET_LENGTH; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/SearchIndexConverterTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/SearchIndexConverterTest.java deleted file mode 100644 index f42f2d36..00000000 --- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/SearchIndexConverterTest.java +++ /dev/null @@ -1,89 +0,0 @@ -package nu.marginalia.wmsa.edge.index.service; - -import lombok.SneakyThrows; -import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndexConverter; -import nu.marginalia.wmsa.edge.index.service.query.SearchIndexPartitioner; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; - -class SearchIndexConverterTest { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Test @Disabled @SneakyThrows - public void test() { - // File dictFile = new File("/home/vlofgren/dictionary.dat"); - File inFile = new File("/home/vlofgren/Work/converter/3/page-index.dat"); - - new SearchIndexConverter(IndexBlock.Title, 0, Path.of("/tmp"), inFile, - new File("/home/vlofgren/Work/converter/words.dat"), - new File("/home/vlofgren/Work/converter/urls.dat"), new SearchIndexPartitioner(null), val -> false); - - // sanityCheck(); - } - - @Test @Disabled - public void sanityCheck() { - File inFile = new File("/home/vlofgren/write/6/page-index.dat"); - -// SearchIndexReader sir = new SearchIndexReader(new SearchIndex[]{ -// new SearchIndex("body", Path.of("/tmp"), -// new File("/home/vlofgren/data/urls.dat"), -// new File("/home/vlofgren/data/words.dat")), -// new SearchIndex("body", Path.of("/tmp"), -// new File("/home/vlofgren/data/urls.dat"), -// new File("/home/vlofgren/data/words.dat")) -// , -// new SearchIndex("body", Path.of("/tmp"), -// new File("/home/vlofgren/data/urls.dat"), -// new File("/home/vlofgren/data/words.dat")) -// , -// new SearchIndex("body", Path.of("/tmp"), -// new File("/home/vlofgren/data/urls.dat"), -// new File("/home/vlofgren/data/words.dat")) -// }); - -// getQuery(sir, new EdgeIndexSearchTerms(List.of(152, 106), Collections.emptyList())).stream().forEach(System.out::println); -// sir.findWord(152).also(106).stream().forEach(System.out::println); -// scanFile(inFile, (url, word) -> { -// //System.out.println(url + " " + word); -// if (!sir.findWord(word).stream().anyMatch(url::equals)) { -// logger.error("Can't find word {} in {}", word, url); -// } -// }); - - - } -/* - private SearchIndexReader.Query getQuery(SearchIndexReader indexReader, EdgeIndexSearchTerms searchTerms) { - var orderedIncludes = searchTerms.includes - .stream() - .sorted(Comparator.comparingLong(indexReader::numHits)) - .distinct() - .mapToInt(Integer::intValue) - .toArray(); - - logger.info("Includes: ({}); excludes: ({})", Arrays. - stream(orderedIncludes) - .mapToObj(String::valueOf) - .collect(Collectors.joining(",")), - searchTerms.excludes.stream().map(String::valueOf).collect(Collectors.joining(","))); - SearchIndexReader.Query query = indexReader.findWord(orderedIncludes[0]); - for (int i = 1; i < orderedIncludes.length; i++) { - query = query.also(orderedIncludes[i]); - } - for (int term : searchTerms.excludes) { - query = query.not(term); - } - return query; - } - -*/ -} \ No newline at end of file diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/SearchIndexWriterTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/SearchIndexWriterTest.java index 5f1d2a0f..edcfa71f 100644 --- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/SearchIndexWriterTest.java +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/SearchIndexWriterTest.java @@ -1,14 +1,14 @@ package nu.marginalia.wmsa.edge.index.service; import lombok.SneakyThrows; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexPartitioner; import nu.marginalia.wmsa.edge.index.model.IndexBlock; -import nu.marginalia.wmsa.edge.index.service.dictionary.DictionaryWriter; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndex; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndexConverter; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndexReader; -import nu.marginalia.wmsa.edge.index.service.index.SearchIndexWriterImpl; -import nu.marginalia.wmsa.edge.index.service.query.IndexSearchBudget; -import nu.marginalia.wmsa.edge.index.service.query.SearchIndexPartitioner; +import nu.marginalia.wmsa.edge.index.dictionary.DictionaryWriter; +import nu.marginalia.wmsa.edge.index.reader.SearchIndex; +import nu.marginalia.wmsa.edge.index.conversion.SearchIndexConverter; +import nu.marginalia.wmsa.edge.index.reader.SearchIndexReader; +import nu.marginalia.wmsa.edge.index.journal.SearchIndexWriterImpl; +import nu.marginalia.wmsa.edge.index.reader.query.IndexSearchBudget; import nu.marginalia.wmsa.edge.model.EdgeId; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/TokenCompressorTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/TokenCompressorTest.java index ee84472e..e780ed62 100644 --- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/TokenCompressorTest.java +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/index/service/TokenCompressorTest.java @@ -1,6 +1,6 @@ package nu.marginalia.wmsa.edge.index.service; -import nu.marginalia.wmsa.edge.index.service.dictionary.TokenCompressor; +import nu.marginalia.wmsa.edge.index.dictionary.TokenCompressor; import org.junit.jupiter.api.Test; import java.util.Arrays; diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/QueryVariantsTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/QueryVariantsTest.java index 4aa9bceb..65b1ad57 100644 --- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/QueryVariantsTest.java +++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/QueryVariantsTest.java @@ -13,6 +13,7 @@ class QueryVariantsTest { QueryVariants variants; QueryParser parser; SentenceExtractor se; + @BeforeEach public void setUp() { LanguageModels lm = TestLanguageModels.getLanguageModels(); @@ -24,7 +25,7 @@ class QueryVariantsTest { parser = new QueryParser(new EnglishDictionary(dict), variants); } - @Test + @Test @SuppressWarnings("unchecked") void getQueryVariants() { System.out.println(se.extractSentence("we are alone")); testCase("DOS", List.of("DOS")); @@ -50,7 +51,5 @@ class QueryVariantsTest { private void testCase(String input, List... expected) { var tokens = variants.getQueryVariants(parser.extractBasicTokens(input)); System.out.println(tokens); -// var result = tokens.stream().map(lst -> lst.terms).collect(Collectors.toSet()); -// assertEquals(Set.of(expected), result, "Case failed: " + input); } } \ No newline at end of file