Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix serialization issue (#121)

Browse Source 
Co-authored-by: vlofgren <vlofgren@gmail.com>
Co-authored-by: vlofgren <vlofgren@marginalia.nu>
Reviewed-on: https://git.marginalia.nu/marginalia/marginalia.nu/pulls/121
This commit is contained in:
Viktor Lofgren 2022-10-22 15:01:41 +02:00
parent 0a35a7c1d0
commit c6abbc12f6
5 changed files with 48 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLong.java
View File

@ -376,11 +376,11 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice {
}
private void swap(long idx1, long idx2) {
LongBuffer buff1 = buffers.get((int)(idx1) / bufferSize);
final int o1 = (int) (idx1) % bufferSize;
LongBuffer buff1 = buffers.get((int)(idx1 / bufferSize));
final int o1 = (int) (idx1 % bufferSize);
LongBuffer buff2 = buffers.get((int)(idx2) / bufferSize);
final int o2 = (int) (idx2) % bufferSize;
LongBuffer buff2 = buffers.get((int)(idx2 / bufferSize));
final int o2 = (int) (idx2 % bufferSize);
long tmp = buff1.get(o1);
buff1.put(o1, buff2.get(o2));
@ -436,13 +436,13 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice {
}
if (i0 != iN) {
int startBuf0 = (int) ((destOffset) % bufferSize) * 8;
int endBuf0 = buffers[0].capacity() - (int) ((destOffset) % bufferSize) * 8;
int endBuf0 = buffers[0].capacity() - (int) (destOffset % bufferSize) * 8;
int endBufN = (int)((destOffset + length) % bufferSize)*8;
buffers[0] = buffers[0].slice(startBuf0, endBuf0);
buffers[numBuffers-1] = buffers[numBuffers-1].slice(0, endBufN);
}
else {
buffers[0] = buffers[0].slice((int) ((destOffset) % bufferSize) * 8, 8*length);
buffers[0] = buffers[0].slice((int) (destOffset % bufferSize) * 8, 8*length);
}
sourceChannel.position(sourceStart*8);
@ -465,13 +465,14 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice {
long low = 0;
long high = n - 1;
if (fromIndex/bufferSize == (fromIndex+step*n)/bufferSize) {
if (isSameBuffer(fromIndex, fromIndex+step*n)) {
int idx = (int)(fromIndex / bufferSize);
var buffer = buffers.get(idx);
while (low <= high) {
long mid = (low + high) >>> 1;
long off = fromIndex + mid*step;
long midVal = buffers.get(idx).get((int)(off % bufferSize)) & mask;
long midVal = buffer.get((int)(off % bufferSize)) & mask;
if (midVal < key)
low = mid + 1;
@ -507,13 +508,14 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice {
long low = 0;
long high = n - 1;
if (fromIndex/bufferSize == (fromIndex+n)/bufferSize) {
if (isSameBuffer(fromIndex, fromIndex+n)) {
int idx = (int)(fromIndex / bufferSize);
var buffer = buffers.get(idx);
while (low <= high) {
long mid = (low + high) >>> 1;
long off = fromIndex + mid;
long midVal = buffers.get(idx).get((int)(off % bufferSize)) & mask;
long midVal = buffer.get((int)(off % bufferSize)) & mask;
if (midVal < key)
low = mid + 1;
@ -551,13 +553,15 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice {
long low = 0;
long high = n - 1;
if (fromIndex/bufferSize == (fromIndex+n)/bufferSize) {
if (isSameBuffer(fromIndex, fromIndex+n)) {
int idx = (int)(fromIndex / bufferSize);
var buffer = buffers.get(idx);
while (low <= high) {
long mid = (low + high) >>> 1;
long off = fromIndex + mid;
long midVal = buffers.get(idx).get((int)(off % bufferSize));
long midVal = buffer.get((int)(off % bufferSize));
if (midVal < key)
low = mid + 1;
@ -594,13 +598,15 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice {
long low = 0;
long high = n - 1;
if (fromIndex/bufferSize == (fromIndex+n)/bufferSize) {
if (isSameBuffer(fromIndex, fromIndex+n)) {
int idx = (int)(fromIndex / bufferSize);
var buffer = buffers.get(idx);
while (low <= high) {
long mid = (low + high) >>> 1;
long off = fromIndex + mid;
long midVal = buffers.get(idx).get((int)(off % bufferSize));
long midVal = buffer.get((int)(off % bufferSize));
if (midVal < key)
low = mid + 1;
@ -716,11 +722,13 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice {
final var buffer = buffers.get((int) (low / bufferSize));
int pivotPoint = (int) ((low + high) / (2L*wordSize)) * wordSize % bufferSize;
long pivot = buffer.get(pivotPoint);
final long pivotPointLong = ((low + high) / (2L*wordSize)) * wordSize;
final int pivotPoint = (int) (pivotPointLong % bufferSize);
int j = (int) (high) % bufferSize + wordSize;
int i = (int) (low) % bufferSize - wordSize;
final long pivot = buffer.get(pivotPoint);
int j = (int) (high % bufferSize) + wordSize;
int i = (int) (low % bufferSize) - wordSize;
long j0 = high + wordSize - j;

2
marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSorter.java
View File

@ -38,6 +38,7 @@ public class MultimapSorter {
mergeSort(start, (int) (end - start));
}
if (MultimapSorter.class.desiredAssertionStatus()) {
for (long lp = start + wordSize; lp < end; lp += wordSize) {
if (multimapFileLong.get(lp - wordSize) > multimapFileLong.get(lp)) {
@ -50,6 +51,7 @@ public class MultimapSorter {
}
}
}
}
public void mergeSort(long start, int lengthLongs) throws IOException {
if (lengthLongs == 1)

1
marginalia_nu/src/main/java/nu/marginalia/wmsa/client/GsonFactory.java
View File

@ -24,6 +24,7 @@ public class GsonFactory {
.registerTypeAdapter(EdgeDomain.class, (JsonDeserializer<EdgeDomain>) (json, typeOfT, context) -> new EdgeDomain(json.getAsString()))
.registerTypeAdapter(EdgeId.class, (JsonDeserializer<EdgeId<?>>) (json, typeOfT, context) -> new EdgeId<>(json.getAsInt()))
.registerTypeAdapter(EdgeId.class, (JsonSerializer<EdgeId<?>>) (src, typeOfSrc, context) -> new JsonPrimitive(src.id()))
.serializeSpecialFloatingPointValues()
.create();
}
}

7
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/UpdateDomainStatistics.java
View File

@ -2,6 +2,7 @@ package nu.marginalia.wmsa.edge.converting;
import com.zaxxer.hikari.HikariDataSource;
import gnu.trove.map.hash.TIntIntHashMap;
import nu.marginalia.wmsa.configuration.module.DatabaseModule;
import java.sql.SQLException;
@ -12,6 +13,10 @@ public class UpdateDomainStatistics {
this.dataSource = dataSource;
}
public static void main(String... args) throws SQLException {
new UpdateDomainStatistics(new DatabaseModule().provideConnection()).run();
}
public void run() throws SQLException {
// This looks weird, but it's actually much faster than doing the computations with SQL queries
@ -22,7 +27,7 @@ public class UpdateDomainStatistics {
try (var conn = dataSource.getConnection();
var stmt = conn.createStatement();
var domainInfoQuery = conn.prepareStatement("SELECT DOMAIN_ID, VISITED, STATE='ok' FROM EC_URL");
var insertDomainInfo = conn.prepareStatement("INSERT INTO DOMAIN_METADATA(ID,KNOWN_URLS,GOOD_URLS,VISITED_URLS) VALUES (?, ?, ?, ?)")
var insertDomainInfo = conn.prepareStatement("INSERT INTO DOMAIN_METADATA(ID,KNOWN_URLS,VISITED_URLS,GOOD_URLS) VALUES (?, ?, ?, ?)")
) {
stmt.executeUpdate("DELETE FROM DOMAIN_METADATA");

6
marginalia_nu/src/main/resources/static/edge/style-new.css
View File

@ -647,6 +647,12 @@ a.underline {
background-color: #222;
color: #fff;
}
.query-samples dt {
color :#000;
}
.query-samples dd sample {
color: #ee8 !important;
}
}