From c6abbc12f67fbcbcfd0f4b1f334fa8e948d5722c Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sat, 22 Oct 2022 15:01:41 +0200 Subject: [PATCH] fix serialization issue (#121) Co-authored-by: vlofgren Co-authored-by: vlofgren Reviewed-on: https://git.marginalia.nu/marginalia/marginalia.nu/pulls/121 --- .../util/multimap/MultimapFileLong.java | 44 +++++++++++-------- .../util/multimap/MultimapSorter.java | 16 ++++--- .../marginalia/wmsa/client/GsonFactory.java | 1 + .../converting/UpdateDomainStatistics.java | 7 ++- .../main/resources/static/edge/style-new.css | 6 +++ 5 files changed, 48 insertions(+), 26 deletions(-) diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLong.java b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLong.java index d3e1376d..a43e1694 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLong.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapFileLong.java @@ -376,11 +376,11 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice { } private void swap(long idx1, long idx2) { - LongBuffer buff1 = buffers.get((int)(idx1) / bufferSize); - final int o1 = (int) (idx1) % bufferSize; + LongBuffer buff1 = buffers.get((int)(idx1 / bufferSize)); + final int o1 = (int) (idx1 % bufferSize); - LongBuffer buff2 = buffers.get((int)(idx2) / bufferSize); - final int o2 = (int) (idx2) % bufferSize; + LongBuffer buff2 = buffers.get((int)(idx2 / bufferSize)); + final int o2 = (int) (idx2 % bufferSize); long tmp = buff1.get(o1); buff1.put(o1, buff2.get(o2)); @@ -436,13 +436,13 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice { } if (i0 != iN) { int startBuf0 = (int) ((destOffset) % bufferSize) * 8; - int endBuf0 = buffers[0].capacity() - (int) ((destOffset) % bufferSize) * 8; + int endBuf0 = buffers[0].capacity() - (int) (destOffset % bufferSize) * 8; int endBufN = (int)((destOffset + length) % bufferSize)*8; buffers[0] = buffers[0].slice(startBuf0, endBuf0); buffers[numBuffers-1] = buffers[numBuffers-1].slice(0, endBufN); } else { - buffers[0] = buffers[0].slice((int) ((destOffset) % bufferSize) * 8, 8*length); + buffers[0] = buffers[0].slice((int) (destOffset % bufferSize) * 8, 8*length); } sourceChannel.position(sourceStart*8); @@ -465,13 +465,14 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice { long low = 0; long high = n - 1; - if (fromIndex/bufferSize == (fromIndex+step*n)/bufferSize) { + if (isSameBuffer(fromIndex, fromIndex+step*n)) { int idx = (int)(fromIndex / bufferSize); + var buffer = buffers.get(idx); while (low <= high) { long mid = (low + high) >>> 1; long off = fromIndex + mid*step; - long midVal = buffers.get(idx).get((int)(off % bufferSize)) & mask; + long midVal = buffer.get((int)(off % bufferSize)) & mask; if (midVal < key) low = mid + 1; @@ -507,13 +508,14 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice { long low = 0; long high = n - 1; - if (fromIndex/bufferSize == (fromIndex+n)/bufferSize) { + if (isSameBuffer(fromIndex, fromIndex+n)) { int idx = (int)(fromIndex / bufferSize); + var buffer = buffers.get(idx); while (low <= high) { long mid = (low + high) >>> 1; long off = fromIndex + mid; - long midVal = buffers.get(idx).get((int)(off % bufferSize)) & mask; + long midVal = buffer.get((int)(off % bufferSize)) & mask; if (midVal < key) low = mid + 1; @@ -551,13 +553,15 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice { long low = 0; long high = n - 1; - if (fromIndex/bufferSize == (fromIndex+n)/bufferSize) { + if (isSameBuffer(fromIndex, fromIndex+n)) { int idx = (int)(fromIndex / bufferSize); + var buffer = buffers.get(idx); + while (low <= high) { long mid = (low + high) >>> 1; long off = fromIndex + mid; - long midVal = buffers.get(idx).get((int)(off % bufferSize)); + long midVal = buffer.get((int)(off % bufferSize)); if (midVal < key) low = mid + 1; @@ -594,13 +598,15 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice { long low = 0; long high = n - 1; - if (fromIndex/bufferSize == (fromIndex+n)/bufferSize) { + if (isSameBuffer(fromIndex, fromIndex+n)) { int idx = (int)(fromIndex / bufferSize); + var buffer = buffers.get(idx); + while (low <= high) { long mid = (low + high) >>> 1; long off = fromIndex + mid; - long midVal = buffers.get(idx).get((int)(off % bufferSize)); + long midVal = buffer.get((int)(off % bufferSize)); if (midVal < key) low = mid + 1; @@ -716,11 +722,13 @@ public class MultimapFileLong implements AutoCloseable, MultimapFileLongSlice { final var buffer = buffers.get((int) (low / bufferSize)); - int pivotPoint = (int) ((low + high) / (2L*wordSize)) * wordSize % bufferSize; - long pivot = buffer.get(pivotPoint); + final long pivotPointLong = ((low + high) / (2L*wordSize)) * wordSize; + final int pivotPoint = (int) (pivotPointLong % bufferSize); - int j = (int) (high) % bufferSize + wordSize; - int i = (int) (low) % bufferSize - wordSize; + final long pivot = buffer.get(pivotPoint); + + int j = (int) (high % bufferSize) + wordSize; + int i = (int) (low % bufferSize) - wordSize; long j0 = high + wordSize - j; diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSorter.java b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSorter.java index 88d873a3..abdeb52f 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSorter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/util/multimap/MultimapSorter.java @@ -38,15 +38,17 @@ public class MultimapSorter { mergeSort(start, (int) (end - start)); } - for (long lp = start + wordSize; lp < end; lp += wordSize) { - if (multimapFileLong.get(lp - wordSize) > multimapFileLong.get(lp)) { + if (MultimapSorter.class.desiredAssertionStatus()) { + for (long lp = start + wordSize; lp < end; lp += wordSize) { + if (multimapFileLong.get(lp - wordSize) > multimapFileLong.get(lp)) { - logger.error("Sort contract breached [{}:{} ({}), ws={}, ) (json, typeOfT, context) -> new EdgeDomain(json.getAsString())) .registerTypeAdapter(EdgeId.class, (JsonDeserializer>) (json, typeOfT, context) -> new EdgeId<>(json.getAsInt())) .registerTypeAdapter(EdgeId.class, (JsonSerializer>) (src, typeOfSrc, context) -> new JsonPrimitive(src.id())) + .serializeSpecialFloatingPointValues() .create(); } } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/UpdateDomainStatistics.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/UpdateDomainStatistics.java index 162c2d8b..428bd902 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/UpdateDomainStatistics.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/UpdateDomainStatistics.java @@ -2,6 +2,7 @@ package nu.marginalia.wmsa.edge.converting; import com.zaxxer.hikari.HikariDataSource; import gnu.trove.map.hash.TIntIntHashMap; +import nu.marginalia.wmsa.configuration.module.DatabaseModule; import java.sql.SQLException; @@ -12,6 +13,10 @@ public class UpdateDomainStatistics { this.dataSource = dataSource; } + public static void main(String... args) throws SQLException { + new UpdateDomainStatistics(new DatabaseModule().provideConnection()).run(); + } + public void run() throws SQLException { // This looks weird, but it's actually much faster than doing the computations with SQL queries @@ -22,7 +27,7 @@ public class UpdateDomainStatistics { try (var conn = dataSource.getConnection(); var stmt = conn.createStatement(); var domainInfoQuery = conn.prepareStatement("SELECT DOMAIN_ID, VISITED, STATE='ok' FROM EC_URL"); - var insertDomainInfo = conn.prepareStatement("INSERT INTO DOMAIN_METADATA(ID,KNOWN_URLS,GOOD_URLS,VISITED_URLS) VALUES (?, ?, ?, ?)") + var insertDomainInfo = conn.prepareStatement("INSERT INTO DOMAIN_METADATA(ID,KNOWN_URLS,VISITED_URLS,GOOD_URLS) VALUES (?, ?, ?, ?)") ) { stmt.executeUpdate("DELETE FROM DOMAIN_METADATA"); diff --git a/marginalia_nu/src/main/resources/static/edge/style-new.css b/marginalia_nu/src/main/resources/static/edge/style-new.css index 23da585b..20cf707f 100644 --- a/marginalia_nu/src/main/resources/static/edge/style-new.css +++ b/marginalia_nu/src/main/resources/static/edge/style-new.css @@ -647,6 +647,12 @@ a.underline { background-color: #222; color: #fff; } + .query-samples dt { + color :#000; + } + .query-samples dd sample { + color: #ee8 !important; + } }