Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(*) Clean up

Browse Source
This commit is contained in:
Viktor Lofgren 2024-08-14 11:43:45 +02:00
parent 623ee5570f
commit 2ad93ad41a
2 changed files with 0 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
code/common/model/java/nu/marginalia/util/QueryParams.java
View File

@ -10,7 +10,6 @@ import java.util.StringJoiner;
public class QueryParams {
@Nullable
public static String queryParamsSanitizer(String path, @Nullable String queryParams) {
if (queryParams == null) {

70
code/common/model/java/nu/marginalia/util/StringPool.java
View File

@ -1,70 +0,0 @@
package nu.marginalia.util;
import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Objects;
public class StringPool {
private final HashMap<String, String> words;
private final Object2LongOpenHashMap<String> ages;
private final int maxCap;
long idx;
private StringPool(int capacity, int maxCap) {
this.ages = new Object2LongOpenHashMap<>(capacity);
this.words = new HashMap<>(capacity);
this.maxCap = maxCap;
}
public static StringPool create(int capacity) {
return new StringPool(capacity, capacity * 10);
}
public String internalize(String str) {
prune();
final String ret = words.putIfAbsent(str, str);
ages.put(ret, idx++);
return Objects.requireNonNullElse(ret, str);
}
public String[] internalize(String[] str) {
for (int i = 0; i < str.length; i++) {
str[i] = internalize(str[i]);
}
return str;
}
public void prune() {
if (words.size() < maxCap)
return;
long[] ageValues = ages.values().toLongArray();
Arrays.sort(ageValues);
long cutoff = ageValues[ageValues.length - maxCap / 10];
words.clear();
ages.forEach((word, cnt) -> {
if (cnt >= cutoff) {
words.put(word, word);
}
});
ages.clear();
words.forEach((w,w2) -> {
ages.put(w, idx);
});
}
public void flush() {
words.clear();
}
}