Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(converter) Penalize chatgpt content farm spam

Browse Source
This commit is contained in:
Viktor Lofgren 2024-01-03 17:04:38 +01:00
parent f599944942
commit 41a540a629
1 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/DocumentValuator.java
View File

@ -37,17 +37,31 @@ public class DocumentValuator {
} }
private double getChatGptContentFarmPenalty(Document parsedDocument) { private double getChatGptContentFarmPenalty(Document parsedDocument) {
// easily 90% of modern AI-authored content farm spam have this exact string in one of the headings // easily 90% of modern AI-authored content farm spam has these nonsense headers
boolean benefitsOf = false, keyBenefits = false, keyTakeaways = false;
outer:
for (String tagName : List.of("h1", "h2", "h3")) { for (String tagName : List.of("h1", "h2", "h3")) {
for (var elem : parsedDocument.getElementsByTag(tagName)) { for (var elem : parsedDocument.getElementsByTag(tagName)) {
if (elem.text().startsWith("Benefits of")) { if (benefitsOf && keyBenefits && keyTakeaways)
return 10; break outer;
}
String text = elem.text().toLowerCase();
benefitsOf = benefitsOf || text.startsWith("benefits of");
keyBenefits = keyBenefits || text.startsWith("key benefits");
keyTakeaways = keyTakeaways || text.startsWith("key takeaways");
} }
} }
return 0; double penalty = 0;
if (benefitsOf) penalty += 10;
if (keyBenefits) penalty += 5;
if (keyTakeaways) penalty += 5;
return penalty;
} }