Add relay-spam-filter.js
This commit is contained in:
parent
82bfcf60da
commit
1fbf260b09
457
relay-spam-filter.js
Normal file
457
relay-spam-filter.js
Normal file
@ -0,0 +1,457 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
|
||||
// Configuration
|
||||
const config = {
|
||||
fastReplyThreshold: 3, // 3 seconds
|
||||
maxRepliesPerMinute: 50,
|
||||
maxRepliesPerHour: 400,
|
||||
recentEventWindow: 3600, // 1 hour in seconds
|
||||
contentSimilarityThreshold: 0.8,
|
||||
relayUrl: "freelay.sovbit.host",
|
||||
|
||||
// Periodic cleanup intervals
|
||||
cleanupInterval: 300, // 5 minutes
|
||||
statsRetentionPeriod: 86400, // 24 hours
|
||||
|
||||
// Enhanced reputation system
|
||||
reputationConfig: {
|
||||
initialScore: 100,
|
||||
goodEventBonus: 1,
|
||||
spamPenalty: -15, // Base penalty
|
||||
recoveryRate: 3, // Base recovery rate
|
||||
minScore: -100, // Absolute minimum
|
||||
maxScore: 1000,
|
||||
trustThreshold: 50,
|
||||
blockThreshold: -50, // Start blocking at this point
|
||||
blockRecoveryThreshold: -25, // Must recover to this to post again
|
||||
|
||||
// Progressive penalties
|
||||
penaltyScaling: {
|
||||
0: -15, // Normal penalty above 0 reputation
|
||||
"-25": -20, // Harsher penalty below -25
|
||||
"-50": -25 // Even harsher below -50
|
||||
},
|
||||
|
||||
// Recovery scaling
|
||||
recoveryScaling: {
|
||||
0: 3, // Normal recovery above 0
|
||||
"-25": 2, // Slower recovery below -25
|
||||
"-50": 1 // Very slow recovery below -50
|
||||
}
|
||||
},
|
||||
|
||||
// Kind-specific limits
|
||||
kindSpecificLimits: {
|
||||
0: { // Profile updates
|
||||
maxPerHour: 10
|
||||
},
|
||||
3: { // Contacts
|
||||
maxPerHour: 5
|
||||
},
|
||||
1: { // Regular posts
|
||||
maxPerMinute: 30,
|
||||
maxPerHour: 200
|
||||
}
|
||||
},
|
||||
|
||||
// New user limits
|
||||
newPubkeyReplyThreshold: 60,
|
||||
newPubkeyMaxPostsIn5Min: 10,
|
||||
|
||||
allowedKinds: [3, 5, 10001, 10002, 30311, 10050],
|
||||
bypassAllChecksKinds: [38383],
|
||||
};
|
||||
|
||||
// In-memory stores with cleanup
|
||||
const recentEvents = [];
|
||||
const authorStats = new Map();
|
||||
const authorReputation = new Map();
|
||||
const newPubkeyPostCount = new Map();
|
||||
|
||||
function log(message) {
|
||||
console.error(`[${new Date().toISOString()}] ${message}`);
|
||||
}
|
||||
|
||||
function getAuthorReputation(pubkey) {
|
||||
if (!authorReputation.has(pubkey)) {
|
||||
authorReputation.set(pubkey, {
|
||||
score: config.reputationConfig.initialScore,
|
||||
lastUpdate: Date.now()
|
||||
});
|
||||
}
|
||||
return authorReputation.get(pubkey);
|
||||
}
|
||||
|
||||
function updateReputation(pubkey, isSpam) {
|
||||
const reputation = getAuthorReputation(pubkey);
|
||||
const hoursSinceLastUpdate = (Date.now() - reputation.lastUpdate) / (1000 * 60 * 60);
|
||||
|
||||
// Get appropriate recovery rate based on current score
|
||||
let recoveryRate = config.reputationConfig.recoveryRate;
|
||||
for (const [threshold, rate] of Object.entries(config.reputationConfig.recoveryScaling)
|
||||
.sort((a, b) => Number(b[0]) - Number(a[0]))) {
|
||||
if (reputation.score <= Number(threshold)) {
|
||||
recoveryRate = rate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply scaled recovery
|
||||
reputation.score = Math.min(
|
||||
config.reputationConfig.maxScore,
|
||||
reputation.score + (hoursSinceLastUpdate * recoveryRate)
|
||||
);
|
||||
|
||||
if (isSpam) {
|
||||
// Get appropriate penalty based on current score
|
||||
let penalty = config.reputationConfig.spamPenalty;
|
||||
for (const [threshold, pen] of Object.entries(config.reputationConfig.penaltyScaling)
|
||||
.sort((a, b) => Number(b[0]) - Number(a[0]))) {
|
||||
if (reputation.score <= Number(threshold)) {
|
||||
penalty = pen;
|
||||
break;
|
||||
}
|
||||
}
|
||||
reputation.score += penalty;
|
||||
} else {
|
||||
reputation.score += config.reputationConfig.goodEventBonus;
|
||||
}
|
||||
|
||||
reputation.score = Math.max(
|
||||
config.reputationConfig.minScore,
|
||||
Math.min(config.reputationConfig.maxScore, reputation.score)
|
||||
);
|
||||
|
||||
reputation.lastUpdate = Date.now();
|
||||
return reputation.score;
|
||||
}
|
||||
|
||||
function canUserPost(pubkey) {
|
||||
const reputation = getAuthorReputation(pubkey);
|
||||
|
||||
// If below block threshold, check if they've recovered enough
|
||||
if (reputation.score <= config.reputationConfig.blockThreshold) {
|
||||
if (reputation.score < config.reputationConfig.blockRecoveryThreshold) {
|
||||
log(`User ${pubkey} blocked from posting: reputation (${reputation.score.toFixed(2)}) below recovery threshold`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function getKindSpecificLimits(kind) {
|
||||
return config.kindSpecificLimits[kind] || {
|
||||
maxPerMinute: config.maxRepliesPerMinute,
|
||||
maxPerHour: config.maxRepliesPerHour
|
||||
};
|
||||
}
|
||||
|
||||
function updateAuthorStats(pubkey, timestamp, kind) {
|
||||
if (config.bypassAllChecksKinds.includes(kind)) return;
|
||||
|
||||
const now = Date.now();
|
||||
if (!authorStats.has(pubkey)) {
|
||||
authorStats.set(pubkey, {
|
||||
lastReply: now,
|
||||
repliesLastMinute: 0,
|
||||
repliesLastHour: 0,
|
||||
lastReset: now,
|
||||
kinds: new Map()
|
||||
});
|
||||
}
|
||||
|
||||
const stats = authorStats.get(pubkey);
|
||||
const timeSinceLastReset = now - stats.lastReset;
|
||||
|
||||
// Reset counters if needed
|
||||
if (timeSinceLastReset >= 3600000) { // 1 hour
|
||||
stats.repliesLastHour = 0;
|
||||
stats.repliesLastMinute = 0;
|
||||
stats.lastReset = now;
|
||||
stats.kinds.clear();
|
||||
} else if (timeSinceLastReset >= 60000) { // 1 minute
|
||||
stats.repliesLastMinute = 0;
|
||||
}
|
||||
|
||||
// Update kind-specific counters
|
||||
if (!stats.kinds.has(kind)) {
|
||||
stats.kinds.set(kind, { hourly: 0, lastReset: now });
|
||||
}
|
||||
const kindStats = stats.kinds.get(kind);
|
||||
if (now - kindStats.lastReset >= 3600000) {
|
||||
kindStats.hourly = 0;
|
||||
kindStats.lastReset = now;
|
||||
}
|
||||
kindStats.hourly++;
|
||||
|
||||
stats.repliesLastMinute++;
|
||||
stats.repliesLastHour++;
|
||||
stats.lastReply = now;
|
||||
}
|
||||
|
||||
function isRateLimitExceeded(pubkey, kind) {
|
||||
const stats = authorStats.get(pubkey);
|
||||
if (!stats) return false;
|
||||
|
||||
const kindLimits = getKindSpecificLimits(kind);
|
||||
const kindStats = stats.kinds.get(kind);
|
||||
|
||||
// Consider reputation in rate limiting
|
||||
const reputation = getAuthorReputation(pubkey);
|
||||
const reputationMultiplier = Math.max(0.5, Math.min(2, reputation.score / 100));
|
||||
|
||||
const adjustedMinuteLimit = kindLimits.maxPerMinute * reputationMultiplier;
|
||||
const adjustedHourLimit = kindLimits.maxPerHour * reputationMultiplier;
|
||||
|
||||
const exceeded =
|
||||
stats.repliesLastMinute > adjustedMinuteLimit ||
|
||||
stats.repliesLastHour > adjustedHourLimit ||
|
||||
(kindStats && kindStats.hourly > kindLimits.maxPerHour);
|
||||
|
||||
if (exceeded) {
|
||||
log(`Rate limit exceeded for ${pubkey} (reputation: ${reputation.score.toFixed(2)})`);
|
||||
}
|
||||
|
||||
return exceeded;
|
||||
}
|
||||
|
||||
function isFastReply(event, recentEvents) {
|
||||
const replyTo = event.tags.find(tag => tag[0] === 'e');
|
||||
if (replyTo) {
|
||||
const originalEvent = recentEvents.find(e => e.id === replyTo[1]);
|
||||
if (originalEvent && event.created_at - originalEvent.created_at <= config.fastReplyThreshold) {
|
||||
log(`Fast reply detected: ${event.created_at - originalEvent.created_at} seconds`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function calculateSimilarity(str1, str2) {
|
||||
const len1 = str1.length;
|
||||
const len2 = str2.length;
|
||||
const maxLength = Math.max(len1, len2);
|
||||
if (maxLength === 0) return 1.0;
|
||||
return (maxLength - levenshteinDistance(str1, str2)) / maxLength;
|
||||
}
|
||||
|
||||
function levenshteinDistance(str1, str2) {
|
||||
const m = str1.length;
|
||||
const n = str2.length;
|
||||
const dp = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0));
|
||||
|
||||
for (let i = 0; i <= m; i++) dp[i][0] = i;
|
||||
for (let j = 0; j <= n; j++) dp[0][j] = j;
|
||||
|
||||
for (let i = 1; i <= m; i++) {
|
||||
for (let j = 1; j <= n; j++) {
|
||||
if (str1[i - 1] === str2[j - 1]) {
|
||||
dp[i][j] = dp[i - 1][j - 1];
|
||||
} else {
|
||||
dp[i][j] = Math.min(
|
||||
dp[i - 1][j - 1],
|
||||
dp[i - 1][j],
|
||||
dp[i][j - 1]
|
||||
) + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return dp[m][n];
|
||||
}
|
||||
|
||||
function isContentCopy(event, recentEvents) {
|
||||
if (config.allowedKinds.includes(event.kind)) {
|
||||
log(`Skipping content copy check for allowed kind ${event.kind}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const recentEvent of recentEvents) {
|
||||
if (recentEvent.pubkey !== event.pubkey) {
|
||||
const similarity = calculateSimilarity(recentEvent.content, event.content);
|
||||
if (similarity >= config.contentSimilarityThreshold) {
|
||||
log(`Content copy detected: ${similarity.toFixed(2)} similarity with event ${recentEvent.id}`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isBotBehavior(event) {
|
||||
const isBotLike = event.content.endsWith(config.relayUrl);
|
||||
if (isBotLike) {
|
||||
log(`Bot behavior detected: content ends with relay URL`);
|
||||
}
|
||||
return isBotLike;
|
||||
}
|
||||
|
||||
function isNewPubkeySpam(event) {
|
||||
const isNewPubkey = !authorStats.has(event.pubkey);
|
||||
if (isNewPubkey) {
|
||||
const replyTo = event.tags.find(tag => tag[0] === 'e');
|
||||
if (replyTo) {
|
||||
const originalEvent = recentEvents.find(e => e.id === replyTo[1]);
|
||||
if (originalEvent && event.created_at - originalEvent.created_at <= config.newPubkeyReplyThreshold) {
|
||||
log(`New pubkey replied to recent event within ${config.newPubkeyReplyThreshold} seconds`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isRapidNewPubkeyPosting(event) {
|
||||
if (authorStats.has(event.pubkey)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const count = newPubkeyPostCount.get(event.pubkey) || 0;
|
||||
newPubkeyPostCount.set(event.pubkey, count + 1);
|
||||
|
||||
if (count >= config.newPubkeyMaxPostsIn5Min && now - event.created_at <= 300) {
|
||||
log(`Rapid new pubkey posting detected: ${count} posts in 5 minutes`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (now - event.created_at > 300) {
|
||||
newPubkeyPostCount.delete(event.pubkey);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function isSpam(event, recentEvents) {
|
||||
const pubkey = event.pubkey;
|
||||
|
||||
log(`Checking event ${event.id} from pubkey ${pubkey} (kind: ${event.kind})`);
|
||||
|
||||
if (config.bypassAllChecksKinds.includes(event.kind)) {
|
||||
log(`Bypassing all checks for kind ${event.kind}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isRateLimitExceeded(pubkey, event.kind)) {
|
||||
log(`Rate limit exceeded for pubkey ${pubkey}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isNewPubkeySpam(event)) {
|
||||
log(`New pubkey spam detected`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isRapidNewPubkeyPosting(event)) {
|
||||
log(`Rapid-fire posting from new pubkey detected`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isFastReply(event, recentEvents)) {
|
||||
log(`Fast reply detected`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isContentCopy(event, recentEvents)) {
|
||||
log(`Content copy detected`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isBotBehavior(event)) {
|
||||
log(`Bot behavior detected`);
|
||||
return true;
|
||||
}
|
||||
|
||||
log(`Event ${event.id} passed all spam checks`);
|
||||
return false;
|
||||
}
|
||||
|
||||
function periodicCleanup() {
|
||||
const now = Date.now();
|
||||
|
||||
// Clean up old events
|
||||
while (recentEvents.length > 0 &&
|
||||
(now - recentEvents[recentEvents.length - 1].created_at * 1000) > config.recentEventWindow * 1000) {
|
||||
recentEvents.pop();
|
||||
}
|
||||
|
||||
// Clean up old author stats
|
||||
for (const [pubkey, stats] of authorStats.entries()) {
|
||||
if (now - stats.lastReply > config.statsRetentionPeriod * 1000) {
|
||||
authorStats.delete(pubkey);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up old reputation data
|
||||
for (const [pubkey, rep] of authorReputation.entries()) {
|
||||
if (now - rep.lastUpdate > config.statsRetentionPeriod * 1000) {
|
||||
authorReputation.delete(pubkey);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up newPubkeyPostCount
|
||||
for (const [pubkey, timestamp] of newPubkeyPostCount.entries()) {
|
||||
if (now - timestamp > 300000) { // 5 minutes
|
||||
newPubkeyPostCount.delete(pubkey);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const rl = require('readline').createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
terminal: false
|
||||
});
|
||||
|
||||
// Set up periodic cleanup
|
||||
setInterval(periodicCleanup, config.cleanupInterval * 1000);
|
||||
|
||||
rl.on('line', (line) => {
|
||||
let req;
|
||||
try {req = JSON.parse(line);
|
||||
} catch (error) {
|
||||
log(`Error parsing JSON: ${error.message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.type !== 'new') {
|
||||
log("unexpected request type");
|
||||
return;
|
||||
}
|
||||
|
||||
let res = { id: req.event.id };
|
||||
|
||||
// Check if user is allowed to post before any other processing
|
||||
if (!canUserPost(req.event.pubkey)) {
|
||||
res.action = 'reject';
|
||||
res.msg = 'rejected: reputation too low to post';
|
||||
log(`Rejected event ${req.event.id}: reputation too low`);
|
||||
console.log(JSON.stringify(res));
|
||||
return;
|
||||
}
|
||||
|
||||
// Update recent events
|
||||
recentEvents.push(req.event);
|
||||
recentEvents.sort((a, b) => b.created_at - a.created_at);
|
||||
|
||||
// Process event
|
||||
updateAuthorStats(req.event.pubkey, req.event.created_at, req.event.kind);
|
||||
|
||||
const isSpamEvent = isSpam(req.event, recentEvents);
|
||||
const reputationScore = updateReputation(req.event.pubkey, isSpamEvent);
|
||||
|
||||
if (isSpamEvent) {
|
||||
res.action = 'reject';
|
||||
res.msg = 'rejected: spam or bot-like behavior detected';
|
||||
log(`Rejected event ${req.event.id}: ${res.msg} (reputation: ${reputationScore.toFixed(2)})`);
|
||||
} else {
|
||||
res.action = 'accept';
|
||||
log(`Accepted event ${req.event.id} (reputation: ${reputationScore.toFixed(2)})`);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(res));
|
||||
});
|
||||
|
||||
log("Strfry filter started with config: " + JSON.stringify(config, null, 2));
|
||||
Loading…
Reference in New Issue
Block a user