From a126ff0e4f66f4e286b03b21ade9ef3cd9329e94 Mon Sep 17 00:00:00 2001 From: Enki Date: Thu, 19 Dec 2024 22:28:30 +0000 Subject: [PATCH] Better logging update Also added kind 10050 to default whitelist. Noticed it was triggering false spam reports. --- relay-spam-filter.js | 104 +++++++++++++++++++++++++++++-------------- 1 file changed, 71 insertions(+), 33 deletions(-) diff --git a/relay-spam-filter.js b/relay-spam-filter.js index 3d9292d..685d43c 100644 --- a/relay-spam-filter.js +++ b/relay-spam-filter.js @@ -69,11 +69,32 @@ const recentEvents = []; const authorStats = new Map(); const authorReputation = new Map(); const newPubkeyPostCount = new Map(); +const userViolationHistory = new Map(); // Track violation history function log(message) { console.error(`[${new Date().toISOString()}] ${message}`); } +function recordViolation(pubkey, eventKind, reason, penalty) { + if (!userViolationHistory.has(pubkey)) { + userViolationHistory.set(pubkey, []); + } + + const history = userViolationHistory.get(pubkey); + history.push({ + timestamp: Date.now(), + eventKind: eventKind, + reason: reason, + penalty: penalty, + currentReputation: getAuthorReputation(pubkey).score + }); + + // Keep only last 50 violations + if (history.length > 50) { + history.shift(); + } +} + function getAuthorReputation(pubkey) { if (!authorReputation.has(pubkey)) { authorReputation.set(pubkey, { @@ -84,7 +105,7 @@ function getAuthorReputation(pubkey) { return authorReputation.get(pubkey); } -function updateReputation(pubkey, isSpam) { +function updateReputation(pubkey, isSpam, eventKind, spamReason) { const reputation = getAuthorReputation(pubkey); const hoursSinceLastUpdate = (Date.now() - reputation.lastUpdate) / (1000 * 60 * 60); @@ -115,6 +136,9 @@ function updateReputation(pubkey, isSpam) { } } reputation.score += penalty; + + // Record the violation + recordViolation(pubkey, eventKind, spamReason, penalty); } else { reputation.score += config.reputationConfig.goodEventBonus; } @@ -128,13 +152,20 @@ function updateReputation(pubkey, isSpam) { return reputation.score; } -function canUserPost(pubkey) { +function canUserPost(pubkey, eventKind) { const reputation = getAuthorReputation(pubkey); - // If below block threshold, check if they've recovered enough if (reputation.score <= config.reputationConfig.blockThreshold) { if (reputation.score < config.reputationConfig.blockRecoveryThreshold) { - log(`User ${pubkey} blocked from posting: reputation (${reputation.score.toFixed(2)}) below recovery threshold`); + // Get violation history + const history = userViolationHistory.get(pubkey) || []; + const recentViolations = history + .slice(-5) // Get last 5 violations + .map(v => `\n - ${new Date(v.timestamp).toISOString()}: kind ${v.eventKind}, ${v.reason} (penalty: ${v.penalty})`) + .join(''); + + log(`User ${pubkey} blocked from posting: reputation (${reputation.score.toFixed(2)}) below recovery threshold +Recent violations:${recentViolations || '\n No recent violations recorded'}`); return false; } } @@ -211,7 +242,7 @@ function isRateLimitExceeded(pubkey, kind) { (kindStats && kindStats.hourly > kindLimits.maxPerHour); if (exceeded) { - log(`Rate limit exceeded for ${pubkey} (reputation: ${reputation.score.toFixed(2)})`); + log(`Rate limit exceeded for ${pubkey} (reputation: ${reputation.score.toFixed(2)}, kind: ${kind})`); } return exceeded; @@ -222,7 +253,7 @@ function isFastReply(event, recentEvents) { if (replyTo) { const originalEvent = recentEvents.find(e => e.id === replyTo[1]); if (originalEvent && event.created_at - originalEvent.created_at <= config.fastReplyThreshold) { - log(`Fast reply detected: ${event.created_at - originalEvent.created_at} seconds`); + log(`Fast reply detected: ${event.created_at - originalEvent.created_at} seconds to ${originalEvent.id}`); return true; } } @@ -272,7 +303,7 @@ function isContentCopy(event, recentEvents) { if (recentEvent.pubkey !== event.pubkey) { const similarity = calculateSimilarity(recentEvent.content, event.content); if (similarity >= config.contentSimilarityThreshold) { - log(`Content copy detected: ${similarity.toFixed(2)} similarity with event ${recentEvent.id}`); + log(`Content copy detected: ${similarity.toFixed(2)} similarity between ${event.id} and ${recentEvent.id}`); return true; } } @@ -283,7 +314,7 @@ function isContentCopy(event, recentEvents) { function isBotBehavior(event) { const isBotLike = event.content.endsWith(config.relayUrl); if (isBotLike) { - log(`Bot behavior detected: content ends with relay URL`); + log(`Bot behavior detected in event ${event.id}: content ends with relay URL`); } return isBotLike; } @@ -295,7 +326,7 @@ function isNewPubkeySpam(event) { if (replyTo) { const originalEvent = recentEvents.find(e => e.id === replyTo[1]); if (originalEvent && event.created_at - originalEvent.created_at <= config.newPubkeyReplyThreshold) { - log(`New pubkey replied to recent event within ${config.newPubkeyReplyThreshold} seconds`); + log(`New pubkey ${event.pubkey} replied too quickly to ${originalEvent.id}`); return true; } } @@ -313,7 +344,7 @@ function isRapidNewPubkeyPosting(event) { newPubkeyPostCount.set(event.pubkey, count + 1); if (count >= config.newPubkeyMaxPostsIn5Min && now - event.created_at <= 300) { - log(`Rapid new pubkey posting detected: ${count} posts in 5 minutes`); + log(`Rapid new pubkey posting detected for ${event.pubkey}: ${count} posts in 5 minutes`); return true; } @@ -331,41 +362,35 @@ function isSpam(event, recentEvents) { if (config.bypassAllChecksKinds.includes(event.kind)) { log(`Bypassing all checks for kind ${event.kind}`); - return false; + return { isSpam: false }; } if (isRateLimitExceeded(pubkey, event.kind)) { - log(`Rate limit exceeded for pubkey ${pubkey}`); - return true; + return { isSpam: true, reason: `Rate limit exceeded for kind ${event.kind}` }; } if (isNewPubkeySpam(event)) { - log(`New pubkey spam detected`); - return true; + return { isSpam: true, reason: 'New pubkey replying too quickly' }; } if (isRapidNewPubkeyPosting(event)) { - log(`Rapid-fire posting from new pubkey detected`); - return true; + return { isSpam: true, reason: 'Rapid-fire posting from new pubkey' }; } if (isFastReply(event, recentEvents)) { - log(`Fast reply detected`); - return true; + return { isSpam: true, reason: 'Fast reply to recent event' }; } if (isContentCopy(event, recentEvents)) { - log(`Content copy detected`); - return true; + return { isSpam: true, reason: 'Content copy detected' }; } if (isBotBehavior(event)) { - log(`Bot behavior detected`); - return true; + return { isSpam: true, reason: 'Bot-like behavior detected' }; } log(`Event ${event.id} passed all spam checks`); - return false; + return { isSpam: false }; } function periodicCleanup() { @@ -397,6 +422,13 @@ function periodicCleanup() { newPubkeyPostCount.delete(pubkey); } } + + // Clean up old violation history + for (const [pubkey, history] of userViolationHistory.entries()) { + if (history.length === 0 || now - history[history.length - 1].timestamp > config.statsRetentionPeriod * 1000) { + userViolationHistory.delete(pubkey); + } + } } const rl = require('readline').createInterface({ @@ -410,7 +442,8 @@ setInterval(periodicCleanup, config.cleanupInterval * 1000); rl.on('line', (line) => { let req; - try {req = JSON.parse(line); + try { + req = JSON.parse(line); } catch (error) { log(`Error parsing JSON: ${error.message}`); return; @@ -424,10 +457,10 @@ rl.on('line', (line) => { let res = { id: req.event.id }; // Check if user is allowed to post before any other processing - if (!canUserPost(req.event.pubkey)) { + if (!canUserPost(req.event.pubkey, req.event.kind)) { res.action = 'reject'; res.msg = 'rejected: reputation too low to post'; - log(`Rejected event ${req.event.id}: reputation too low`); + log(`Rejected event ${req.event.id} (kind: ${req.event.kind}): reputation too low`); console.log(JSON.stringify(res)); return; } @@ -439,16 +472,21 @@ rl.on('line', (line) => { // Process event updateAuthorStats(req.event.pubkey, req.event.created_at, req.event.kind); - const isSpamEvent = isSpam(req.event, recentEvents); - const reputationScore = updateReputation(req.event.pubkey, isSpamEvent); + const spamCheck = isSpam(req.event, recentEvents); + const reputationScore = updateReputation( + req.event.pubkey, + spamCheck.isSpam, + req.event.kind, + spamCheck.reason + ); - if (isSpamEvent) { + if (spamCheck.isSpam) { res.action = 'reject'; - res.msg = 'rejected: spam or bot-like behavior detected'; - log(`Rejected event ${req.event.id}: ${res.msg} (reputation: ${reputationScore.toFixed(2)})`); + res.msg = `rejected: ${spamCheck.reason}`; + log(`Rejected event ${req.event.id} (kind: ${req.event.kind}): ${res.msg} (reputation: ${reputationScore.toFixed(2)})`); } else { res.action = 'accept'; - log(`Accepted event ${req.event.id} (reputation: ${reputationScore.toFixed(2)})`); + log(`Accepted event ${req.event.id} (kind: ${req.event.kind}, reputation: ${reputationScore.toFixed(2)})`); } console.log(JSON.stringify(res));