#!/usr/bin/env node const fs = require('fs'); const path = require('path'); const KindAwareContentChecker = require(path.join(__dirname, 'kind-aware-content-checker.js')); const contentChecker = new KindAwareContentChecker(); // Configuration const config = { fastReplyThreshold: 3, // 3 seconds maxRepliesPerMinute: 50, maxRepliesPerHour: 400, recentEventWindow: 3600, // 1 hour in seconds contentSimilarityThreshold: 0.8, relayUrl: "freelay.sovbit.host", useBlacklistWhitelist: false, globalMaxEventsPerMinute: 50, globalMaxEventsPerHour: 600, newPubkeyReplyThreshold: 60, newPubkeyMaxPostsIn5Min: 10, // Kind-specific limits with new burst handling kindSpecificLimits: { 0: { maxPerHour: 10, // Profile updates minTimeBetween: 300 // 5 minutes between updates }, 1: { // Regular posts maxPerMinute: 30, maxPerHour: 200 }, 3: { // Follow lists maxPerMinute: 120, // Allow more frequent updates for bulk operations maxPerHour: 360, // One update every 10 seconds average minTimeBetween: 2, // Reduced from 30 to 2 seconds ignoreSimilarity: true, // Don't check content similarity burstAllowance: { // New burst handling maxBurst: 500, // Maximum events in burst burstWindow: 300, // Window for burst (5 minutes) cooldownPeriod: 3600 // Cooldown after burst (1 hour) } }, 7: { // Reactions maxPerMinute: 60, maxPerHour: 300, ignoreSimilarity: true }, 10002: { maxPerHour: 12, // Relay List updates minTimeBetween: 300 // 5 minutes between updates } }, // Enhanced reputation system config reputationConfig: { initialScore: 100, goodEventBonus: 1, spamPenalty: -15, recoveryRate: 3, minScore: -100, maxScore: 1000, trustThreshold: 50, blockThreshold: -50, blockRecoveryThreshold: -25, penaltyScaling: { 0: -15, // Normal penalty above 0 reputation "-25": -20, // Harsher penalty below -25 "-50": -25 // Even harsher below -50 }, recoveryScaling: { 0: 3, // Normal recovery above 0 "-25": 2, // Slower recovery below -25 "-50": 1 // Very slow recovery below -50 } }, // Cleanup settings cleanupInterval: 300, // 5 minutes statsRetentionPeriod: 86400, // 24 hours allowedKinds: [3, 5, 10001, 10002, 30311], // Kinds that bypass content similarity check bypassAllChecksKinds: [38383] }; // In-memory stores const recentEvents = []; const authorStats = new Map(); const authorReputation = new Map(); const newPubkeyPostCount = new Map(); const userViolationHistory = new Map(); const lastKindUpdates = new Map(); // Track last update time per kind per user function log(message) { console.error(`[${new Date().toISOString()}] ${message}`); } function logWithReputation(pubkey, message) { const reputation = getAuthorReputation(pubkey).score; log(`${message} (reputation: ${reputation.toFixed(2)})`); } function recordViolation(pubkey, eventKind, reason, penalty) { if (!userViolationHistory.has(pubkey)) { userViolationHistory.set(pubkey, []); } const history = userViolationHistory.get(pubkey); const currentReputation = getAuthorReputation(pubkey).score; history.push({ timestamp: Date.now(), eventKind: eventKind, reason: reason, penalty: penalty, currentReputation: currentReputation }); logWithReputation(pubkey, `Violation recorded for kind ${eventKind}: ${reason} (penalty: ${penalty})`); if (history.length > 50) { history.shift(); // Keep only last 50 violations } } function getAuthorReputation(pubkey) { if (!authorReputation.has(pubkey)) { authorReputation.set(pubkey, { score: config.reputationConfig.initialScore, lastUpdate: Date.now() }); log(`New reputation created for ${pubkey} with initial score ${config.reputationConfig.initialScore}`); } return authorReputation.get(pubkey); } function updateReputation(pubkey, isSpam, eventKind, spamReason) { const reputation = getAuthorReputation(pubkey); const oldScore = reputation.score; const hoursSinceLastUpdate = (Date.now() - reputation.lastUpdate) / (1000 * 60 * 60); let recoveryRate = config.reputationConfig.recoveryRate; for (const [threshold, rate] of Object.entries(config.reputationConfig.recoveryScaling) .sort((a, b) => Number(b[0]) - Number(a[0]))) { if (reputation.score <= Number(threshold)) { recoveryRate = rate; break; } } reputation.score = Math.min( config.reputationConfig.maxScore, reputation.score + (hoursSinceLastUpdate * recoveryRate) ); if (isSpam) { let penalty = config.reputationConfig.spamPenalty; // Reduce penalty for follow list violations if (eventKind === 3) { penalty = penalty / 2; } for (const [threshold, pen] of Object.entries(config.reputationConfig.penaltyScaling) .sort((a, b) => Number(b[0]) - Number(a[0]))) { if (reputation.score <= Number(threshold)) { penalty = pen; break; } } reputation.score += penalty; recordViolation(pubkey, eventKind, spamReason, penalty); } else { reputation.score += config.reputationConfig.goodEventBonus; } reputation.score = Math.max( config.reputationConfig.minScore, Math.min(config.reputationConfig.maxScore, reputation.score) ); reputation.lastUpdate = Date.now(); // Log reputation changes const change = reputation.score - oldScore; if (change !== 0) { log(`Reputation update for ${pubkey}: ${oldScore.toFixed(2)} -> ${reputation.score.toFixed(2)} (change: ${change >= 0 ? '+' : ''}${change.toFixed(2)})`); } return reputation.score; } function canUserPost(pubkey, eventKind) { const reputation = getAuthorReputation(pubkey); // Special case for follow lists - be more lenient if (eventKind === 3 && reputation.score > -75) { logWithReputation(pubkey, `Follow list exception applied for kind 3`); return true; } if (reputation.score <= config.reputationConfig.blockThreshold) { if (reputation.score < config.reputationConfig.blockRecoveryThreshold) { const history = userViolationHistory.get(pubkey) || []; const recentViolations = history .slice(-5) .map(v => `\n - ${new Date(v.timestamp).toISOString()}: kind ${v.eventKind}, ${v.reason} (penalty: ${v.penalty})`) .join(''); log(`User ${pubkey} blocked: reputation (${reputation.score.toFixed(2)}) below recovery threshold Recent violations:${recentViolations || '\n No recent violations recorded'}`); return false; } } return true; } function updateAuthorStats(pubkey, timestamp, kind) { if (!authorStats.has(pubkey)) { authorStats.set(pubkey, { lastReply: Date.now(), repliesLastMinute: 0, repliesLastHour: 0, lastReset: Date.now(), kinds: new Map(), burstTracking: new Map() }); logWithReputation(pubkey, `New author stats created`); } const stats = authorStats.get(pubkey); const now = Date.now(); // Initialize burst tracking for kind 3 if (kind === 3 && !stats.burstTracking.has(kind)) { stats.burstTracking.set(kind, { burstCount: 0, burstStart: now, lastBurstEnd: 0 }); logWithReputation(pubkey, `Initialized burst tracking for kind 3`); } if (now - stats.lastReset >= 3600000) { stats.repliesLastHour = 0; stats.repliesLastMinute = 0; stats.lastReset = now; stats.kinds.clear(); logWithReputation(pubkey, `Stats reset (hourly)`); } else if (now - stats.lastReset >= 60000) { stats.repliesLastMinute = 0; logWithReputation(pubkey, `Stats reset (minute)`); } if (!stats.kinds.has(kind)) { stats.kinds.set(kind, { hourly: 0, lastReset: now }); } const kindStats = stats.kinds.get(kind); if (now - kindStats.lastReset >= 3600000) { kindStats.hourly = 0; kindStats.lastReset = now; } kindStats.hourly++; stats.repliesLastMinute++; stats.repliesLastHour++; stats.lastReply = now; logWithReputation(pubkey, `Stats updated - minute: ${stats.repliesLastMinute}, hour: ${stats.repliesLastHour}, kind ${kind}: ${kindStats.hourly}`); } function isRateLimitExceeded(pubkey, kind) { const stats = authorStats.get(pubkey); if (!stats) return false; const now = Date.now(); const kindLimits = config.kindSpecificLimits[kind] || { maxPerMinute: config.maxRepliesPerMinute, maxPerHour: config.maxRepliesPerHour }; // Special handling for follow list bursts if (kind === 3 && kindLimits.burstAllowance) { const burstTracking = stats.burstTracking.get(kind); if (burstTracking) { const { burstCount, burstStart, lastBurstEnd } = burstTracking; const { maxBurst, burstWindow, cooldownPeriod } = kindLimits.burstAllowance; // Check if we're in cooldown period if (lastBurstEnd > 0 && now - lastBurstEnd < cooldownPeriod * 1000) { logWithReputation(pubkey, `In burst cooldown period for kind 3`); return isNormalRateLimitExceeded(stats, kindLimits); } // Check if we're in an active burst else if (now - burstStart < burstWindow * 1000) { if (burstCount >= maxBurst) { burstTracking.lastBurstEnd = now; burstTracking.burstCount = 0; logWithReputation(pubkey, `Burst limit reached (${maxBurst} events), entering cooldown`); return true; } burstTracking.burstCount++; logWithReputation(pubkey, `In active burst - count: ${burstCount + 1}`); return false; } // Start new burst else { burstTracking.burstStart = now; burstTracking.burstCount = 1; logWithReputation(pubkey, `Starting new burst for kind 3`); return false; } } } // Check minimum time between updates if specified if (kindLimits.minTimeBetween) { const lastUpdate = lastKindUpdates.get(`${pubkey}:${kind}`) || 0; if (now - lastUpdate < kindLimits.minTimeBetween * 1000) { logWithReputation(pubkey, `Too frequent updates for kind ${kind}`); return true; } } return isNormalRateLimitExceeded(stats, kindLimits); } function isNormalRateLimitExceeded(stats, kindLimits) { const reputation = getAuthorReputation(stats.pubkey); const reputationMultiplier = Math.max(0.5, Math.min(2, reputation.score / 100)); if (kindLimits.maxPerMinute) { const adjustedMinuteLimit = kindLimits.maxPerMinute * reputationMultiplier; if (stats.repliesLastMinute > adjustedMinuteLimit) { logWithReputation(stats.pubkey, `Per-minute rate limit exceeded: ${stats.repliesLastMinute} > ${adjustedMinuteLimit} (base: ${kindLimits.maxPerMinute}, multiplier: ${reputationMultiplier.toFixed(2)})`); return true; } } if (kindLimits.maxPerHour) { const adjustedHourLimit = kindLimits.maxPerHour * reputationMultiplier; if (stats.repliesLastHour > adjustedHourLimit) { logWithReputation(stats.pubkey, `Hourly rate limit exceeded: ${stats.repliesLastHour} > ${adjustedHourLimit} (base: ${kindLimits.maxPerHour}, multiplier: ${reputationMultiplier.toFixed(2)})`); return true; } } return false; } function isFastReply(event, recentEvents) { if (event.kind !== 1) return false; const replyTo = event.tags.find(tag => tag[0] === 'e'); if (replyTo) { const originalEvent = recentEvents.find(e => e.id === replyTo[1]); if (originalEvent && event.created_at - originalEvent.created_at <= config.fastReplyThreshold) { logWithReputation(event.pubkey, `Fast reply detected: ${event.created_at - originalEvent.created_at} seconds to ${originalEvent.id}`); return true; } } return false; } function isContentCopy(event, recentEvents) { // Get kind-specific limits const kindLimits = config.kindSpecificLimits[event.kind]; if (kindLimits && kindLimits.ignoreSimilarity) { return false; } const checkResult = contentChecker.checkContent(event, recentEvents); if (checkResult.isSpam) { logWithReputation(event.pubkey, `Content spam detected for kind ${event.kind}: ${checkResult.reason}`); return true; } return false; } function isNewPubkeySpam(event) { const isNewPubkey = !authorStats.has(event.pubkey); if (isNewPubkey) { const replyTo = event.tags.find(tag => tag[0] === 'e'); if (replyTo) { const originalEvent = recentEvents.find(e => e.id === replyTo[1]); if (originalEvent && event.created_at - originalEvent.created_at <= config.newPubkeyReplyThreshold) { logWithReputation(event.pubkey, `New pubkey replied too quickly to ${originalEvent.id}`); return true; } } } return false; } function isRapidNewPubkeyPosting(event) { // Skip rapid posting check for standardized kinds if (contentChecker.standardizedContentKinds.has(event.kind)) { return false; } if (authorStats.has(event.pubkey)) { return false; // This is not a new pubkey } const now = Math.floor(Date.now() / 1000); const count = newPubkeyPostCount.get(event.pubkey) || 0; newPubkeyPostCount.set(event.pubkey, count + 1); if (count >= config.newPubkeyMaxPostsIn5Min && now - event.created_at <= 300) { logWithReputation(event.pubkey, `Rapid new pubkey posting detected: ${count + 1} posts in 5 minutes`); return true; } if (now - event.created_at > 300) { newPubkeyPostCount.delete(event.pubkey); } return false; } function isBotBehavior(event) { // Skip bot behavior check for standardized kinds if (contentChecker.standardizedContentKinds.has(event.kind)) { return false; } const isBotLike = event.content.endsWith(config.relayUrl); if (isBotLike) { logWithReputation(event.pubkey, `Bot behavior detected in event ${event.id}`); } return isBotLike; } function isSpam(event, recentEvents) { const pubkey = event.pubkey; logWithReputation(pubkey, `Checking event ${event.id} (kind: ${event.kind})`); if (config.bypassAllChecksKinds.includes(event.kind)) { logWithReputation(pubkey, `Bypassing all checks for kind ${event.kind}`); return false; } if (!canUserPost(pubkey, event.kind)) { return { isSpam: true, reason: 'User reputation too low to post' }; } if (isRateLimitExceeded(pubkey, event.kind)) { return { isSpam: true, reason: `Rate limit exceeded for kind ${event.kind}` }; } // Special handling for follow lists (kind 3) if (event.kind === 3) { const checkResult = contentChecker.checkContent(event, recentEvents); if (checkResult.isSpam) { return { isSpam: true, reason: checkResult.reason }; } logWithReputation(pubkey, `Follow list check passed`); return { isSpam: false }; } if (isNewPubkeySpam(event)) { return { isSpam: true, reason: 'New pubkey replying too quickly' }; } if (isRapidNewPubkeyPosting(event)) { return { isSpam: true, reason: 'Rapid-fire posting from new pubkey' }; } if (isFastReply(event, recentEvents)) { return { isSpam: true, reason: 'Fast reply to recent event' }; } if (isContentCopy(event, recentEvents)) { return { isSpam: true, reason: 'Content copy detected' }; } if (isBotBehavior(event)) { return { isSpam: true, reason: 'Bot-like behavior detected' }; } logWithReputation(pubkey, `Event ${event.id} passed all spam checks`); return { isSpam: false }; } function periodicCleanup() { const now = Date.now(); let cleanupCount = 0; // Clean up old events const oldEventCount = recentEvents.length; while (recentEvents.length > 0 && (now - recentEvents[recentEvents.length - 1].created_at * 1000) > config.recentEventWindow * 1000) { recentEvents.pop(); cleanupCount++; } // Clean up old author stats let statsCleanupCount = 0; for (const [pubkey, stats] of authorStats.entries()) { if (now - stats.lastReply > config.statsRetentionPeriod * 1000) { authorStats.delete(pubkey); statsCleanupCount++; } } // Clean up old reputation data let reputationCleanupCount = 0; for (const [pubkey, rep] of authorReputation.entries()) { if (now - rep.lastUpdate > config.statsRetentionPeriod * 1000) { authorReputation.delete(pubkey); reputationCleanupCount++; } } // Clean up newPubkeyPostCount let newPubkeyCleanupCount = 0; for (const [pubkey, timestamp] of newPubkeyPostCount.entries()) { if (now - timestamp > 300000) { // 5 minutes newPubkeyPostCount.delete(pubkey); newPubkeyCleanupCount++; } } // Clean up old violation history let violationCleanupCount = 0; for (const [pubkey, history] of userViolationHistory.entries()) { if (history.length === 0 || now - history[history.length - 1].timestamp > config.statsRetentionPeriod * 1000) { userViolationHistory.delete(pubkey); violationCleanupCount++; } } // Clean up last kind updates let kindUpdateCleanupCount = 0; for (const [key, timestamp] of lastKindUpdates.entries()) { if (now - timestamp > config.statsRetentionPeriod * 1000) { lastKindUpdates.delete(key); kindUpdateCleanupCount++; } } // Log cleanup summary if anything was cleaned const totalCleanup = cleanupCount + statsCleanupCount + reputationCleanupCount + newPubkeyCleanupCount + violationCleanupCount + kindUpdateCleanupCount; if (totalCleanup > 0) { log(`Cleanup summary: Events: ${cleanupCount} (${recentEvents.length} remaining) Author stats: ${statsCleanupCount} (${authorStats.size} remaining) Reputation data: ${reputationCleanupCount} (${authorReputation.size} remaining) New pubkey counts: ${newPubkeyCleanupCount} (${newPubkeyPostCount.size} remaining) Violation histories: ${violationCleanupCount} (${userViolationHistory.size} remaining) Kind updates: ${kindUpdateCleanupCount} (${lastKindUpdates.size} remaining)`); } } // Set up periodic cleanup setInterval(periodicCleanup, config.cleanupInterval * 1000); // Set up stdin/stdout interface const rl = require('readline').createInterface({ input: process.stdin, output: process.stdout, terminal: false }); rl.on('line', (line) => { let req; try { req = JSON.parse(line); } catch (error) { log(`Error parsing JSON: ${error.message}`); return; } if (req.type !== 'new') { log("unexpected request type"); return; } let res = { id: req.event.id }; // Get initial reputation for comparison const initialReputation = getAuthorReputation(req.event.pubkey).score; // Update recent events recentEvents.push(req.event); recentEvents.sort((a, b) => b.created_at - a.created_at); // Update author stats updateAuthorStats(req.event.pubkey, req.event.created_at, req.event.kind); const spamCheck = isSpam(req.event, recentEvents); if (spamCheck.isSpam) { res.action = 'reject'; res.msg = `rejected: ${spamCheck.reason}`; const finalReputation = updateReputation(req.event.pubkey, true, req.event.kind, spamCheck.reason); log(`Rejected event ${req.event.id}: ${res.msg} (reputation: ${initialReputation.toFixed(2)} -> ${finalReputation.toFixed(2)})`); } else { res.action = 'accept'; const finalReputation = updateReputation(req.event.pubkey, false, req.event.kind, null); log(`Accepted event ${req.event.id} (reputation: ${initialReputation.toFixed(2)} -> ${finalReputation.toFixed(2)})`); } console.log(JSON.stringify(res)); }); log("Strfry filter started with config: " + JSON.stringify(config, null, 2));