stirfry-spam-filter/relay-spam-filter.js

#!/usr/bin/env node

const fs = require('fs');
const path = require('path');
const KindAwareContentChecker = require(path.join(__dirname, 'kind-aware-content-checker.js'));
const contentChecker = new KindAwareContentChecker();

// Configuration
const config = {
    fastReplyThreshold: 3, // 3 seconds
    maxRepliesPerMinute: 50,
    maxRepliesPerHour: 400,
    recentEventWindow: 3600, // 1 hour in seconds
    contentSimilarityThreshold: 0.8,
    relayUrl: "freelay.sovbit.host",
    useBlacklistWhitelist: false,
    
    globalMaxEventsPerMinute: 50,
    globalMaxEventsPerHour: 600,
    
    newPubkeyReplyThreshold: 60,
    newPubkeyMaxPostsIn5Min: 10,

    // Kind-specific limits with new burst handling
    kindSpecificLimits: {
        0: { 
            maxPerHour: 10,         // Profile updates
            minTimeBetween: 300     // 5 minutes between updates
        },
        1: {                        // Regular posts
            maxPerMinute: 30,
            maxPerHour: 200
        },
        3: {                        // Follow lists
            maxPerMinute: 120,      // Allow more frequent updates for bulk operations
            maxPerHour: 360,        // One update every 10 seconds average
            minTimeBetween: 2,      // Reduced from 30 to 2 seconds
            ignoreSimilarity: true,  // Don't check content similarity
            burstAllowance: {        // New burst handling
                maxBurst: 500,       // Maximum events in burst
                burstWindow: 300,    // Window for burst (5 minutes)
                cooldownPeriod: 3600 // Cooldown after burst (1 hour)
            }
        },
        7: {                        // Reactions
            maxPerMinute: 60,
            maxPerHour: 300,
            ignoreSimilarity: true
        },
        10002: { 
            maxPerHour: 12,         // Relay List updates
            minTimeBetween: 300     // 5 minutes between updates
        }
    },

    // Enhanced reputation system config
    reputationConfig: {
        initialScore: 100,
        goodEventBonus: 1,
        spamPenalty: -15,
        recoveryRate: 3,
        minScore: -100,
        maxScore: 1000,
        trustThreshold: 50,
        blockThreshold: -50,
        blockRecoveryThreshold: -25,
        
        penaltyScaling: {
            0: -15,    // Normal penalty above 0 reputation
            "-25": -20, // Harsher penalty below -25
            "-50": -25  // Even harsher below -50
        },
        
        recoveryScaling: {
            0: 3,      // Normal recovery above 0
            "-25": 2,  // Slower recovery below -25
            "-50": 1   // Very slow recovery below -50
        }
    },

    // Cleanup settings
    cleanupInterval: 300,           // 5 minutes
    statsRetentionPeriod: 86400,    // 24 hours
    
    allowedKinds: [3, 5, 10001, 10002, 30311], // Kinds that bypass content similarity check
    bypassAllChecksKinds: [38383]
};

// In-memory stores
const recentEvents = [];
const authorStats = new Map();
const authorReputation = new Map();
const newPubkeyPostCount = new Map();
const userViolationHistory = new Map();
const lastKindUpdates = new Map(); // Track last update time per kind per user

function log(message) {
    console.error(`[${new Date().toISOString()}] ${message}`);
}

function logWithReputation(pubkey, message) {
    const reputation = getAuthorReputation(pubkey).score;
    log(`${message} (reputation: ${reputation.toFixed(2)})`);
}

function recordViolation(pubkey, eventKind, reason, penalty) {
    if (!userViolationHistory.has(pubkey)) {
        userViolationHistory.set(pubkey, []);
    }
    
    const history = userViolationHistory.get(pubkey);
    const currentReputation = getAuthorReputation(pubkey).score;
    
    history.push({
        timestamp: Date.now(),
        eventKind: eventKind,
        reason: reason,
        penalty: penalty,
        currentReputation: currentReputation
    });

    logWithReputation(pubkey, `Violation recorded for kind ${eventKind}: ${reason} (penalty: ${penalty})`);

    if (history.length > 50) {
        history.shift(); // Keep only last 50 violations
    }
}

function getAuthorReputation(pubkey) {
    if (!authorReputation.has(pubkey)) {
        authorReputation.set(pubkey, {
            score: config.reputationConfig.initialScore,
            lastUpdate: Date.now()
        });
        log(`New reputation created for ${pubkey} with initial score ${config.reputationConfig.initialScore}`);
    }
    return authorReputation.get(pubkey);
}

function updateReputation(pubkey, isSpam, eventKind, spamReason) {
    const reputation = getAuthorReputation(pubkey);
    const oldScore = reputation.score;
    const hoursSinceLastUpdate = (Date.now() - reputation.lastUpdate) / (1000 * 60 * 60);
    
    let recoveryRate = config.reputationConfig.recoveryRate;
    for (const [threshold, rate] of Object.entries(config.reputationConfig.recoveryScaling)
        .sort((a, b) => Number(b[0]) - Number(a[0]))) {
        if (reputation.score <= Number(threshold)) {
            recoveryRate = rate;
            break;
        }
    }
    
    reputation.score = Math.min(
        config.reputationConfig.maxScore,
        reputation.score + (hoursSinceLastUpdate * recoveryRate)
    );
    
    if (isSpam) {
        let penalty = config.reputationConfig.spamPenalty;
        // Reduce penalty for follow list violations
        if (eventKind === 3) {
            penalty = penalty / 2;
        }
        
        for (const [threshold, pen] of Object.entries(config.reputationConfig.penaltyScaling)
            .sort((a, b) => Number(b[0]) - Number(a[0]))) {
            if (reputation.score <= Number(threshold)) {
                penalty = pen;
                break;
            }
        }
        reputation.score += penalty;
        recordViolation(pubkey, eventKind, spamReason, penalty);
    } else {
        reputation.score += config.reputationConfig.goodEventBonus;
    }
    
    reputation.score = Math.max(
        config.reputationConfig.minScore,
        Math.min(config.reputationConfig.maxScore, reputation.score)
    );
    
    reputation.lastUpdate = Date.now();

    // Log reputation changes
    const change = reputation.score - oldScore;
    if (change !== 0) {
        log(`Reputation update for ${pubkey}: ${oldScore.toFixed(2)} -> ${reputation.score.toFixed(2)} (change: ${change >= 0 ? '+' : ''}${change.toFixed(2)})`);
    }

    return reputation.score;
}

function canUserPost(pubkey, eventKind) {
    const reputation = getAuthorReputation(pubkey);
    
    // Special case for follow lists - be more lenient
    if (eventKind === 3 && reputation.score > -75) {
        logWithReputation(pubkey, `Follow list exception applied for kind 3`);
        return true;
    }
    
    if (reputation.score <= config.reputationConfig.blockThreshold) {
        if (reputation.score < config.reputationConfig.blockRecoveryThreshold) {
            const history = userViolationHistory.get(pubkey) || [];
            const recentViolations = history
                .slice(-5)
                .map(v => `\n    - ${new Date(v.timestamp).toISOString()}: kind ${v.eventKind}, ${v.reason} (penalty: ${v.penalty})`)
                .join('');

            log(`User ${pubkey} blocked: reputation (${reputation.score.toFixed(2)}) below recovery threshold
Recent violations:${recentViolations || '\n    No recent violations recorded'}`);
            return false;
        }
    }
    return true;
}

function updateAuthorStats(pubkey, timestamp, kind) {
    if (!authorStats.has(pubkey)) {
        authorStats.set(pubkey, {
            lastReply: Date.now(),
            repliesLastMinute: 0,
            repliesLastHour: 0,
            lastReset: Date.now(),
            kinds: new Map(),
            burstTracking: new Map()
        });
        logWithReputation(pubkey, `New author stats created`);
    }

    const stats = authorStats.get(pubkey);
    const now = Date.now();

    // Initialize burst tracking for kind 3
    if (kind === 3 && !stats.burstTracking.has(kind)) {
        stats.burstTracking.set(kind, {
            burstCount: 0,
            burstStart: now,
            lastBurstEnd: 0
        });
        logWithReputation(pubkey, `Initialized burst tracking for kind 3`);
    }

    if (now - stats.lastReset >= 3600000) {
        stats.repliesLastHour = 0;
        stats.repliesLastMinute = 0;
        stats.lastReset = now;
        stats.kinds.clear();
        logWithReputation(pubkey, `Stats reset (hourly)`);
    } else if (now - stats.lastReset >= 60000) {
        stats.repliesLastMinute = 0;
        logWithReputation(pubkey, `Stats reset (minute)`);
    }

    if (!stats.kinds.has(kind)) {
        stats.kinds.set(kind, { hourly: 0, lastReset: now });
    }
    
    const kindStats = stats.kinds.get(kind);
    if (now - kindStats.lastReset >= 3600000) {
        kindStats.hourly = 0;
        kindStats.lastReset = now;
    }
    kindStats.hourly++;

    stats.repliesLastMinute++;
    stats.repliesLastHour++;
    stats.lastReply = now;

    logWithReputation(pubkey, 
        `Stats updated - minute: ${stats.repliesLastMinute}, hour: ${stats.repliesLastHour}, kind ${kind}: ${kindStats.hourly}`);
}

function isRateLimitExceeded(pubkey, kind) {
    const stats = authorStats.get(pubkey);
    if (!stats) return false;

    const now = Date.now();
    const kindLimits = config.kindSpecificLimits[kind] || {
        maxPerMinute: config.maxRepliesPerMinute,
        maxPerHour: config.maxRepliesPerHour
    };
    
    // Special handling for follow list bursts
    if (kind === 3 && kindLimits.burstAllowance) {
        const burstTracking = stats.burstTracking.get(kind);
        if (burstTracking) {
            const { burstCount, burstStart, lastBurstEnd } = burstTracking;
            const { maxBurst, burstWindow, cooldownPeriod } = kindLimits.burstAllowance;

            // Check if we're in cooldown period
            if (lastBurstEnd > 0 && now - lastBurstEnd < cooldownPeriod * 1000) {
                logWithReputation(pubkey, `In burst cooldown period for kind 3`);
                return isNormalRateLimitExceeded(stats, kindLimits);
            }
            // Check if we're in an active burst
            else if (now - burstStart < burstWindow * 1000) {
                if (burstCount >= maxBurst) {
                    burstTracking.lastBurstEnd = now;
                    burstTracking.burstCount = 0;
                    logWithReputation(pubkey, `Burst limit reached (${maxBurst} events), entering cooldown`);
                    return true;
                }
                burstTracking.burstCount++;
                logWithReputation(pubkey, `In active burst - count: ${burstCount + 1}`);
                return false;
            }
            // Start new burst
            else {
                burstTracking.burstStart = now;
                burstTracking.burstCount = 1;
                logWithReputation(pubkey, `Starting new burst for kind 3`);
                return false;
            }
        }
    }

    // Check minimum time between updates if specified
    if (kindLimits.minTimeBetween) {
        const lastUpdate = lastKindUpdates.get(`${pubkey}:${kind}`) || 0;
        if (now - lastUpdate < kindLimits.minTimeBetween * 1000) {
            logWithReputation(pubkey, `Too frequent updates for kind ${kind}`);
            return true;
        }
    }

    return isNormalRateLimitExceeded(stats, kindLimits);
}

function isNormalRateLimitExceeded(stats, kindLimits) {
    const reputation = getAuthorReputation(stats.pubkey);
    const reputationMultiplier = Math.max(0.5, Math.min(2, reputation.score / 100));
    
    if (kindLimits.maxPerMinute) {
        const adjustedMinuteLimit = kindLimits.maxPerMinute * reputationMultiplier;
        if (stats.repliesLastMinute > adjustedMinuteLimit) {
            logWithReputation(stats.pubkey, 
                `Per-minute rate limit exceeded: ${stats.repliesLastMinute} > ${adjustedMinuteLimit} (base: ${kindLimits.maxPerMinute}, multiplier: ${reputationMultiplier.toFixed(2)})`);
            return true;
        }
    }

    if (kindLimits.maxPerHour) {
        const adjustedHourLimit = kindLimits.maxPerHour * reputationMultiplier;
        if (stats.repliesLastHour > adjustedHourLimit) {
            logWithReputation(stats.pubkey, 
                `Hourly rate limit exceeded: ${stats.repliesLastHour} > ${adjustedHourLimit} (base: ${kindLimits.maxPerHour}, multiplier: ${reputationMultiplier.toFixed(2)})`);
            return true;
        }
    }

    return false;
}

function isFastReply(event, recentEvents) {
    if (event.kind !== 1) return false;
    
    const replyTo = event.tags.find(tag => tag[0] === 'e');
    if (replyTo) {
        const originalEvent = recentEvents.find(e => e.id === replyTo[1]);
        if (originalEvent && event.created_at - originalEvent.created_at <= config.fastReplyThreshold) {
            logWithReputation(event.pubkey, 
                `Fast reply detected: ${event.created_at - originalEvent.created_at} seconds to ${originalEvent.id}`);
            return true;
        }
    }
    return false;
}

function isContentCopy(event, recentEvents) {
    // Get kind-specific limits
    const kindLimits = config.kindSpecificLimits[event.kind];
    if (kindLimits && kindLimits.ignoreSimilarity) {
        return false;
    }

    const checkResult = contentChecker.checkContent(event, recentEvents);
    if (checkResult.isSpam) {
        logWithReputation(event.pubkey, `Content spam detected for kind ${event.kind}: ${checkResult.reason}`);
        return true;
    }
    return false;
}

function isNewPubkeySpam(event) {
    const isNewPubkey = !authorStats.has(event.pubkey);
    if (isNewPubkey) {
        const replyTo = event.tags.find(tag => tag[0] === 'e');
        if (replyTo) {
            const originalEvent = recentEvents.find(e => e.id === replyTo[1]);
            if (originalEvent && event.created_at - originalEvent.created_at <= config.newPubkeyReplyThreshold) {
                logWithReputation(event.pubkey, `New pubkey replied too quickly to ${originalEvent.id}`);
                return true;
            }
        }
    }
    return false;
}

function isRapidNewPubkeyPosting(event) {
    // Skip rapid posting check for standardized kinds
    if (contentChecker.standardizedContentKinds.has(event.kind)) {
        return false;
    }

    if (authorStats.has(event.pubkey)) {
        return false;  // This is not a new pubkey
    }

    const now = Math.floor(Date.now() / 1000);
    const count = newPubkeyPostCount.get(event.pubkey) || 0;
    newPubkeyPostCount.set(event.pubkey, count + 1);
    
    if (count >= config.newPubkeyMaxPostsIn5Min && now - event.created_at <= 300) {
        logWithReputation(event.pubkey, `Rapid new pubkey posting detected: ${count + 1} posts in 5 minutes`);
        return true;
    }
    
    if (now - event.created_at > 300) {
        newPubkeyPostCount.delete(event.pubkey);
    }
    
    return false;
}

function isBotBehavior(event) {
    // Skip bot behavior check for standardized kinds
    if (contentChecker.standardizedContentKinds.has(event.kind)) {
        return false;
    }
    
    const isBotLike = event.content.endsWith(config.relayUrl);
    if (isBotLike) {
        logWithReputation(event.pubkey, `Bot behavior detected in event ${event.id}`);
    }
    return isBotLike;
}

function isSpam(event, recentEvents) {
    const pubkey = event.pubkey;
    
    logWithReputation(pubkey, `Checking event ${event.id} (kind: ${event.kind})`);

    if (config.bypassAllChecksKinds.includes(event.kind)) {
        logWithReputation(pubkey, `Bypassing all checks for kind ${event.kind}`);
        return false;
    }

    if (!canUserPost(pubkey, event.kind)) {
        return { 
            isSpam: true, 
            reason: 'User reputation too low to post' 
        };
    }

    if (isRateLimitExceeded(pubkey, event.kind)) {
        return { 
            isSpam: true, 
            reason: `Rate limit exceeded for kind ${event.kind}` 
        };
    }

    // Special handling for follow lists (kind 3)
    if (event.kind === 3) {
        const checkResult = contentChecker.checkContent(event, recentEvents);
        if (checkResult.isSpam) {
            return {
                isSpam: true,
                reason: checkResult.reason
            };
        }
        logWithReputation(pubkey, `Follow list check passed`);
        return { isSpam: false };
    }

    if (isNewPubkeySpam(event)) {
        return { 
            isSpam: true, 
            reason: 'New pubkey replying too quickly' 
        };
    }

    if (isRapidNewPubkeyPosting(event)) {
        return { 
            isSpam: true, 
            reason: 'Rapid-fire posting from new pubkey' 
        };
    }

    if (isFastReply(event, recentEvents)) {
        return { 
            isSpam: true, 
            reason: 'Fast reply to recent event' 
        };
    }
    
    if (isContentCopy(event, recentEvents)) {
        return { 
            isSpam: true, 
            reason: 'Content copy detected' 
        };
    }
    
    if (isBotBehavior(event)) {
        return { 
            isSpam: true, 
            reason: 'Bot-like behavior detected' 
        };
    }

    logWithReputation(pubkey, `Event ${event.id} passed all spam checks`);
    return { isSpam: false };
}

function periodicCleanup() {
    const now = Date.now();
    let cleanupCount = 0;
    
    // Clean up old events
    const oldEventCount = recentEvents.length;
    while (recentEvents.length > 0 && 
           (now - recentEvents[recentEvents.length - 1].created_at * 1000) > config.recentEventWindow * 1000) {
        recentEvents.pop();
        cleanupCount++;
    }
    
    // Clean up old author stats
    let statsCleanupCount = 0;
    for (const [pubkey, stats] of authorStats.entries()) {
        if (now - stats.lastReply > config.statsRetentionPeriod * 1000) {
            authorStats.delete(pubkey);
            statsCleanupCount++;
        }
    }
    
    // Clean up old reputation data
    let reputationCleanupCount = 0;
    for (const [pubkey, rep] of authorReputation.entries()) {
        if (now - rep.lastUpdate > config.statsRetentionPeriod * 1000) {
            authorReputation.delete(pubkey);
            reputationCleanupCount++;
        }
    }
    
    // Clean up newPubkeyPostCount
    let newPubkeyCleanupCount = 0;
    for (const [pubkey, timestamp] of newPubkeyPostCount.entries()) {
        if (now - timestamp > 300000) { // 5 minutes
            newPubkeyPostCount.delete(pubkey);
            newPubkeyCleanupCount++;
        }
    }

    // Clean up old violation history
    let violationCleanupCount = 0;
    for (const [pubkey, history] of userViolationHistory.entries()) {
        if (history.length === 0 || now - history[history.length - 1].timestamp > config.statsRetentionPeriod * 1000) {
            userViolationHistory.delete(pubkey);
            violationCleanupCount++;
        }
    }

    // Clean up last kind updates
    let kindUpdateCleanupCount = 0;
    for (const [key, timestamp] of lastKindUpdates.entries()) {
        if (now - timestamp > config.statsRetentionPeriod * 1000) {
            lastKindUpdates.delete(key);
            kindUpdateCleanupCount++;
        }
    }

    // Log cleanup summary if anything was cleaned
    const totalCleanup = cleanupCount + statsCleanupCount + reputationCleanupCount + 
                        newPubkeyCleanupCount + violationCleanupCount + kindUpdateCleanupCount;
    
    if (totalCleanup > 0) {
        log(`Cleanup summary:
    Events: ${cleanupCount} (${recentEvents.length} remaining)
    Author stats: ${statsCleanupCount} (${authorStats.size} remaining)
    Reputation data: ${reputationCleanupCount} (${authorReputation.size} remaining)
    New pubkey counts: ${newPubkeyCleanupCount} (${newPubkeyPostCount.size} remaining)
    Violation histories: ${violationCleanupCount} (${userViolationHistory.size} remaining)
    Kind updates: ${kindUpdateCleanupCount} (${lastKindUpdates.size} remaining)`);
    }
}

// Set up periodic cleanup
setInterval(periodicCleanup, config.cleanupInterval * 1000);

// Set up stdin/stdout interface
const rl = require('readline').createInterface({
    input: process.stdin,
    output: process.stdout,
    terminal: false
});

rl.on('line', (line) => {
    let req;
    try {
        req = JSON.parse(line);
    } catch (error) {
        log(`Error parsing JSON: ${error.message}`);
        return;
    }

    if (req.type !== 'new') {
        log("unexpected request type");
        return;
    }

    let res = { id: req.event.id };

    // Get initial reputation for comparison
    const initialReputation = getAuthorReputation(req.event.pubkey).score;

    // Update recent events
    recentEvents.push(req.event);
    recentEvents.sort((a, b) => b.created_at - a.created_at);
    
    // Update author stats
    updateAuthorStats(req.event.pubkey, req.event.created_at, req.event.kind);
    
    const spamCheck = isSpam(req.event, recentEvents);
    
    if (spamCheck.isSpam) {
        res.action = 'reject';
        res.msg = `rejected: ${spamCheck.reason}`;
        const finalReputation = updateReputation(req.event.pubkey, true, req.event.kind, spamCheck.reason);
        log(`Rejected event ${req.event.id}: ${res.msg} (reputation: ${initialReputation.toFixed(2)} -> ${finalReputation.toFixed(2)})`);
    } else {
        res.action = 'accept';
        const finalReputation = updateReputation(req.event.pubkey, false, req.event.kind, null);
        log(`Accepted event ${req.event.id} (reputation: ${initialReputation.toFixed(2)} -> ${finalReputation.toFixed(2)})`);
    }

    console.log(JSON.stringify(res));
});

log("Strfry filter started with config: " + JSON.stringify(config, null, 2));