update main spam checker for kind awarenes
Needed to make things more robust and take individual kinds into account
This commit is contained in:
parent
0908385008
commit
084aab7847
@ -1,6 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const KindAwareContentChecker = require(path.join(__dirname, 'kind-aware-content-checker.js'));
|
||||
const contentChecker = new KindAwareContentChecker();
|
||||
|
||||
// Configuration
|
||||
const config = {
|
||||
@ -10,31 +13,64 @@ const config = {
|
||||
recentEventWindow: 3600, // 1 hour in seconds
|
||||
contentSimilarityThreshold: 0.8,
|
||||
relayUrl: "freelay.sovbit.host",
|
||||
useBlacklistWhitelist: false,
|
||||
|
||||
// Periodic cleanup intervals
|
||||
cleanupInterval: 300, // 5 minutes
|
||||
statsRetentionPeriod: 86400, // 24 hours
|
||||
globalMaxEventsPerMinute: 50,
|
||||
globalMaxEventsPerHour: 600,
|
||||
|
||||
// Enhanced reputation system
|
||||
newPubkeyReplyThreshold: 60,
|
||||
newPubkeyMaxPostsIn5Min: 10,
|
||||
|
||||
// Kind-specific limits with new burst handling
|
||||
kindSpecificLimits: {
|
||||
0: {
|
||||
maxPerHour: 10, // Profile updates
|
||||
minTimeBetween: 300 // 5 minutes between updates
|
||||
},
|
||||
1: { // Regular posts
|
||||
maxPerMinute: 30,
|
||||
maxPerHour: 200
|
||||
},
|
||||
3: { // Follow lists
|
||||
maxPerMinute: 120, // Allow more frequent updates for bulk operations
|
||||
maxPerHour: 360, // One update every 10 seconds average
|
||||
minTimeBetween: 2, // Reduced from 30 to 2 seconds
|
||||
ignoreSimilarity: true, // Don't check content similarity
|
||||
burstAllowance: { // New burst handling
|
||||
maxBurst: 500, // Maximum events in burst
|
||||
burstWindow: 300, // Window for burst (5 minutes)
|
||||
cooldownPeriod: 3600 // Cooldown after burst (1 hour)
|
||||
}
|
||||
},
|
||||
7: { // Reactions
|
||||
maxPerMinute: 60,
|
||||
maxPerHour: 300,
|
||||
ignoreSimilarity: true
|
||||
},
|
||||
10002: {
|
||||
maxPerHour: 12, // Relay List updates
|
||||
minTimeBetween: 300 // 5 minutes between updates
|
||||
}
|
||||
},
|
||||
|
||||
// Enhanced reputation system config
|
||||
reputationConfig: {
|
||||
initialScore: 100,
|
||||
goodEventBonus: 1,
|
||||
spamPenalty: -15, // Base penalty
|
||||
recoveryRate: 3, // Base recovery rate
|
||||
minScore: -100, // Absolute minimum
|
||||
spamPenalty: -15,
|
||||
recoveryRate: 3,
|
||||
minScore: -100,
|
||||
maxScore: 1000,
|
||||
trustThreshold: 50,
|
||||
blockThreshold: -50, // Start blocking at this point
|
||||
blockRecoveryThreshold: -25, // Must recover to this to post again
|
||||
blockThreshold: -50,
|
||||
blockRecoveryThreshold: -25,
|
||||
|
||||
// Progressive penalties
|
||||
penaltyScaling: {
|
||||
0: -15, // Normal penalty above 0 reputation
|
||||
"-25": -20, // Harsher penalty below -25
|
||||
"-50": -25 // Even harsher below -50
|
||||
},
|
||||
|
||||
// Recovery scaling
|
||||
recoveryScaling: {
|
||||
0: 3, // Normal recovery above 0
|
||||
"-25": 2, // Slower recovery below -25
|
||||
@ -42,56 +78,51 @@ const config = {
|
||||
}
|
||||
},
|
||||
|
||||
// Kind-specific limits
|
||||
kindSpecificLimits: {
|
||||
0: { // Profile updates
|
||||
maxPerHour: 10
|
||||
},
|
||||
3: { // Contacts
|
||||
maxPerHour: 5
|
||||
},
|
||||
1: { // Regular posts
|
||||
maxPerMinute: 30,
|
||||
maxPerHour: 200
|
||||
}
|
||||
},
|
||||
|
||||
// New user limits
|
||||
newPubkeyReplyThreshold: 60,
|
||||
newPubkeyMaxPostsIn5Min: 10,
|
||||
|
||||
allowedKinds: [3, 5, 10001, 10002, 30311, 10050],
|
||||
bypassAllChecksKinds: [38383],
|
||||
// Cleanup settings
|
||||
cleanupInterval: 300, // 5 minutes
|
||||
statsRetentionPeriod: 86400, // 24 hours
|
||||
|
||||
allowedKinds: [3, 5, 10001, 10002, 30311], // Kinds that bypass content similarity check
|
||||
bypassAllChecksKinds: [38383]
|
||||
};
|
||||
|
||||
// In-memory stores with cleanup
|
||||
// In-memory stores
|
||||
const recentEvents = [];
|
||||
const authorStats = new Map();
|
||||
const authorReputation = new Map();
|
||||
const newPubkeyPostCount = new Map();
|
||||
const userViolationHistory = new Map(); // Track violation history
|
||||
const userViolationHistory = new Map();
|
||||
const lastKindUpdates = new Map(); // Track last update time per kind per user
|
||||
|
||||
function log(message) {
|
||||
console.error(`[${new Date().toISOString()}] ${message}`);
|
||||
}
|
||||
|
||||
function logWithReputation(pubkey, message) {
|
||||
const reputation = getAuthorReputation(pubkey).score;
|
||||
log(`${message} (reputation: ${reputation.toFixed(2)})`);
|
||||
}
|
||||
|
||||
function recordViolation(pubkey, eventKind, reason, penalty) {
|
||||
if (!userViolationHistory.has(pubkey)) {
|
||||
userViolationHistory.set(pubkey, []);
|
||||
}
|
||||
|
||||
const history = userViolationHistory.get(pubkey);
|
||||
const currentReputation = getAuthorReputation(pubkey).score;
|
||||
|
||||
history.push({
|
||||
timestamp: Date.now(),
|
||||
eventKind: eventKind,
|
||||
reason: reason,
|
||||
penalty: penalty,
|
||||
currentReputation: getAuthorReputation(pubkey).score
|
||||
currentReputation: currentReputation
|
||||
});
|
||||
|
||||
// Keep only last 50 violations
|
||||
logWithReputation(pubkey, `Violation recorded for kind ${eventKind}: ${reason} (penalty: ${penalty})`);
|
||||
|
||||
if (history.length > 50) {
|
||||
history.shift();
|
||||
history.shift(); // Keep only last 50 violations
|
||||
}
|
||||
}
|
||||
|
||||
@ -101,15 +132,16 @@ function getAuthorReputation(pubkey) {
|
||||
score: config.reputationConfig.initialScore,
|
||||
lastUpdate: Date.now()
|
||||
});
|
||||
log(`New reputation created for ${pubkey} with initial score ${config.reputationConfig.initialScore}`);
|
||||
}
|
||||
return authorReputation.get(pubkey);
|
||||
}
|
||||
|
||||
function updateReputation(pubkey, isSpam, eventKind, spamReason) {
|
||||
const reputation = getAuthorReputation(pubkey);
|
||||
const oldScore = reputation.score;
|
||||
const hoursSinceLastUpdate = (Date.now() - reputation.lastUpdate) / (1000 * 60 * 60);
|
||||
|
||||
// Get appropriate recovery rate based on current score
|
||||
let recoveryRate = config.reputationConfig.recoveryRate;
|
||||
for (const [threshold, rate] of Object.entries(config.reputationConfig.recoveryScaling)
|
||||
.sort((a, b) => Number(b[0]) - Number(a[0]))) {
|
||||
@ -119,15 +151,18 @@ function updateReputation(pubkey, isSpam, eventKind, spamReason) {
|
||||
}
|
||||
}
|
||||
|
||||
// Apply scaled recovery
|
||||
reputation.score = Math.min(
|
||||
config.reputationConfig.maxScore,
|
||||
reputation.score + (hoursSinceLastUpdate * recoveryRate)
|
||||
);
|
||||
|
||||
if (isSpam) {
|
||||
// Get appropriate penalty based on current score
|
||||
let penalty = config.reputationConfig.spamPenalty;
|
||||
// Reduce penalty for follow list violations
|
||||
if (eventKind === 3) {
|
||||
penalty = penalty / 2;
|
||||
}
|
||||
|
||||
for (const [threshold, pen] of Object.entries(config.reputationConfig.penaltyScaling)
|
||||
.sort((a, b) => Number(b[0]) - Number(a[0]))) {
|
||||
if (reputation.score <= Number(threshold)) {
|
||||
@ -136,8 +171,6 @@ function updateReputation(pubkey, isSpam, eventKind, spamReason) {
|
||||
}
|
||||
}
|
||||
reputation.score += penalty;
|
||||
|
||||
// Record the violation
|
||||
recordViolation(pubkey, eventKind, spamReason, penalty);
|
||||
} else {
|
||||
reputation.score += config.reputationConfig.goodEventBonus;
|
||||
@ -149,22 +182,34 @@ function updateReputation(pubkey, isSpam, eventKind, spamReason) {
|
||||
);
|
||||
|
||||
reputation.lastUpdate = Date.now();
|
||||
|
||||
// Log reputation changes
|
||||
const change = reputation.score - oldScore;
|
||||
if (change !== 0) {
|
||||
log(`Reputation update for ${pubkey}: ${oldScore.toFixed(2)} -> ${reputation.score.toFixed(2)} (change: ${change >= 0 ? '+' : ''}${change.toFixed(2)})`);
|
||||
}
|
||||
|
||||
return reputation.score;
|
||||
}
|
||||
|
||||
function canUserPost(pubkey, eventKind) {
|
||||
const reputation = getAuthorReputation(pubkey);
|
||||
|
||||
// Special case for follow lists - be more lenient
|
||||
if (eventKind === 3 && reputation.score > -75) {
|
||||
logWithReputation(pubkey, `Follow list exception applied for kind 3`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (reputation.score <= config.reputationConfig.blockThreshold) {
|
||||
if (reputation.score < config.reputationConfig.blockRecoveryThreshold) {
|
||||
// Get violation history
|
||||
const history = userViolationHistory.get(pubkey) || [];
|
||||
const recentViolations = history
|
||||
.slice(-5) // Get last 5 violations
|
||||
.slice(-5)
|
||||
.map(v => `\n - ${new Date(v.timestamp).toISOString()}: kind ${v.eventKind}, ${v.reason} (penalty: ${v.penalty})`)
|
||||
.join('');
|
||||
|
||||
log(`User ${pubkey} blocked from posting: reputation (${reputation.score.toFixed(2)}) below recovery threshold
|
||||
log(`User ${pubkey} blocked: reputation (${reputation.score.toFixed(2)}) below recovery threshold
|
||||
Recent violations:${recentViolations || '\n No recent violations recorded'}`);
|
||||
return false;
|
||||
}
|
||||
@ -172,44 +217,47 @@ Recent violations:${recentViolations || '\n No recent violations recorded'}`)
|
||||
return true;
|
||||
}
|
||||
|
||||
function getKindSpecificLimits(kind) {
|
||||
return config.kindSpecificLimits[kind] || {
|
||||
maxPerMinute: config.maxRepliesPerMinute,
|
||||
maxPerHour: config.maxRepliesPerHour
|
||||
};
|
||||
}
|
||||
|
||||
function updateAuthorStats(pubkey, timestamp, kind) {
|
||||
if (config.bypassAllChecksKinds.includes(kind)) return;
|
||||
|
||||
const now = Date.now();
|
||||
if (!authorStats.has(pubkey)) {
|
||||
authorStats.set(pubkey, {
|
||||
lastReply: now,
|
||||
lastReply: Date.now(),
|
||||
repliesLastMinute: 0,
|
||||
repliesLastHour: 0,
|
||||
lastReset: now,
|
||||
kinds: new Map()
|
||||
lastReset: Date.now(),
|
||||
kinds: new Map(),
|
||||
burstTracking: new Map()
|
||||
});
|
||||
logWithReputation(pubkey, `New author stats created`);
|
||||
}
|
||||
|
||||
const stats = authorStats.get(pubkey);
|
||||
const timeSinceLastReset = now - stats.lastReset;
|
||||
|
||||
// Reset counters if needed
|
||||
if (timeSinceLastReset >= 3600000) { // 1 hour
|
||||
const now = Date.now();
|
||||
|
||||
// Initialize burst tracking for kind 3
|
||||
if (kind === 3 && !stats.burstTracking.has(kind)) {
|
||||
stats.burstTracking.set(kind, {
|
||||
burstCount: 0,
|
||||
burstStart: now,
|
||||
lastBurstEnd: 0
|
||||
});
|
||||
logWithReputation(pubkey, `Initialized burst tracking for kind 3`);
|
||||
}
|
||||
|
||||
if (now - stats.lastReset >= 3600000) {
|
||||
stats.repliesLastHour = 0;
|
||||
stats.repliesLastMinute = 0;
|
||||
stats.lastReset = now;
|
||||
stats.kinds.clear();
|
||||
} else if (timeSinceLastReset >= 60000) { // 1 minute
|
||||
logWithReputation(pubkey, `Stats reset (hourly)`);
|
||||
} else if (now - stats.lastReset >= 60000) {
|
||||
stats.repliesLastMinute = 0;
|
||||
logWithReputation(pubkey, `Stats reset (minute)`);
|
||||
}
|
||||
|
||||
// Update kind-specific counters
|
||||
if (!stats.kinds.has(kind)) {
|
||||
stats.kinds.set(kind, { hourly: 0, lastReset: now });
|
||||
}
|
||||
|
||||
const kindStats = stats.kinds.get(kind);
|
||||
if (now - kindStats.lastReset >= 3600000) {
|
||||
kindStats.hourly = 0;
|
||||
@ -220,105 +268,122 @@ function updateAuthorStats(pubkey, timestamp, kind) {
|
||||
stats.repliesLastMinute++;
|
||||
stats.repliesLastHour++;
|
||||
stats.lastReply = now;
|
||||
|
||||
logWithReputation(pubkey,
|
||||
`Stats updated - minute: ${stats.repliesLastMinute}, hour: ${stats.repliesLastHour}, kind ${kind}: ${kindStats.hourly}`);
|
||||
}
|
||||
|
||||
function isRateLimitExceeded(pubkey, kind) {
|
||||
const stats = authorStats.get(pubkey);
|
||||
if (!stats) return false;
|
||||
|
||||
const kindLimits = getKindSpecificLimits(kind);
|
||||
const kindStats = stats.kinds.get(kind);
|
||||
const now = Date.now();
|
||||
const kindLimits = config.kindSpecificLimits[kind] || {
|
||||
maxPerMinute: config.maxRepliesPerMinute,
|
||||
maxPerHour: config.maxRepliesPerHour
|
||||
};
|
||||
|
||||
// Consider reputation in rate limiting
|
||||
const reputation = getAuthorReputation(pubkey);
|
||||
const reputationMultiplier = Math.max(0.5, Math.min(2, reputation.score / 100));
|
||||
|
||||
const adjustedMinuteLimit = kindLimits.maxPerMinute * reputationMultiplier;
|
||||
const adjustedHourLimit = kindLimits.maxPerHour * reputationMultiplier;
|
||||
// Special handling for follow list bursts
|
||||
if (kind === 3 && kindLimits.burstAllowance) {
|
||||
const burstTracking = stats.burstTracking.get(kind);
|
||||
if (burstTracking) {
|
||||
const { burstCount, burstStart, lastBurstEnd } = burstTracking;
|
||||
const { maxBurst, burstWindow, cooldownPeriod } = kindLimits.burstAllowance;
|
||||
|
||||
const exceeded =
|
||||
stats.repliesLastMinute > adjustedMinuteLimit ||
|
||||
stats.repliesLastHour > adjustedHourLimit ||
|
||||
(kindStats && kindStats.hourly > kindLimits.maxPerHour);
|
||||
|
||||
if (exceeded) {
|
||||
log(`Rate limit exceeded for ${pubkey} (reputation: ${reputation.score.toFixed(2)}, kind: ${kind})`);
|
||||
// Check if we're in cooldown period
|
||||
if (lastBurstEnd > 0 && now - lastBurstEnd < cooldownPeriod * 1000) {
|
||||
logWithReputation(pubkey, `In burst cooldown period for kind 3`);
|
||||
return isNormalRateLimitExceeded(stats, kindLimits);
|
||||
}
|
||||
// Check if we're in an active burst
|
||||
else if (now - burstStart < burstWindow * 1000) {
|
||||
if (burstCount >= maxBurst) {
|
||||
burstTracking.lastBurstEnd = now;
|
||||
burstTracking.burstCount = 0;
|
||||
logWithReputation(pubkey, `Burst limit reached (${maxBurst} events), entering cooldown`);
|
||||
return true;
|
||||
}
|
||||
burstTracking.burstCount++;
|
||||
logWithReputation(pubkey, `In active burst - count: ${burstCount + 1}`);
|
||||
return false;
|
||||
}
|
||||
// Start new burst
|
||||
else {
|
||||
burstTracking.burstStart = now;
|
||||
burstTracking.burstCount = 1;
|
||||
logWithReputation(pubkey, `Starting new burst for kind 3`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return exceeded;
|
||||
// Check minimum time between updates if specified
|
||||
if (kindLimits.minTimeBetween) {
|
||||
const lastUpdate = lastKindUpdates.get(`${pubkey}:${kind}`) || 0;
|
||||
if (now - lastUpdate < kindLimits.minTimeBetween * 1000) {
|
||||
logWithReputation(pubkey, `Too frequent updates for kind ${kind}`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return isNormalRateLimitExceeded(stats, kindLimits);
|
||||
}
|
||||
|
||||
function isNormalRateLimitExceeded(stats, kindLimits) {
|
||||
const reputation = getAuthorReputation(stats.pubkey);
|
||||
const reputationMultiplier = Math.max(0.5, Math.min(2, reputation.score / 100));
|
||||
|
||||
if (kindLimits.maxPerMinute) {
|
||||
const adjustedMinuteLimit = kindLimits.maxPerMinute * reputationMultiplier;
|
||||
if (stats.repliesLastMinute > adjustedMinuteLimit) {
|
||||
logWithReputation(stats.pubkey,
|
||||
`Per-minute rate limit exceeded: ${stats.repliesLastMinute} > ${adjustedMinuteLimit} (base: ${kindLimits.maxPerMinute}, multiplier: ${reputationMultiplier.toFixed(2)})`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (kindLimits.maxPerHour) {
|
||||
const adjustedHourLimit = kindLimits.maxPerHour * reputationMultiplier;
|
||||
if (stats.repliesLastHour > adjustedHourLimit) {
|
||||
logWithReputation(stats.pubkey,
|
||||
`Hourly rate limit exceeded: ${stats.repliesLastHour} > ${adjustedHourLimit} (base: ${kindLimits.maxPerHour}, multiplier: ${reputationMultiplier.toFixed(2)})`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function isFastReply(event, recentEvents) {
|
||||
if (event.kind !== 1) return false;
|
||||
|
||||
const replyTo = event.tags.find(tag => tag[0] === 'e');
|
||||
if (replyTo) {
|
||||
const originalEvent = recentEvents.find(e => e.id === replyTo[1]);
|
||||
if (originalEvent && event.created_at - originalEvent.created_at <= config.fastReplyThreshold) {
|
||||
log(`Fast reply detected: ${event.created_at - originalEvent.created_at} seconds to ${originalEvent.id}`);
|
||||
logWithReputation(event.pubkey,
|
||||
`Fast reply detected: ${event.created_at - originalEvent.created_at} seconds to ${originalEvent.id}`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function calculateSimilarity(str1, str2) {
|
||||
const len1 = str1.length;
|
||||
const len2 = str2.length;
|
||||
const maxLength = Math.max(len1, len2);
|
||||
if (maxLength === 0) return 1.0;
|
||||
return (maxLength - levenshteinDistance(str1, str2)) / maxLength;
|
||||
}
|
||||
|
||||
function levenshteinDistance(str1, str2) {
|
||||
const m = str1.length;
|
||||
const n = str2.length;
|
||||
const dp = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0));
|
||||
|
||||
for (let i = 0; i <= m; i++) dp[i][0] = i;
|
||||
for (let j = 0; j <= n; j++) dp[0][j] = j;
|
||||
|
||||
for (let i = 1; i <= m; i++) {
|
||||
for (let j = 1; j <= n; j++) {
|
||||
if (str1[i - 1] === str2[j - 1]) {
|
||||
dp[i][j] = dp[i - 1][j - 1];
|
||||
} else {
|
||||
dp[i][j] = Math.min(
|
||||
dp[i - 1][j - 1],
|
||||
dp[i - 1][j],
|
||||
dp[i][j - 1]
|
||||
) + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return dp[m][n];
|
||||
}
|
||||
|
||||
function isContentCopy(event, recentEvents) {
|
||||
if (config.allowedKinds.includes(event.kind)) {
|
||||
log(`Skipping content copy check for allowed kind ${event.kind}`);
|
||||
// Get kind-specific limits
|
||||
const kindLimits = config.kindSpecificLimits[event.kind];
|
||||
if (kindLimits && kindLimits.ignoreSimilarity) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const recentEvent of recentEvents) {
|
||||
if (recentEvent.pubkey !== event.pubkey) {
|
||||
const similarity = calculateSimilarity(recentEvent.content, event.content);
|
||||
if (similarity >= config.contentSimilarityThreshold) {
|
||||
log(`Content copy detected: ${similarity.toFixed(2)} similarity between ${event.id} and ${recentEvent.id}`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
const checkResult = contentChecker.checkContent(event, recentEvents);
|
||||
if (checkResult.isSpam) {
|
||||
logWithReputation(event.pubkey, `Content spam detected for kind ${event.kind}: ${checkResult.reason}`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isBotBehavior(event) {
|
||||
const isBotLike = event.content.endsWith(config.relayUrl);
|
||||
if (isBotLike) {
|
||||
log(`Bot behavior detected in event ${event.id}: content ends with relay URL`);
|
||||
}
|
||||
return isBotLike;
|
||||
}
|
||||
|
||||
function isNewPubkeySpam(event) {
|
||||
const isNewPubkey = !authorStats.has(event.pubkey);
|
||||
if (isNewPubkey) {
|
||||
@ -326,7 +391,7 @@ function isNewPubkeySpam(event) {
|
||||
if (replyTo) {
|
||||
const originalEvent = recentEvents.find(e => e.id === replyTo[1]);
|
||||
if (originalEvent && event.created_at - originalEvent.created_at <= config.newPubkeyReplyThreshold) {
|
||||
log(`New pubkey ${event.pubkey} replied too quickly to ${originalEvent.id}`);
|
||||
logWithReputation(event.pubkey, `New pubkey replied too quickly to ${originalEvent.id}`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -335,16 +400,21 @@ function isNewPubkeySpam(event) {
|
||||
}
|
||||
|
||||
function isRapidNewPubkeyPosting(event) {
|
||||
if (authorStats.has(event.pubkey)) {
|
||||
// Skip rapid posting check for standardized kinds
|
||||
if (contentChecker.standardizedContentKinds.has(event.kind)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (authorStats.has(event.pubkey)) {
|
||||
return false; // This is not a new pubkey
|
||||
}
|
||||
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const count = newPubkeyPostCount.get(event.pubkey) || 0;
|
||||
newPubkeyPostCount.set(event.pubkey, count + 1);
|
||||
|
||||
if (count >= config.newPubkeyMaxPostsIn5Min && now - event.created_at <= 300) {
|
||||
log(`Rapid new pubkey posting detected for ${event.pubkey}: ${count} posts in 5 minutes`);
|
||||
logWithReputation(event.pubkey, `Rapid new pubkey posting detected: ${count + 1} posts in 5 minutes`);
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -355,91 +425,177 @@ function isRapidNewPubkeyPosting(event) {
|
||||
return false;
|
||||
}
|
||||
|
||||
function isBotBehavior(event) {
|
||||
// Skip bot behavior check for standardized kinds
|
||||
if (contentChecker.standardizedContentKinds.has(event.kind)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const isBotLike = event.content.endsWith(config.relayUrl);
|
||||
if (isBotLike) {
|
||||
logWithReputation(event.pubkey, `Bot behavior detected in event ${event.id}`);
|
||||
}
|
||||
return isBotLike;
|
||||
}
|
||||
|
||||
function isSpam(event, recentEvents) {
|
||||
const pubkey = event.pubkey;
|
||||
|
||||
log(`Checking event ${event.id} from pubkey ${pubkey} (kind: ${event.kind})`);
|
||||
logWithReputation(pubkey, `Checking event ${event.id} (kind: ${event.kind})`);
|
||||
|
||||
if (config.bypassAllChecksKinds.includes(event.kind)) {
|
||||
log(`Bypassing all checks for kind ${event.kind}`);
|
||||
return { isSpam: false };
|
||||
logWithReputation(pubkey, `Bypassing all checks for kind ${event.kind}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!canUserPost(pubkey, event.kind)) {
|
||||
return {
|
||||
isSpam: true,
|
||||
reason: 'User reputation too low to post'
|
||||
};
|
||||
}
|
||||
|
||||
if (isRateLimitExceeded(pubkey, event.kind)) {
|
||||
return { isSpam: true, reason: `Rate limit exceeded for kind ${event.kind}` };
|
||||
return {
|
||||
isSpam: true,
|
||||
reason: `Rate limit exceeded for kind ${event.kind}`
|
||||
};
|
||||
}
|
||||
|
||||
// Special handling for follow lists (kind 3)
|
||||
if (event.kind === 3) {
|
||||
const checkResult = contentChecker.checkContent(event, recentEvents);
|
||||
if (checkResult.isSpam) {
|
||||
return {
|
||||
isSpam: true,
|
||||
reason: checkResult.reason
|
||||
};
|
||||
}
|
||||
logWithReputation(pubkey, `Follow list check passed`);
|
||||
return { isSpam: false };
|
||||
}
|
||||
|
||||
if (isNewPubkeySpam(event)) {
|
||||
return { isSpam: true, reason: 'New pubkey replying too quickly' };
|
||||
return {
|
||||
isSpam: true,
|
||||
reason: 'New pubkey replying too quickly'
|
||||
};
|
||||
}
|
||||
|
||||
if (isRapidNewPubkeyPosting(event)) {
|
||||
return { isSpam: true, reason: 'Rapid-fire posting from new pubkey' };
|
||||
return {
|
||||
isSpam: true,
|
||||
reason: 'Rapid-fire posting from new pubkey'
|
||||
};
|
||||
}
|
||||
|
||||
if (isFastReply(event, recentEvents)) {
|
||||
return { isSpam: true, reason: 'Fast reply to recent event' };
|
||||
return {
|
||||
isSpam: true,
|
||||
reason: 'Fast reply to recent event'
|
||||
};
|
||||
}
|
||||
|
||||
if (isContentCopy(event, recentEvents)) {
|
||||
return { isSpam: true, reason: 'Content copy detected' };
|
||||
return {
|
||||
isSpam: true,
|
||||
reason: 'Content copy detected'
|
||||
};
|
||||
}
|
||||
|
||||
if (isBotBehavior(event)) {
|
||||
return { isSpam: true, reason: 'Bot-like behavior detected' };
|
||||
return {
|
||||
isSpam: true,
|
||||
reason: 'Bot-like behavior detected'
|
||||
};
|
||||
}
|
||||
|
||||
log(`Event ${event.id} passed all spam checks`);
|
||||
logWithReputation(pubkey, `Event ${event.id} passed all spam checks`);
|
||||
return { isSpam: false };
|
||||
}
|
||||
|
||||
function periodicCleanup() {
|
||||
const now = Date.now();
|
||||
let cleanupCount = 0;
|
||||
|
||||
// Clean up old events
|
||||
const oldEventCount = recentEvents.length;
|
||||
while (recentEvents.length > 0 &&
|
||||
(now - recentEvents[recentEvents.length - 1].created_at * 1000) > config.recentEventWindow * 1000) {
|
||||
recentEvents.pop();
|
||||
cleanupCount++;
|
||||
}
|
||||
|
||||
// Clean up old author stats
|
||||
let statsCleanupCount = 0;
|
||||
for (const [pubkey, stats] of authorStats.entries()) {
|
||||
if (now - stats.lastReply > config.statsRetentionPeriod * 1000) {
|
||||
authorStats.delete(pubkey);
|
||||
statsCleanupCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up old reputation data
|
||||
let reputationCleanupCount = 0;
|
||||
for (const [pubkey, rep] of authorReputation.entries()) {
|
||||
if (now - rep.lastUpdate > config.statsRetentionPeriod * 1000) {
|
||||
authorReputation.delete(pubkey);
|
||||
reputationCleanupCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up newPubkeyPostCount
|
||||
let newPubkeyCleanupCount = 0;
|
||||
for (const [pubkey, timestamp] of newPubkeyPostCount.entries()) {
|
||||
if (now - timestamp > 300000) { // 5 minutes
|
||||
newPubkeyPostCount.delete(pubkey);
|
||||
newPubkeyCleanupCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up old violation history
|
||||
let violationCleanupCount = 0;
|
||||
for (const [pubkey, history] of userViolationHistory.entries()) {
|
||||
if (history.length === 0 || now - history[history.length - 1].timestamp > config.statsRetentionPeriod * 1000) {
|
||||
userViolationHistory.delete(pubkey);
|
||||
violationCleanupCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up last kind updates
|
||||
let kindUpdateCleanupCount = 0;
|
||||
for (const [key, timestamp] of lastKindUpdates.entries()) {
|
||||
if (now - timestamp > config.statsRetentionPeriod * 1000) {
|
||||
lastKindUpdates.delete(key);
|
||||
kindUpdateCleanupCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Log cleanup summary if anything was cleaned
|
||||
const totalCleanup = cleanupCount + statsCleanupCount + reputationCleanupCount +
|
||||
newPubkeyCleanupCount + violationCleanupCount + kindUpdateCleanupCount;
|
||||
|
||||
if (totalCleanup > 0) {
|
||||
log(`Cleanup summary:
|
||||
Events: ${cleanupCount} (${recentEvents.length} remaining)
|
||||
Author stats: ${statsCleanupCount} (${authorStats.size} remaining)
|
||||
Reputation data: ${reputationCleanupCount} (${authorReputation.size} remaining)
|
||||
New pubkey counts: ${newPubkeyCleanupCount} (${newPubkeyPostCount.size} remaining)
|
||||
Violation histories: ${violationCleanupCount} (${userViolationHistory.size} remaining)
|
||||
Kind updates: ${kindUpdateCleanupCount} (${lastKindUpdates.size} remaining)`);
|
||||
}
|
||||
}
|
||||
|
||||
// Set up periodic cleanup
|
||||
setInterval(periodicCleanup, config.cleanupInterval * 1000);
|
||||
|
||||
// Set up stdin/stdout interface
|
||||
const rl = require('readline').createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
terminal: false
|
||||
});
|
||||
|
||||
// Set up periodic cleanup
|
||||
setInterval(periodicCleanup, config.cleanupInterval * 1000);
|
||||
|
||||
rl.on('line', (line) => {
|
||||
let req;
|
||||
try {
|
||||
@ -456,37 +612,27 @@ rl.on('line', (line) => {
|
||||
|
||||
let res = { id: req.event.id };
|
||||
|
||||
// Check if user is allowed to post before any other processing
|
||||
if (!canUserPost(req.event.pubkey, req.event.kind)) {
|
||||
res.action = 'reject';
|
||||
res.msg = 'rejected: reputation too low to post';
|
||||
log(`Rejected event ${req.event.id} (kind: ${req.event.kind}): reputation too low`);
|
||||
console.log(JSON.stringify(res));
|
||||
return;
|
||||
}
|
||||
// Get initial reputation for comparison
|
||||
const initialReputation = getAuthorReputation(req.event.pubkey).score;
|
||||
|
||||
// Update recent events
|
||||
recentEvents.push(req.event);
|
||||
recentEvents.sort((a, b) => b.created_at - a.created_at);
|
||||
|
||||
// Process event
|
||||
// Update author stats
|
||||
updateAuthorStats(req.event.pubkey, req.event.created_at, req.event.kind);
|
||||
|
||||
const spamCheck = isSpam(req.event, recentEvents);
|
||||
const reputationScore = updateReputation(
|
||||
req.event.pubkey,
|
||||
spamCheck.isSpam,
|
||||
req.event.kind,
|
||||
spamCheck.reason
|
||||
);
|
||||
|
||||
|
||||
if (spamCheck.isSpam) {
|
||||
res.action = 'reject';
|
||||
res.msg = `rejected: ${spamCheck.reason}`;
|
||||
log(`Rejected event ${req.event.id} (kind: ${req.event.kind}): ${res.msg} (reputation: ${reputationScore.toFixed(2)})`);
|
||||
const finalReputation = updateReputation(req.event.pubkey, true, req.event.kind, spamCheck.reason);
|
||||
log(`Rejected event ${req.event.id}: ${res.msg} (reputation: ${initialReputation.toFixed(2)} -> ${finalReputation.toFixed(2)})`);
|
||||
} else {
|
||||
res.action = 'accept';
|
||||
log(`Accepted event ${req.event.id} (kind: ${req.event.kind}, reputation: ${reputationScore.toFixed(2)})`);
|
||||
const finalReputation = updateReputation(req.event.pubkey, false, req.event.kind, null);
|
||||
log(`Accepted event ${req.event.id} (reputation: ${initialReputation.toFixed(2)} -> ${finalReputation.toFixed(2)})`);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(res));
|
||||
|
||||
Loading…
Reference in New Issue
Block a user